mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
Merge remote-tracking branch 'origin/main' into litellm_least_busy_burst_reservation
# Conflicts: # tests/unit/decisions/test_main.py
This commit is contained in:
commit
ce90f4c10f
76 changed files with 3916 additions and 439 deletions
|
|
@ -177,6 +177,8 @@ MID_STREAM_FAILURE_LOG_WAIT_SECONDS: Final = 1.0
|
|||
# Set to 0 to disable truncation.
|
||||
MAX_BASE64_LENGTH_FOR_LOGGING: Final = get_env_int("MAX_BASE64_LENGTH_FOR_LOGGING", 64)
|
||||
BASE64_TRUNCATION_OFFLOAD_THRESHOLD_CHARS: Final = 256 * 1024
|
||||
BASE64_TRUNCATION_OFFLOAD_THRESHOLD_STRINGS: Final = 256
|
||||
BASE64_TRUNCATION_INLINE_COPY_MAX_CHARS: Final = 8 * 1024 * 1024
|
||||
REDACTED_BY_LITELLM: Final = "redacted-by-litellm"
|
||||
# in-memory stand-in handed to provider converters for redacted arguments; never stored
|
||||
REDACTED_TOOL_CALL_ARGUMENTS_PLACEHOLDER: Final = "{}"
|
||||
|
|
@ -634,6 +636,8 @@ STS_CREDENTIAL_EXPIRY_SAFETY_MARGIN_SECONDS: Final = 60
|
|||
BEDROCK_MIN_THINKING_BUDGET_TOKENS: Final = get_env_int("BEDROCK_MIN_THINKING_BUDGET_TOKENS", 1024)
|
||||
# Anthropic's Messages API rejects thinking.budget_tokens < 1024.
|
||||
ANTHROPIC_MIN_THINKING_BUDGET_TOKENS: Final = 1024
|
||||
ANTHROPIC_MAX_USER_ID_LENGTH: Final = 512
|
||||
ANTHROPIC_COMPATIBLE_MAX_USER_ID_LENGTH: Final = 255
|
||||
ANTHROPIC_BILLING_METADATA_PREFIX: Final = "x-anthropic-billing-header:"
|
||||
REPLICATE_POLLING_DELAY_SECONDS: Final = get_env_float("REPLICATE_POLLING_DELAY_SECONDS", 0.5)
|
||||
DEFAULT_ANTHROPIC_CHAT_MAX_TOKENS: Final = get_env_int("DEFAULT_ANTHROPIC_CHAT_MAX_TOKENS", 128000)
|
||||
|
|
@ -1163,6 +1167,7 @@ openai_compatible_providers: Final[list[str]] = [
|
|||
"cortecs",
|
||||
"scx-ai",
|
||||
"prism",
|
||||
"hostinger_router",
|
||||
"sail",
|
||||
"reka",
|
||||
"streamlake",
|
||||
|
|
|
|||
|
|
@ -3,10 +3,10 @@ from collections.abc import Mapping, Sequence
|
|||
from dataclasses import dataclass, field, replace
|
||||
from functools import partial
|
||||
from types import MappingProxyType
|
||||
from typing import Final, Literal, TypeAlias
|
||||
from typing import Annotated, Final, Literal, TypeAlias
|
||||
|
||||
import httpx
|
||||
from pydantic import BaseModel, ConfigDict, StrictBool, TypeAdapter, ValidationError
|
||||
from pydantic import BaseModel, BeforeValidator, ConfigDict, TypeAdapter, ValidationError
|
||||
from pydantic_core import to_jsonable_python
|
||||
from typing_extensions import assert_never
|
||||
|
||||
|
|
@ -23,6 +23,7 @@ from litellm.llms.base_llm.chat.transformation import BaseLLMException
|
|||
from litellm.llms.base_llm.decisions.transformation import (
|
||||
BaseDecisionsConfig,
|
||||
DecisionsResponseValidationError,
|
||||
catalog_mode,
|
||||
image_media_type,
|
||||
image_param,
|
||||
input_images,
|
||||
|
|
@ -61,7 +62,7 @@ from litellm.types.decisions import (
|
|||
UnsupportedDecisionsRequest,
|
||||
)
|
||||
from litellm.types.llms.openai import ChatCompletionUserMessage
|
||||
from litellm.types.utils import LlmProviders, ProviderSpecificHeader
|
||||
from litellm.types.utils import LlmProviders, ProviderSpecificHeader, is_decisions_model_mode
|
||||
from litellm.utils import ProviderConfigManager, get_model_cost_key
|
||||
|
||||
SystemOneQuestions: TypeAlias = Mapping[str, DecisionQuestion | Mapping[str, object]]
|
||||
|
|
@ -257,19 +258,105 @@ def _request_safety_identifier(
|
|||
return None
|
||||
|
||||
|
||||
def _fields_without_unsupported_media(
|
||||
fields: RequestFields,
|
||||
def _bool_or_none(value: object) -> bool | None:
|
||||
return value if isinstance(value, bool) else None
|
||||
|
||||
|
||||
def _str_or_none(value: object) -> str | None:
|
||||
return value if isinstance(value, str) else None
|
||||
|
||||
|
||||
_DeploymentFlag: TypeAlias = Annotated[bool | None, BeforeValidator(_bool_or_none)]
|
||||
|
||||
|
||||
class _DeploymentCapabilities(BaseModel):
|
||||
model_config = ConfigDict(frozen=True, from_attributes=True)
|
||||
|
||||
mode: Annotated[str | None, BeforeValidator(_str_or_none)] = None
|
||||
supports_box_questions: _DeploymentFlag = None
|
||||
supports_audio_input: _DeploymentFlag = None
|
||||
supports_video_input: _DeploymentFlag = None
|
||||
|
||||
def decisions_media_flag(self, flag: bool | None, model: str, custom_llm_provider: str) -> bool | None:
|
||||
if flag is None:
|
||||
return None
|
||||
mode: Final = self.mode if self.mode is not None else catalog_mode(model, custom_llm_provider)
|
||||
return flag if mode is None or is_decisions_model_mode(mode) else None
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _ResolvedSupport:
|
||||
box_questions: bool
|
||||
audio: bool
|
||||
videos: bool
|
||||
|
||||
|
||||
def _takes_audio(
|
||||
provider_config: BaseDecisionsConfig, model: str, custom_llm_provider: str, deployment: _DeploymentCapabilities
|
||||
) -> bool:
|
||||
return provider_config.supports_audio_input(
|
||||
model,
|
||||
custom_llm_provider,
|
||||
deployment.decisions_media_flag(deployment.supports_audio_input, model, custom_llm_provider),
|
||||
)
|
||||
|
||||
|
||||
def _takes_videos(
|
||||
provider_config: BaseDecisionsConfig, model: str, custom_llm_provider: str, deployment: _DeploymentCapabilities
|
||||
) -> bool:
|
||||
return provider_config.supports_video_input(
|
||||
model,
|
||||
custom_llm_provider,
|
||||
deployment.decisions_media_flag(deployment.supports_video_input, model, custom_llm_provider),
|
||||
)
|
||||
|
||||
|
||||
def deployment_refuses_media(
|
||||
model: str,
|
||||
custom_llm_provider: str,
|
||||
model_info: Mapping[str, object],
|
||||
call_kwargs: Mapping[str, object],
|
||||
*,
|
||||
audio: bool,
|
||||
videos: bool,
|
||||
) -> bool:
|
||||
if _drops_params(call_kwargs):
|
||||
return False
|
||||
provider_config: Final = get_decisions_provider_config(model, custom_llm_provider)
|
||||
canonical_model: Final = provider_config.canonical_model(model)
|
||||
deployment: Final = _DeploymentCapabilities.model_validate(model_info)
|
||||
return (audio and not _takes_audio(provider_config, canonical_model, custom_llm_provider, deployment)) or (
|
||||
videos and not _takes_videos(provider_config, canonical_model, custom_llm_provider, deployment)
|
||||
)
|
||||
|
||||
|
||||
def _resolved_support(
|
||||
provider_config: BaseDecisionsConfig,
|
||||
model: str,
|
||||
custom_llm_provider: str,
|
||||
headers: Mapping[str, str],
|
||||
kwargs: Mapping[str, object],
|
||||
fields: RequestFields,
|
||||
) -> _ResolvedSupport:
|
||||
deployment: Final = _DeploymentCapabilities.model_validate(kwargs.get("model_info") or {})
|
||||
return _ResolvedSupport(
|
||||
box_questions=provider_config.supports_box_questions(
|
||||
model, custom_llm_provider, headers, deployment.supports_box_questions
|
||||
),
|
||||
audio=bool(fields.audio) and _takes_audio(provider_config, model, custom_llm_provider, deployment),
|
||||
videos=bool(fields.videos) and _takes_videos(provider_config, model, custom_llm_provider, deployment),
|
||||
)
|
||||
|
||||
|
||||
def _fields_without_unsupported_media(
|
||||
fields: RequestFields, support: _ResolvedSupport, kwargs: Mapping[str, object]
|
||||
) -> RequestFields:
|
||||
if not _drops_params(kwargs):
|
||||
return fields
|
||||
return replace(
|
||||
fields,
|
||||
audio=fields.audio if provider_config.supports_audio_input(model, custom_llm_provider) else None,
|
||||
videos=fields.videos if provider_config.supports_video_input(model, custom_llm_provider) else None,
|
||||
audio=fields.audio if support.audio else None,
|
||||
videos=fields.videos if support.videos else None,
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -292,10 +379,8 @@ class _MalformedImages:
|
|||
def _unsupported_params(
|
||||
ir_request: DecisionsIRRequest,
|
||||
provider_config: BaseDecisionsConfig,
|
||||
model: str,
|
||||
custom_llm_provider: str,
|
||||
headers: Mapping[str, str],
|
||||
supports_box_questions: bool,
|
||||
support: _ResolvedSupport,
|
||||
) -> tuple[str, ...]:
|
||||
refuses_safety_identifier: Final = (
|
||||
ir_request.safety_identifier is not None and not provider_config.supports_safety_identifier
|
||||
|
|
@ -304,13 +389,9 @@ def _unsupported_params(
|
|||
None if provider_config.supports_images_in(ir_request.input, headers) else image_param(ir_request.input)
|
||||
)
|
||||
refuses_assets: Final = bool(ir_request.assets) and not provider_config.supports_assets(headers)
|
||||
refuses_box_questions: Final = _has_box_question(ir_request.questions) and not supports_box_questions
|
||||
refuses_audio: Final = bool(ir_request.audio) and not provider_config.supports_audio_input(
|
||||
model, custom_llm_provider
|
||||
)
|
||||
refuses_videos: Final = bool(ir_request.videos) and not provider_config.supports_video_input(
|
||||
model, custom_llm_provider
|
||||
)
|
||||
refuses_box_questions: Final = _has_box_question(ir_request.questions) and not support.box_questions
|
||||
refuses_audio: Final = bool(ir_request.audio) and not support.audio
|
||||
refuses_videos: Final = bool(ir_request.videos) and not support.videos
|
||||
return (
|
||||
*(("safety_identifier",) if refuses_safety_identifier else ()),
|
||||
*(() if refused_image_param is None else (refused_image_param,)),
|
||||
|
|
@ -333,7 +414,7 @@ def _without_unsupported_params(
|
|||
ir_request: DecisionsIRRequest,
|
||||
provider_config: BaseDecisionsConfig,
|
||||
headers: Mapping[str, str],
|
||||
supports_box_questions: bool,
|
||||
support: _ResolvedSupport,
|
||||
) -> DecisionsIRRequest:
|
||||
return replace(
|
||||
ir_request,
|
||||
|
|
@ -345,9 +426,7 @@ def _without_unsupported_params(
|
|||
safety_identifier=ir_request.safety_identifier if provider_config.supports_safety_identifier else None,
|
||||
assets=ir_request.assets if provider_config.supports_assets(headers) else MappingProxyType({}),
|
||||
questions=(
|
||||
ir_request.questions
|
||||
if supports_box_questions
|
||||
else tuple(filter(_is_not_box_question, ir_request.questions))
|
||||
ir_request.questions if support.box_questions else tuple(filter(_is_not_box_question, ir_request.questions))
|
||||
),
|
||||
)
|
||||
|
||||
|
|
@ -368,39 +447,20 @@ def _without_malformed_images(
|
|||
return well_formed
|
||||
|
||||
|
||||
class _DeploymentBoxQuestions(BaseModel):
|
||||
model_config = ConfigDict(frozen=True, from_attributes=True)
|
||||
|
||||
supports_box_questions: StrictBool | None = None
|
||||
|
||||
|
||||
def _deployment_supports_box_questions(kwargs: Mapping[str, object]) -> bool | None:
|
||||
try:
|
||||
return _DeploymentBoxQuestions.model_validate(kwargs.get("model_info") or {}).supports_box_questions
|
||||
except ValidationError:
|
||||
return None
|
||||
|
||||
|
||||
def _provider_ir_request(
|
||||
ir_request: DecisionsIRRequest,
|
||||
*,
|
||||
provider_config: BaseDecisionsConfig,
|
||||
model: str,
|
||||
custom_llm_provider: str,
|
||||
headers: Mapping[str, str],
|
||||
support: _ResolvedSupport,
|
||||
kwargs: Mapping[str, object],
|
||||
) -> DecisionsIRRequest | _UnsupportedParams | _NothingLeftAfterDrop:
|
||||
supports_box_questions: Final = provider_config.supports_box_questions(
|
||||
model, custom_llm_provider, headers, _deployment_supports_box_questions(kwargs)
|
||||
)
|
||||
unsupported: Final = _unsupported_params(
|
||||
ir_request, provider_config, model, custom_llm_provider, headers, supports_box_questions
|
||||
)
|
||||
unsupported: Final = _unsupported_params(ir_request, provider_config, headers, support)
|
||||
if not unsupported:
|
||||
return ir_request
|
||||
if not _drops_params(kwargs):
|
||||
return _UnsupportedParams(names=unsupported)
|
||||
supported: Final = _without_unsupported_params(ir_request, provider_config, headers, supports_box_questions)
|
||||
supported: Final = _without_unsupported_params(ir_request, provider_config, headers, support)
|
||||
if _leaves_no_input(supported.input):
|
||||
return _NothingLeftAfterDrop(names=unsupported, missing="input to decide on")
|
||||
if not supported.questions:
|
||||
|
|
@ -479,10 +539,11 @@ def prepare_call(
|
|||
raise litellm.BadRequestError(message=str(error), model=model, llm_provider=provider) from error
|
||||
try:
|
||||
headers: Final = decisions_outbound_headers(extra_headers, provider, kwargs)
|
||||
support: Final = _resolved_support(provider_config, canonical_model, provider, headers, kwargs, fields)
|
||||
request_safety_identifier: Final = _request_safety_identifier(safety_identifier, provider_config, kwargs)
|
||||
request: Final = _fields_without_unsupported_media(
|
||||
fields, provider_config, canonical_model, provider, kwargs
|
||||
).validate(safety_identifier=request_safety_identifier, drops_params=_drops_params(kwargs))
|
||||
request: Final = _fields_without_unsupported_media(fields, support, kwargs).validate(
|
||||
safety_identifier=request_safety_identifier, drops_params=_drops_params(kwargs)
|
||||
)
|
||||
except ValidationError as error:
|
||||
raise litellm.BadRequestError(
|
||||
message=f"Invalid Decisions request: {error}",
|
||||
|
|
@ -523,12 +584,7 @@ def prepare_call(
|
|||
llm_provider=provider,
|
||||
)
|
||||
ir_request: Final = _provider_ir_request(
|
||||
caller_ir_request,
|
||||
provider_config=provider_config,
|
||||
model=canonical_model,
|
||||
custom_llm_provider=provider,
|
||||
headers=headers,
|
||||
kwargs=kwargs,
|
||||
caller_ir_request, provider_config=provider_config, headers=headers, support=support, kwargs=kwargs
|
||||
)
|
||||
if isinstance(ir_request, _UnsupportedParams):
|
||||
raise litellm.UnsupportedParamsError(
|
||||
|
|
|
|||
|
|
@ -14,7 +14,7 @@ import itertools
|
|||
import os
|
||||
import re
|
||||
from collections.abc import Callable, Container, Iterable, Mapping, Sequence
|
||||
from typing import TYPE_CHECKING, Any, Final, cast
|
||||
from typing import TYPE_CHECKING, Any, Final, TypeAlias, TypeVar, cast
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from pydantic import TypeAdapter, ValidationError
|
||||
|
|
@ -102,6 +102,192 @@ def _validated_object_list(value: object) -> list[object] | None:
|
|||
return None
|
||||
|
||||
|
||||
_MessageT: Final = TypeVar("_MessageT")
|
||||
_CacheControlRewrite: TypeAlias = Callable[[object], ChatCompletionCachedContent | None]
|
||||
_MARK_CARRYING_MESSAGE_FIELDS: Final = ("content", "tool_calls")
|
||||
|
||||
|
||||
def _mark_carrying_blocks(fields: Mapping[str, object]) -> Sequence[object]:
|
||||
lists: Final = tuple(fields.get(key) for key in _MARK_CARRYING_MESSAGE_FIELDS)
|
||||
return tuple(
|
||||
itertools.chain.from_iterable(
|
||||
_OBJECT_LIST_ADAPTER.validate_python(value) for value in lists if isinstance(value, list)
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _block_carries_cache_control(block: object) -> bool:
|
||||
return isinstance(block, dict) and _CONTENT_PART_ADAPTER.validate_python(block).get("cache_control") is not None
|
||||
|
||||
|
||||
_ROLES_SENDING_MESSAGE_MARK_WITH_LIST_CONTENT: Final = frozenset({"tool", "function"})
|
||||
|
||||
|
||||
def _message_mark_reaches_provider(fields: Mapping[str, object]) -> bool:
|
||||
if fields.get("cache_control") is None:
|
||||
return False
|
||||
return (
|
||||
not isinstance(fields.get("content"), list)
|
||||
or fields.get("role") in _ROLES_SENDING_MESSAGE_MARK_WITH_LIST_CONTENT
|
||||
)
|
||||
|
||||
|
||||
def cache_control_count(message: object) -> int:
|
||||
if not isinstance(message, dict):
|
||||
return 0
|
||||
fields: Final = _CONTENT_PART_ADAPTER.validate_python(message)
|
||||
block_marks: Final = sum(1 for block in _mark_carrying_blocks(fields) if _block_carries_cache_control(block))
|
||||
return int(_message_mark_reaches_provider(fields)) + block_marks
|
||||
|
||||
|
||||
def _fields_with_cache_control(fields: Mapping[str, object], rewrite: _CacheControlRewrite) -> Mapping[str, object]:
|
||||
others: Final = {key: value for key, value in fields.items() if key != "cache_control"}
|
||||
if "cache_control" not in fields:
|
||||
return others
|
||||
cache_control: Final = rewrite(fields["cache_control"])
|
||||
return others if cache_control is None else {**others, "cache_control": cache_control}
|
||||
|
||||
|
||||
def block_with_cache_control(block: object, rewrite: _CacheControlRewrite) -> object:
|
||||
if not isinstance(block, dict):
|
||||
return block
|
||||
return _fields_with_cache_control(_CONTENT_PART_ADAPTER.validate_python(block), rewrite)
|
||||
|
||||
|
||||
def with_cache_control(message: _MessageT, rewrite: _CacheControlRewrite) -> _MessageT:
|
||||
if not isinstance(message, dict):
|
||||
return message
|
||||
fields: Final = _CONTENT_PART_ADAPTER.validate_python(message)
|
||||
blocks_rewritten: Final = {
|
||||
key: (
|
||||
[block_with_cache_control(block, rewrite) for block in _OBJECT_LIST_ADAPTER.validate_python(value)]
|
||||
if key in _MARK_CARRYING_MESSAGE_FIELDS and isinstance(value, list)
|
||||
else value
|
||||
)
|
||||
for key, value in fields.items()
|
||||
}
|
||||
return cast( # cast-ok: the same message shape with each cache marker rewritten or dropped
|
||||
_MessageT, _fields_with_cache_control(blocks_rewritten, rewrite)
|
||||
)
|
||||
|
||||
|
||||
def dropped_cache_control(_: object) -> None:
|
||||
return None
|
||||
|
||||
|
||||
def _first_marked_index(value: object) -> int | None:
|
||||
if not isinstance(value, list):
|
||||
return None
|
||||
blocks: Final = _OBJECT_LIST_ADAPTER.validate_python(value)
|
||||
return next((index for index, block in enumerate(blocks) if _block_carries_cache_control(block)), None)
|
||||
|
||||
|
||||
_MarkSlot: TypeAlias = tuple[str, int]
|
||||
_MESSAGE_MARK_SLOT: Final[_MarkSlot] = ("cache_control", -1)
|
||||
|
||||
|
||||
def _block_mark_slots(fields: Mapping[str, object], key: str) -> tuple[_MarkSlot, ...]:
|
||||
value: Final = fields.get(key)
|
||||
if not isinstance(value, list):
|
||||
return ()
|
||||
blocks: Final = _OBJECT_LIST_ADAPTER.validate_python(value)
|
||||
return tuple((key, index) for index, block in enumerate(blocks) if _block_carries_cache_control(block))
|
||||
|
||||
|
||||
def _mark_slots_in_wire_order(fields: Mapping[str, object]) -> tuple[_MarkSlot, ...]:
|
||||
content_slots: Final = _block_mark_slots(fields, "content")
|
||||
tool_call_slots: Final = _block_mark_slots(fields, "tool_calls")
|
||||
if not _message_mark_reaches_provider(fields):
|
||||
return content_slots + tool_call_slots
|
||||
if isinstance(fields.get("content"), list):
|
||||
return content_slots + (_MESSAGE_MARK_SLOT,) + tool_call_slots
|
||||
return (_MESSAGE_MARK_SLOT,) + content_slots + tool_call_slots
|
||||
|
||||
|
||||
def _blocks_without_marks(key: str, value: object, dropped_slots: frozenset[_MarkSlot]) -> object:
|
||||
if key not in _MARK_CARRYING_MESSAGE_FIELDS or not isinstance(value, list):
|
||||
return value
|
||||
return [
|
||||
block_with_cache_control(block, dropped_cache_control) if (key, index) in dropped_slots else block
|
||||
for index, block in enumerate(_OBJECT_LIST_ADAPTER.validate_python(value))
|
||||
]
|
||||
|
||||
|
||||
def _without_oldest_marks(message: _MessageT, dropped: int) -> _MessageT:
|
||||
if dropped == 0 or not isinstance(message, dict):
|
||||
return message
|
||||
fields: Final = _CONTENT_PART_ADAPTER.validate_python(message)
|
||||
dropped_slots: Final = frozenset(_mark_slots_in_wire_order(fields)[:dropped])
|
||||
blocks_unmarked: Final = {key: _blocks_without_marks(key, value, dropped_slots) for key, value in fields.items()}
|
||||
return cast( # cast-ok: the same message with its oldest wire-order marks dropped
|
||||
_MessageT,
|
||||
_fields_with_cache_control(blocks_unmarked, dropped_cache_control)
|
||||
if _MESSAGE_MARK_SLOT in dropped_slots
|
||||
else blocks_unmarked,
|
||||
)
|
||||
|
||||
|
||||
def tool_message_with_single_cache_mark(message: _MessageT) -> _MessageT:
|
||||
if not isinstance(message, dict):
|
||||
return message
|
||||
fields: Final = _CONTENT_PART_ADAPTER.validate_python(message)
|
||||
content: Final = fields.get("content")
|
||||
if fields.get("role") != "tool" or not isinstance(content, list):
|
||||
return message
|
||||
kept_index: Final = None if fields.get("cache_control") is not None else _first_marked_index(content)
|
||||
return cast( # cast-ok: the same tool message keeping only the mark its single cache point reads
|
||||
_MessageT,
|
||||
{
|
||||
**fields,
|
||||
"content": [
|
||||
block if index == kept_index else block_with_cache_control(block, dropped_cache_control)
|
||||
for index, block in enumerate(_OBJECT_LIST_ADAPTER.validate_python(content))
|
||||
],
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
_BudgetSpend: TypeAlias = Callable[[tuple[int, int], int], tuple[int, int]]
|
||||
|
||||
|
||||
def _newest_marks_kept(state: tuple[int, int], marks: int) -> tuple[int, int]:
|
||||
remaining, _ = state
|
||||
kept: Final = min(marks, remaining)
|
||||
return remaining - kept, marks - kept
|
||||
|
||||
|
||||
def _whole_message_kept(state: tuple[int, int], marks: int) -> tuple[int, int]:
|
||||
remaining, _ = state
|
||||
if marks > remaining:
|
||||
return remaining, marks
|
||||
return remaining - marks, 0
|
||||
|
||||
|
||||
def _messages_within_budget(
|
||||
messages: Sequence[_MessageT], reserved_marks: int, spend: _BudgetSpend
|
||||
) -> Sequence[_MessageT]:
|
||||
budget: Final = max(MAX_CACHE_CONTROL_BLOCKS - reserved_marks, 0)
|
||||
newest_first: Final = tuple(reversed(messages))
|
||||
budget_after_each: Final = itertools.accumulate(
|
||||
(cache_control_count(message) for message in newest_first), spend, initial=(budget, 0)
|
||||
)
|
||||
dropped_counts: Final = tuple(dropped for _, dropped in itertools.islice(budget_after_each, 1, None))
|
||||
kept_newest_first: Final = tuple(
|
||||
_without_oldest_marks(message, dropped) for message, dropped in zip(newest_first, dropped_counts)
|
||||
)
|
||||
return tuple(reversed(kept_newest_first))
|
||||
|
||||
|
||||
def newest_marks_within_cache_control_limit(messages: Sequence[_MessageT], reserved_marks: int) -> Sequence[_MessageT]:
|
||||
return _messages_within_budget(messages, reserved_marks, _newest_marks_kept)
|
||||
|
||||
|
||||
def whole_messages_within_cache_control_limit(
|
||||
messages: Sequence[_MessageT], reserved_marks: int
|
||||
) -> Sequence[_MessageT]:
|
||||
return _messages_within_budget(messages, reserved_marks, _whole_message_kept)
|
||||
|
||||
|
||||
def _part_with_cache_control_renamed(part: object) -> object:
|
||||
if not _accepts_prompt_cache_breakpoint(part):
|
||||
return part
|
||||
|
|
@ -1439,3 +1625,25 @@ class AnthropicCacheControlHook(CustomPromptManagement):
|
|||
llm_router=None,
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
def messages_within_cache_control_cap(
|
||||
messages: Sequence[_MessageT], tools: object, cache_control: object = None, request_kwargs: object = None
|
||||
) -> Sequence[_MessageT]:
|
||||
"""Under drop_params, drop the oldest message hints the provider's cache_control cap has no room for.
|
||||
|
||||
Tool hints and a top-level ``cache_control`` stay on the wire, so they are reserved first.
|
||||
"""
|
||||
reserved_marks: Final = AnthropicCacheControlHook.count_external_cache_breakpoints(
|
||||
_validated_object_list(tools), cache_control, request_kwargs
|
||||
)
|
||||
kept: Final = newest_marks_within_cache_control_limit(messages, reserved_marks)
|
||||
dropped: Final = sum(map(cache_control_count, messages)) - sum(map(cache_control_count, kept))
|
||||
if dropped > 0:
|
||||
verbose_logger.warning(
|
||||
"Dropped %d cache_control hint(s) from the oldest messages because drop_params is on and the "
|
||||
"provider accepts at most %d cache_control blocks per request",
|
||||
dropped,
|
||||
MAX_CACHE_CONTROL_BLOCKS,
|
||||
)
|
||||
return kept
|
||||
|
|
|
|||
|
|
@ -3,13 +3,15 @@ import functools
|
|||
import inspect
|
||||
import re
|
||||
import time
|
||||
from collections.abc import Iterator, Mapping, Sequence
|
||||
from collections.abc import Callable, Iterator, Mapping, Sequence
|
||||
from datetime import datetime
|
||||
from typing import TYPE_CHECKING, Any, Final, TypeAlias, cast
|
||||
|
||||
from litellm._logging import format_base64_size, verbose_logger
|
||||
from litellm.constants import (
|
||||
BASE64_TRUNCATION_INLINE_COPY_MAX_CHARS,
|
||||
BASE64_TRUNCATION_OFFLOAD_THRESHOLD_CHARS,
|
||||
BASE64_TRUNCATION_OFFLOAD_THRESHOLD_STRINGS,
|
||||
MAX_BASE64_LENGTH_FOR_LOGGING,
|
||||
)
|
||||
from litellm.litellm_core_utils.core_helpers import get_litellm_metadata_from_kwargs
|
||||
|
|
@ -44,6 +46,8 @@ Helper utils used for logging callbacks
|
|||
# Regex matching data-URI base64 content: "data:<mime>;base64,<payload>"
|
||||
# Captures: group(1)=mime_type, group(2)=base64_payload
|
||||
_DATA_URI_RE: Final = re.compile(r"data:([^;,\s]{1,255});base64,([A-Za-z0-9+/=]+)")
|
||||
_DATA_URI_PREFIX: Final = "data:"
|
||||
_DATA_URI_BASE64_MARKER: Final = ";base64,"
|
||||
|
||||
# Maximum nesting depth for _truncate_base64_in_value to guard against
|
||||
# pathological payloads. OpenAI message format is typically 3-4 levels deep.
|
||||
|
|
@ -60,15 +64,20 @@ def _base64_data_uri_replacer(match: re.Match[str]) -> str:
|
|||
return f"data:{mime_type};base64,[base64_data truncated: {size_str}]"
|
||||
|
||||
|
||||
def _may_contain_base64_data_uri(value: str) -> bool:
|
||||
return ";" in value and _DATA_URI_PREFIX in value and _DATA_URI_BASE64_MARKER in value
|
||||
|
||||
|
||||
def _truncate_base64_in_string(value: str) -> str:
|
||||
"""Replace long base64 data-URI payloads in a string with a size placeholder."""
|
||||
if MAX_BASE64_LENGTH_FOR_LOGGING <= 0:
|
||||
if MAX_BASE64_LENGTH_FOR_LOGGING <= 0 or not _may_contain_base64_data_uri(value):
|
||||
return value
|
||||
return _DATA_URI_RE.sub(_base64_data_uri_replacer, value)
|
||||
|
||||
|
||||
def _truncate_base64_in_value(
|
||||
value: str | dict[str, object] | list[object] | None,
|
||||
truncate_string: Callable[[str], str],
|
||||
) -> str | dict[str, object] | list[object] | None:
|
||||
"""Iteratively truncate base64 data URIs in a JSON-like value (str/list/dict).
|
||||
|
||||
|
|
@ -78,13 +87,13 @@ def _truncate_base64_in_value(
|
|||
# Stack entries: (source_value, depth, parent_container, key_or_index)
|
||||
# We mutate *copies* of dicts/lists in-place via parent references.
|
||||
if isinstance(value, str):
|
||||
return _truncate_base64_in_string(value)
|
||||
return truncate_string(value)
|
||||
if not isinstance(value, (dict, list)):
|
||||
return value
|
||||
|
||||
# Shallow-copy the root so we don't mutate the caller's data.
|
||||
root: Final = {k: v for k, v in value.items()} if isinstance(value, dict) else list(value)
|
||||
stack: Final[list] = [(root, 0)]
|
||||
stack: Final[list[tuple[dict[str, object] | list[object], int]]] = [(root, 0)] # mutable-ok: explicit stack
|
||||
|
||||
while stack:
|
||||
container, depth = stack.pop()
|
||||
|
|
@ -93,19 +102,19 @@ def _truncate_base64_in_value(
|
|||
if isinstance(container, dict):
|
||||
for k, v in container.items():
|
||||
if isinstance(v, str):
|
||||
container[k] = _truncate_base64_in_string(v)
|
||||
container[k] = truncate_string(v)
|
||||
elif isinstance(v, dict):
|
||||
copy: dict | list = {ck: cv for ck, cv in v.items()}
|
||||
copy: dict[str, object] | list[object] = {ck: cv for ck, cv in v.items()}
|
||||
container[k] = copy
|
||||
stack.append((copy, depth + 1))
|
||||
elif isinstance(v, list):
|
||||
copy = list(v)
|
||||
container[k] = copy
|
||||
stack.append((copy, depth + 1))
|
||||
elif isinstance(container, list):
|
||||
else:
|
||||
for i, v in enumerate(container):
|
||||
if isinstance(v, str):
|
||||
container[i] = _truncate_base64_in_string(v)
|
||||
container[i] = truncate_string(v)
|
||||
elif isinstance(v, dict):
|
||||
copy = {ck: cv for ck, cv in v.items()}
|
||||
container[i] = copy
|
||||
|
|
@ -127,8 +136,15 @@ def truncate_base64_in_messages(
|
|||
"""
|
||||
if messages is None or MAX_BASE64_LENGTH_FOR_LOGGING <= 0:
|
||||
return messages
|
||||
return _copy_messages(messages, _truncate_base64_in_string)
|
||||
|
||||
|
||||
def _copy_messages(
|
||||
messages: str | dict[str, object] | list[object], # mutable-ok: same contract as truncate_base64_in_messages
|
||||
truncate_string: Callable[[str], str],
|
||||
) -> str | dict[str, object] | list[object] | None: # mutable-ok: same contract as truncate_base64_in_messages
|
||||
try:
|
||||
return _truncate_base64_in_value(messages)
|
||||
return _truncate_base64_in_value(messages, truncate_string)
|
||||
except Exception as e:
|
||||
verbose_logger.debug("Failed to truncate base64 in messages: %s", e)
|
||||
return messages
|
||||
|
|
@ -151,19 +167,31 @@ def _iter_string_leaves(value: _StringTree) -> Iterator[str]:
|
|||
pass
|
||||
|
||||
|
||||
def _is_few_plain_strings(leaves: tuple[str, ...]) -> bool:
|
||||
return len(leaves) < BASE64_TRUNCATION_OFFLOAD_THRESHOLD_STRINGS and not any(
|
||||
_may_contain_base64_data_uri(leaf) for leaf in leaves
|
||||
)
|
||||
|
||||
|
||||
async def truncate_base64_in_messages_async(
|
||||
messages: str | list | dict | None, # mutable-ok: same contract as truncate_base64_in_messages
|
||||
) -> str | list | dict | None: # mutable-ok: same contract as truncate_base64_in_messages
|
||||
"""
|
||||
Same result as truncate_base64_in_messages, but payloads whose string content
|
||||
reaches BASE64_TRUNCATION_OFFLOAD_THRESHOLD_CHARS are scanned in a worker
|
||||
thread so the regex pass over multi-MB base64 images does not block the event loop.
|
||||
reaches BASE64_TRUNCATION_OFFLOAD_THRESHOLD_CHARS are copied in a worker thread,
|
||||
so neither the regex pass over multi-MB base64 images nor the copy of a long
|
||||
history blocks the event loop. A few strings with no data URI, under
|
||||
BASE64_TRUNCATION_INLINE_COPY_MAX_CHARS in total, are copied inline, since
|
||||
checking them for data URIs and copying them costs less than the thread hop.
|
||||
"""
|
||||
if messages is None or MAX_BASE64_LENGTH_FOR_LOGGING <= 0:
|
||||
return messages
|
||||
total_chars: Final = sum(len(leaf) for leaf in _iter_string_leaves(messages))
|
||||
leaves: Final = tuple(_iter_string_leaves(messages))
|
||||
total_chars: Final = sum(len(leaf) for leaf in leaves)
|
||||
if total_chars < BASE64_TRUNCATION_OFFLOAD_THRESHOLD_CHARS:
|
||||
return truncate_base64_in_messages(messages)
|
||||
if total_chars < BASE64_TRUNCATION_INLINE_COPY_MAX_CHARS and _is_few_plain_strings(leaves):
|
||||
return _copy_messages(messages, lambda text: text)
|
||||
return await asyncio.to_thread(truncate_base64_in_messages, messages)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -88,17 +88,22 @@ def handle_any_messages_to_chat_completion_str_messages_conversion(
|
|||
return [{"input": str(messages)}]
|
||||
|
||||
|
||||
def _message_with_str_content(message: AllMessageValues) -> AllMessageValues:
|
||||
texts: Final = convert_content_list_to_str(message=message)
|
||||
if not texts:
|
||||
return message
|
||||
return cast( # cast-ok: the same message with its content flattened to one string
|
||||
AllMessageValues, {**message, "content": texts}
|
||||
)
|
||||
|
||||
|
||||
def handle_messages_with_content_list_to_str_conversion(
|
||||
messages: list[AllMessageValues],
|
||||
) -> list[AllMessageValues]:
|
||||
"""
|
||||
Handles messages with content list conversion
|
||||
Returns new messages with each content list flattened to a string, leaving the caller's messages untouched
|
||||
"""
|
||||
for message in messages:
|
||||
texts = convert_content_list_to_str(message=message)
|
||||
if texts:
|
||||
message["content"] = texts
|
||||
return messages
|
||||
return [_message_with_str_content(message) for message in messages]
|
||||
|
||||
|
||||
def strip_name_from_message(message: AllMessageValues, allowed_name_roles: list[str] = ["user"]) -> AllMessageValues:
|
||||
|
|
|
|||
|
|
@ -167,6 +167,7 @@ _DEFER_LOADING_ADAPTER: Final = TypeAdapter(bool)
|
|||
_TOOL_FIELDS_ADAPTER: Final = TypeAdapter(Mapping[str, object])
|
||||
_TOOLS_ADAPTER: Final = TypeAdapter(list[object])
|
||||
_LITELLM_PARAMS_ADAPTER: Final = TypeAdapter(Mapping[str, object])
|
||||
_REQUEST_PARAMS_ADAPTER: Final = TypeAdapter(Mapping[str, object])
|
||||
_NATIVE_OUTPUT_STRING_FORMATS: Final = frozenset(
|
||||
("date-time", "time", "date", "duration", "email", "hostname", "uri", "ipv4", "ipv6", "uuid")
|
||||
)
|
||||
|
|
@ -2254,6 +2255,16 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
|
|||
headers=headers,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _messages_within_cache_control_cap(
|
||||
messages: Sequence[AllMessageValues], optional_params: Mapping[str, object]
|
||||
) -> Sequence[AllMessageValues]:
|
||||
from litellm.integrations.anthropic_cache_control_hook import messages_within_cache_control_cap
|
||||
|
||||
return messages_within_cache_control_cap(
|
||||
messages, optional_params.get("tools"), optional_params.get("cache_control"), optional_params
|
||||
)
|
||||
|
||||
def transform_request(
|
||||
self,
|
||||
model: str,
|
||||
|
|
@ -2271,9 +2282,18 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
|
|||
anthropic_messages_pt,
|
||||
)
|
||||
|
||||
drop_params: Final = litellm.drop_params is True or litellm_params.get("drop_params") is True
|
||||
drop_params: Final = drops_unsupported_params(litellm_params.get("drop_params"))
|
||||
capped_messages: Final = (
|
||||
list(
|
||||
self._messages_within_cache_control_cap(
|
||||
messages, _REQUEST_PARAMS_ADAPTER.validate_python(optional_params)
|
||||
)
|
||||
)
|
||||
if drop_params
|
||||
else messages
|
||||
)
|
||||
|
||||
if "tools" not in optional_params and messages is not None and has_tool_call_blocks(messages):
|
||||
if "tools" not in optional_params and has_tool_call_blocks(capped_messages):
|
||||
optional_params["tools"], _ = self._map_tools(add_dummy_tool(custom_llm_provider="anthropic"))
|
||||
|
||||
# Drop thinking param if thinking is enabled but thinking_blocks are missing
|
||||
|
|
@ -2285,9 +2305,8 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
|
|||
# Related issue: https://github.com/BerriAI/litellm/issues/18926
|
||||
if (
|
||||
optional_params.get("thinking") is not None
|
||||
and messages is not None
|
||||
and last_assistant_with_tool_calls_has_no_thinking_blocks(messages)
|
||||
and not any_assistant_message_has_thinking_blocks(messages)
|
||||
and last_assistant_with_tool_calls_has_no_thinking_blocks(capped_messages)
|
||||
and not any_assistant_message_has_thinking_blocks(capped_messages)
|
||||
):
|
||||
if litellm.modify_params:
|
||||
optional_params.pop("thinking", None)
|
||||
|
|
@ -2332,15 +2351,18 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
|
|||
_name_forward_map, _name_reverse_map = self._sanitize_tool_names_in_request(
|
||||
optional_params=optional_params,
|
||||
)
|
||||
if _name_forward_map:
|
||||
messages = self._rewrite_tool_names_in_messages(messages, _name_forward_map)
|
||||
request_messages: Final = (
|
||||
self._rewrite_tool_names_in_messages(capped_messages, _name_forward_map)
|
||||
if _name_forward_map
|
||||
else capped_messages
|
||||
)
|
||||
if _name_reverse_map and isinstance(litellm_params, dict):
|
||||
litellm_params[ANTHROPIC_TOOL_NAME_REVERSE_MAP_KEY] = _name_reverse_map
|
||||
|
||||
# Only the leading system run becomes the top-level system prompt. A later
|
||||
# system message stays in the conversation: hoisting it rewrites the cached
|
||||
# prefix and re-bills the whole history at cache-write pricing (#36559).
|
||||
leading_system_run, later_messages = split_leading_system_run(messages)
|
||||
leading_system_run, later_messages = split_leading_system_run(request_messages)
|
||||
anthropic_system_message_list: Final = self.translate_system_message(messages=list(leading_system_run))
|
||||
# Handling anthropic API Prompt Caching
|
||||
if len(anthropic_system_message_list) > 0:
|
||||
|
|
@ -2362,7 +2384,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
|
|||
except Exception as e:
|
||||
raise AnthropicError(
|
||||
status_code=400,
|
||||
message=f"{e}\nReceived Messages={messages}",
|
||||
message=f"{e}\nReceived Messages={request_messages}",
|
||||
) # don't use verbose_logger.exception, if exception is raised
|
||||
|
||||
self.update_headers_with_optional_anthropic_beta(
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ This file contains common utils for anthropic calls.
|
|||
|
||||
import copy
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
from collections.abc import Container, Mapping, MutableMapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
|
|
@ -19,6 +20,8 @@ from typing_extensions import ReadOnly, TypedDict
|
|||
|
||||
import litellm
|
||||
from litellm.constants import (
|
||||
ANTHROPIC_COMPATIBLE_MAX_USER_ID_LENGTH,
|
||||
ANTHROPIC_MAX_USER_ID_LENGTH,
|
||||
DEFAULT_MODEL_CREATED_AT_TIME,
|
||||
DEFAULT_REASONING_EFFORT_HIGH_THINKING_BUDGET,
|
||||
DEFAULT_REASONING_EFFORT_MEDIUM_THINKING_BUDGET,
|
||||
|
|
@ -178,6 +181,40 @@ def tools_without_custom_field(tools: Sequence[object], hoist_defer_loading: boo
|
|||
return [_tool_without_custom_field(tool, hoist_defer_loading, drop_custom) for tool in tools]
|
||||
|
||||
|
||||
def anthropic_pass_through_api_base() -> str:
|
||||
return os.getenv("ANTHROPIC_API_BASE") or os.getenv("ANTHROPIC_BASE_URL") or "https://api.anthropic.com"
|
||||
|
||||
|
||||
def _same_host_and_port(url: httpx.URL, other: str) -> bool:
|
||||
try:
|
||||
other_url: Final = httpx.URL(other)
|
||||
except httpx.InvalidURL:
|
||||
return False
|
||||
return (url.host, url.port) == (other_url.host, other_url.port)
|
||||
|
||||
|
||||
def anthropic_upstream_user_id_limit(url: str) -> int | None:
|
||||
upstream: Final = httpx.URL(url)
|
||||
if not upstream.path.removesuffix("/").endswith("/v1/messages"):
|
||||
return None
|
||||
if upstream.host == "api.anthropic.com":
|
||||
return ANTHROPIC_MAX_USER_ID_LENGTH
|
||||
return (
|
||||
ANTHROPIC_COMPATIBLE_MAX_USER_ID_LENGTH
|
||||
if _same_host_and_port(upstream, anthropic_pass_through_api_base())
|
||||
else None
|
||||
)
|
||||
|
||||
|
||||
def _has_email_shape(token: str) -> bool:
|
||||
at_sign: Final = token.find("@", 1)
|
||||
return at_sign != -1 and token.find(".", at_sign + 2, -1) != -1
|
||||
|
||||
|
||||
def anthropic_accepts_user_id(user_id: str, max_length: int) -> bool:
|
||||
return len(user_id) <= max_length and user_id.isprintable() and not any(map(_has_email_shape, user_id.split()))
|
||||
|
||||
|
||||
def is_anthropic_messages_url(url: str) -> bool:
|
||||
"""Check whether a URL addresses Anthropic's Messages API."""
|
||||
parsed_url: Final = urlparse(url)
|
||||
|
|
|
|||
|
|
@ -12,6 +12,7 @@ from pydantic_core import ErrorDetails
|
|||
from typing_extensions import assert_never
|
||||
|
||||
import litellm
|
||||
from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap
|
||||
from litellm.litellm_core_utils.url_utils import join_api_base
|
||||
from litellm.llms.base_llm.chat.transformation import BaseLLMException
|
||||
from litellm.secret_managers.main import get_secret_str
|
||||
|
|
@ -297,13 +298,7 @@ def _systemone_question(question: DecisionsIRQuestion) -> Mapping[str, object]:
|
|||
assert_never(question)
|
||||
|
||||
|
||||
def ir_to_systemone_request(
|
||||
model: str, request: DecisionsIRRequest, *, supports_audio: bool = False, supports_video: bool = False
|
||||
) -> Mapping[str, object] | UnsupportedDecisionsRequest:
|
||||
if request.audio and not supports_audio:
|
||||
return UnsupportedDecisionsRequest(reason="audio input is not supported by this Decisions model")
|
||||
if request.videos and not supports_video:
|
||||
return UnsupportedDecisionsRequest(reason="video input is not supported by this Decisions model")
|
||||
def ir_to_systemone_request(model: str, request: DecisionsIRRequest) -> Mapping[str, object]:
|
||||
state: Final = _systemone_state(request.input)
|
||||
keyed_questions: Final = zip(systemone_keys(request.questions), request.questions, strict=True)
|
||||
images: Final = input_images(request.input)
|
||||
|
|
@ -503,6 +498,16 @@ def ir_to_systemone_response(response: DecisionsIRResponse, request: DecisionsIR
|
|||
)
|
||||
|
||||
|
||||
def catalog_mode(model: str, custom_llm_provider: str) -> str | None:
|
||||
try:
|
||||
key: Final = litellm.get_model_info(model=model, custom_llm_provider=custom_llm_provider)["key"]
|
||||
except Exception: # noqa: BLE001 # get_model_info raises bare Exception for an unmapped model, which the catalog does not list
|
||||
return None
|
||||
entry: Final = GetModelCostMap.loaded_model_cost_map().get(key)
|
||||
mode: Final = entry.get("mode") if entry is not None else None
|
||||
return mode if isinstance(mode, str) else None
|
||||
|
||||
|
||||
def _is_decisions_model(model: str, custom_llm_provider: str) -> bool:
|
||||
try:
|
||||
model_mode: Final = litellm.get_model_info(model=model, custom_llm_provider=custom_llm_provider).get("mode")
|
||||
|
|
@ -640,12 +645,16 @@ class BaseDecisionsConfig(ABC):
|
|||
model=model, custom_llm_provider=custom_llm_provider
|
||||
)
|
||||
|
||||
def supports_audio_input(self, model: str, custom_llm_provider: str) -> bool:
|
||||
def supports_audio_input(self, model: str, custom_llm_provider: str, deployment_flag: bool | None) -> bool:
|
||||
if deployment_flag is not None:
|
||||
return deployment_flag
|
||||
return _is_decisions_model(model, custom_llm_provider) and litellm.supports_audio_input(
|
||||
model=model, custom_llm_provider=custom_llm_provider
|
||||
)
|
||||
|
||||
def supports_video_input(self, model: str, custom_llm_provider: str) -> bool:
|
||||
def supports_video_input(self, model: str, custom_llm_provider: str, deployment_flag: bool | None) -> bool:
|
||||
if deployment_flag is not None:
|
||||
return deployment_flag
|
||||
return _is_decisions_model(model, custom_llm_provider) and litellm.supports_video_input(
|
||||
model=model, custom_llm_provider=custom_llm_provider
|
||||
)
|
||||
|
|
@ -656,12 +665,7 @@ class BaseDecisionsConfig(ABC):
|
|||
request: DecisionsIRRequest,
|
||||
custom_llm_provider: str,
|
||||
) -> Mapping[str, object] | UnsupportedDecisionsRequest:
|
||||
return ir_to_systemone_request(
|
||||
self.request_model(model),
|
||||
request,
|
||||
supports_audio=self.supports_audio_input(model, custom_llm_provider),
|
||||
supports_video=self.supports_video_input(model, custom_llm_provider),
|
||||
)
|
||||
return ir_to_systemone_request(self.request_model(model), request)
|
||||
|
||||
def unwrap_response(self, payload: object) -> object:
|
||||
return payload
|
||||
|
|
|
|||
|
|
@ -25,6 +25,7 @@ from litellm.constants import (
|
|||
)
|
||||
from litellm.litellm_core_utils.core_helpers import (
|
||||
as_str_mapping,
|
||||
drops_unsupported_params,
|
||||
filter_exceptions_from_params,
|
||||
filter_internal_params,
|
||||
map_finish_reason,
|
||||
|
|
@ -2129,6 +2130,24 @@ class AmazonConverseConfig(BaseConfig):
|
|||
|
||||
return data
|
||||
|
||||
@staticmethod
|
||||
def _claude_messages_within_cache_control_cap(
|
||||
messages: Sequence[AllMessageValues],
|
||||
capability_model: str,
|
||||
optional_params: Mapping[str, object],
|
||||
litellm_params: Mapping[str, object],
|
||||
) -> Sequence[AllMessageValues]:
|
||||
if not drops_unsupported_params(litellm_params.get("drop_params")) or "claude" not in capability_model.lower():
|
||||
return messages
|
||||
from litellm.integrations.anthropic_cache_control_hook import (
|
||||
messages_within_cache_control_cap,
|
||||
tool_message_with_single_cache_mark,
|
||||
)
|
||||
|
||||
return messages_within_cache_control_cap(
|
||||
tuple(map(tool_message_with_single_cache_mark, messages)), optional_params.get("tools")
|
||||
)
|
||||
|
||||
async def _async_transform_request(
|
||||
self,
|
||||
model: str,
|
||||
|
|
@ -2140,7 +2159,13 @@ class AmazonConverseConfig(BaseConfig):
|
|||
base_model: Final = litellm_params.get("base_model")
|
||||
capability_model: Final = bedrock_capability_model(model, base_model)
|
||||
messages, system_content_blocks = self._transform_system_message(
|
||||
messages, model=capability_model, deployment_model=model
|
||||
list(
|
||||
self._claude_messages_within_cache_control_cap(
|
||||
messages, capability_model, _JSON_OBJECT.validate_python(optional_params), litellm_params
|
||||
)
|
||||
),
|
||||
model=capability_model,
|
||||
deployment_model=model,
|
||||
)
|
||||
|
||||
# Convert last user message to guarded_text if guardrailConfig is present
|
||||
|
|
@ -2221,7 +2246,13 @@ class AmazonConverseConfig(BaseConfig):
|
|||
base_model: Final = litellm_params.get("base_model")
|
||||
capability_model: Final = bedrock_capability_model(model, base_model)
|
||||
messages, system_content_blocks = self._transform_system_message(
|
||||
messages, model=capability_model, deployment_model=model
|
||||
list(
|
||||
self._claude_messages_within_cache_control_cap(
|
||||
messages, capability_model, _JSON_OBJECT.validate_python(optional_params), litellm_params
|
||||
)
|
||||
),
|
||||
model=capability_model,
|
||||
deployment_model=model,
|
||||
)
|
||||
|
||||
# Convert last user message to guarded_text if guardrailConfig is present
|
||||
|
|
|
|||
|
|
@ -26,6 +26,7 @@ class DatabricksEmbeddingHandler(OpenAILikeEmbeddingHandler, DatabricksBase):
|
|||
aembedding=None,
|
||||
custom_endpoint: bool | None = None,
|
||||
headers: dict | None = None,
|
||||
drop_params: bool = False,
|
||||
) -> EmbeddingResponse:
|
||||
# Check for custom user agent in optional_params or environment
|
||||
# This allows partners building on LiteLLM to set their own telemetry
|
||||
|
|
@ -58,4 +59,5 @@ class DatabricksEmbeddingHandler(OpenAILikeEmbeddingHandler, DatabricksBase):
|
|||
aembedding=aembedding,
|
||||
custom_endpoint=True,
|
||||
headers=headers,
|
||||
drop_params=drop_params,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -130,7 +130,8 @@ class DeepInfraConfig(OpenAIGPTConfig):
|
|||
- Input: {"role": "tool", "content": [{"type": "text", "text": "result"}]}
|
||||
- Output: {"role": "tool", "content": "[{\"type\": \"text\", \"text\": \"result\"}]"}
|
||||
"""
|
||||
for message in messages:
|
||||
copied_messages: Final = [message.copy() for message in messages]
|
||||
for message in copied_messages:
|
||||
if message.get("role") == "tool":
|
||||
content = message.get("content")
|
||||
|
||||
|
|
@ -149,7 +150,7 @@ class DeepInfraConfig(OpenAIGPTConfig):
|
|||
# For complex content, serialize the entire array as JSON string
|
||||
message["content"] = json.dumps(content)
|
||||
|
||||
return messages
|
||||
return copied_messages
|
||||
|
||||
@overload
|
||||
def _transform_messages(
|
||||
|
|
|
|||
|
|
@ -23,6 +23,7 @@ from litellm.secret_managers.main import get_secret_str
|
|||
from litellm.types.llms.openai import (
|
||||
AllMessageValues,
|
||||
ChatCompletionToolParam,
|
||||
ChatCompletionToolParamFunctionChunk,
|
||||
OpenAIChatCompletionToolParam,
|
||||
)
|
||||
from litellm.types.utils import (
|
||||
|
|
@ -45,6 +46,7 @@ from ...openai.chat.gpt_transformation import (
|
|||
OpenAIChatCompletionStreamingHandler,
|
||||
OpenAIGPTConfig,
|
||||
)
|
||||
from ...openai.common_utils import without_cache_control
|
||||
from ..common_utils import (
|
||||
FIREROUTER,
|
||||
FireworksAIException,
|
||||
|
|
@ -480,15 +482,23 @@ class FireworksAIConfig(FireworksAIMixin, OpenAIGPTConfig):
|
|||
return (("response_format", _json_schema_response_format(choice_schema, "choice")),)
|
||||
|
||||
def _transform_tools(self, tools: list[OpenAIChatCompletionToolParam]) -> list[OpenAIChatCompletionToolParam]:
|
||||
for tool in tools:
|
||||
if tool.get("type") != "function":
|
||||
continue
|
||||
function = tool["function"]
|
||||
function.pop("strict", None)
|
||||
params = function.get("parameters")
|
||||
if isinstance(params, dict):
|
||||
unpack_legacy_defs(params)
|
||||
return tools
|
||||
return [self._fireworks_tool(tool) for tool in tools]
|
||||
|
||||
@staticmethod
|
||||
def _fireworks_tool(tool: OpenAIChatCompletionToolParam) -> OpenAIChatCompletionToolParam:
|
||||
if tool.get("type") != "function":
|
||||
return tool
|
||||
function: Final = tool["function"]
|
||||
params: Final = function.get("parameters")
|
||||
unstrict_function: Final = cast( # cast-ok: the same function definition minus `strict`
|
||||
ChatCompletionToolParamFunctionChunk, {key: value for key, value in function.items() if key != "strict"}
|
||||
)
|
||||
new_function: Final[ChatCompletionToolParamFunctionChunk] = (
|
||||
{**unstrict_function, "parameters": unpack_legacy_defs(params, copy=True)}
|
||||
if isinstance(params, dict)
|
||||
else unstrict_function
|
||||
)
|
||||
return {**tool, "function": new_function}
|
||||
|
||||
def _transform_messages_helper(
|
||||
self, messages: list[AllMessageValues], model: str, litellm_params: dict
|
||||
|
|
@ -496,10 +506,6 @@ class FireworksAIConfig(FireworksAIMixin, OpenAIGPTConfig):
|
|||
"""
|
||||
Strip fields not permitted by FireworksAI from messages.
|
||||
"""
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
||||
filter_value_from_dict,
|
||||
)
|
||||
|
||||
supports_vision_value: Final = self._get_model_cost_capability_exact(model=model, capability="supports_vision")
|
||||
for message in messages:
|
||||
if message["role"] == "user":
|
||||
|
|
@ -530,9 +536,11 @@ class FireworksAIConfig(FireworksAIMixin, OpenAIGPTConfig):
|
|||
model=model,
|
||||
llm_provider="fireworks_ai",
|
||||
)
|
||||
filter_value_from_dict(cast(dict, message), "cache_control")
|
||||
|
||||
return [strip_litellm_response_only_fields(message) for message in messages]
|
||||
return [
|
||||
strip_litellm_response_only_fields(cast(AllMessageValues, without_cache_control(message)))
|
||||
for message in messages
|
||||
]
|
||||
|
||||
# Cached index of fireworks_ai/* entries from litellm.model_cost. Building
|
||||
# this index requires a full scan of model_cost (tens of thousands of
|
||||
|
|
|
|||
|
|
@ -140,7 +140,8 @@ class GroqChatConfig(OpenAILikeChatConfig):
|
|||
def _transform_messages(
|
||||
self, messages: list[AllMessageValues], model: str, is_async: bool = False
|
||||
) -> list[AllMessageValues] | Coroutine[object, object, list[AllMessageValues]]:
|
||||
for idx, message in enumerate(messages):
|
||||
transformed_messages: Final = list(messages)
|
||||
for idx, message in enumerate(transformed_messages):
|
||||
"""
|
||||
1. Don't pass 'null' function_call assistant message to groq - https://github.com/BerriAI/litellm/issues/5839
|
||||
"""
|
||||
|
|
@ -154,12 +155,12 @@ class GroqChatConfig(OpenAILikeChatConfig):
|
|||
for k, v in _message.items():
|
||||
if v is not None:
|
||||
new_message[k] = v
|
||||
messages[idx] = strip_litellm_internal_message_fields(new_message)
|
||||
transformed_messages[idx] = strip_litellm_internal_message_fields(new_message)
|
||||
|
||||
if is_async:
|
||||
return super()._transform_messages(messages=messages, model=model, is_async=True)
|
||||
return super()._transform_messages(messages=transformed_messages, model=model, is_async=True)
|
||||
else:
|
||||
return super()._transform_messages(messages=messages, model=model, is_async=False)
|
||||
return super()._transform_messages(messages=transformed_messages, model=model, is_async=False)
|
||||
|
||||
def transform_messages(
|
||||
self,
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@
|
|||
Translate from OpenAI's `/v1/chat/completions` to VLLM's `/v1/chat/completions`
|
||||
"""
|
||||
|
||||
import copy
|
||||
import json
|
||||
from collections.abc import Coroutine
|
||||
from typing import Final, Literal, cast, overload
|
||||
|
|
@ -88,7 +89,7 @@ class HostedVLLMChatConfig(OpenAIGPTConfig):
|
|||
) -> dict:
|
||||
_tools = non_default_params.pop("tools", None)
|
||||
if _tools is not None:
|
||||
_tools = remove_additional_properties(_tools)
|
||||
_tools = remove_additional_properties(copy.deepcopy(_tools)) # pyright: ignore[reportUnknownArgumentType] # tools arrive untyped through the bare-dict params
|
||||
_tools = remove_strict_from_schema(_tools)
|
||||
if isinstance(_tools, list):
|
||||
_tools = self._convert_custom_tools_to_function_tools(_tools)
|
||||
|
|
|
|||
|
|
@ -755,7 +755,8 @@ class MistralConfig(OpenAIGPTConfig):
|
|||
if _name is not None:
|
||||
# Remove name if not a tool message
|
||||
if message["role"] != "tool" or isinstance(_name, str) and len(_name.strip()) == 0:
|
||||
message.pop("name", None)
|
||||
message_without_name: Final = {k: v for k, v in message.items() if k != "name"}
|
||||
return cast(AllMessageValues, message_without_name) # cast-ok: the same message minus name
|
||||
|
||||
return message
|
||||
|
||||
|
|
|
|||
|
|
@ -14,7 +14,7 @@ from pydantic import TypeAdapter, ValidationError
|
|||
|
||||
import litellm
|
||||
from litellm.constants import OPENAI_SYSTEM_MESSAGES_FIRST_PROVIDERS
|
||||
from litellm.litellm_core_utils.core_helpers import map_finish_reason
|
||||
from litellm.litellm_core_utils.core_helpers import drops_unsupported_params, map_finish_reason
|
||||
from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import (
|
||||
extract_reasoning_content,
|
||||
handle_invalid_parallel_tool_calls,
|
||||
|
|
@ -64,7 +64,7 @@ from litellm.types.utils import (
|
|||
)
|
||||
from litellm.utils import convert_to_model_response_object, declared_mid_conversation_system_support
|
||||
|
||||
from ..common_utils import OpenAIError
|
||||
from ..common_utils import OpenAIError, without_cache_control
|
||||
from ..workload_identity import get_workload_identity_bearer_token, resolve_openai_workload_identity_config
|
||||
|
||||
if TYPE_CHECKING:
|
||||
|
|
@ -128,6 +128,29 @@ def _litellm_params_str(litellm_params: Mapping[str, object] | None, key: str) -
|
|||
return value if isinstance(value, str) else None
|
||||
|
||||
|
||||
def _drops_params(litellm_params: Mapping[str, object]) -> bool:
|
||||
return drops_unsupported_params(litellm_params.get("drop_params"))
|
||||
|
||||
|
||||
def _names_claude(model: str, litellm_params: Mapping[str, object]) -> bool:
|
||||
base_model: Final = litellm_params.get("base_model")
|
||||
return "claude" in model.lower() or (isinstance(base_model, str) and "claude" in base_model.lower())
|
||||
|
||||
|
||||
def _claude_messages_within_cache_control_cap(
|
||||
model: str,
|
||||
messages: Sequence[AllMessageValues],
|
||||
tools: Sequence["ChatCompletionToolParam"] | None,
|
||||
litellm_params: Mapping[str, object],
|
||||
request_params: Mapping[str, object],
|
||||
) -> Sequence[AllMessageValues]:
|
||||
if not _names_claude(model, litellm_params) or not _drops_params(litellm_params):
|
||||
return messages
|
||||
from litellm.integrations.anthropic_cache_control_hook import messages_within_cache_control_cap
|
||||
|
||||
return list(messages_within_cache_control_cap(messages, tools, request_params.get("cache_control"), request_params))
|
||||
|
||||
|
||||
def _prompt_cache_options(optional_params: Mapping[str, object]) -> object | None:
|
||||
try:
|
||||
extra_body: Final = _OBJECT_MAPPING.validate_python(optional_params.get("extra_body"))
|
||||
|
|
@ -311,52 +334,58 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig):
|
|||
return content_copy
|
||||
|
||||
async def _async_handle_pdf_url(self, content_item: ChatCompletionFileObjectFile) -> ChatCompletionFileObjectFile:
|
||||
file_id: Final = content_item.get("file_id")
|
||||
content_copy: Final = content_item.copy()
|
||||
file_id: Final = content_copy.get("file_id")
|
||||
if file_id is not None: # check for file id being url done in _handle_pdf_url
|
||||
base64_data: Final = await async_convert_url_to_base64(file_id)
|
||||
content_item["file_data"] = base64_data
|
||||
content_item["filename"] = "my_file.pdf"
|
||||
content_item.pop("file_id")
|
||||
return content_item
|
||||
content_copy["file_data"] = base64_data
|
||||
content_copy["filename"] = "my_file.pdf"
|
||||
content_copy.pop("file_id")
|
||||
return content_copy
|
||||
|
||||
def _common_file_data_check(self, content_item: ChatCompletionFileObjectFile) -> ChatCompletionFileObjectFile:
|
||||
file_data: Final = content_item.get("file_data")
|
||||
filename: Final = content_item.get("filename")
|
||||
if file_data is not None and filename is None:
|
||||
content_item["filename"] = "my_file.pdf"
|
||||
return {**content_item, "filename": "my_file.pdf"}
|
||||
return content_item
|
||||
|
||||
@staticmethod
|
||||
def _image_item_for_openai(content_item: ChatCompletionImageObject) -> ChatCompletionImageObject:
|
||||
image_url: Final = content_item["image_url"]
|
||||
if isinstance(image_url, str):
|
||||
return {**content_item, "image_url": {"url": image_url}}
|
||||
if not isinstance(image_url, dict):
|
||||
return content_item
|
||||
new_image_url_obj: Final = cast(
|
||||
ChatCompletionImageUrlObject,
|
||||
{k: v for k, v in image_url.items() if k not in {"format", "video_metadata"}},
|
||||
)
|
||||
return {**content_item, "image_url": new_image_url_obj}
|
||||
|
||||
@staticmethod
|
||||
def _file_item_for_openai(content_item: ChatCompletionFileObject) -> ChatCompletionFileObject:
|
||||
litellm_specific_params: Final = {"format", "detail", "video_metadata"}
|
||||
file_obj: Final = content_item.get("file")
|
||||
if file_obj is None:
|
||||
raise litellm.BadRequestError(
|
||||
message="Content block has type='file' but is missing the required 'file' field",
|
||||
model=None,
|
||||
llm_provider="openai",
|
||||
)
|
||||
new_file_obj: Final = ChatCompletionFileObjectFile(
|
||||
**{k: v for k, v in file_obj.items() if k not in litellm_specific_params}
|
||||
)
|
||||
return {**content_item, "file": new_file_obj}
|
||||
|
||||
def _apply_common_transform_content_item(
|
||||
self,
|
||||
content_item: OpenAIMessageContentListBlock,
|
||||
) -> OpenAIMessageContentListBlock:
|
||||
litellm_specific_params: Final = {"format", "detail", "video_metadata"}
|
||||
if content_item.get("type") == "image_url":
|
||||
content_item = cast(ChatCompletionImageObject, content_item)
|
||||
if isinstance(content_item["image_url"], str):
|
||||
content_item["image_url"] = {
|
||||
"url": content_item["image_url"],
|
||||
}
|
||||
elif isinstance(content_item["image_url"], dict):
|
||||
new_image_url_obj: Final = cast(
|
||||
ChatCompletionImageUrlObject,
|
||||
{k: v for k, v in content_item["image_url"].items() if k not in {"format", "video_metadata"}},
|
||||
)
|
||||
content_item["image_url"] = new_image_url_obj
|
||||
elif content_item.get("type") == "file":
|
||||
content_item = cast(ChatCompletionFileObject, content_item)
|
||||
file_obj: Final = content_item.get("file")
|
||||
if file_obj is None:
|
||||
raise litellm.BadRequestError(
|
||||
message="Content block has type='file' but is missing the required 'file' field",
|
||||
model=None,
|
||||
llm_provider="openai",
|
||||
)
|
||||
new_file_obj: Final = ChatCompletionFileObjectFile(
|
||||
**{k: v for k, v in file_obj.items() if k not in litellm_specific_params}
|
||||
)
|
||||
content_item["file"] = new_file_obj
|
||||
|
||||
return self._image_item_for_openai(cast(ChatCompletionImageObject, content_item))
|
||||
if content_item.get("type") == "file":
|
||||
return self._file_item_for_openai(cast(ChatCompletionFileObject, content_item))
|
||||
return content_item
|
||||
|
||||
def _transform_content_item(
|
||||
|
|
@ -419,31 +448,34 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig):
|
|||
hoisted_messages: Final = hoist_images_from_tool_messages(stripped_messages)
|
||||
|
||||
async def _async_transform():
|
||||
for message in hoisted_messages:
|
||||
message_content = message.get("content")
|
||||
message_role = message.get("role")
|
||||
|
||||
if message_role == "user" and message_content and isinstance(message_content, list):
|
||||
message_content_types = cast(list[OpenAIMessageContentListBlock], message_content)
|
||||
for i, content_item in enumerate(message_content_types):
|
||||
message_content_types[i] = await self._async_transform_content_item(
|
||||
cast(OpenAIMessageContentListBlock, content_item),
|
||||
)
|
||||
return hoisted_messages
|
||||
return [await self._async_user_message_for_openai(message) for message in hoisted_messages]
|
||||
|
||||
if is_async:
|
||||
return _async_transform()
|
||||
else:
|
||||
for message in hoisted_messages:
|
||||
message_content = message.get("content")
|
||||
message_role = message.get("role")
|
||||
if message_role == "user" and message_content and isinstance(message_content, list):
|
||||
message_content_types = cast(list[OpenAIMessageContentListBlock], message_content)
|
||||
for i, content_item in enumerate(message_content):
|
||||
message_content_types[i] = self._transform_content_item(
|
||||
cast(OpenAIMessageContentListBlock, content_item)
|
||||
)
|
||||
return hoisted_messages
|
||||
return [self._user_message_for_openai(message) for message in hoisted_messages]
|
||||
|
||||
@staticmethod
|
||||
def _user_content_parts(message: AllMessageValues) -> Sequence[OpenAIMessageContentListBlock] | None:
|
||||
message_content: Final = message.get("content")
|
||||
if message.get("role") != "user" or not message_content or not isinstance(message_content, list):
|
||||
return None
|
||||
return cast(Sequence[OpenAIMessageContentListBlock], message_content)
|
||||
|
||||
def _user_message_for_openai(self, message: AllMessageValues) -> AllMessageValues:
|
||||
parts: Final = self._user_content_parts(message)
|
||||
if parts is None:
|
||||
return message
|
||||
return cast( # cast-ok: the same user message with each content part rebuilt for OpenAI
|
||||
AllMessageValues, {**message, "content": [self._transform_content_item(part) for part in parts]}
|
||||
)
|
||||
|
||||
async def _async_user_message_for_openai(self, message: AllMessageValues) -> AllMessageValues:
|
||||
parts: Final = self._user_content_parts(message)
|
||||
if parts is None:
|
||||
return message
|
||||
return cast( # cast-ok: the same user message with each content part rebuilt for OpenAI
|
||||
AllMessageValues, {**message, "content": [await self._async_transform_content_item(part) for part in parts]}
|
||||
)
|
||||
|
||||
def transform_messages(
|
||||
self,
|
||||
|
|
@ -462,23 +494,15 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig):
|
|||
messages: list[AllMessageValues],
|
||||
tools: list["ChatCompletionToolParam"] | None = None,
|
||||
) -> tuple[list[AllMessageValues], list["ChatCompletionToolParam"] | None]:
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
||||
filter_value_from_dict,
|
||||
)
|
||||
from litellm.types.llms.openai import ChatCompletionToolParam
|
||||
|
||||
for i, message in enumerate(messages):
|
||||
messages[i] = cast(
|
||||
AllMessageValues,
|
||||
filter_value_from_dict(message, "cache_control"),
|
||||
)
|
||||
if tools is not None:
|
||||
for i, tool in enumerate(tools):
|
||||
tools[i] = cast(
|
||||
ChatCompletionToolParam,
|
||||
filter_value_from_dict(tool, "cache_control"),
|
||||
)
|
||||
return messages, tools
|
||||
new_messages: Final = [cast(AllMessageValues, without_cache_control(message)) for message in messages]
|
||||
new_tools: Final = (
|
||||
[cast(ChatCompletionToolParam, without_cache_control(tool)) for tool in tools]
|
||||
if tools is not None
|
||||
else None
|
||||
)
|
||||
return new_messages, new_tools
|
||||
|
||||
def translate_developer_role_to_system_role(
|
||||
self,
|
||||
|
|
@ -527,6 +551,7 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig):
|
|||
tools: Sequence["ChatCompletionToolParam"] | None,
|
||||
prompt_cache_options: object,
|
||||
litellm_params: Mapping[str, object],
|
||||
request_params: Mapping[str, object],
|
||||
) -> tuple[Sequence[AllMessageValues], Sequence["ChatCompletionToolParam"] | None]:
|
||||
"""
|
||||
A deployment that reads OpenAI's ``prompt_cache_breakpoint`` (gpt-5.6+ on
|
||||
|
|
@ -543,10 +568,11 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig):
|
|||
model, custom_llm_provider, api_base, prompt_cache_options
|
||||
)
|
||||
if not reads_breakpoints and self._should_preserve_cache_control_for_endpoint(custom_llm_provider, api_base):
|
||||
return messages, tools
|
||||
drops_breakpoints: Final = not reads_breakpoints and (
|
||||
litellm.drop_params is True or litellm_params.get("drop_params") is True
|
||||
)
|
||||
return (
|
||||
_claude_messages_within_cache_control_cap(model, messages, tools, litellm_params, request_params),
|
||||
tools,
|
||||
)
|
||||
drops_breakpoints: Final = not reads_breakpoints and _drops_params(litellm_params)
|
||||
renamed: Final = (
|
||||
[AnthropicCacheControlHook.cache_control_as_prompt_cache_breakpoints(message) for message in messages]
|
||||
if reads_breakpoints
|
||||
|
|
@ -561,7 +587,13 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig):
|
|||
if drops_breakpoints
|
||||
else list(tools)
|
||||
)
|
||||
return self.remove_cache_control_flag_from_messages_and_tools(model=model, messages=renamed, tools=kept_tools)
|
||||
sent_messages, sent_tools = self.remove_cache_control_flag_from_messages_and_tools(
|
||||
model=model, messages=renamed, tools=kept_tools
|
||||
)
|
||||
return (
|
||||
_claude_messages_within_cache_control_cap(model, sent_messages, sent_tools, litellm_params, request_params),
|
||||
sent_tools,
|
||||
)
|
||||
|
||||
def _sanitized_tools_update_for_openai(
|
||||
self,
|
||||
|
|
@ -623,6 +655,7 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig):
|
|||
tools=optional_params.get("tools", []),
|
||||
prompt_cache_options=_prompt_cache_options(optional_params),
|
||||
litellm_params=litellm_params,
|
||||
request_params=optional_params,
|
||||
)
|
||||
if tools is not None and len(tools) > 0:
|
||||
optional_params["tools"] = tools
|
||||
|
|
@ -654,6 +687,7 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig):
|
|||
tools=optional_params.get("tools", []),
|
||||
prompt_cache_options=_prompt_cache_options(optional_params),
|
||||
litellm_params=litellm_params,
|
||||
request_params=optional_params,
|
||||
)
|
||||
if tools is not None and len(tools) > 0:
|
||||
optional_params["tools"] = tools
|
||||
|
|
|
|||
|
|
@ -153,12 +153,13 @@ class OpenAIOSeriesConfig(OpenAIGPTConfig):
|
|||
- role: system ==> translate to role 'user'
|
||||
"""
|
||||
_supports_system_messages: Final = supports_system_messages(model, "openai")
|
||||
for i, message in enumerate(messages):
|
||||
transformed_messages: Final = list(messages)
|
||||
for i, message in enumerate(transformed_messages):
|
||||
if message["role"] == "system" and not _supports_system_messages:
|
||||
new_message = ChatCompletionUserMessage(content=message["content"], role="user")
|
||||
messages[i] = new_message # Replace the old message with the new one
|
||||
transformed_messages[i] = new_message # Replace the old message with the new one
|
||||
|
||||
if is_async:
|
||||
return super()._transform_messages(messages, model, is_async=cast(Literal[True], True))
|
||||
return super()._transform_messages(transformed_messages, model, is_async=cast(Literal[True], True))
|
||||
else:
|
||||
return super()._transform_messages(messages, model, is_async=cast(Literal[False], False))
|
||||
return super()._transform_messages(transformed_messages, model, is_async=cast(Literal[False], False))
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ Common helpers / utils across al OpenAI endpoints
|
|||
"""
|
||||
|
||||
import asyncio
|
||||
import copy
|
||||
import hashlib
|
||||
import inspect
|
||||
import json
|
||||
|
|
@ -31,7 +32,7 @@ if TYPE_CHECKING:
|
|||
import litellm
|
||||
from litellm.exceptions import error_response_without_credentials
|
||||
from litellm.litellm_core_utils.completion_timeout import CompletionTimeout
|
||||
from litellm.litellm_core_utils.core_helpers import as_str_mapping, decoded_json_object
|
||||
from litellm.litellm_core_utils.core_helpers import as_str_mapping, decoded_json_object, iter_items
|
||||
from litellm.litellm_core_utils.reasoning_effort_utils import warn_dropped_reasoning_params
|
||||
from litellm.litellm_core_utils.token_counter import token_counter, usage_tokenizer_model
|
||||
from litellm.llms.base_llm.chat.transformation import BaseLLMException
|
||||
|
|
@ -43,6 +44,47 @@ from litellm.llms.custom_httpx.http_handler import (
|
|||
)
|
||||
|
||||
|
||||
def _is_plain_dict(value: object) -> bool:
|
||||
return isinstance(value, dict)
|
||||
|
||||
|
||||
def _is_list(value: object) -> bool:
|
||||
return isinstance(value, list)
|
||||
|
||||
|
||||
def _contains_cache_control(value: object) -> bool:
|
||||
pending: Final[list[object]] = [value] # mutable-ok: a local worklist, never returned or stored
|
||||
while pending:
|
||||
node = pending.pop()
|
||||
mapping = as_str_mapping(node)
|
||||
if mapping is not None:
|
||||
if "cache_control" in mapping:
|
||||
return True
|
||||
pending.extend(mapping.values())
|
||||
elif _is_list(node):
|
||||
pending.extend(iter_items(node))
|
||||
return False
|
||||
|
||||
|
||||
def without_cache_control(item: object) -> object:
|
||||
"""Return `item` with every `cache_control` key removed, without touching the original.
|
||||
|
||||
`filter_value_from_dict` deletes the key in place and recurses into nested dicts and
|
||||
lists, so the caller's own message, tool or input object has to be copied first: it
|
||||
may reuse the same list on a provider that does support prompt caching. The copy is
|
||||
skipped when there is nothing to strip, which keeps the object identity the
|
||||
pass-through paths rely on and avoids a deep copy on every request.
|
||||
"""
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import filter_value_from_dict
|
||||
|
||||
if not _is_plain_dict(item) or not _contains_cache_control(item):
|
||||
return item
|
||||
copied: Final = copy.deepcopy(item)
|
||||
if isinstance(copied, dict):
|
||||
filter_value_from_dict(copied, "cache_control")
|
||||
return copied
|
||||
|
||||
|
||||
def _get_client_init_params(cls: type) -> tuple[str, ...]:
|
||||
"""Extract __init__ parameter names (excluding 'self') from a class."""
|
||||
return tuple(p for p in inspect.signature(cls.__init__).parameters if p != "self")
|
||||
|
|
|
|||
|
|
@ -354,10 +354,10 @@ class OpenAIDecisionsConfig(BaseDecisionsConfig):
|
|||
) -> str | None:
|
||||
return resolve_openai_bearer_token(api_key=api_key, api_base=api_base, litellm_params=litellm_params)
|
||||
|
||||
def supports_audio_input(self, model: str, custom_llm_provider: str) -> bool:
|
||||
def supports_audio_input(self, model: str, custom_llm_provider: str, deployment_flag: bool | None) -> bool:
|
||||
return False
|
||||
|
||||
def supports_video_input(self, model: str, custom_llm_provider: str) -> bool:
|
||||
def supports_video_input(self, model: str, custom_llm_provider: str, deployment_flag: bool | None) -> bool:
|
||||
return False
|
||||
|
||||
def transform_decisions_request(
|
||||
|
|
|
|||
|
|
@ -52,7 +52,7 @@ from litellm.types.router import GenericLiteLLMParams
|
|||
from litellm.types.utils import LlmProviders
|
||||
from litellm.types.workload_identity import OPENAI_WIF_KWARGS_KEYS
|
||||
|
||||
from ..common_utils import OpenAIError
|
||||
from ..common_utils import OpenAIError, without_cache_control
|
||||
from ..workload_identity import get_workload_identity_bearer_token, resolve_openai_workload_identity_config
|
||||
|
||||
OPENAI_RESPONSES_API_MIN_MAX_OUTPUT_TOKENS: Final = 16
|
||||
|
|
@ -658,13 +658,6 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig):
|
|||
input=self._replayable_input(marker_safe_input, reads_breakpoints, maps_anthropic_tool_search),
|
||||
tools=mapped_tools,
|
||||
)
|
||||
if marker_safe_input is not validated_input:
|
||||
self._strip_caller_input_as_if_unmarked(
|
||||
model=model,
|
||||
caller_input=validated_input,
|
||||
reads_breakpoints=reads_breakpoints,
|
||||
maps_anthropic_tool_search=maps_anthropic_tool_search,
|
||||
)
|
||||
object_schema_tools: Final = self._tools_with_object_parameters(model=model, tools=stripped_tools)
|
||||
sanitized_tools: Final = self._sanitized_tool_schemas_for_openai(
|
||||
model=model, tools=object_schema_tools, litellm_params=litellm_params
|
||||
|
|
@ -679,17 +672,6 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig):
|
|||
return marked_input
|
||||
return input_without_unanswered_anthropic_tool_search_calls(marked_input)
|
||||
|
||||
def _strip_caller_input_as_if_unmarked(
|
||||
self,
|
||||
model: str,
|
||||
caller_input: str | ResponseInputParam,
|
||||
reads_breakpoints: bool,
|
||||
maps_anthropic_tool_search: bool,
|
||||
) -> None:
|
||||
self.remove_cache_control_flag_from_input_and_tools(
|
||||
model=model, input=self._replayable_input(caller_input, reads_breakpoints, maps_anthropic_tool_search)
|
||||
)
|
||||
|
||||
def _input_with_wire_safe_prompt_cache_breakpoints(
|
||||
self,
|
||||
model: str,
|
||||
|
|
@ -800,24 +782,29 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig):
|
|||
the chat path. Strips Anthropic-only `cache_control` markers from
|
||||
Responses API input content blocks and tools.
|
||||
|
||||
`filter_value_from_dict` mutates each dict in place, so the same
|
||||
objects are returned.
|
||||
`filter_value_from_dict` deletes the key in place and recurses, so each
|
||||
item is copied first: the caller keeps its own input list and may reuse
|
||||
it on a provider that does support prompt caching.
|
||||
"""
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
||||
filter_value_from_dict,
|
||||
new_input: Final = (
|
||||
cast( # cast-ok: the comprehension rebuilds the input item for item
|
||||
ResponseInputParam,
|
||||
[without_cache_control(item) for item in input],
|
||||
)
|
||||
if isinstance(input, list)
|
||||
else input
|
||||
)
|
||||
|
||||
if isinstance(input, list):
|
||||
for item in input:
|
||||
if isinstance(item, dict):
|
||||
filter_value_from_dict(cast(dict, item), "cache_control")
|
||||
new_tools: Final = (
|
||||
cast( # cast-ok: the comprehension rebuilds the tool list item for item
|
||||
Sequence[ALL_RESPONSES_API_TOOL_PARAMS],
|
||||
[without_cache_control(tool) for tool in tools],
|
||||
)
|
||||
if tools is not None
|
||||
else None
|
||||
)
|
||||
|
||||
if tools is not None:
|
||||
for tool in tools:
|
||||
if isinstance(tool, dict):
|
||||
filter_value_from_dict(cast(dict, tool), "cache_control")
|
||||
|
||||
return input, tools
|
||||
return new_input, new_tools
|
||||
|
||||
def _drop_bridge_minted_reasoning_items(self, input: str | ResponseInputParam) -> str | ResponseInputParam:
|
||||
if self.custom_llm_provider not in _PROVIDERS_REPLAYING_ONLY_THEIR_OWN_REASONING or not isinstance(input, list):
|
||||
|
|
|
|||
|
|
@ -125,6 +125,20 @@ The provider inherits all request/response handling from OpenAI's Responses API
|
|||
|
||||
If `supported_endpoints` is omitted, it defaults to `[]` (only chat completions, which is always enabled for JSON providers).
|
||||
|
||||
When only some models serve an endpoint natively, list them under `model_supported_endpoints`. Every other model reaches `/v1/responses` and `/v1/messages` through chat completions:
|
||||
|
||||
```json
|
||||
{
|
||||
"your_provider": {
|
||||
"supported_endpoints": ["/v1/chat/completions"],
|
||||
"model_supported_endpoints": {
|
||||
"model-a": ["/v1/messages"],
|
||||
"model-b": ["/v1/responses"]
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### How It Works
|
||||
|
||||
1. `json_loader.py` checks `supported_endpoints` for `/v1/responses`
|
||||
|
|
|
|||
|
|
@ -3,11 +3,16 @@
|
|||
## Allows jina ai embedding calls - which don't allow 'encoding_format' in payload.
|
||||
|
||||
import json
|
||||
import re
|
||||
from collections.abc import Mapping
|
||||
from types import MappingProxyType
|
||||
from typing import Final
|
||||
|
||||
import httpx
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
import litellm
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.llms.custom_httpx.http_handler import (
|
||||
AsyncHTTPHandler,
|
||||
HTTPHandler,
|
||||
|
|
@ -17,6 +22,57 @@ from litellm.types.utils import EmbeddingResponse
|
|||
|
||||
from ..common_utils import OpenAILikeBase, OpenAILikeError
|
||||
|
||||
_KEY_HEADERS_ADAPTER: Final = TypeAdapter(Mapping[str, str])
|
||||
_EXTRA_HEADERS_ADAPTER: Final = TypeAdapter(Mapping[str, object])
|
||||
_TRANSPORT_HEADER_NAMES: Final = frozenset(
|
||||
{
|
||||
"host",
|
||||
"content-length",
|
||||
"transfer-encoding",
|
||||
"connection",
|
||||
"keep-alive",
|
||||
"proxy-connection",
|
||||
"te",
|
||||
"trailer",
|
||||
"upgrade",
|
||||
"expect",
|
||||
}
|
||||
)
|
||||
_HEADER_NAME: Final = re.compile(r"[!#$%&'*+\-.^_`|~0-9A-Za-z]+")
|
||||
_HEADER_VALUE: Final = re.compile(r"[\t\x20-\x7e]*")
|
||||
|
||||
|
||||
def _header_text(name: str, value: object) -> str | None:
|
||||
if isinstance(value, bool) or not isinstance(value, str | int | float):
|
||||
return None
|
||||
if _HEADER_NAME.fullmatch(name) is None:
|
||||
return None
|
||||
return str(value).strip(" \t\r\n")
|
||||
|
||||
|
||||
def _extra_headers_as_text(extra_headers: Mapping[str, object], model: str, drop_params: bool) -> Mapping[str, str]:
|
||||
caller_set: Final = {
|
||||
name: value for name, value in extra_headers.items() if name.lower() not in _TRANSPORT_HEADER_NAMES
|
||||
}
|
||||
texts: Final = MappingProxyType({name: _header_text(name, value) for name, value in caller_set.items()})
|
||||
refused: Final = sorted(name for name, text in texts.items() if text is None)
|
||||
if refused and not drop_params:
|
||||
raise litellm.UnsupportedParamsError(
|
||||
message=f"extra_headers {', '.join(map(repr, refused))} cannot be sent. A header needs a name made of "
|
||||
"letters, digits or !#$%&'*+-.^_`|~ and a text or number value. "
|
||||
"To drop them from the call, set `litellm.drop_params = True`.",
|
||||
model=model,
|
||||
)
|
||||
named: Final = MappingProxyType({name: text for name, text in texts.items() if text is not None})
|
||||
unsendable: Final = sorted(name for name, text in named.items() if _HEADER_VALUE.fullmatch(text) is None)
|
||||
if unsendable:
|
||||
verbose_logger.warning(
|
||||
"extra_headers %s dropped for model %r, since a header value must be printable ASCII on one line",
|
||||
unsendable,
|
||||
model,
|
||||
)
|
||||
return MappingProxyType({name: text for name, text in named.items() if name not in unsendable})
|
||||
|
||||
|
||||
class OpenAILikeEmbeddingHandler(OpenAILikeBase):
|
||||
def __init__(self, **kwargs):
|
||||
|
|
@ -94,6 +150,7 @@ class OpenAILikeEmbeddingHandler(OpenAILikeBase):
|
|||
aembedding=None,
|
||||
custom_endpoint: bool | None = None,
|
||||
headers: dict | None = None,
|
||||
drop_params: bool = False,
|
||||
) -> EmbeddingResponse:
|
||||
api_base, headers = self._validate_environment(
|
||||
api_base=api_base,
|
||||
|
|
@ -103,8 +160,20 @@ class OpenAILikeEmbeddingHandler(OpenAILikeBase):
|
|||
custom_endpoint=custom_endpoint,
|
||||
)
|
||||
model = model
|
||||
filtered_optional_params: Final = {k: v for k, v in optional_params.items() if v not in (None, "")}
|
||||
data: Final = {"model": model, "input": input, **filtered_optional_params}
|
||||
data: Final = {
|
||||
"model": model,
|
||||
"input": input,
|
||||
**{k: v for k, v in optional_params.items() if k != "extra_headers" and v not in (None, "")},
|
||||
}
|
||||
extra_headers: Final = _extra_headers_as_text(
|
||||
_EXTRA_HEADERS_ADAPTER.validate_python(optional_params.get("extra_headers") or {}), model, drop_params
|
||||
)
|
||||
key_headers: Final = _KEY_HEADERS_ADAPTER.validate_python(headers)
|
||||
extra_header_names: Final = frozenset(name.lower() for name in extra_headers)
|
||||
request_headers: Final = {
|
||||
**{name: value for name, value in key_headers.items() if name.lower() not in extra_header_names},
|
||||
**extra_headers,
|
||||
}
|
||||
|
||||
## LOGGING
|
||||
logging_obj.pre_call(
|
||||
|
|
@ -123,7 +192,7 @@ class OpenAILikeEmbeddingHandler(OpenAILikeBase):
|
|||
api_key=api_key,
|
||||
timeout=timeout,
|
||||
client=client,
|
||||
headers=headers,
|
||||
headers=request_headers,
|
||||
)
|
||||
if client is None or isinstance(client, AsyncHTTPHandler):
|
||||
self.client = HTTPHandler(timeout=timeout)
|
||||
|
|
@ -134,7 +203,7 @@ class OpenAILikeEmbeddingHandler(OpenAILikeBase):
|
|||
try:
|
||||
response: Final = self.client.post(
|
||||
api_base,
|
||||
headers=headers,
|
||||
headers=request_headers,
|
||||
data=json.dumps(data),
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@ JSON-based provider configuration loader for OpenAI-compatible providers.
|
|||
"""
|
||||
|
||||
import json
|
||||
from collections.abc import Mapping
|
||||
from collections.abc import Mapping, Sequence
|
||||
from pathlib import Path
|
||||
from typing import Final
|
||||
|
||||
|
|
@ -14,6 +14,7 @@ from litellm.types.utils import ProviderSpecificModelInfo
|
|||
|
||||
_PROVIDER_INFO_ADAPTER: Final = TypeAdapter(ProviderSpecificModelInfo)
|
||||
_SPECIAL_HANDLING_ADAPTER: Final = TypeAdapter(Mapping[str, bool])
|
||||
_MODEL_ENDPOINTS_ADAPTER: Final = TypeAdapter(Mapping[str, tuple[str, ...]])
|
||||
|
||||
|
||||
class SimpleProviderConfig:
|
||||
|
|
@ -30,7 +31,10 @@ class SimpleProviderConfig:
|
|||
self.special_handling: Mapping[str, bool] = _SPECIAL_HANDLING_ADAPTER.validate_python(
|
||||
data.get("special_handling", {})
|
||||
)
|
||||
self.supported_endpoints = data.get("supported_endpoints", [])
|
||||
self.supported_endpoints: Sequence[str] = data.get("supported_endpoints", [])
|
||||
self.model_supported_endpoints: Mapping[str, tuple[str, ...]] = _MODEL_ENDPOINTS_ADAPTER.validate_python(
|
||||
data.get("model_supported_endpoints", {})
|
||||
)
|
||||
self.provider_info: ProviderSpecificModelInfo = (
|
||||
_PROVIDER_INFO_ADAPTER.validate_python(data["provider_info"]) if "provider_info" in data else {}
|
||||
)
|
||||
|
|
@ -90,6 +94,23 @@ class JSONProviderRegistry:
|
|||
return False
|
||||
return "/v1/responses" in provider.supported_endpoints
|
||||
|
||||
@classmethod
|
||||
def supports_embeddings(cls, slug: str) -> bool:
|
||||
provider: Final = cls._providers.get(slug)
|
||||
if provider is None:
|
||||
return False
|
||||
return "/v1/embeddings" in provider.supported_endpoints
|
||||
|
||||
@classmethod
|
||||
def serves_endpoint(cls, slug: str, model: str | None, endpoint: str) -> bool:
|
||||
provider: Final = cls._providers.get(slug)
|
||||
if provider is None:
|
||||
return False
|
||||
model_endpoints: Final = (
|
||||
provider.model_supported_endpoints.get(model.removeprefix(f"{slug}/"), ()) if model else ()
|
||||
)
|
||||
return endpoint in provider.supported_endpoints or endpoint in model_endpoints
|
||||
|
||||
@classmethod
|
||||
def list_providers(cls) -> list:
|
||||
"""List all registered provider slugs"""
|
||||
|
|
|
|||
|
|
@ -211,7 +211,7 @@
|
|||
"param_mappings": {
|
||||
"max_completion_tokens": "max_tokens"
|
||||
},
|
||||
"supported_endpoints": ["/v1/chat/completions", "/v1/responses", "/v1/embeddings"]
|
||||
"supported_endpoints": ["/v1/chat/completions", "/v1/responses"]
|
||||
},
|
||||
"scx-ai": {
|
||||
"base_url": "https://api.scx.ai/v1",
|
||||
|
|
@ -231,6 +231,16 @@
|
|||
"api_base_env": "PRISM_API_BASE",
|
||||
"supported_endpoints": ["/v1/chat/completions", "/v1/responses", "/v1/messages"]
|
||||
},
|
||||
"hostinger_router": {
|
||||
"base_url": "https://router.hostinger.com/v1",
|
||||
"api_key_env": "HOSTINGER_ROUTER_API_KEY",
|
||||
"api_base_env": "HOSTINGER_ROUTER_API_BASE",
|
||||
"supported_endpoints": ["/v1/chat/completions", "/v1/embeddings"],
|
||||
"model_supported_endpoints": {
|
||||
"claude-sonnet-5": ["/v1/messages"],
|
||||
"gpt-5.2": ["/v1/responses"]
|
||||
}
|
||||
},
|
||||
"sail": {
|
||||
"base_url": "https://api.sailresearch.com/v1",
|
||||
"api_key_env": "SAIL_API_KEY",
|
||||
|
|
|
|||
|
|
@ -110,38 +110,23 @@ def separate_cached_messages(
|
|||
messages: list[AllMessageValues],
|
||||
) -> tuple[list[AllMessageValues], list[AllMessageValues]]:
|
||||
"""
|
||||
Returns separated cached and non-cached messages.
|
||||
|
||||
Args:
|
||||
messages: List of messages to be separated.
|
||||
|
||||
Returns:
|
||||
Tuple containing:
|
||||
- cached_messages: List of cached messages.
|
||||
- non_cached_messages: List of non-cached messages.
|
||||
Returns the first continuous block of cached messages, plus every system message once
|
||||
any message is cached, since generateContent rejects a system_instruction beside a
|
||||
cachedContent, and the remaining messages.
|
||||
"""
|
||||
cached_messages: list[AllMessageValues] = []
|
||||
non_cached_messages: list[AllMessageValues] = []
|
||||
filtered_messages: Final = [(idx, message) for idx, message in enumerate(messages) if is_cached_message(message)]
|
||||
if not filtered_messages:
|
||||
return [], messages
|
||||
|
||||
# Extract cached messages and their indices
|
||||
filtered_messages: Final[list[tuple[int, AllMessageValues]]] = []
|
||||
for idx, message in enumerate(messages):
|
||||
if is_cached_message(message=message):
|
||||
filtered_messages.append((idx, message))
|
||||
|
||||
# Validate only one block of continuous cached messages
|
||||
last_continuous_block_idx: Final = get_first_continuous_block_idx(filtered_messages)
|
||||
# Separate messages based on the block of cached messages
|
||||
if filtered_messages and last_continuous_block_idx is not None:
|
||||
first_cached_idx: Final = filtered_messages[0][0]
|
||||
last_cached_idx: Final = filtered_messages[last_continuous_block_idx][0]
|
||||
|
||||
cached_messages = messages[first_cached_idx : last_cached_idx + 1]
|
||||
non_cached_messages = messages[:first_cached_idx] + messages[last_cached_idx + 1 :]
|
||||
else:
|
||||
non_cached_messages = messages
|
||||
|
||||
return cached_messages, non_cached_messages
|
||||
cached_block: Final = range(filtered_messages[0][0], filtered_messages[last_continuous_block_idx][0] + 1)
|
||||
goes_in_cache: Final = tuple(
|
||||
idx in cached_block or message.get("role") == "system" for idx, message in enumerate(messages)
|
||||
)
|
||||
return (
|
||||
[message for message, cached in zip(messages, goes_in_cache) if cached],
|
||||
[message for message, cached in zip(messages, goes_in_cache) if not cached],
|
||||
)
|
||||
|
||||
|
||||
def cached_messages_end_on_supported_turn(cached_messages: Sequence[AllMessageValues]) -> bool:
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ Translation from OpenAI's `/chat/completions` endpoint to IBM WatsonX's `/text/c
|
|||
Docs: https://cloud.ibm.com/apidocs/watsonx-ai#text-chat
|
||||
"""
|
||||
|
||||
import copy
|
||||
from typing import Final
|
||||
|
||||
from litellm import verbose_logger
|
||||
|
|
@ -57,7 +58,7 @@ class IBMWatsonXChatConfig(IBMWatsonXMixin, OpenAIGPTConfig):
|
|||
_tools = non_default_params.pop("tools", None)
|
||||
if _tools is not None:
|
||||
# remove 'additionalProperties' from tools
|
||||
_tools = remove_additional_properties(_tools)
|
||||
_tools = remove_additional_properties(copy.deepcopy(_tools)) # pyright: ignore[reportUnknownArgumentType] # tools arrive untyped through the bare-dict params
|
||||
# remove 'strict' from tools
|
||||
_tools = remove_strict_from_schema(_tools)
|
||||
if _tools is not None:
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
import copy
|
||||
from collections.abc import AsyncIterator, Iterator, Mapping
|
||||
from types import MappingProxyType
|
||||
from typing import Any, Final
|
||||
|
|
@ -208,7 +209,8 @@ class XAIChatConfig(OpenAIGPTConfig):
|
|||
tools = [
|
||||
filtered_tool
|
||||
for filtered_tool in (
|
||||
_FILTERED_TOOL.validate_python(filter_value_from_dict(tool, "strict")) for tool in value
|
||||
_FILTERED_TOOL.validate_python(filter_value_from_dict(copy.deepcopy(tool), "strict")) # pyright: ignore[reportUnknownArgumentType] # tools arrive untyped through the bare-dict params
|
||||
for tool in value
|
||||
)
|
||||
if filtered_tool is not None
|
||||
]
|
||||
|
|
|
|||
|
|
@ -6809,6 +6809,23 @@ async def aembedding(*args, **kwargs) -> EmbeddingResponse:
|
|||
)
|
||||
|
||||
|
||||
def _embedding_headers(param: str, configured: object, model: str, drop_params: bool) -> Mapping[str, object]:
|
||||
if not configured:
|
||||
return {}
|
||||
if isinstance(configured, Mapping):
|
||||
return dict(cast(Mapping[str, object], configured)) # cast-ok: header names from untyped request JSON
|
||||
try:
|
||||
return dict(cast(Iterable[tuple[str, object]], configured)) # cast-ok: name and value pairs from request JSON
|
||||
except (TypeError, ValueError):
|
||||
if drop_params:
|
||||
return {}
|
||||
raise litellm.UnsupportedParamsError(
|
||||
message=f"{param} must be a JSON object of header names to values. "
|
||||
"To drop it from the call, set `litellm.drop_params = True`.",
|
||||
model=model,
|
||||
)
|
||||
|
||||
|
||||
# fmt: off
|
||||
|
||||
# Overload for when aembedding=True (returns coroutine)
|
||||
|
|
@ -6923,11 +6940,10 @@ def embedding(
|
|||
azure_ad_token_provider: Final = kwargs.get("azure_ad_token_provider", None)
|
||||
aembedding: Final[bool | None] = kwargs.get("aembedding", None)
|
||||
extra_headers: Final = kwargs.get("extra_headers", None)
|
||||
headers = kwargs.get("headers", None) or extra_headers
|
||||
if headers is None:
|
||||
headers = {}
|
||||
if extra_headers is not None:
|
||||
headers.update(extra_headers)
|
||||
headers = {
|
||||
**_embedding_headers("headers", kwargs.get("headers"), model, drop_unsupported_params),
|
||||
**_embedding_headers("extra_headers", extra_headers, model, drop_unsupported_params),
|
||||
}
|
||||
# Inject proxy auth headers if configured
|
||||
if litellm.proxy_auth is not None:
|
||||
try:
|
||||
|
|
@ -7091,7 +7107,7 @@ def embedding(
|
|||
# set API KEY
|
||||
api_key = api_key or litellm.api_key or litellm.openai_key or get_secret_str("OPENAI_API_KEY")
|
||||
|
||||
if headers is not None and headers != {}:
|
||||
if headers:
|
||||
optional_params["extra_headers"] = headers
|
||||
|
||||
requested_encoding_format: Final = (
|
||||
|
|
@ -7183,14 +7199,15 @@ def embedding(
|
|||
custom_llm_provider == "openai_like"
|
||||
or custom_llm_provider == "llamafile"
|
||||
or custom_llm_provider == "lm_studio"
|
||||
or JSONProviderRegistry.supports_embeddings(custom_llm_provider)
|
||||
):
|
||||
api_base = api_base or litellm.api_base or get_secret_str("OPENAI_LIKE_API_BASE")
|
||||
|
||||
# set API KEY
|
||||
if api_key is None:
|
||||
api_key = api_key or litellm.api_key or litellm.openai_like_key or get_secret_str("OPENAI_LIKE_API_KEY")
|
||||
if api_key is None and not JSONProviderRegistry.supports_embeddings(custom_llm_provider):
|
||||
api_key = litellm.api_key or litellm.openai_like_key or get_secret_str("OPENAI_LIKE_API_KEY")
|
||||
|
||||
if headers is not None and headers != {}:
|
||||
if headers:
|
||||
optional_params["extra_headers"] = headers
|
||||
|
||||
## EMBEDDING CALL
|
||||
|
|
@ -7205,10 +7222,9 @@ def embedding(
|
|||
optional_params=optional_params,
|
||||
client=client,
|
||||
aembedding=aembedding,
|
||||
drop_params=drop_unsupported_params,
|
||||
)
|
||||
elif custom_llm_provider == "oci":
|
||||
if headers is None:
|
||||
headers = {}
|
||||
response = base_llm_http_handler.embedding(
|
||||
model=model,
|
||||
input=input,
|
||||
|
|
@ -7233,11 +7249,6 @@ def embedding(
|
|||
or litellm.api_key
|
||||
)
|
||||
|
||||
# Use the merged headers variable (already merged at the top of the function)
|
||||
# Don't overwrite it with just extra_headers
|
||||
if headers is None:
|
||||
headers = {}
|
||||
|
||||
response = base_llm_http_handler.embedding(
|
||||
model=model,
|
||||
input=input,
|
||||
|
|
|
|||
|
|
@ -39687,6 +39687,57 @@
|
|||
"/v1/images/generations"
|
||||
]
|
||||
},
|
||||
"hostinger_router/gpt-5.2": {
|
||||
"cache_read_input_token_cost": 0.00000021,
|
||||
"input_cost_per_token": 0.0000021,
|
||||
"litellm_provider": "hostinger_router",
|
||||
"max_input_tokens": 272000,
|
||||
"max_output_tokens": 128000,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 0.0000168,
|
||||
"supports_function_calling": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_vision": true,
|
||||
"supported_endpoints": ["/v1/chat/completions", "/v1/responses"],
|
||||
"source": "https://router.hostinger.com/v1/models"
|
||||
},
|
||||
"hostinger_router/claude-sonnet-5": {
|
||||
"cache_creation_input_token_cost": 3.3e-06,
|
||||
"cache_creation_input_token_cost_above_1hr": 5.28e-06,
|
||||
"cache_read_input_token_cost": 2.64e-07,
|
||||
"input_cost_per_token": 2.64e-06,
|
||||
"litellm_provider": "hostinger_router",
|
||||
"max_input_tokens": 1000000,
|
||||
"max_output_tokens": 128000,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 1.32e-05,
|
||||
"supports_function_calling": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": true,
|
||||
"supports_response_schema": true,
|
||||
"supported_endpoints": ["/v1/chat/completions", "/v1/messages"],
|
||||
"source": "https://router.hostinger.com/v1/models"
|
||||
},
|
||||
"hostinger_router/text-embedding-3-large": {
|
||||
"input_cost_per_token": 1.716e-07,
|
||||
"litellm_provider": "hostinger_router",
|
||||
"max_input_tokens": 8191,
|
||||
"mode": "embedding",
|
||||
"output_cost_per_token": 0.0,
|
||||
"source": "https://router.hostinger.com/v1/models"
|
||||
},
|
||||
"hostinger_router/text-embedding-3-small": {
|
||||
"input_cost_per_token": 3e-08,
|
||||
"litellm_provider": "hostinger_router",
|
||||
"max_input_tokens": 8191,
|
||||
"mode": "embedding",
|
||||
"output_cost_per_token": 0.0,
|
||||
"source": "https://router.hostinger.com/v1/models"
|
||||
},
|
||||
"hyperbolic/NousResearch/Hermes-3-Llama-3.1-70B": {
|
||||
"input_cost_per_token": 1.2e-07,
|
||||
"litellm_provider": "hyperbolic",
|
||||
|
|
|
|||
|
|
@ -1087,6 +1087,32 @@
|
|||
"generateContent": false
|
||||
}
|
||||
},
|
||||
"hostinger_router": {
|
||||
"display_name": "Hostinger Router (`hostinger_router`)",
|
||||
"url": "https://docs.litellm.ai/docs/providers/hostinger_router",
|
||||
"endpoints": {
|
||||
"chat_completions": true,
|
||||
"responses": true,
|
||||
"text_completion": true,
|
||||
"embeddings": true,
|
||||
"image_generations": false,
|
||||
"image_edits": false,
|
||||
"audio_transcriptions": false,
|
||||
"audio_speech": false,
|
||||
"rerank": false,
|
||||
"ocr": false,
|
||||
"search": false,
|
||||
"moderations": false,
|
||||
"realtime": false,
|
||||
"video_generations": false,
|
||||
"batches": false,
|
||||
"messages": true,
|
||||
"interactions": true,
|
||||
"systemone": false,
|
||||
"a2a": true,
|
||||
"generateContent": true
|
||||
}
|
||||
},
|
||||
"crusoe": {
|
||||
"display_name": "Crusoe (`crusoe`)",
|
||||
"url": "https://docs.litellm.ai/docs/providers/crusoe",
|
||||
|
|
|
|||
|
|
@ -52,7 +52,11 @@ from litellm.exceptions import BlockedPiiEntityError, GuardrailRaisedException
|
|||
from litellm.integrations.custom_guardrail import guardrail_request_data_with_streaming, is_guardrail_intervention
|
||||
from litellm.litellm_core_utils.aws_partition import get_aws_dns_suffix
|
||||
from litellm.litellm_core_utils.fallback_generalizations import match_routing_generalization
|
||||
from litellm.llms.anthropic.common_utils import AnthropicModelInfo, merge_anthropic_beta_headers
|
||||
from litellm.llms.anthropic.common_utils import (
|
||||
AnthropicModelInfo,
|
||||
anthropic_pass_through_api_base,
|
||||
merge_anthropic_beta_headers,
|
||||
)
|
||||
from litellm.llms.azure.passthrough.transformation import (
|
||||
foreign_azure_deployment,
|
||||
is_azure_body_model_inference_endpoint,
|
||||
|
|
@ -1047,7 +1051,7 @@ async def anthropic_proxy_route(
|
|||
"""
|
||||
[Docs](https://docs.litellm.ai/docs/pass_through/anthropic_completion)
|
||||
"""
|
||||
base_target_url = os.getenv("ANTHROPIC_API_BASE") or os.getenv("ANTHROPIC_BASE_URL") or "https://api.anthropic.com"
|
||||
base_target_url = anthropic_pass_through_api_base()
|
||||
encoded_endpoint = httpx.URL(endpoint).path
|
||||
|
||||
# Ensure endpoint starts with '/' for proper URL construction
|
||||
|
|
|
|||
|
|
@ -71,7 +71,11 @@ from litellm.litellm_core_utils.litellm_logging import get_masked_values
|
|||
from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
|
||||
from litellm.litellm_core_utils.redact_messages import should_redact_message_logging
|
||||
from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
|
||||
from litellm.llms.anthropic.common_utils import is_anthropic_messages_url
|
||||
from litellm.llms.anthropic.common_utils import (
|
||||
anthropic_accepts_user_id,
|
||||
anthropic_upstream_user_id_limit,
|
||||
is_anthropic_messages_url,
|
||||
)
|
||||
from litellm.llms.base_llm.managed_resources.utils import (
|
||||
resolve_passthrough_managed_id_provider,
|
||||
)
|
||||
|
|
@ -503,7 +507,7 @@ class HttpPassThroughEndpointHelpers(BasePassthroughUtils):
|
|||
url: httpx.URL,
|
||||
headers: dict,
|
||||
requested_query_params: dict | None = None,
|
||||
_parsed_body: dict | None = None,
|
||||
_parsed_body: Mapping[str, object] | None = None,
|
||||
forward_multipart: bool = False,
|
||||
) -> httpx.Response:
|
||||
"""
|
||||
|
|
@ -717,8 +721,8 @@ class HttpPassThroughEndpointHelpers(BasePassthroughUtils):
|
|||
# unenforced. Set after the client merge so a request body cannot supply its own budget.
|
||||
#
|
||||
# Only for the built-in provider routes. `get_model_from_request` returns
|
||||
# None for a user-defined pass-through, deliberately: its body is forwarded
|
||||
# verbatim, so `model` there names an UPSTREAM model rather than a
|
||||
# None for a user-defined pass-through, deliberately: the client's `model`
|
||||
# goes upstream unchanged, so it names an UPSTREAM model rather than a
|
||||
# LiteLLM-managed one. Enforcement is therefore skipped on those routes, and
|
||||
# charging a counter anyway would track spend that nothing can refuse, and
|
||||
# would attribute it to a budget the operator scoped to a LiteLLM model that
|
||||
|
|
@ -1187,6 +1191,36 @@ def _unfiltered_headers(headers: Mapping[str, str]) -> Mapping[str, str]:
|
|||
return headers
|
||||
|
||||
|
||||
def _json_object(value: object) -> Mapping[str, object]:
|
||||
return cast("Mapping[str, object]", value) if isinstance(value, Mapping) else MappingProxyType({}) # cast-ok: json
|
||||
|
||||
|
||||
def _anthropic_user_id_to_forward(
|
||||
body: Mapping[str, object],
|
||||
url: str,
|
||||
has_pre_call_guardrails: Callable[[Mapping[str, object]], bool],
|
||||
) -> str | None:
|
||||
user_id: Final = _json_object(body.get("metadata")).get("user_id")
|
||||
if not isinstance(user_id, str):
|
||||
return None
|
||||
max_length: Final = anthropic_upstream_user_id_limit(url)
|
||||
if max_length is None:
|
||||
return None
|
||||
if not anthropic_accepts_user_id(user_id, max_length):
|
||||
verbose_proxy_logger.warning("Not forwarding metadata.user_id because the upstream would refuse it")
|
||||
return None
|
||||
if has_pre_call_guardrails(body):
|
||||
verbose_proxy_logger.debug("Not forwarding metadata.user_id because a pre-call guardrail cannot scan it")
|
||||
return None
|
||||
return user_id
|
||||
|
||||
|
||||
def _with_anthropic_user_id(body: Mapping[str, object] | None, user_id: str | None) -> Mapping[str, object] | None:
|
||||
if body is None or user_id is None:
|
||||
return body
|
||||
return {**body, "metadata": {"user_id": user_id}}
|
||||
|
||||
|
||||
def _eu_jev_client_body(custom_llm_provider: str | None, body: Mapping[str, object]) -> Mapping[str, object] | None:
|
||||
return MappingProxyType(dict(body)) if custom_llm_provider == "eu_jev" else None
|
||||
|
||||
|
|
@ -1423,6 +1457,10 @@ async def pass_through_request(
|
|||
request_method=getattr(request, "method", None),
|
||||
cost_per_request=cost_per_request,
|
||||
)
|
||||
hooked_body: Final = _json_object(_parsed_body)
|
||||
client_user_id: Final = _anthropic_user_id_to_forward(
|
||||
hooked_body, str(url), proxy_logging_obj.request_has_pre_call_guardrails
|
||||
)
|
||||
kwargs = HttpPassThroughEndpointHelpers.init_kwargs_for_pass_through_endpoint( # rebind-ok: pre-existing rebinding on a rename-only line
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
_parsed_body=_parsed_body,
|
||||
|
|
@ -1590,11 +1628,13 @@ async def pass_through_request(
|
|||
else:
|
||||
logging_url = str(url) + "?" + requested_query_params_str
|
||||
|
||||
outgoing_body: Final = cast("Mapping[str, object] | None", _parsed_body) # cast-ok: json
|
||||
upstream_body: Final = _with_anthropic_user_id(outgoing_body, client_user_id)
|
||||
logging_obj.pre_call(
|
||||
input=[{"role": "user", "content": safe_dumps(_parsed_body)}],
|
||||
api_key="",
|
||||
additional_args={
|
||||
"complete_input_dict": _parsed_body,
|
||||
"complete_input_dict": upstream_body,
|
||||
"api_base": str(logging_url),
|
||||
"headers": redact_credential_headers(upstream_headers),
|
||||
},
|
||||
|
|
@ -1634,7 +1674,7 @@ async def pass_through_request(
|
|||
url,
|
||||
params=requested_query_params,
|
||||
headers=upstream_headers,
|
||||
json=_parsed_body,
|
||||
json=upstream_body,
|
||||
)
|
||||
)
|
||||
|
||||
|
|
@ -1733,7 +1773,7 @@ async def pass_through_request(
|
|||
url=url,
|
||||
headers=upstream_headers,
|
||||
requested_query_params=requested_query_params,
|
||||
_parsed_body=_parsed_body,
|
||||
_parsed_body=upstream_body,
|
||||
forward_multipart=is_multipart,
|
||||
)
|
||||
verbose_proxy_logger.debug("response.headers= %s", response.headers)
|
||||
|
|
|
|||
|
|
@ -3343,6 +3343,34 @@
|
|||
],
|
||||
"default_model_placeholder": "prism/deepseek-v4.1-flash"
|
||||
},
|
||||
{
|
||||
"provider": "HOSTINGER_ROUTER",
|
||||
"provider_display_name": "Hostinger Router",
|
||||
"litellm_provider": "hostinger_router",
|
||||
"credential_fields": [
|
||||
{
|
||||
"key": "api_base",
|
||||
"label": "API Base",
|
||||
"placeholder": "https://router.hostinger.com/v1",
|
||||
"tooltip": null,
|
||||
"required": false,
|
||||
"field_type": "text",
|
||||
"options": null,
|
||||
"default_value": null
|
||||
},
|
||||
{
|
||||
"key": "api_key",
|
||||
"label": "API Key",
|
||||
"placeholder": null,
|
||||
"tooltip": null,
|
||||
"required": true,
|
||||
"field_type": "password",
|
||||
"options": null,
|
||||
"default_value": null
|
||||
}
|
||||
],
|
||||
"default_model_placeholder": "hostinger_router/claude-sonnet-5"
|
||||
},
|
||||
{
|
||||
"provider": "RECRAFT",
|
||||
"provider_display_name": "Recraft",
|
||||
|
|
|
|||
|
|
@ -3031,14 +3031,21 @@ class ProxyLogging:
|
|||
hook to consult, but it judges the payload the same way, so a proxy configured only with
|
||||
one of those still has something to say about every record.
|
||||
"""
|
||||
if request_metadata.get("_guardrail_pipelines"):
|
||||
return self.request_has_pre_call_guardrails({"metadata": dict(request_metadata)}, event_type)
|
||||
|
||||
def request_has_pre_call_guardrails(
|
||||
self,
|
||||
request_data: Mapping[str, object],
|
||||
event_type: GuardrailEventHooks = GuardrailEventHooks.pre_call,
|
||||
) -> bool:
|
||||
if _policy_state_metadata(request_data).get("_guardrail_pipelines"):
|
||||
return True
|
||||
caps: Final = ProxyLogging.callback_capabilities()
|
||||
if caps.has_content_enforcer:
|
||||
return True
|
||||
probe: Final = {"metadata": dict(request_metadata)}
|
||||
return any(
|
||||
isinstance(callback, CustomGuardrail) and callback.should_run_guardrail(data=probe, event_type=event_type)
|
||||
isinstance(callback, CustomGuardrail)
|
||||
and callback.should_run_guardrail(data=request_data, event_type=event_type)
|
||||
for callback in caps.resolved_callbacks
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@ import uuid
|
|||
from collections.abc import Awaitable, Callable, Iterable, Iterator, Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from functools import partial
|
||||
from itertools import accumulate, chain, groupby, islice
|
||||
from itertools import chain, groupby
|
||||
from types import MappingProxyType
|
||||
from typing import (
|
||||
TYPE_CHECKING,
|
||||
|
|
@ -45,7 +45,15 @@ from litellm.constants import (
|
|||
REDACTED_BY_LITELLM,
|
||||
REDACTED_TOOL_CALL_ARGUMENTS_PLACEHOLDER,
|
||||
)
|
||||
from litellm.integrations.anthropic_cache_control_hook import MAX_CACHE_CONTROL_BLOCKS, AnthropicCacheControlHook
|
||||
from litellm.integrations.anthropic_cache_control_hook import (
|
||||
AnthropicCacheControlHook,
|
||||
block_with_cache_control,
|
||||
cache_control_count,
|
||||
dropped_cache_control,
|
||||
newest_marks_within_cache_control_limit,
|
||||
whole_messages_within_cache_control_limit,
|
||||
with_cache_control,
|
||||
)
|
||||
from litellm.litellm_core_utils.core_helpers import drops_unsupported_params, normalize_drop_params
|
||||
from litellm.litellm_core_utils.get_supported_openai_params import (
|
||||
get_supported_openai_params,
|
||||
|
|
@ -603,10 +611,6 @@ def _validated_cache_control(value: object) -> ChatCompletionCachedContent | Non
|
|||
return None
|
||||
|
||||
|
||||
def _dropped_cache_control(_: object) -> None:
|
||||
return None
|
||||
|
||||
|
||||
def _without_cache_control_key(item: object) -> object:
|
||||
if not isinstance(item, dict):
|
||||
return item
|
||||
|
|
@ -694,83 +698,6 @@ def _content_blocks_carrying(content: object, cache_control: ChatCompletionCache
|
|||
return _with_cache_control_on_last_block(_OBJECT_LIST_ADAPTER.validate_python(content), cache_control)
|
||||
|
||||
|
||||
_MARK_CARRYING_MESSAGE_FIELDS: Final = ("content", "tool_calls")
|
||||
|
||||
|
||||
def _mark_carrying_blocks(fields: Mapping[str, object]) -> Sequence[object]:
|
||||
lists: Final = tuple(fields.get(key) for key in _MARK_CARRYING_MESSAGE_FIELDS)
|
||||
return tuple(
|
||||
chain.from_iterable(_OBJECT_LIST_ADAPTER.validate_python(value) for value in lists if isinstance(value, list))
|
||||
)
|
||||
|
||||
|
||||
def _block_carries_cache_control(block: object) -> bool:
|
||||
return isinstance(block, dict) and _STR_KEY_DICT_ADAPTER.validate_python(block).get("cache_control") is not None
|
||||
|
||||
|
||||
def _cache_control_count(message: object) -> int:
|
||||
if not isinstance(message, dict):
|
||||
return 0
|
||||
fields: Final = _STR_KEY_DICT_ADAPTER.validate_python(message)
|
||||
block_marks: Final = sum(1 for block in _mark_carrying_blocks(fields) if _block_carries_cache_control(block))
|
||||
return int(fields.get("cache_control") is not None) + block_marks
|
||||
|
||||
|
||||
_CacheControlRewrite: TypeAlias = Callable[[object], ChatCompletionCachedContent | None]
|
||||
|
||||
|
||||
def _fields_with_cache_control(fields: Mapping[str, object], rewrite: _CacheControlRewrite) -> Mapping[str, object]:
|
||||
others: Final = {key: value for key, value in fields.items() if key != "cache_control"}
|
||||
if "cache_control" not in fields:
|
||||
return others
|
||||
cache_control: Final = rewrite(fields["cache_control"])
|
||||
return others if cache_control is None else {**others, "cache_control": cache_control}
|
||||
|
||||
|
||||
def _block_with_cache_control(block: object, rewrite: _CacheControlRewrite) -> object:
|
||||
if not isinstance(block, dict):
|
||||
return block
|
||||
return _fields_with_cache_control(_STR_KEY_DICT_ADAPTER.validate_python(block), rewrite)
|
||||
|
||||
|
||||
def _with_cache_control(message: _MessageT, rewrite: _CacheControlRewrite) -> _MessageT:
|
||||
if not isinstance(message, dict):
|
||||
return message
|
||||
fields: Final = _STR_KEY_DICT_ADAPTER.validate_python(message)
|
||||
blocks_rewritten: Final = {
|
||||
key: (
|
||||
[_block_with_cache_control(block, rewrite) for block in _OBJECT_LIST_ADAPTER.validate_python(value)]
|
||||
if key in _MARK_CARRYING_MESSAGE_FIELDS and isinstance(value, list)
|
||||
else value
|
||||
)
|
||||
for key, value in fields.items()
|
||||
}
|
||||
return cast( # cast-ok: the same message shape with each cache marker rewritten or dropped
|
||||
_MessageT, _fields_with_cache_control(blocks_rewritten, rewrite)
|
||||
)
|
||||
|
||||
|
||||
def _kept_within_budget(state: tuple[int, bool], marks: int) -> tuple[int, bool]:
|
||||
remaining, _ = state
|
||||
if marks > remaining:
|
||||
return remaining, False
|
||||
return remaining - marks, True
|
||||
|
||||
|
||||
def _messages_within_cache_control_limit(messages: Sequence[_MessageT], reserved_marks: int) -> Sequence[_MessageT]:
|
||||
budget: Final = max(MAX_CACHE_CONTROL_BLOCKS - reserved_marks, 0)
|
||||
newest_first: Final = tuple(reversed(messages))
|
||||
budget_after_each: Final = accumulate(
|
||||
(_cache_control_count(message) for message in newest_first), _kept_within_budget, initial=(budget, True)
|
||||
)
|
||||
kept_flags: Final = tuple(kept for _, kept in islice(budget_after_each, 1, None))
|
||||
kept_newest_first: Final = tuple(
|
||||
message if kept else _with_cache_control(message, _dropped_cache_control)
|
||||
for message, kept in zip(newest_first, kept_flags)
|
||||
)
|
||||
return tuple(reversed(kept_newest_first))
|
||||
|
||||
|
||||
_SYSTEM_MESSAGE_FIELDS: Final = frozenset(("content", "cache_control"))
|
||||
|
||||
|
||||
|
|
@ -799,7 +726,7 @@ def _without_tool_block_marks(message: _MessageT) -> _MessageT:
|
|||
if fields is None:
|
||||
return message
|
||||
if fields.get("role") == "tool":
|
||||
return _with_cache_control(message, _dropped_cache_control)
|
||||
return with_cache_control(message, dropped_cache_control)
|
||||
tool_calls: Final = fields.get("tool_calls")
|
||||
if not isinstance(tool_calls, list):
|
||||
return message
|
||||
|
|
@ -808,7 +735,7 @@ def _without_tool_block_marks(message: _MessageT) -> _MessageT:
|
|||
{
|
||||
**fields,
|
||||
"tool_calls": [
|
||||
_block_with_cache_control(tool_call, _dropped_cache_control)
|
||||
block_with_cache_control(tool_call, dropped_cache_control)
|
||||
for tool_call in _OBJECT_LIST_ADAPTER.validate_python(tool_calls)
|
||||
],
|
||||
},
|
||||
|
|
@ -822,7 +749,7 @@ class _ItemMarkRules:
|
|||
external_marks: int
|
||||
|
||||
def sanitized(self, message: _MessageT) -> _MessageT:
|
||||
valid: Final = _with_cache_control(message, _validated_cache_control)
|
||||
valid: Final = with_cache_control(message, _validated_cache_control)
|
||||
return _without_tool_block_marks(valid) if self.refuses_tool_block_marks else valid
|
||||
|
||||
def checked(self, message: _MessageT) -> _MessageT:
|
||||
|
|
@ -845,15 +772,15 @@ def _request_messages_for_cache_points(messages: Sequence[_MessageT], rules: _It
|
|||
as_system: Final = tuple(_developer_message_as_system_message(message) for message in messages)
|
||||
if not rules.drop_params:
|
||||
return as_system
|
||||
return _messages_within_cache_control_limit(tuple(map(rules.checked, as_system)), rules.external_marks)
|
||||
return newest_marks_within_cache_control_limit(tuple(map(rules.checked, as_system)), rules.external_marks)
|
||||
|
||||
|
||||
def _replayed_messages_for_cache_points(
|
||||
replayed_messages: Sequence[_MessageT], current_messages: Sequence[object], rules: _ItemMarkRules
|
||||
) -> Sequence[_MessageT]:
|
||||
as_system: Final = tuple(_developer_message_as_system_message(message) for message in replayed_messages)
|
||||
current_marks: Final = sum(_cache_control_count(message) for message in current_messages)
|
||||
return _messages_within_cache_control_limit(
|
||||
current_marks: Final = sum(cache_control_count(message) for message in current_messages)
|
||||
return whole_messages_within_cache_control_limit(
|
||||
tuple(map(rules.sanitized, as_system)), current_marks + rules.external_marks
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -643,6 +643,7 @@ _PRE_CALL_CHECK_FILTERED_PARAMS: Final = frozenset({"response_format"})
|
|||
_STREAM_TIMEOUT_AWARE_GENERIC_HANDLERS: Final = frozenset({"anthropic_messages", "aresponses", "responses"})
|
||||
_video_call_deployment_id: Final[ContextVar[str | None]] = ContextVar("video_call_deployment_id", default=None)
|
||||
_routed_call_type: Final[ContextVar[str | None]] = ContextVar("routed_call_type", default=None)
|
||||
_DECISIONS_MEDIA_CALL_TYPES: Final = frozenset({"adecisions", "decisions", "asystemone", "systemone"})
|
||||
_ROUTE_CAPABILITY_RESOLVERS: Final[Mapping[str, Callable[[str, LlmProviders], object | None]]] = MappingProxyType(
|
||||
{
|
||||
"adecisions": ProviderConfigManager.get_provider_decisions_config,
|
||||
|
|
@ -4174,13 +4175,16 @@ class Router(DeprecatedRouterMembers):
|
|||
raise e.original_exception from e
|
||||
raise e
|
||||
|
||||
def _active_default_litellm_params(self) -> Mapping[str, object]:
|
||||
default_params: Final = _REQUEST_KWARGS_ADAPTER.validate_python(self._core_default_litellm_params)
|
||||
return {key: value for key, value in default_params.items() if value is not None}
|
||||
|
||||
def _params_sent_to_deployment(
|
||||
self, request_kwargs: Mapping[str, object], deployment: Deployment | None
|
||||
) -> Mapping[str, object]:
|
||||
"""The request as the deployment received it, merged in the order _update_kwargs_with_deployment applies."""
|
||||
deployment_params: Final = deployment.litellm_params.model_dump(exclude_none=True) if deployment else {}
|
||||
default_params: Final = _REQUEST_KWARGS_ADAPTER.validate_python(self._core_default_litellm_params)
|
||||
defaults: Final = {key: value for key, value in default_params.items() if value is not None}
|
||||
defaults: Final = self._active_default_litellm_params()
|
||||
deployment_tool_choice: Final = (
|
||||
{"tool_choice": deployment_params["tool_choice"]}
|
||||
if deployment_params.get("tool_choice") is not None
|
||||
|
|
@ -14945,6 +14949,62 @@ class Router(DeprecatedRouterMembers):
|
|||
return healthy_deployments
|
||||
return cast(_RouteDeploymentsT, filtered_deployments) # cast-ok: same list shape as the input candidates
|
||||
|
||||
@staticmethod
|
||||
def _decisions_deployment_refuses_media(
|
||||
deployment: DeploymentTypedDict, group: str, request_kwargs: Mapping[str, object], *, audio: bool, videos: bool
|
||||
) -> bool:
|
||||
from litellm.decisions.call import deployment_refuses_media
|
||||
|
||||
plain_deployment: Final = cast(Mapping[str, object], deployment) # cast-ok: router deployments are plain dicts
|
||||
model_info: Final = cast(Mapping[str, object], plain_deployment.get("model_info") or {}) # cast-ok: plain dict
|
||||
deployment_params: Final = cast(Mapping[str, object], deployment["litellm_params"]) # cast-ok: plain dict
|
||||
try:
|
||||
model, custom_llm_provider = Router._capability_model_and_provider(deployment["litellm_params"], group)
|
||||
return deployment_refuses_media(
|
||||
model,
|
||||
custom_llm_provider,
|
||||
model_info,
|
||||
{**deployment_params, **request_kwargs},
|
||||
audio=audio,
|
||||
videos=videos,
|
||||
)
|
||||
except Exception as e: # noqa: BLE001 # best-effort filter: a deployment it cannot resolve answers for itself
|
||||
verbose_router_logger.debug(
|
||||
"litellm.router.py::_decisions_deployment_refuses_media: keeping a deployment of model=%s. Got - %s",
|
||||
group,
|
||||
e,
|
||||
)
|
||||
return False
|
||||
|
||||
def _without_decisions_deployments_refusing_media(
|
||||
self, healthy_deployments: _RouteDeploymentsT, group: str, request_kwargs: Mapping[str, object] | None
|
||||
) -> _RouteDeploymentsT:
|
||||
call_kwargs: Final = request_kwargs or {}
|
||||
audio: Final = bool(call_kwargs.get("audio"))
|
||||
videos: Final = bool(call_kwargs.get("videos"))
|
||||
if (
|
||||
_routed_call_type.get() not in _DECISIONS_MEDIA_CALL_TYPES
|
||||
or not (audio or videos)
|
||||
or isinstance(healthy_deployments, Mapping)
|
||||
):
|
||||
return healthy_deployments
|
||||
|
||||
sent_kwargs: Final = {**self._active_default_litellm_params(), **call_kwargs}
|
||||
deployments_taking_media: Final = [
|
||||
deployment
|
||||
for deployment in healthy_deployments
|
||||
if not Router._decisions_deployment_refuses_media(
|
||||
cast(DeploymentTypedDict, deployment), # cast-ok: router candidates use the deployment shape
|
||||
group,
|
||||
sent_kwargs,
|
||||
audio=audio,
|
||||
videos=videos,
|
||||
)
|
||||
]
|
||||
if not deployments_taking_media:
|
||||
return healthy_deployments
|
||||
return cast(_RouteDeploymentsT, deployments_taking_media) # cast-ok: same list shape as the input candidates
|
||||
|
||||
@staticmethod
|
||||
def _without_declared_text_only_deployments(
|
||||
healthy_deployments: _RouteDeploymentsT,
|
||||
|
|
@ -15669,10 +15729,14 @@ class Router(DeprecatedRouterMembers):
|
|||
|
||||
## ORDER FILTERING ## -> if user set 'order' in deployments, return deployments with lowest order (e.g. order=1 > order=2)
|
||||
_target_order: Final = (request_kwargs or {}).pop("_target_order", None)
|
||||
batch_candidates: Final = self._without_declared_text_only_deployments(
|
||||
cast(list[dict[str, object]], healthy_deployments), # cast-ok: router deployments are plain dicts
|
||||
batch_candidates: Final = self._without_decisions_deployments_refusing_media(
|
||||
self._without_declared_text_only_deployments(
|
||||
cast(list[dict[str, object]], healthy_deployments), # cast-ok: router deployments are plain dicts
|
||||
model,
|
||||
messages,
|
||||
cast(Mapping[str, object] | None, request_kwargs), # cast-ok: legacy signature uses a bare dict
|
||||
),
|
||||
model,
|
||||
messages,
|
||||
cast(Mapping[str, object] | None, request_kwargs), # cast-ok: legacy signature uses a bare dict
|
||||
)
|
||||
embedding_batch_admitting: Final = deployments_admitting_embedding_batch(
|
||||
|
|
@ -16916,10 +16980,14 @@ class Router(DeprecatedRouterMembers):
|
|||
|
||||
## ORDER FILTERING ## -> if user set 'order' in deployments, return deployments with lowest order (e.g. order=1 > order=2)
|
||||
_target_order: Final = (request_kwargs or {}).pop("_target_order", None)
|
||||
batch_candidates: Final = self._without_declared_text_only_deployments(
|
||||
cast(list[dict[str, object]], healthy_deployments), # cast-ok: router deployments are plain dicts
|
||||
batch_candidates: Final = self._without_decisions_deployments_refusing_media(
|
||||
self._without_declared_text_only_deployments(
|
||||
cast(list[dict[str, object]], healthy_deployments), # cast-ok: router deployments are plain dicts
|
||||
model,
|
||||
messages,
|
||||
cast(Mapping[str, object] | None, request_kwargs), # cast-ok: legacy signature uses a bare dict
|
||||
),
|
||||
model,
|
||||
messages,
|
||||
cast(Mapping[str, object] | None, request_kwargs), # cast-ok: legacy signature uses a bare dict
|
||||
)
|
||||
embedding_batch_admitting: Final = deployments_admitting_embedding_batch(
|
||||
|
|
|
|||
|
|
@ -4683,6 +4683,7 @@ class LlmProviders(str, Enum):
|
|||
PRISM = "prism"
|
||||
OPENINFER = "openinfer"
|
||||
INFERENCEAPIS = "inferenceapis"
|
||||
HOSTINGER_ROUTER = "hostinger_router"
|
||||
ZEROGPU = "zerogpu"
|
||||
DARKBLOOM = "darkbloom"
|
||||
META = "meta"
|
||||
|
|
|
|||
|
|
@ -10159,7 +10159,7 @@ class ProviderConfigManager:
|
|||
from litellm.llms.openai_like.json_loader import JSONProviderRegistry
|
||||
|
||||
json_provider: Final = JSONProviderRegistry.get(provider.value)
|
||||
if json_provider is not None and "/v1/messages" in json_provider.supported_endpoints:
|
||||
if json_provider is not None and JSONProviderRegistry.serves_endpoint(provider.value, model, "/v1/messages"):
|
||||
from litellm.llms.openai_like.messages.transformation import (
|
||||
JSONProviderAnthropicMessagesConfig,
|
||||
)
|
||||
|
|
@ -10298,7 +10298,7 @@ class ProviderConfigManager:
|
|||
return result
|
||||
|
||||
# Fall back to JSON providers (generic OpenAI-compatible)
|
||||
if JSONProviderRegistry.exists(provider_str) and JSONProviderRegistry.supports_responses_api(provider_str):
|
||||
if JSONProviderRegistry.serves_endpoint(provider_str, model, "/v1/responses"):
|
||||
provider_config: Final = JSONProviderRegistry.get(provider_str)
|
||||
if provider_config is not None:
|
||||
return create_responses_config_class(provider_config)()
|
||||
|
|
|
|||
|
|
@ -39687,6 +39687,57 @@
|
|||
"/v1/images/generations"
|
||||
]
|
||||
},
|
||||
"hostinger_router/gpt-5.2": {
|
||||
"cache_read_input_token_cost": 0.00000021,
|
||||
"input_cost_per_token": 0.0000021,
|
||||
"litellm_provider": "hostinger_router",
|
||||
"max_input_tokens": 272000,
|
||||
"max_output_tokens": 128000,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 0.0000168,
|
||||
"supports_function_calling": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_vision": true,
|
||||
"supported_endpoints": ["/v1/chat/completions", "/v1/responses"],
|
||||
"source": "https://router.hostinger.com/v1/models"
|
||||
},
|
||||
"hostinger_router/claude-sonnet-5": {
|
||||
"cache_creation_input_token_cost": 3.3e-06,
|
||||
"cache_creation_input_token_cost_above_1hr": 5.28e-06,
|
||||
"cache_read_input_token_cost": 2.64e-07,
|
||||
"input_cost_per_token": 2.64e-06,
|
||||
"litellm_provider": "hostinger_router",
|
||||
"max_input_tokens": 1000000,
|
||||
"max_output_tokens": 128000,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 1.32e-05,
|
||||
"supports_function_calling": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": true,
|
||||
"supports_response_schema": true,
|
||||
"supported_endpoints": ["/v1/chat/completions", "/v1/messages"],
|
||||
"source": "https://router.hostinger.com/v1/models"
|
||||
},
|
||||
"hostinger_router/text-embedding-3-large": {
|
||||
"input_cost_per_token": 1.716e-07,
|
||||
"litellm_provider": "hostinger_router",
|
||||
"max_input_tokens": 8191,
|
||||
"mode": "embedding",
|
||||
"output_cost_per_token": 0.0,
|
||||
"source": "https://router.hostinger.com/v1/models"
|
||||
},
|
||||
"hostinger_router/text-embedding-3-small": {
|
||||
"input_cost_per_token": 3e-08,
|
||||
"litellm_provider": "hostinger_router",
|
||||
"max_input_tokens": 8191,
|
||||
"mode": "embedding",
|
||||
"output_cost_per_token": 0.0,
|
||||
"source": "https://router.hostinger.com/v1/models"
|
||||
},
|
||||
"hyperbolic/NousResearch/Hermes-3-Llama-3.1-70B": {
|
||||
"input_cost_per_token": 1.2e-07,
|
||||
"litellm_provider": "hyperbolic",
|
||||
|
|
|
|||
|
|
@ -1087,6 +1087,32 @@
|
|||
"generateContent": false
|
||||
}
|
||||
},
|
||||
"hostinger_router": {
|
||||
"display_name": "Hostinger Router (`hostinger_router`)",
|
||||
"url": "https://docs.litellm.ai/docs/providers/hostinger_router",
|
||||
"endpoints": {
|
||||
"chat_completions": true,
|
||||
"responses": true,
|
||||
"text_completion": true,
|
||||
"embeddings": true,
|
||||
"image_generations": false,
|
||||
"image_edits": false,
|
||||
"audio_transcriptions": false,
|
||||
"audio_speech": false,
|
||||
"rerank": false,
|
||||
"ocr": false,
|
||||
"search": false,
|
||||
"moderations": false,
|
||||
"realtime": false,
|
||||
"video_generations": false,
|
||||
"batches": false,
|
||||
"messages": true,
|
||||
"interactions": true,
|
||||
"systemone": false,
|
||||
"a2a": true,
|
||||
"generateContent": true
|
||||
}
|
||||
},
|
||||
"crusoe": {
|
||||
"display_name": "Crusoe (`crusoe`)",
|
||||
"url": "https://docs.litellm.ai/docs/providers/crusoe",
|
||||
|
|
|
|||
|
|
@ -383,6 +383,8 @@ exclude-newer = "3 days"
|
|||
[tool.uv.exclude-newer-package]
|
||||
# TODO: remove after 2026-10-25, once brotli 1.2.1 (CVE-2026-111497) clears the 3-day window
|
||||
brotli = "2026-10-22T14:01:00Z"
|
||||
# TODO: remove after 2026-10-31, once pypdf 6.21.4 clears the 3-day window
|
||||
pypdf = "2026-10-28T17:01:00Z"
|
||||
|
||||
[tool.uv.sources]
|
||||
litellm-proxy-extras = { workspace = true }
|
||||
|
|
|
|||
|
|
@ -183,6 +183,9 @@ ignored_function_names = [
|
|||
"_aresponses_background_stream_can_release", # Tested through the background Responses stream tests in test_router.py
|
||||
"_raise_released_stream_error", # Tested through the background Responses stream content-policy tests in test_router.py
|
||||
"_mid_stream_error_before_output", # Tested through the Responses mid-stream fallback and background stream tests
|
||||
"_without_decisions_deployments_refusing_media", # Tested through the mixed media-flag group tests in tests/unit/decisions/test_main.py
|
||||
"_decisions_deployment_refuses_media", # Read by _without_decisions_deployments_refusing_media, tested through the same group tests
|
||||
"_active_default_litellm_params", # Tested through the router default drop_params test in tests/unit/decisions/test_main.py
|
||||
]
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -317,7 +317,6 @@ def test_credentialless_proxy_forwards_the_caller_anthropic_credential_and_never
|
|||
|
||||
|
||||
def test_anthropic_sdk_metadata_user_id_reaches_upstream(api_key_proxy: Gateway, wire: Wire) -> None:
|
||||
pytest.skip("BUG: /anthropic/v1/messages drops the native metadata.user_id field before forwarding upstream")
|
||||
with api_key_proxy.scenario() as scenario:
|
||||
key: Final = scenario.key()
|
||||
sent: Final[list[httpx.Request]] = []
|
||||
|
|
|
|||
|
|
@ -0,0 +1,123 @@
|
|||
import json
|
||||
import uuid
|
||||
from collections.abc import Mapping
|
||||
from dataclasses import dataclass
|
||||
from typing import Final, Literal
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
from integration._support.client import Gateway, eventually, object_value
|
||||
from integration._support.database import read_rows
|
||||
from integration._support.wire import Reply, Request, wire_server
|
||||
from pydantic import JsonValue, TypeAdapter
|
||||
|
||||
_API_KEY: Final = "synthetic-decisions-key"
|
||||
_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue])
|
||||
_INPUT: Final = "Listen to the attached clip."
|
||||
_AUDIO: Final = ("data:audio/wav;base64,UklGRiQAAABXQVZF",)
|
||||
_VIDEOS: Final = ("data:video/mp4;base64,AAAAIGZ0eXBpc29t",)
|
||||
_QUESTION: Final[dict[str, JsonValue]] = {
|
||||
"type": "predicate",
|
||||
"name": "banana",
|
||||
"instructions": "Does the clip name a banana?",
|
||||
}
|
||||
_SPEND_QUERY: Final = 'SELECT status, model_group FROM "LiteLLM_SpendLogs" WHERE request_id = %s'
|
||||
|
||||
Route = Literal["/v1/decisions", "/v1/systemone"]
|
||||
Media = Literal["audio", "videos"]
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _Upstream:
|
||||
model: str
|
||||
|
||||
def reply(self, request: Request) -> Reply:
|
||||
questions: Final = object_value(_JSON_OBJECT.validate_json(request.body)["questions"])
|
||||
plain: Final[dict[str, JsonValue]] = {
|
||||
"model": self.model,
|
||||
"answers": {name: {"type": "noul", "noul": 0.9} for name in questions},
|
||||
"usage": {"input_tokens": 41, "output_tokens": 0},
|
||||
}
|
||||
return Reply(body=json.dumps(plain).encode())
|
||||
|
||||
|
||||
def _unlisted_openrouter_model() -> _Upstream:
|
||||
return _Upstream(f"openrouter/example/omni-{uuid.uuid4().hex[:12]}")
|
||||
|
||||
|
||||
def _decide(gateway: Gateway, model: str, route: Route, media: Mapping[str, JsonValue]) -> httpx.Response:
|
||||
body: Final[dict[str, JsonValue]] = (
|
||||
{"model": model, "input": _INPUT, "questions": [_QUESTION], "num_retries": 0, **media}
|
||||
if route == "/v1/decisions"
|
||||
else {
|
||||
"model": model,
|
||||
"state": _INPUT,
|
||||
"questions": {"banana": {"type": "noul", "instructions": _QUESTION["instructions"]}},
|
||||
"num_retries": 0,
|
||||
**media,
|
||||
}
|
||||
)
|
||||
return gateway.request("POST", route, body)
|
||||
|
||||
|
||||
def _sent_media(request: Request) -> dict[str, JsonValue]:
|
||||
body: Final = _JSON_OBJECT.validate_json(request.body)
|
||||
return {kind: body[kind] for kind in ("audio", "videos") if kind in body}
|
||||
|
||||
|
||||
def _assert_refused(response: httpx.Response, kinds: tuple[Media, ...]) -> None:
|
||||
assert response.status_code == 400, response.text
|
||||
assert f"does not support parameters: {list(kinds)}" in response.text, response.text
|
||||
|
||||
|
||||
_BOTH: Final[dict[str, JsonValue]] = {"audio": list(_AUDIO), "videos": list(_VIDEOS)}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("route", ("/v1/decisions", "/v1/systemone"))
|
||||
def test_a_deployment_flagged_for_audio_and_video_sends_both_clips_for_a_model_the_cost_map_does_not_list(
|
||||
gateway: Gateway, route: Route
|
||||
) -> None:
|
||||
upstream: Final = _unlisted_openrouter_model()
|
||||
with wire_server(upstream.reply) as wire, gateway.scenario() as scenario:
|
||||
flags: Final[dict[str, JsonValue]] = {"supports_audio_input": True, "supports_video_input": True}
|
||||
flagged: Final = scenario.model(model=upstream.model, api_base=wire.url, api_key=_API_KEY, model_info=flags)
|
||||
flagged_dropping: Final = scenario.model(
|
||||
model=upstream.model, api_base=wire.url, api_key=_API_KEY, model_info=flags, drop_params=True
|
||||
)
|
||||
audio_only: Final = scenario.model(
|
||||
model=upstream.model, api_base=wire.url, api_key=_API_KEY, model_info={"supports_audio_input": True}
|
||||
)
|
||||
unflagged: Final = scenario.model(model=upstream.model, api_base=wire.url, api_key=_API_KEY)
|
||||
|
||||
_assert_refused(_decide(gateway, unflagged, route, _BOTH), ("audio", "videos"))
|
||||
_assert_refused(_decide(gateway, audio_only, route, _BOTH), ("videos",))
|
||||
assert wire.drain() == ()
|
||||
|
||||
answered: Final = _decide(gateway, flagged, route, _BOTH)
|
||||
assert answered.status_code == 200, answered.text
|
||||
(call,) = wire.drain()
|
||||
assert _sent_media(call) == _BOTH
|
||||
rows: Final = eventually(
|
||||
lambda: read_rows(_SPEND_QUERY, (answered.headers["x-litellm-call-id"],)),
|
||||
lambda found: len(found) == 1,
|
||||
seconds=70,
|
||||
)
|
||||
assert (rows[0]["status"], rows[0]["model_group"]) == ("success", flagged)
|
||||
|
||||
kept: Final = _decide(gateway, flagged_dropping, route, _BOTH)
|
||||
assert kept.status_code == 200, kept.text
|
||||
(kept_call,) = wire.drain()
|
||||
assert _sent_media(kept_call) == _BOTH
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"value",
|
||||
({"supports_audio_input": True, "supports_video_input": True}, 5, [{"supports_audio_input": True}], "x" * 5000),
|
||||
ids=("flags", "int", "list", "5kb"),
|
||||
)
|
||||
def test_a_caller_sent_model_info_never_turns_media_on(gateway: Gateway, value: JsonValue) -> None:
|
||||
upstream: Final = _unlisted_openrouter_model()
|
||||
with wire_server(upstream.reply) as wire, gateway.scenario() as scenario:
|
||||
model: Final = scenario.model(model=upstream.model, api_base=wire.url, api_key=_API_KEY)
|
||||
_assert_refused(_decide(gateway, model, "/v1/decisions", {**_BOTH, "model_info": value}), ("audio", "videos"))
|
||||
assert wire.drain() == ()
|
||||
|
|
@ -7,11 +7,12 @@ import json
|
|||
import re
|
||||
import threading
|
||||
import uuid
|
||||
from datetime import datetime
|
||||
from collections.abc import Mapping, Sequence
|
||||
from collections.abc import Awaitable, Callable, Iterator, Mapping, Sequence
|
||||
from concurrent.futures import wait
|
||||
from datetime import datetime
|
||||
from functools import partial
|
||||
from types import MappingProxyType
|
||||
from typing import Final
|
||||
from typing import Final, TypeAlias
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
|
|
@ -25,6 +26,7 @@ import litellm
|
|||
from litellm.constants import AWS_SIGNING_MAX_THREADS
|
||||
from litellm.cost_calculator import get_response_cost_from_hidden_params
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.litellm_core_utils.get_model_cost_map import ModelCostMapReloaded, refetch_model_cost_map
|
||||
from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
|
||||
from litellm.llms.azure_ai.common_utils import get_azure_ai_auth_headers
|
||||
from litellm.llms.bedrock.base_aws_llm import AWS_SIGNING_EXECUTOR
|
||||
|
|
@ -3741,6 +3743,387 @@ async def test_a_model_info_object_instead_of_a_dict_still_reaches_the_provider(
|
|||
assert response.answers[0].model_dump(mode="json")["name"] == "is_defect"
|
||||
|
||||
|
||||
_AUDIO_CLIPS: Final = ("data:audio/wav;base64,YQ==",)
|
||||
_VIDEO_CLIPS: Final = ("data:video/mp4;base64,YQ==",)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_deployment_that_advertises_media_sends_it_for_a_model_the_cost_map_does_not_list(
|
||||
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, local_model_cost_map: None
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "drop_params", False)
|
||||
backend: Final = "openrouter/example/omni-preview"
|
||||
route: Final = respx_mock.post("https://openrouter.ai/api/alpha/decisions").respond(json=_RESPONSE)
|
||||
router: Final = litellm.Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "hears",
|
||||
"litellm_params": {"model": backend, "api_key": "k"},
|
||||
"model_info": {"id": "media-hears", "supports_audio_input": True},
|
||||
},
|
||||
{
|
||||
"model_name": "hears-and-sees",
|
||||
"litellm_params": {"model": backend, "api_key": "k"},
|
||||
"model_info": {"id": "media-both", "supports_audio_input": True, "supports_video_input": True},
|
||||
},
|
||||
]
|
||||
)
|
||||
send: Final = partial(
|
||||
router.adecisions, input="review", questions=_PREDICATE_QUESTIONS, audio=_AUDIO_CLIPS, videos=_VIDEO_CLIPS
|
||||
)
|
||||
|
||||
with pytest.raises(litellm.UnsupportedParamsError, match=re.escape("['videos']")):
|
||||
await send(model="hears")
|
||||
assert not route.called
|
||||
|
||||
await send(model="hears-and-sees")
|
||||
|
||||
sent: Final = json.loads(route.calls.last.request.content)
|
||||
assert (sent["audio"], sent["videos"]) == (list(_AUDIO_CLIPS), list(_VIDEO_CLIPS))
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_media_flags_on_a_deployment_whose_own_mode_is_not_decisions_never_send_clips(
|
||||
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, local_model_cost_map: None
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "drop_params", False)
|
||||
route: Final = respx_mock.post("https://openrouter.ai/api/alpha/decisions").respond(json=_RESPONSE)
|
||||
advertised: Final = {"supports_audio_input": True, "supports_video_input": True}
|
||||
router: Final = litellm.Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": mode,
|
||||
"litellm_params": {"model": f"openrouter/example/omni-{mode}-preview", "api_key": "k"},
|
||||
"model_info": {"id": f"media-{mode}", "mode": mode, **advertised},
|
||||
}
|
||||
for mode in ("chat", "decisions")
|
||||
]
|
||||
)
|
||||
send: Final = partial(
|
||||
router.adecisions, input="review", questions=_PREDICATE_QUESTIONS, audio=_AUDIO_CLIPS, videos=_VIDEO_CLIPS
|
||||
)
|
||||
|
||||
with pytest.raises(litellm.UnsupportedParamsError, match=re.escape("['audio', 'videos']")):
|
||||
await send(model="chat")
|
||||
assert not route.called
|
||||
|
||||
await send(model="decisions")
|
||||
|
||||
sent: Final = json.loads(route.calls.last.request.content)
|
||||
assert (sent["audio"], sent["videos"]) == (list(_AUDIO_CLIPS), list(_VIDEO_CLIPS))
|
||||
|
||||
|
||||
CatalogLoader: TypeAlias = Callable[[Mapping[str, Mapping[str, object]]], Awaitable[None]]
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def load_catalog_with(monkeypatch: pytest.MonkeyPatch, local_model_cost_map: None) -> Iterator[CatalogLoader]:
|
||||
async def load(rows: Mapping[str, Mapping[str, object]]) -> None:
|
||||
catalog: Final = {**litellm.get_model_cost_map(url=""), **rows}
|
||||
transport: Final = httpx.MockTransport(lambda _: httpx.Response(200, json=catalog))
|
||||
monkeypatch.delenv("LITELLM_LOCAL_MODEL_COST_MAP")
|
||||
async with httpx.AsyncClient(transport=transport) as client:
|
||||
loaded: Final = await refetch_model_cost_map(url="https://catalog.example/prices.json", client=client)
|
||||
monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True")
|
||||
assert isinstance(loaded, ModelCostMapReloaded)
|
||||
monkeypatch.setattr(litellm, "model_cost", loaded.model_cost_map)
|
||||
litellm.get_model_info.cache_clear()
|
||||
|
||||
yield load
|
||||
litellm.get_model_cost_map(url="")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_media_flags_on_a_deployment_without_its_own_mode_follow_the_mode_the_catalog_lists(
|
||||
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, load_catalog_with: CatalogLoader
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "drop_params", False)
|
||||
await load_catalog_with(
|
||||
{
|
||||
f"openrouter/example/omni-{mode}": {"litellm_provider": "openrouter", "mode": mode}
|
||||
for mode in ("chat", "decisions")
|
||||
}
|
||||
)
|
||||
route: Final = respx_mock.post("https://openrouter.ai/api/alpha/decisions").respond(json=_RESPONSE)
|
||||
router: Final = litellm.Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": mode,
|
||||
"litellm_params": {"model": f"openrouter/example/omni-{mode}", "api_key": "k"},
|
||||
"model_info": {"id": f"listed-{mode}", "supports_audio_input": True, "supports_video_input": True},
|
||||
}
|
||||
for mode in ("chat", "decisions")
|
||||
]
|
||||
)
|
||||
send: Final = partial(
|
||||
router.adecisions, input="review", questions=_PREDICATE_QUESTIONS, audio=_AUDIO_CLIPS, videos=_VIDEO_CLIPS
|
||||
)
|
||||
|
||||
with pytest.raises(litellm.UnsupportedParamsError, match=re.escape("['audio', 'videos']")):
|
||||
await send(model="chat")
|
||||
assert not route.called
|
||||
|
||||
monkeypatch.setattr(litellm, "drop_params", True)
|
||||
await send(model="chat")
|
||||
dropped: Final = json.loads(route.calls.last.request.content)
|
||||
assert ("audio" in dropped, "videos" in dropped) == (False, False)
|
||||
|
||||
await send(model="decisions")
|
||||
sent: Final = json.loads(route.calls.last.request.content)
|
||||
assert (sent["audio"], sent["videos"]) == (list(_AUDIO_CLIPS), list(_VIDEO_CLIPS))
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("sibling_mode", ("responses", "embedding"))
|
||||
async def test_another_deployments_mode_never_switches_off_a_deployments_own_media_flags(
|
||||
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, local_model_cost_map: None, sibling_mode: str
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "drop_params", False)
|
||||
route: Final = respx_mock.post("https://openrouter.ai/api/alpha/decisions").respond(json=_RESPONSE)
|
||||
shared_params: Final = {"model": "openrouter/example/omni-shared", "api_key": "k"}
|
||||
router: Final = litellm.Router(
|
||||
model_list=[
|
||||
{"model_name": "sibling", "litellm_params": shared_params, "model_info": {"id": "s", "mode": sibling_mode}},
|
||||
{
|
||||
"model_name": "flagged",
|
||||
"litellm_params": shared_params,
|
||||
"model_info": {"id": "f", "supports_audio_input": True, "supports_video_input": True},
|
||||
},
|
||||
]
|
||||
)
|
||||
|
||||
await router.adecisions(
|
||||
model="flagged", input="review", questions=_PREDICATE_QUESTIONS, audio=_AUDIO_CLIPS, videos=_VIDEO_CLIPS
|
||||
)
|
||||
|
||||
sent: Final = json.loads(route.calls.last.request.content)
|
||||
assert (sent["audio"], sent["videos"]) == (list(_AUDIO_CLIPS), list(_VIDEO_CLIPS))
|
||||
|
||||
|
||||
_MEDIA_SURFACES: Final[Mapping[str, Mapping[str, object]]] = MappingProxyType(
|
||||
{
|
||||
"adecisions": {"input": "review", "questions": _PREDICATE_QUESTIONS},
|
||||
"asystemone": {"state": "review", "questions": _QUESTIONS},
|
||||
"systemone": {"state": "review", "questions": _QUESTIONS},
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _group_member(
|
||||
group: str, model: str, key: str, weight: int, model_info: Mapping[str, object], **params: object
|
||||
) -> dict[str, object]:
|
||||
return {
|
||||
"model_name": group,
|
||||
"litellm_params": {"model": model, "api_key": key, "weight": weight, **params},
|
||||
"model_info": {"id": f"{group}-{key}", **model_info},
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("surface", tuple(_MEDIA_SURFACES))
|
||||
async def test_a_group_sends_clips_to_a_deployment_that_takes_them_and_refuses_only_when_none_does(
|
||||
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, local_model_cost_map: None, surface: str
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "drop_params", False)
|
||||
monkeypatch.setenv("CLOUDFLARE_ACCOUNT_ID", "acct")
|
||||
monkeypatch.delenv("CLOUDFLARE_API_BASE", raising=False)
|
||||
monkeypatch.setitem(
|
||||
litellm.model_cost,
|
||||
"cloudflare/@cf/cloudflare/clef-omni",
|
||||
{"litellm_provider": "cloudflare", "mode": "evaluation", "supports_audio_input": True},
|
||||
)
|
||||
respx_mock.post("https://openrouter.ai/api/alpha/decisions").respond(json=_RESPONSE)
|
||||
respx_mock.post("https://api.cloudflare.com/client/v4/accounts/acct/ai/run/@cf/cloudflare/clef-omni").respond(
|
||||
json=_RESPONSE
|
||||
)
|
||||
unlisted: Final = "openrouter/example/omni-preview"
|
||||
listed: Final = "cloudflare/clef-omni"
|
||||
router: Final = litellm.Router(
|
||||
model_list=[
|
||||
_group_member("unlisted", unlisted, "refuses", 1, {}),
|
||||
_group_member("unlisted", unlisted, "takes", 0, {"supports_audio_input": True}),
|
||||
_group_member("listed", listed, "refuses", 1, {"supports_audio_input": False}),
|
||||
_group_member("listed", listed, "takes", 0, {}),
|
||||
_group_member("none", unlisted, "refuses", 1, {}),
|
||||
_group_member("none", listed, "refuses", 1, {"supports_audio_input": False}),
|
||||
_group_member("drops", unlisted, "drops", 1, {}, drop_params=True),
|
||||
_group_member("drops", listed, "takes", 0, {}),
|
||||
_group_member("unresolvable", "databricks/not a serving endpoint", "unresolvable", 0, {}),
|
||||
_group_member("unresolvable", listed, "takes", 1, {}),
|
||||
]
|
||||
)
|
||||
call: Final = getattr(router, surface)
|
||||
|
||||
async def send(group: str, **media: object) -> object:
|
||||
response: Final = call(model=group, **_MEDIA_SURFACES[surface], **media)
|
||||
return await response if asyncio.iscoroutine(response) else response
|
||||
|
||||
for group in ("unlisted", "listed", "unresolvable"):
|
||||
await send(group, audio=_AUDIO_CLIPS)
|
||||
assert respx_mock.calls.last.request.headers["authorization"] == "Bearer takes"
|
||||
assert json.loads(respx_mock.calls.last.request.content)["audio"] == list(_AUDIO_CLIPS)
|
||||
|
||||
await send("drops", audio=_AUDIO_CLIPS)
|
||||
assert respx_mock.calls.last.request.headers["authorization"] == "Bearer drops"
|
||||
assert "audio" not in json.loads(respx_mock.calls.last.request.content)
|
||||
|
||||
await send("unlisted")
|
||||
assert respx_mock.calls.last.request.headers["authorization"] == "Bearer refuses"
|
||||
|
||||
sent_before_refusal: Final = len(respx_mock.calls)
|
||||
with pytest.raises(litellm.UnsupportedParamsError, match=re.escape("['audio']")):
|
||||
await send("none", audio=_AUDIO_CLIPS)
|
||||
assert len(respx_mock.calls) == sent_before_refusal
|
||||
|
||||
|
||||
def _answer_unless_key_is(rejected_key: str) -> Callable[[httpx.Request], httpx.Response]:
|
||||
def answer(request: httpx.Request) -> httpx.Response:
|
||||
if request.headers["authorization"] == f"Bearer {rejected_key}":
|
||||
return httpx.Response(401, json={"error": {"message": "invalid key"}})
|
||||
return httpx.Response(200, json=_RESPONSE)
|
||||
|
||||
return answer
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_clips_to_a_group_whose_only_taker_is_cooling_down_are_refused_rather_than_rate_limited(
|
||||
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, local_model_cost_map: None
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "drop_params", False)
|
||||
route: Final = respx_mock.post("https://openrouter.ai/api/alpha/decisions").mock(
|
||||
side_effect=_answer_unless_key_is("takes")
|
||||
)
|
||||
unlisted: Final = "openrouter/example/omni-preview"
|
||||
router: Final = litellm.Router(
|
||||
model_list=[
|
||||
_group_member("g", unlisted, "refuses", 1, {}),
|
||||
_group_member("g", unlisted, "takes", 1, {"supports_audio_input": True}),
|
||||
],
|
||||
num_retries=0,
|
||||
cooldown_time=120,
|
||||
)
|
||||
with pytest.raises(litellm.AuthenticationError):
|
||||
await router.adecisions(model="g-takes", input="review", questions=_PREDICATE_QUESTIONS)
|
||||
sent_before_clips: Final = len(route.calls)
|
||||
|
||||
with pytest.raises(litellm.UnsupportedParamsError, match=re.escape("['audio']")):
|
||||
await router.adecisions(model="g", input="review", questions=_PREDICATE_QUESTIONS, audio=_AUDIO_CLIPS)
|
||||
|
||||
assert len(route.calls) == sent_before_clips
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("surface", tuple(_MEDIA_SURFACES))
|
||||
async def test_a_router_default_drop_params_overrides_a_deployment_when_choosing_who_takes_clips(
|
||||
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, local_model_cost_map: None, surface: str
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "drop_params", False)
|
||||
respx_mock.post("https://openrouter.ai/api/alpha/decisions").respond(json=_RESPONSE)
|
||||
unlisted: Final = "openrouter/example/omni-preview"
|
||||
router: Final = litellm.Router(
|
||||
model_list=[
|
||||
_group_member("g", unlisted, "drops", 1, {}, drop_params=True),
|
||||
_group_member("g", unlisted, "takes", 0, {"supports_audio_input": True}),
|
||||
],
|
||||
default_litellm_params={"drop_params": False},
|
||||
)
|
||||
call: Final = getattr(router, surface)
|
||||
|
||||
async def send(**params: object) -> object:
|
||||
response: Final = call(model="g", **_MEDIA_SURFACES[surface], audio=_AUDIO_CLIPS, **params)
|
||||
return await response if asyncio.iscoroutine(response) else response
|
||||
|
||||
await send()
|
||||
assert respx_mock.calls.last.request.headers["authorization"] == "Bearer takes"
|
||||
assert json.loads(respx_mock.calls.last.request.content)["audio"] == list(_AUDIO_CLIPS)
|
||||
|
||||
await send(drop_params=True)
|
||||
assert respx_mock.calls.last.request.headers["authorization"] == "Bearer drops"
|
||||
assert "audio" not in json.loads(respx_mock.calls.last.request.content)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_deployment_that_turns_audio_off_is_refused_or_dropped_even_when_the_cost_map_allows_it(
|
||||
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, local_model_cost_map: None
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "drop_params", False)
|
||||
monkeypatch.setitem(
|
||||
litellm.model_cost,
|
||||
"typesafe/jev-omni",
|
||||
{
|
||||
"litellm_provider": "typesafe",
|
||||
"mode": "decisions",
|
||||
"supports_audio_input": True,
|
||||
"supports_video_input": True,
|
||||
},
|
||||
)
|
||||
route: Final = respx_mock.post("https://example.com/v1/systemone").respond(json=_RESPONSE)
|
||||
send: Final = partial(
|
||||
litellm.asystemone,
|
||||
model="typesafe/jev-omni",
|
||||
state="test",
|
||||
questions=_QUESTIONS,
|
||||
api_base="https://example.com",
|
||||
api_key="test",
|
||||
audio=_AUDIO_CLIPS,
|
||||
videos=_VIDEO_CLIPS,
|
||||
model_info={"supports_audio_input": False},
|
||||
)
|
||||
|
||||
with pytest.raises(litellm.UnsupportedParamsError, match=re.escape("does not support parameters: ['audio']")):
|
||||
await send()
|
||||
assert not route.called
|
||||
|
||||
await send(drop_params=True)
|
||||
|
||||
sent: Final = json.loads(route.calls.last.request.content)
|
||||
assert "audio" not in sent
|
||||
assert sent["videos"] == list(_VIDEO_CLIPS)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_media_flag_that_is_not_a_boolean_leaves_the_cost_map_in_charge_of_that_kind_only(
|
||||
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "drop_params", False)
|
||||
route: Final = respx_mock.post("https://example.com/v1/systemone").respond(json=_RESPONSE)
|
||||
|
||||
with pytest.raises(litellm.UnsupportedParamsError, match=re.escape("does not support parameters: ['audio']")):
|
||||
await litellm.asystemone(
|
||||
model="typesafe/jev-unlisted",
|
||||
state="test",
|
||||
questions=_QUESTIONS,
|
||||
api_base="https://example.com",
|
||||
api_key="test",
|
||||
audio=_AUDIO_CLIPS,
|
||||
videos=_VIDEO_CLIPS,
|
||||
model_info={"supports_audio_input": "true", "supports_video_input": True},
|
||||
)
|
||||
|
||||
assert not route.called
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_an_openai_format_deployment_that_advertises_media_is_still_refused_it(
|
||||
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "drop_params", False)
|
||||
route: Final = respx_mock.post("https://api.openai.com/v1/decisions").respond(json=_OPENAI_RESPONSE)
|
||||
|
||||
with pytest.raises(litellm.UnsupportedParamsError, match=re.escape("['audio', 'videos']")):
|
||||
await litellm.adecisions(
|
||||
model="openai/gpt-6-luna",
|
||||
input="review",
|
||||
questions=_PREDICATE_QUESTIONS,
|
||||
api_key="deployment-key",
|
||||
audio=_AUDIO_CLIPS,
|
||||
videos=_VIDEO_CLIPS,
|
||||
model_info={"supports_audio_input": True, "supports_video_input": True},
|
||||
)
|
||||
|
||||
assert not route.called
|
||||
|
||||
|
||||
def test_dropping_every_question_answers_400_before_http(respx_mock: respx.MockRouter) -> None:
|
||||
route: Final = respx_mock.post(_PERPLEXITY_DECISIONS_URL).respond(json=_RESPONSE)
|
||||
|
||||
|
|
|
|||
|
|
@ -25,6 +25,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
|||
get_semantic_cache_prompt_from_messages,
|
||||
get_str_from_messages,
|
||||
handle_any_messages_to_chat_completion_str_messages_conversion,
|
||||
handle_messages_with_content_list_to_str_conversion,
|
||||
hoist_images_from_tool_messages,
|
||||
is_encrypted_reasoning_block,
|
||||
merge_consecutive_system_messages,
|
||||
|
|
@ -2675,3 +2676,16 @@ def test_tpm_projection_preserves_non_media_identity_and_opaque_values() -> None
|
|||
projected, media = split_media_for_tpm(request)
|
||||
assert projected is request
|
||||
assert media == ()
|
||||
|
||||
|
||||
def test_content_list_flattening_returns_new_messages_and_leaves_the_callers_untouched() -> None:
|
||||
messages: Final = [
|
||||
{"role": "user", "content": [{"type": "text", "text": "Hello"}, {"type": "text", "text": " world"}]},
|
||||
{"role": "assistant", "content": "Hi"},
|
||||
]
|
||||
messages_before: Final = copy.deepcopy(messages)
|
||||
|
||||
flattened: Final = handle_messages_with_content_list_to_str_conversion(messages)
|
||||
|
||||
assert flattened == [{"role": "user", "content": "Hello world"}, {"role": "assistant", "content": "Hi"}]
|
||||
assert messages == messages_before
|
||||
|
|
|
|||
|
|
@ -3,7 +3,10 @@ Tests for litellm.litellm_core_utils.logging_utils — base64 truncation helpers
|
|||
"""
|
||||
|
||||
import asyncio, datetime, importlib, litellm, os, pytest_asyncio
|
||||
import sys
|
||||
import threading
|
||||
from itertools import chain
|
||||
from types import FrameType
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
|
|
@ -17,7 +20,7 @@ from litellm.litellm_core_utils.logging_utils import (
|
|||
truncate_base64_in_messages,
|
||||
truncate_base64_in_messages_async,
|
||||
)
|
||||
from collections.abc import AsyncIterator
|
||||
from collections.abc import AsyncIterator, Callable, Coroutine, Sequence
|
||||
from datetime import datetime as datetime_assemble_streaming
|
||||
from litellm import(
|
||||
Choices,
|
||||
|
|
@ -28,6 +31,7 @@ from litellm import(
|
|||
)
|
||||
from litellm.constants import LOGGING_WORKER_MAX_TIME_PER_COROUTINE
|
||||
from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
|
||||
from litellm.types.llms.openai import AllMessageValues
|
||||
from tests._vcr_conftest_common import install_live_call_probe, record_vcr_outcome
|
||||
from typing import Final
|
||||
|
||||
|
|
@ -99,8 +103,8 @@ class TestTruncateBase64InString:
|
|||
@pytest.mark.parametrize(
|
||||
"text",
|
||||
[
|
||||
'data: {"choices": [{"delta": {"content": "hi"}}]}\n\n' * 50_000,
|
||||
"data:" * 200_000,
|
||||
'data: {"choices": [{"delta": {"content": "hi"}}]}\n\n' * 50_000 + ";base64,",
|
||||
"data:" * 200_000 + ";base64,",
|
||||
],
|
||||
ids=["sse_lines", "whitespace_free_prefixes"],
|
||||
)
|
||||
|
|
@ -241,6 +245,49 @@ def scan_threads(monkeypatch):
|
|||
return threads
|
||||
|
||||
|
||||
def _run_without_yielding(coroutine: Coroutine[object, object, object]) -> object:
|
||||
try:
|
||||
coroutine.send(None)
|
||||
except StopIteration as finished:
|
||||
return finished.value
|
||||
coroutine.close()
|
||||
raise AssertionError("the coroutine yielded to the event loop")
|
||||
|
||||
|
||||
def _regex_calls_while(run: Callable[[], object]) -> tuple[object, tuple[str, ...]]:
|
||||
calls: Final[list[str]] = [] # mutable-ok: sys.setprofile reports each call to a callback that has to record it
|
||||
previous: Final = sys.getprofile()
|
||||
|
||||
def record(frame: FrameType, event: str, arg: object) -> None:
|
||||
name: Final = getattr(arg, "__qualname__", "")
|
||||
if event == "c_call" and name.startswith("Pattern."):
|
||||
calls.append(name)
|
||||
|
||||
sys.setprofile(record)
|
||||
try:
|
||||
result: Final = run()
|
||||
finally:
|
||||
sys.setprofile(previous)
|
||||
return result, tuple(calls)
|
||||
|
||||
|
||||
def _tool_result_messages(tool_output: str) -> list[AllMessageValues]: # mutable-ok: the truncator only walks lists
|
||||
return [
|
||||
{"role": "user", "content": "Summarize the rows the tool returned."},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": None,
|
||||
"tool_calls": [
|
||||
{"id": "call_1", "type": "function", "function": {"name": "dump_rows", "arguments": "{}"}}
|
||||
],
|
||||
},
|
||||
{"role": "tool", "tool_call_id": "call_1", "content": tool_output},
|
||||
]
|
||||
|
||||
|
||||
_PLAIN_TOOL_OUTPUT_CHARS: Final = 400 * 1024
|
||||
|
||||
|
||||
class TestTruncateBase64InMessagesAsync:
|
||||
@pytest.mark.asyncio
|
||||
async def test_large_payload_is_scanned_off_the_event_loop(self, monkeypatch, scan_threads):
|
||||
|
|
@ -268,6 +315,83 @@ class TestTruncateBase64InMessagesAsync:
|
|||
assert scan_threads
|
||||
assert set(scan_threads) == {threading.get_ident()}
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"row",
|
||||
[
|
||||
'{"id": 7, "name": "widget", "status": "ok"}\n',
|
||||
"data: {\"choices\": [{\"delta\": {\"content\": \"hi\"}}]}\n\n",
|
||||
"const total = rows.map((r) => r.price).reduce((a, b) => a + b, 0);\n",
|
||||
'data: {"choices": [{"delta": {"content": "a; b"}}]}\n\n',
|
||||
"Content-Type: text/plain;base64,aGVsbG8=\n",
|
||||
],
|
||||
ids=["json_rows", "sse_lines", "code_with_semicolons", "sse_lines_with_semicolons", "base64_marker_without_data"],
|
||||
)
|
||||
def test_tool_message_without_data_uri_is_copied_without_a_regex_scan(self, row: str):
|
||||
tool_output: Final = (row * (_PLAIN_TOOL_OUTPUT_CHARS // len(row) + 1))[:_PLAIN_TOOL_OUTPUT_CHARS]
|
||||
messages: Final = _tool_result_messages(tool_output)
|
||||
|
||||
result, regex_calls = _regex_calls_while(lambda: truncate_base64_in_messages(messages))
|
||||
|
||||
assert result == messages
|
||||
assert result is not messages
|
||||
assert regex_calls == ()
|
||||
|
||||
def test_small_data_uri_is_truncated_inline_with_a_regex_scan(self):
|
||||
payload: Final = "L" * 2_000
|
||||
messages: Final = _image_messages(payload)
|
||||
|
||||
result, regex_calls = _regex_calls_while(
|
||||
lambda: _run_without_yielding(truncate_base64_in_messages_async(messages))
|
||||
)
|
||||
|
||||
assert regex_calls
|
||||
assert payload not in result[0]["content"][1]["image_url"]["url"]
|
||||
assert "base64_data truncated" in result[0]["content"][1]["image_url"]["url"]
|
||||
|
||||
def test_large_plain_text_is_copied_inline_without_rescanning_or_a_thread_hop(
|
||||
self, monkeypatch: pytest.MonkeyPatch, scan_threads: Sequence[int]
|
||||
):
|
||||
monkeypatch.setattr(logging_utils, "BASE64_TRUNCATION_OFFLOAD_THRESHOLD_CHARS", 1_000)
|
||||
messages: Final = _tool_result_messages("plain tool output\n" * 100)
|
||||
|
||||
result: Final = _run_without_yielding(truncate_base64_in_messages_async(messages))
|
||||
|
||||
assert result == messages
|
||||
assert result is not messages
|
||||
assert scan_threads == []
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_long_plain_text_history_is_copied_off_the_event_loop(
|
||||
self, monkeypatch: pytest.MonkeyPatch, scan_threads: Sequence[int]
|
||||
):
|
||||
monkeypatch.setattr(logging_utils, "BASE64_TRUNCATION_OFFLOAD_THRESHOLD_CHARS", 1_000)
|
||||
monkeypatch.setattr(logging_utils, "BASE64_TRUNCATION_OFFLOAD_THRESHOLD_STRINGS", 10)
|
||||
messages: Final = list(chain.from_iterable(_tool_result_messages(f"rows {turn}\n" * 100) for turn in range(5)))
|
||||
|
||||
result: Final = await truncate_base64_in_messages_async(messages)
|
||||
offload_threads: Final = tuple(scan_threads)
|
||||
|
||||
assert result == messages
|
||||
assert result is not messages
|
||||
assert offload_threads
|
||||
assert threading.get_ident() not in offload_threads
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_plain_text_over_the_inline_limit_is_copied_off_the_event_loop(
|
||||
self, monkeypatch: pytest.MonkeyPatch, scan_threads: Sequence[int]
|
||||
):
|
||||
monkeypatch.setattr(logging_utils, "BASE64_TRUNCATION_OFFLOAD_THRESHOLD_CHARS", 1_000)
|
||||
monkeypatch.setattr(logging_utils, "BASE64_TRUNCATION_INLINE_COPY_MAX_CHARS", 10_000)
|
||||
messages: Final = _tool_result_messages('data: {"choices": [{"delta": {"content": "a; b"}}]}\n\n' * 400)
|
||||
|
||||
result: Final = await truncate_base64_in_messages_async(messages)
|
||||
offload_threads: Final = tuple(scan_threads)
|
||||
|
||||
assert result == messages
|
||||
assert result is not messages
|
||||
assert offload_threads
|
||||
assert threading.get_ident() not in offload_threads
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_none_and_disabled_truncation_short_circuit(self, monkeypatch, scan_threads):
|
||||
assert await truncate_base64_in_messages_async(None) is None
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
import copy
|
||||
import json
|
||||
import logging
|
||||
from collections.abc import Iterator
|
||||
from itertools import chain
|
||||
from typing import Final
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
|
@ -9849,3 +9850,186 @@ def test_transform_request_tool_config_point_without_tools_drops_the_key():
|
|||
assert "cache_control_injection_points" not in data
|
||||
assert "tools" not in data
|
||||
assert litellm_params["metadata"] == {}
|
||||
|
||||
|
||||
def _five_cache_hint_conversation() -> list[dict[str, object]]:
|
||||
return [
|
||||
{"role": "system", "content": [{"type": "text", "text": "system prompt", "cache_control": {"type": "ephemeral"}}]},
|
||||
{"role": "user", "content": [{"type": "text", "text": "turn 1", "cache_control": {"type": "ephemeral"}}]},
|
||||
{"role": "assistant", "content": [{"type": "text", "text": "answer 1", "cache_control": {"type": "ephemeral"}}]},
|
||||
{"role": "user", "content": [{"type": "text", "text": "turn 2", "cache_control": {"type": "ephemeral"}}]},
|
||||
{"role": "assistant", "content": "answer 2"},
|
||||
{"role": "user", "content": [{"type": "text", "text": "turn 3", "cache_control": {"type": "ephemeral"}}]},
|
||||
]
|
||||
|
||||
|
||||
def _cached_texts(node: object) -> Iterator[str]:
|
||||
if isinstance(node, dict):
|
||||
yield from ((node["text"],) if "cache_control" in node and "text" in node else ())
|
||||
yield from chain.from_iterable(map(_cached_texts, node.values()))
|
||||
elif isinstance(node, list):
|
||||
yield from chain.from_iterable(map(_cached_texts, node))
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("config", "model"),
|
||||
[
|
||||
(AnthropicConfig(), "claude-haiku-5-5"),
|
||||
(VertexAIAnthropicConfig(), "claude-haiku-5-5"),
|
||||
(AmazonAnthropicClaudeConfig(), "anthropic.claude-haiku-5-5"),
|
||||
(AzureAnthropicConfig(), "claude-haiku-5-5"),
|
||||
],
|
||||
ids=["anthropic", "vertex_ai", "bedrock_invoke", "azure_ai"],
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
("drop_params", "expected_cached_texts"),
|
||||
[
|
||||
(True, ["answer 1", "turn 1", "turn 2", "turn 3"]),
|
||||
("true", ["answer 1", "turn 1", "turn 2", "turn 3"]),
|
||||
(False, ["answer 1", "system prompt", "turn 1", "turn 2", "turn 3"]),
|
||||
],
|
||||
ids=[
|
||||
"drop_params_keeps_the_newest_four",
|
||||
"string_drop_params_keeps_the_newest_four",
|
||||
"without_drop_params_all_five_pass_through",
|
||||
],
|
||||
)
|
||||
def test_claude_request_over_the_cache_control_cap_drops_the_oldest_hints_only_under_drop_params(
|
||||
config: AnthropicConfig,
|
||||
model: str,
|
||||
drop_params: bool | str,
|
||||
expected_cached_texts: list[str],
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "drop_params", False)
|
||||
messages: Final = _five_cache_hint_conversation()
|
||||
messages_before: Final = copy.deepcopy(messages)
|
||||
|
||||
body: Final = config.transform_request(
|
||||
model=model,
|
||||
messages=messages,
|
||||
optional_params={},
|
||||
litellm_params={"drop_params": drop_params},
|
||||
headers={},
|
||||
)
|
||||
|
||||
assert sorted(_cached_texts(body)) == expected_cached_texts
|
||||
assert json.dumps(body).count('"cache_control"') == len(expected_cached_texts)
|
||||
assert messages == messages_before
|
||||
|
||||
|
||||
def test_claude_cache_control_cap_keeps_a_cached_tool_and_drops_the_oldest_message_hint(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "drop_params", True)
|
||||
messages: Final = _five_cache_hint_conversation()[1:]
|
||||
tool: Final = {
|
||||
"name": "get_weather",
|
||||
"description": "Get the weather",
|
||||
"input_schema": {"type": "object", "properties": {}},
|
||||
"cache_control": {"type": "ephemeral"},
|
||||
}
|
||||
|
||||
body: Final = AnthropicConfig().transform_request(
|
||||
model="claude-haiku-5-5",
|
||||
messages=messages,
|
||||
optional_params={"tools": [tool]},
|
||||
litellm_params={},
|
||||
headers={},
|
||||
)
|
||||
|
||||
assert body["tools"][0]["cache_control"] == {"type": "ephemeral"}
|
||||
assert sorted(_cached_texts(body["messages"])) == ["answer 1", "turn 2", "turn 3"]
|
||||
|
||||
|
||||
def test_claude_cache_control_cap_drops_a_string_assistant_turns_text_hint_before_its_tool_call_hints(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "drop_params", True)
|
||||
hint: Final = {"type": "ephemeral"}
|
||||
tool_calls: Final = [
|
||||
{"id": f"call_{name}", "type": "function", "function": {"name": name, "arguments": "{}"}, "cache_control": hint}
|
||||
for name in ("lookup", "search")
|
||||
]
|
||||
follow_up: Final = [
|
||||
{"type": "text", "text": "follow-up", "cache_control": hint},
|
||||
{"type": "text", "text": "and more", "cache_control": hint},
|
||||
]
|
||||
messages: Final = [
|
||||
{"role": "user", "content": "question"},
|
||||
{"role": "assistant", "content": "checking both", "cache_control": hint, "tool_calls": tool_calls},
|
||||
{"role": "tool", "tool_call_id": "call_lookup", "content": "found"},
|
||||
{"role": "tool", "tool_call_id": "call_search", "content": "none"},
|
||||
{"role": "user", "content": follow_up},
|
||||
]
|
||||
|
||||
body: Final = AnthropicConfig().transform_request(
|
||||
model="claude-haiku-5-5",
|
||||
messages=messages,
|
||||
optional_params={},
|
||||
litellm_params={},
|
||||
headers={},
|
||||
)
|
||||
|
||||
assistant_blocks: Final = body["messages"][1]["content"]
|
||||
assert [block["type"] for block in assistant_blocks] == ["text", "tool_use", "tool_use"]
|
||||
assert ["cache_control" in block for block in assistant_blocks] == [False, True, True]
|
||||
assert json.dumps(body).count('"cache_control"') == 4
|
||||
assert json.dumps(body).count('"cache_control"') == 4
|
||||
|
||||
|
||||
def test_claude_cache_control_cap_keeps_the_newest_hints_inside_one_message(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "drop_params", True)
|
||||
messages: Final = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
*({"type": "text", "text": f"part {n}", "cache_control": {"type": "ephemeral"}} for n in range(1, 6)),
|
||||
{"type": "text", "text": "Reply with the single word OK."},
|
||||
],
|
||||
}
|
||||
]
|
||||
messages_before: Final = copy.deepcopy(messages)
|
||||
|
||||
body: Final = AnthropicConfig().transform_request(
|
||||
model="claude-haiku-5-5",
|
||||
messages=messages,
|
||||
optional_params={},
|
||||
litellm_params={},
|
||||
headers={},
|
||||
)
|
||||
|
||||
assert sorted(_cached_texts(body["messages"])) == ["part 2", "part 3", "part 4", "part 5"]
|
||||
assert json.dumps(body).count('"cache_control"') == 4
|
||||
assert messages == messages_before
|
||||
|
||||
|
||||
def test_claude_cache_control_cap_does_not_count_a_message_level_hint_that_list_content_never_sends(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "drop_params", False)
|
||||
messages: Final = [
|
||||
{"role": "system", "content": "system prompt", "cache_control": {"type": "ephemeral"}},
|
||||
{"role": "user", "content": "turn 1", "cache_control": {"type": "ephemeral"}},
|
||||
{"role": "assistant", "content": "answer 1"},
|
||||
{"role": "user", "content": "turn 2", "cache_control": {"type": "ephemeral"}},
|
||||
{"role": "assistant", "content": "answer 2"},
|
||||
{
|
||||
"role": "user",
|
||||
"content": [{"type": "text", "text": "turn 3", "cache_control": {"type": "ephemeral"}}],
|
||||
"cache_control": {"type": "ephemeral"},
|
||||
},
|
||||
]
|
||||
|
||||
body: Final = AnthropicConfig().transform_request(
|
||||
model="claude-haiku-5-5",
|
||||
messages=messages,
|
||||
optional_params={},
|
||||
litellm_params={"drop_params": True},
|
||||
headers={},
|
||||
)
|
||||
|
||||
assert sorted(_cached_texts(body)) == ["system prompt", "turn 1", "turn 2", "turn 3"]
|
||||
assert json.dumps(body).count('"cache_control"') == 4
|
||||
|
|
|
|||
|
|
@ -56,6 +56,98 @@ def test_is_anthropic_messages_url(url: str, expected: bool) -> None:
|
|||
assert is_anthropic_messages_url(url) is expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("configured_base", "url", "expected"),
|
||||
[
|
||||
pytest.param(None, "https://api.anthropic.com/v1/messages", 512, id="default-upstream"),
|
||||
pytest.param(None, "https://api.anthropic.com:443/v1/messages/", 512, id="default-upstream-explicit-port"),
|
||||
pytest.param("http://relay.test:8080", "https://api.anthropic.com/v1/messages", 512, id="direct-target"),
|
||||
pytest.param("https://api.anthropic.com", "https://api.anthropic.com/v1/messages", 512, id="configured-default"),
|
||||
pytest.param("http://relay.test:8080", "http://relay.test:8080/v1/messages", 255, id="configured-upstream"),
|
||||
pytest.param("http://relay.test:8080/a", "http://relay.test:8080/a/v1/messages", 255, id="configured-prefix"),
|
||||
pytest.param("https://Relay.Test", "https://relay.test/v1/messages", 255, id="configured-upper-case-host"),
|
||||
pytest.param("https://relay.test:443", "https://relay.test/v1/messages", 255, id="configured-https-port"),
|
||||
pytest.param("http://relay.test:80/a", "http://relay.test/a/v1/messages", 255, id="configured-http-port"),
|
||||
pytest.param("http://relay.test:8080", "http://relay.test:8081/v1/messages", None, id="other-port"),
|
||||
pytest.param("http://relay.test:notaport", "http://relay.test/v1/messages", None, id="invalid-configured-base"),
|
||||
pytest.param(None, "https://other-vendor.test/v1/messages", None, id="other-vendor"),
|
||||
pytest.param(None, "https://api.anthropic.com/v1/messages/count_tokens", None, id="count-tokens"),
|
||||
],
|
||||
)
|
||||
def test_anthropic_upstream_user_id_limit(
|
||||
monkeypatch: pytest.MonkeyPatch, configured_base: str | None, url: str, expected: int | None
|
||||
) -> None:
|
||||
from litellm.llms.anthropic.common_utils import anthropic_upstream_user_id_limit
|
||||
|
||||
monkeypatch.delenv("ANTHROPIC_BASE_URL", raising=False)
|
||||
monkeypatch.delenv("ANTHROPIC_API_BASE", raising=False)
|
||||
if configured_base is not None:
|
||||
monkeypatch.setenv("ANTHROPIC_API_BASE", configured_base)
|
||||
|
||||
assert anthropic_upstream_user_id_limit(url) == expected
|
||||
|
||||
|
||||
def test_anthropic_pass_through_api_base_prefers_the_api_base_variable(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
from litellm.llms.anthropic.common_utils import anthropic_pass_through_api_base
|
||||
|
||||
monkeypatch.setenv("ANTHROPIC_BASE_URL", "http://fallback.test")
|
||||
monkeypatch.setenv("ANTHROPIC_API_BASE", "http://preferred.test")
|
||||
assert anthropic_pass_through_api_base() == "http://preferred.test"
|
||||
|
||||
monkeypatch.delenv("ANTHROPIC_API_BASE")
|
||||
assert anthropic_pass_through_api_base() == "http://fallback.test"
|
||||
|
||||
|
||||
# Each row was sent to https://api.anthropic.com/v1/messages on 2026-10-30, False rows answered 400.
|
||||
@pytest.mark.parametrize(
|
||||
("user_id", "expected"),
|
||||
[
|
||||
pytest.param("user-1", True, id="opaque"),
|
||||
pytest.param("", True, id="empty"),
|
||||
pytest.param('{"device_id":"abc","session_id":"def"}', True, id="claude-code-shape"),
|
||||
pytest.param("1234567890", True, id="digits"),
|
||||
pytest.param("+14155550100", True, id="phone-shape"),
|
||||
pytest.param("a@b", True, id="no-dot"),
|
||||
pytest.param("@b.co", True, id="nothing-before-at-sign"),
|
||||
pytest.param("a@.co", True, id="nothing-between-at-sign-and-dot"),
|
||||
pytest.param("a@b.", True, id="nothing-after-dot"),
|
||||
pytest.param("a @ b.co", True, id="spaces-around-at-sign"),
|
||||
pytest.param("a\u00a0@b.co", False, id="no-break-space-before-at-sign"),
|
||||
pytest.param("a@b\n.co", False, id="newline-before-dot"),
|
||||
pytest.param("line one\nline two", False, id="newline"),
|
||||
pytest.param("a\tb", False, id="tab"),
|
||||
pytest.param("a\x1f@b.co", False, id="unit-separator-before-at-sign"),
|
||||
pytest.param("a@\x1cb.co", False, id="file-separator-after-at-sign"),
|
||||
pytest.param("a\u200bb", False, id="zero-width-space"),
|
||||
pytest.param("a@b.co", False, id="email"),
|
||||
pytest.param("user a@b.co here", False, id="email-inside-text"),
|
||||
pytest.param('{"device_id":"a@b.co"}', False, id="email-inside-json"),
|
||||
pytest.param("a@@example.com", False, id="doubled-at-sign"),
|
||||
pytest.param("@@example.com", False, id="leading-doubled-at-sign"),
|
||||
pytest.param("a@b@c.co", False, id="two-at-signs"),
|
||||
pytest.param("a@b..co", False, id="doubled-dot"),
|
||||
pytest.param("a@.b.co", False, id="dot-after-at-sign"),
|
||||
pytest.param("a@b.c d", False, id="email-before-space"),
|
||||
],
|
||||
)
|
||||
def test_anthropic_accepts_user_id(user_id: str, expected: bool) -> None:
|
||||
from litellm.constants import ANTHROPIC_MAX_USER_ID_LENGTH
|
||||
from litellm.llms.anthropic.common_utils import anthropic_accepts_user_id
|
||||
|
||||
assert anthropic_accepts_user_id(user_id, ANTHROPIC_MAX_USER_ID_LENGTH) is expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize("character", ["a", "\u00e9", "\U0001f600"], ids=["ascii", "two-bytes", "four-bytes"])
|
||||
def test_anthropic_accepts_user_id_counts_characters_up_to_the_limit(character: str) -> None:
|
||||
from litellm.constants import ANTHROPIC_MAX_USER_ID_LENGTH
|
||||
from litellm.llms.anthropic.common_utils import anthropic_accepts_user_id
|
||||
|
||||
assert anthropic_accepts_user_id(character * ANTHROPIC_MAX_USER_ID_LENGTH, ANTHROPIC_MAX_USER_ID_LENGTH) is True
|
||||
assert (
|
||||
anthropic_accepts_user_id(character * (ANTHROPIC_MAX_USER_ID_LENGTH + 1), ANTHROPIC_MAX_USER_ID_LENGTH) is False
|
||||
)
|
||||
|
||||
|
||||
def test_anthropic_model_not_found_maps_to_not_found_but_authentication_error_does_not() -> None:
|
||||
with respx.mock(assert_all_called=True) as router:
|
||||
router.post("https://api.anthropic.com/v1/messages").mock(
|
||||
|
|
|
|||
|
|
@ -26,7 +26,6 @@ from litellm.types.decisions import (
|
|||
DecisionsIRScoreAnswer,
|
||||
DecisionsRequestBody,
|
||||
OpenAIDecisionRequestBody,
|
||||
UnsupportedDecisionsRequest,
|
||||
)
|
||||
from litellm.types.utils import LlmProviders
|
||||
from litellm.utils import ProviderConfigManager
|
||||
|
|
@ -279,7 +278,7 @@ def test_a_malformed_systemone_response_names_the_field_at_fault(
|
|||
assert [item["loc"] for item in error.value.errors()] == [field]
|
||||
|
||||
|
||||
def test_each_media_kind_is_gated_by_its_own_support_flag() -> None:
|
||||
def test_each_media_kind_is_sent_in_its_own_field() -> None:
|
||||
audio: Final = ["data:audio/wav;base64,YQ=="]
|
||||
videos: Final = ["data:video/mp4;base64,YQ=="]
|
||||
audio_only: Final = systemone_request_to_ir(_SYSTEMONE_BODY.validate_python({**_SYSTEMONE_REQUEST, "audio": audio}))
|
||||
|
|
@ -287,22 +286,16 @@ def test_each_media_kind_is_gated_by_its_own_support_flag() -> None:
|
|||
_SYSTEMONE_BODY.validate_python({**_SYSTEMONE_REQUEST, "videos": videos})
|
||||
)
|
||||
|
||||
assert ir_to_systemone_request("clef-omni", audio_only, supports_audio=True) == {
|
||||
assert ir_to_systemone_request("clef-omni", audio_only) == {
|
||||
"model": "clef-omni",
|
||||
**_SYSTEMONE_REQUEST,
|
||||
"audio": audio,
|
||||
}
|
||||
assert ir_to_systemone_request("clef-omni", video_only, supports_video=True) == {
|
||||
assert ir_to_systemone_request("clef-omni", video_only) == {
|
||||
"model": "clef-omni",
|
||||
**_SYSTEMONE_REQUEST,
|
||||
"videos": videos,
|
||||
}
|
||||
assert isinstance(
|
||||
ir_to_systemone_request("clef-omni", audio_only, supports_video=True), UnsupportedDecisionsRequest
|
||||
)
|
||||
assert isinstance(
|
||||
ir_to_systemone_request("clef-omni", video_only, supports_audio=True), UnsupportedDecisionsRequest
|
||||
)
|
||||
|
||||
|
||||
_ASSET_REQUEST: Final[Mapping[str, object]] = {
|
||||
|
|
|
|||
|
|
@ -11259,3 +11259,86 @@ def test_converse_sends_only_the_header_betas_when_header_and_body_both_carry_be
|
|||
"thinking-binding-controls-2026-08-01",
|
||||
"thinking-display-updates-2026-08-18",
|
||||
]
|
||||
|
||||
|
||||
def _five_cache_hint_conversation() -> list[dict[str, object]]:
|
||||
return [
|
||||
{"role": "system", "content": [{"type": "text", "text": "system prompt", "cache_control": {"type": "ephemeral"}}]},
|
||||
{"role": "user", "content": [{"type": "text", "text": "turn 1", "cache_control": {"type": "ephemeral"}}]},
|
||||
{"role": "assistant", "content": [{"type": "text", "text": "answer 1", "cache_control": {"type": "ephemeral"}}]},
|
||||
{"role": "user", "content": [{"type": "text", "text": "turn 2", "cache_control": {"type": "ephemeral"}}]},
|
||||
{"role": "assistant", "content": "answer 2"},
|
||||
{"role": "user", "content": [{"type": "text", "text": "turn 3", "cache_control": {"type": "ephemeral"}}]},
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("drop_params", "expected_system_cache_points"),
|
||||
[(True, 0), ("true", 0), (False, 1)],
|
||||
ids=[
|
||||
"drop_params_drops_the_oldest_hint",
|
||||
"string_drop_params_drops_the_oldest_hint",
|
||||
"without_drop_params_all_five_pass_through",
|
||||
],
|
||||
)
|
||||
def test_converse_claude_request_over_the_cache_control_cap_drops_the_oldest_hints_only_under_drop_params(
|
||||
drop_params: bool | str, expected_system_cache_points: int, monkeypatch: pytest.MonkeyPatch, local_model_cost_map: None
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "drop_params", False)
|
||||
messages: Final = _five_cache_hint_conversation()
|
||||
messages_before: Final = copy.deepcopy(messages)
|
||||
|
||||
request: Final = AmazonConverseConfig().transform_request(
|
||||
model="anthropic.claude-haiku-5-5",
|
||||
messages=messages,
|
||||
optional_params={},
|
||||
litellm_params={"drop_params": drop_params},
|
||||
headers={},
|
||||
)
|
||||
|
||||
assert len(list(_cache_points(request["system"]))) == expected_system_cache_points
|
||||
assert [len(list(_cache_points(message))) for message in request["messages"]] == [1, 1, 1, 0, 1]
|
||||
assert messages == messages_before
|
||||
|
||||
|
||||
def test_converse_claude_tool_message_with_several_marked_blocks_counts_as_one_cache_point_against_the_cap(
|
||||
monkeypatch: pytest.MonkeyPatch, local_model_cost_map: None
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "drop_params", False)
|
||||
messages: Final = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "policy", "cache_control": {"type": "ephemeral"}},
|
||||
{"type": "text", "text": "question", "cache_control": {"type": "ephemeral"}},
|
||||
],
|
||||
},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": None,
|
||||
"tool_calls": [{"id": "call_1", "type": "function", "function": {"name": "lookup", "arguments": "{}"}}],
|
||||
},
|
||||
{
|
||||
"role": "tool",
|
||||
"tool_call_id": "call_1",
|
||||
"content": [
|
||||
{"type": "text", "text": "part 1", "cache_control": {"type": "ephemeral"}},
|
||||
{"type": "text", "text": "part 2", "cache_control": {"type": "ephemeral"}},
|
||||
{"type": "text", "text": "part 3", "cache_control": {"type": "ephemeral"}},
|
||||
],
|
||||
},
|
||||
]
|
||||
messages_before: Final = copy.deepcopy(messages)
|
||||
|
||||
request: Final = AmazonConverseConfig().transform_request(
|
||||
model="anthropic.claude-haiku-5-5",
|
||||
messages=messages,
|
||||
optional_params={
|
||||
"tools": [{"type": "function", "function": {"name": "lookup", "parameters": {"type": "object"}}}]
|
||||
},
|
||||
litellm_params={"drop_params": True},
|
||||
headers={},
|
||||
)
|
||||
|
||||
assert len(list(_cache_points(request["messages"]))) == 3
|
||||
assert messages == messages_before
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
|
||||
import asyncio, importlib, litellm, litellm.litellm_core_utils.get_model_cost_map as bedrock_govcloud_model_cost_map, pytest
|
||||
import asyncio, importlib, litellm, pytest
|
||||
from typing import Final
|
||||
|
||||
|
||||
|
|
@ -1563,7 +1563,6 @@ def _pr4_bedrock_env(monkeypatch: pytest.MonkeyPatch) -> None:
|
|||
def _use_local_model_cost_map() -> Iterator[None]:
|
||||
with pytest.MonkeyPatch.context() as monkeypatch:
|
||||
monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True")
|
||||
importlib.reload(bedrock_govcloud_model_cost_map)
|
||||
importlib.reload(litellm)
|
||||
yield
|
||||
|
||||
|
|
|
|||
|
|
@ -2773,3 +2773,66 @@ def test_claude_json_schema_stream_split_across_json_tools_keeps_the_non_stream_
|
|||
choices: Final = _json_schema_stream_choices(respx_mock, call, split_stream)
|
||||
|
||||
assert [choice.finish_reason for choice in choices if choice.finish_reason] == [answer.finish_reason]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("model", "base_model"),
|
||||
[("databricks-claude-haiku-5-5", None), ("support-bot-endpoint", "databricks/databricks-claude-haiku-5-5")],
|
||||
ids=["claude_in_the_endpoint_name", "claude_named_only_by_base_model"],
|
||||
)
|
||||
def test_databricks_claude_request_over_the_cache_control_cap_keeps_the_newest_four_hints_under_drop_params(
|
||||
model: str, base_model: str | None, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "drop_params", False)
|
||||
hint: Final = {"type": "ephemeral"}
|
||||
messages: Final = [
|
||||
{"role": "system", "content": "system prompt", "cache_control": hint},
|
||||
{"role": "user", "content": "turn 1", "cache_control": hint},
|
||||
{"role": "assistant", "content": "answer 1", "cache_control": hint},
|
||||
{"role": "user", "content": "turn 2", "cache_control": hint},
|
||||
{"role": "assistant", "content": "answer 2"},
|
||||
{"role": "user", "content": "turn 3", "cache_control": hint},
|
||||
]
|
||||
messages_before: Final = copy.deepcopy(messages)
|
||||
|
||||
body: Final = DatabricksConfig().transform_request(
|
||||
model=model,
|
||||
messages=messages,
|
||||
optional_params={},
|
||||
litellm_params={"drop_params": True, "custom_llm_provider": "databricks", "base_model": base_model},
|
||||
headers={},
|
||||
)
|
||||
|
||||
assert "cache_control" not in json.dumps(body["messages"][0])
|
||||
assert json.dumps(body["messages"]).count('"cache_control"') == 4
|
||||
assert messages == messages_before
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"optional_params",
|
||||
[{"cache_control": {"type": "ephemeral"}}, {"extra_body": {"cache_control": {"type": "ephemeral"}}}],
|
||||
ids=["top_level_cache_control", "extra_body_cache_control"],
|
||||
)
|
||||
def test_databricks_claude_cache_control_cap_keeps_room_for_a_top_level_cache_control_under_drop_params(
|
||||
optional_params: dict[str, object], monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "drop_params", False)
|
||||
hint: Final = {"type": "ephemeral"}
|
||||
messages: Final = [
|
||||
{"role": "user", "content": "turn 1", "cache_control": hint},
|
||||
{"role": "assistant", "content": "answer 1", "cache_control": hint},
|
||||
{"role": "user", "content": "turn 2", "cache_control": hint},
|
||||
{"role": "assistant", "content": "answer 2", "cache_control": hint},
|
||||
{"role": "user", "content": "turn 3"},
|
||||
]
|
||||
|
||||
body: Final = DatabricksConfig().transform_request(
|
||||
model="databricks-claude-haiku-5-5",
|
||||
messages=messages,
|
||||
optional_params=copy.deepcopy(optional_params),
|
||||
litellm_params={"drop_params": True, "custom_llm_provider": "databricks"},
|
||||
headers={},
|
||||
)
|
||||
|
||||
assert "cache_control" not in json.dumps(body["messages"][0])
|
||||
assert json.dumps(body["messages"]).count('"cache_control"') == 3
|
||||
|
|
|
|||
|
|
@ -1,6 +1,8 @@
|
|||
import asyncio
|
||||
import copy
|
||||
import json
|
||||
import os
|
||||
from typing import Final
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
|
@ -200,3 +202,38 @@ async def test_deepinfra_tool_message_content_transformation_async():
|
|||
print(f"✓ Async test passed: {tool_message['content']}")
|
||||
|
||||
print("\n✅ DeepInfra async tool message transformation test passed!")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("is_async", [False, True], ids=["sync", "async"])
|
||||
async def test_deepinfra_request_stringifies_tool_content_without_touching_the_callers_message(is_async: bool) -> None:
|
||||
from litellm.llms.deepinfra.chat.transformation import DeepInfraConfig
|
||||
|
||||
config: Final = DeepInfraConfig()
|
||||
messages: Final = [
|
||||
{"role": "user", "content": "Calculate 10 + 10"},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": "",
|
||||
"tool_calls": [
|
||||
{"id": "call_1", "type": "function", "function": {"name": "calculator", "arguments": "{}"}}
|
||||
],
|
||||
},
|
||||
{"role": "tool", "tool_call_id": "call_1", "content": [{"type": "text", "text": "20"}]},
|
||||
]
|
||||
messages_before: Final = copy.deepcopy(messages)
|
||||
request_kwargs: Final = {
|
||||
"model": "deepseek-ai/DeepSeek-V3.2",
|
||||
"messages": messages,
|
||||
"optional_params": {},
|
||||
"litellm_params": {},
|
||||
"headers": {},
|
||||
}
|
||||
|
||||
body: Final = (
|
||||
await config.async_transform_request(**request_kwargs)
|
||||
if is_async
|
||||
else config.transform_request(**request_kwargs)
|
||||
)
|
||||
|
||||
assert body["messages"][2]["content"] == "20"
|
||||
assert messages == messages_before
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
import copy
|
||||
import json
|
||||
import logging
|
||||
from collections.abc import Callable
|
||||
|
|
@ -537,6 +538,26 @@ def test_tool_search_reply_sent_to_fireworks_drops_litellm_response_fields():
|
|||
}
|
||||
|
||||
|
||||
def test_cache_control_is_stripped_from_the_request_but_kept_on_the_callers_messages():
|
||||
hint: Final = {"type": "ephemeral"}
|
||||
messages: Final = [
|
||||
{"role": "system", "content": [{"type": "text", "text": "a long cached prompt", "cache_control": hint}]},
|
||||
{"role": "user", "content": "hi", "cache_control": hint},
|
||||
]
|
||||
caller_messages_before: Final = json.dumps(messages)
|
||||
|
||||
request: Final = FireworksAIConfig().transform_request(
|
||||
model="accounts/fireworks/models/gpt-oss-120b",
|
||||
messages=messages,
|
||||
optional_params={},
|
||||
litellm_params={},
|
||||
headers={},
|
||||
)
|
||||
|
||||
assert "cache_control" not in json.dumps(request["messages"])
|
||||
assert json.dumps(messages) == caller_messages_before
|
||||
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# Regression tests for legacy / OpenAPI $ref defs in tool parameters.
|
||||
#
|
||||
|
|
@ -2477,3 +2498,43 @@ def test_translate_developer_role_follows_the_deployment_model_info_over_the_cos
|
|||
)
|
||||
|
||||
assert [message["role"] for message in messages] == expected_roles
|
||||
|
||||
|
||||
def test_fireworks_request_rewrites_tools_without_touching_the_callers_tool() -> None:
|
||||
tools: Final = [
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "create_thing",
|
||||
"strict": True,
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {"thing": {"$ref": "#/definitions/Thing"}},
|
||||
"definitions": {"Thing": {"type": "object", "properties": {"id": {"type": "string"}}}},
|
||||
},
|
||||
},
|
||||
}
|
||||
]
|
||||
tools_before: Final = copy.deepcopy(tools)
|
||||
|
||||
request: Final = FireworksAIConfig().transform_request(
|
||||
model="accounts/fireworks/models/gpt-oss-120b",
|
||||
messages=[{"role": "user", "content": "Make a thing"}],
|
||||
optional_params={"tools": tools},
|
||||
litellm_params={},
|
||||
headers={},
|
||||
)
|
||||
|
||||
assert request["tools"] == [
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "create_thing",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {"thing": {"type": "object", "properties": {"id": {"type": "string"}}}},
|
||||
},
|
||||
},
|
||||
}
|
||||
]
|
||||
assert tools == tools_before
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
import copy
|
||||
import logging
|
||||
from typing import Final
|
||||
from unittest.mock import patch
|
||||
|
|
@ -589,3 +590,19 @@ async def test_streamed_reasoning_reply_can_be_replayed(is_async: bool, with_too
|
|||
}
|
||||
]
|
||||
assert reply.reasoning_content == "I should check the weather."
|
||||
|
||||
|
||||
def test_groq_request_drops_null_assistant_fields_without_touching_the_callers_messages() -> None:
|
||||
messages: Final = [
|
||||
{"role": "user", "content": "Hi"},
|
||||
{"role": "assistant", "content": "Hello", "function_call": None, "tool_calls": None},
|
||||
{"role": "user", "content": "Again"},
|
||||
]
|
||||
messages_before: Final = copy.deepcopy(messages)
|
||||
|
||||
body: Final = GroqChatConfig().transform_request(
|
||||
model="openai/gpt-oss-120b", messages=messages, optional_params={}, litellm_params={}, headers={}
|
||||
)
|
||||
|
||||
assert body["messages"][1] == {"role": "assistant", "content": "Hello"}
|
||||
assert messages == messages_before
|
||||
|
|
|
|||
|
|
@ -1,4 +1,6 @@
|
|||
import copy
|
||||
import json
|
||||
from typing import Final
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
|
@ -397,3 +399,43 @@ def test_hosted_vllm_custom_tools_use_top_level_input_schema():
|
|||
assert tools[0]["function"]["name"] == "search"
|
||||
assert tools[0]["function"]["description"] == "Search docs"
|
||||
assert tools[0]["function"]["parameters"] == input_schema
|
||||
|
||||
|
||||
def _strict_weather_tool() -> dict[str, object]:
|
||||
return {
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "get_weather",
|
||||
"strict": True,
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {"city": {"type": "string"}},
|
||||
"required": ["city"],
|
||||
"additionalProperties": False,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def test_hosted_vllm_tools_are_rewritten_without_touching_the_callers_tool() -> None:
|
||||
tools: Final = [_strict_weather_tool()]
|
||||
tools_before: Final = copy.deepcopy(tools)
|
||||
|
||||
optional_params: Final = HostedVLLMChatConfig().map_openai_params(
|
||||
non_default_params={"tools": tools}, optional_params={}, model="gpt-oss-120b", drop_params=False
|
||||
)
|
||||
|
||||
assert optional_params["tools"] == [
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "get_weather",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {"city": {"type": "string"}},
|
||||
"required": ["city"],
|
||||
},
|
||||
},
|
||||
}
|
||||
]
|
||||
assert tools == tools_before
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
import copy
|
||||
import json
|
||||
import logging
|
||||
from collections.abc import Mapping
|
||||
|
|
@ -1212,7 +1213,10 @@ def test_mistral_sends_only_the_text_items_of_a_native_thinking_chunk_beside_an_
|
|||
"content": [
|
||||
{
|
||||
"type": "thinking",
|
||||
"thinking": [{"type": "text", "text": "It has whiskers."}, {"type": "reference", "reference_ids": [1]}],
|
||||
"thinking": [
|
||||
{"type": "text", "text": "It has whiskers."},
|
||||
{"type": "reference", "reference_ids": [1]},
|
||||
],
|
||||
},
|
||||
{"type": "text", "text": "A cat."},
|
||||
],
|
||||
|
|
@ -1672,6 +1676,18 @@ def test_mistral_transform_request_drops_an_empty_tool_message_whose_call_id_is_
|
|||
assert [m.get("role") for m in request["messages"]] == ["user", "assistant"]
|
||||
|
||||
|
||||
def test_mistral_request_drops_a_user_name_without_touching_the_callers_message() -> None:
|
||||
messages: Final = [{"role": "user", "content": "Hi"}, {"role": "user", "content": "", "name": "alice"}]
|
||||
messages_before: Final = copy.deepcopy(messages)
|
||||
|
||||
body: Final = MistralConfig().transform_request(
|
||||
model="mistral-medium-latest", messages=messages, optional_params={}, litellm_params={}, headers={}
|
||||
)
|
||||
|
||||
assert "name" not in json.dumps(body["messages"])
|
||||
assert messages == messages_before
|
||||
|
||||
|
||||
def _warnings_naming(caplog: pytest.LogCaptureFixture, param: str) -> tuple[str, ...]:
|
||||
return tuple(
|
||||
r.getMessage() for r in caplog.records if r.levelno == logging.WARNING and f"Dropping {param}=" in r.getMessage()
|
||||
|
|
|
|||
|
|
@ -1824,6 +1824,94 @@ class TestToolSchemaCombinatorFlatteningForOpenAI:
|
|||
assert request["tools"][0] is tool
|
||||
|
||||
|
||||
class TestCacheControlStrippingDoesNotMutateCallerInput:
|
||||
"""
|
||||
Stripping cache_control for a provider that cannot use it must not reach back
|
||||
into the caller's own message and tool objects.
|
||||
|
||||
filter_value_from_dict deletes the key in place and recurses into nested dicts
|
||||
and lists, and remove_cache_control_flag_from_messages_and_tools assigned the
|
||||
result back into the caller's list, so one call to any OpenAI-compatible
|
||||
provider permanently stripped cache_control from a list the caller still holds.
|
||||
Reusing that list on Anthropic or Bedrock afterwards then silently lost prompt
|
||||
caching, with no error and full-price billing.
|
||||
"""
|
||||
|
||||
def setup_method(self):
|
||||
self.config = OpenAIGPTConfig()
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _clean_openai_base_env(self, monkeypatch):
|
||||
monkeypatch.delenv("OPENAI_BASE_URL", raising=False)
|
||||
monkeypatch.delenv("OPENAI_API_BASE", raising=False)
|
||||
monkeypatch.setattr(litellm, "api_base", None, raising=False)
|
||||
|
||||
@staticmethod
|
||||
def _messages():
|
||||
return [
|
||||
{
|
||||
"role": "system",
|
||||
"content": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": "a long cached system prompt",
|
||||
"cache_control": {"type": "ephemeral"},
|
||||
}
|
||||
],
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": "Hello",
|
||||
"cache_control": {"type": "ephemeral"},
|
||||
},
|
||||
]
|
||||
|
||||
@staticmethod
|
||||
def _tools():
|
||||
return [
|
||||
{
|
||||
"type": "function",
|
||||
"function": {"name": "get_weather", "parameters": {"type": "object"}},
|
||||
"cache_control": {"type": "ephemeral"},
|
||||
}
|
||||
]
|
||||
|
||||
def _transform(self, messages, tools):
|
||||
return self.config.transform_request(
|
||||
model="gpt-4o",
|
||||
messages=messages,
|
||||
optional_params={"tools": tools},
|
||||
litellm_params={"custom_llm_provider": "openai", "api_base": None},
|
||||
headers={},
|
||||
)
|
||||
|
||||
def test_caller_messages_and_tools_keep_cache_control(self):
|
||||
messages, tools = self._messages(), self._tools()
|
||||
messages_before, tools_before = copy.deepcopy(messages), copy.deepcopy(tools)
|
||||
|
||||
request = self._transform(messages, tools)
|
||||
|
||||
assert "cache_control" not in json.dumps(request)
|
||||
assert messages == messages_before
|
||||
assert tools == tools_before
|
||||
|
||||
def test_a_later_anthropic_call_still_sees_cache_control(self):
|
||||
"""The user-visible consequence: the same message list reused on a provider
|
||||
that does support caching must still carry the cache breakpoints."""
|
||||
messages = self._messages()
|
||||
|
||||
self._transform(messages, self._tools())
|
||||
|
||||
anthropic_body = litellm.AnthropicConfig().transform_request(
|
||||
model="claude-3-5-sonnet-20240620",
|
||||
messages=messages,
|
||||
optional_params={},
|
||||
litellm_params={},
|
||||
headers={},
|
||||
)
|
||||
assert "cache_control" in json.dumps(anthropic_body)
|
||||
|
||||
|
||||
def _developer_message_after_a_user_turn() -> list[dict[str, str]]:
|
||||
return [
|
||||
{"role": "system", "content": "You are terse."},
|
||||
|
|
@ -1916,3 +2004,53 @@ def test_completion_leaves_the_callers_closing_developer_message_unchanged_on_a_
|
|||
)
|
||||
assert json.loads(chat.calls.last.request.content)["messages"][-1]["role"] == "user"
|
||||
assert messages == sent
|
||||
|
||||
|
||||
def _user_turn_with_litellm_only_media_fields() -> list[dict[str, object]]:
|
||||
return [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "Describe these"},
|
||||
{"type": "image_url", "image_url": {"url": "https://example.com/cat.png", "format": "image/png"}},
|
||||
{"type": "image_url", "image_url": "https://example.com/dog.png"},
|
||||
{
|
||||
"type": "file",
|
||||
"file": {"file_data": "data:application/pdf;base64,JVBERi0=", "format": "application/pdf"},
|
||||
},
|
||||
],
|
||||
}
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("is_async", [False, True], ids=["sync", "async"])
|
||||
async def test_openai_request_rewrites_media_parts_without_touching_the_callers_message(
|
||||
is_async: bool, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
monkeypatch.delenv("OPENAI_BASE_URL", raising=False)
|
||||
monkeypatch.delenv("OPENAI_API_BASE", raising=False)
|
||||
monkeypatch.setattr(litellm, "api_base", None, raising=False)
|
||||
config: Final = OpenAIGPTConfig()
|
||||
messages: Final = _user_turn_with_litellm_only_media_fields()
|
||||
messages_before: Final = copy.deepcopy(messages)
|
||||
request_kwargs: Final = {
|
||||
"model": "gpt-5.5",
|
||||
"messages": messages,
|
||||
"optional_params": {},
|
||||
"litellm_params": {"custom_llm_provider": "openai", "api_base": None},
|
||||
"headers": {},
|
||||
}
|
||||
|
||||
body: Final = (
|
||||
await config.async_transform_request(**request_kwargs)
|
||||
if is_async
|
||||
else config.transform_request(**request_kwargs)
|
||||
)
|
||||
|
||||
assert body["messages"][0]["content"] == [
|
||||
{"type": "text", "text": "Describe these"},
|
||||
{"type": "image_url", "image_url": {"url": "https://example.com/cat.png"}},
|
||||
{"type": "image_url", "image_url": {"url": "https://example.com/dog.png"}},
|
||||
{"type": "file", "file": {"file_data": "data:application/pdf;base64,JVBERi0=", "filename": "my_file.pdf"}},
|
||||
]
|
||||
assert messages == messages_before
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
import copy
|
||||
import json
|
||||
import logging
|
||||
from collections.abc import Callable, Mapping
|
||||
|
|
@ -541,6 +542,50 @@ class TestOpenAIResponsesAPIConfig:
|
|||
assert "cache_control" not in result["tools"][0]
|
||||
assert result["tools"][0]["name"] == "get_weather"
|
||||
|
||||
def test_transform_does_not_strip_cache_control_from_the_callers_input(self):
|
||||
"""Stripping for OpenAI must not reach back into the caller's own objects.
|
||||
|
||||
`filter_value_from_dict` deletes the key in place and recurses, and
|
||||
`_validate_input_param` passes plain dict items through by reference, so
|
||||
the caller's input list used to lose its cache breakpoints. Reusing that
|
||||
list on Anthropic or Bedrock afterwards then silently lost prompt caching.
|
||||
Same defect as the Chat Completions path.
|
||||
"""
|
||||
input_with_cache_control = [
|
||||
{
|
||||
"role": "system",
|
||||
"content": [
|
||||
{
|
||||
"type": "input_text",
|
||||
"text": "a long cached system prompt",
|
||||
"cache_control": {"type": "ephemeral"},
|
||||
}
|
||||
],
|
||||
}
|
||||
]
|
||||
tools_with_cache_control = [
|
||||
{
|
||||
"type": "function",
|
||||
"name": "get_weather",
|
||||
"parameters": {"type": "object"},
|
||||
"cache_control": {"type": "ephemeral"},
|
||||
}
|
||||
]
|
||||
input_before = copy.deepcopy(input_with_cache_control)
|
||||
tools_before = copy.deepcopy(tools_with_cache_control)
|
||||
|
||||
result = self.config.transform_responses_api_request(
|
||||
model=self.model,
|
||||
input=input_with_cache_control,
|
||||
response_api_optional_request_params={"tools": tools_with_cache_control},
|
||||
litellm_params={},
|
||||
headers={},
|
||||
)
|
||||
|
||||
assert "cache_control" not in json.dumps(result)
|
||||
assert input_with_cache_control == input_before
|
||||
assert tools_with_cache_control == tools_before
|
||||
|
||||
def test_transform_preserves_input_without_cache_control(self):
|
||||
"""Inputs without cache_control must pass through unmodified."""
|
||||
input_clean = [
|
||||
|
|
@ -4607,10 +4652,10 @@ def test_cache_control_still_marks_a_part_whose_own_marker_is_malformed_under_dr
|
|||
}
|
||||
|
||||
|
||||
def _caller_input_after_transform(model: str, marker: object) -> str:
|
||||
def _hinted_caller_input(marker: object) -> list[dict[str, object]]:
|
||||
hint: Final = {"type": "ephemeral"}
|
||||
marked: Final = {} if marker is None else {"prompt_cache_breakpoint": marker}
|
||||
input: Final = [
|
||||
return [
|
||||
{
|
||||
"type": "message",
|
||||
"role": "user",
|
||||
|
|
@ -4626,6 +4671,10 @@ def _caller_input_after_transform(model: str, marker: object) -> str:
|
|||
"content": [{"type": "output_text", "text": "ok", "cache_control": hint, **marked}],
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
def _caller_input_after_transform(model: str, marker: object) -> str:
|
||||
input: Final = _hinted_caller_input(marker)
|
||||
_CONFIGS[0].transform_responses_api_request(
|
||||
model=model,
|
||||
input=cast("ResponseInputParam", input), # cast-ok: plain dicts shaped like the SDK's input items
|
||||
|
|
@ -4647,6 +4696,15 @@ def test_a_marker_leaves_the_caller_request_that_fallbacks_and_logs_read_as_an_u
|
|||
assert _caller_input_after_transform(model, marker).replace(marker_json, "") == unmarked
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", ("gpt-4.1", "gpt-6.1-sol"))
|
||||
@pytest.mark.parametrize("marker", (None, _VALID_MARKERS[0], "yes"), ids=("unmarked", "explicit", "malformed"))
|
||||
def test_the_caller_input_keeps_its_cache_control_hints_for_a_fallback_deployment(
|
||||
monkeypatch: pytest.MonkeyPatch, model: str, marker: object
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "drop_params", False)
|
||||
assert _caller_input_after_transform(model, marker) == json.dumps(_hinted_caller_input(marker))
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.usefixtures(httpx_transport.__name__)
|
||||
async def test_aresponses_forwards_previous_response_id_to_openai() -> None:
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
import copy
|
||||
from typing import Final
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
|
|
@ -198,3 +199,23 @@ def test_translate_developer_role_keeps_developer_messages_untouched() -> None:
|
|||
)
|
||||
== messages
|
||||
)
|
||||
|
||||
|
||||
def test_o_series_request_turns_system_into_user_without_touching_the_callers_messages(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
model: Final = "o-series-test-model-without-system-role"
|
||||
monkeypatch.setitem(
|
||||
litellm.model_cost,
|
||||
model,
|
||||
{"litellm_provider": "openai", "mode": "chat", "supports_system_messages": False},
|
||||
)
|
||||
messages: Final = [{"role": "system", "content": "Be terse"}, {"role": "user", "content": "Hi"}]
|
||||
messages_before: Final = copy.deepcopy(messages)
|
||||
|
||||
body: Final = OpenAIOSeriesConfig().transform_request(
|
||||
model=model, messages=messages, optional_params={}, litellm_params={}, headers={}
|
||||
)
|
||||
|
||||
assert [message["role"] for message in body["messages"]] == ["user", "user"]
|
||||
assert messages == messages_before
|
||||
|
|
|
|||
706
tests/unit/llms/openai_like/test_hostinger_router_provider.py
Normal file
706
tests/unit/llms/openai_like/test_hostinger_router_provider.py
Normal file
|
|
@ -0,0 +1,706 @@
|
|||
import copy
|
||||
import json
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
import respx
|
||||
|
||||
import litellm
|
||||
from litellm.caching.llm_caching_handler import LLMClientCache
|
||||
|
||||
|
||||
def test_hostinger_router_provider_resolution(monkeypatch: pytest.MonkeyPatch):
|
||||
from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
|
||||
|
||||
monkeypatch.setenv("HOSTINGER_ROUTER_API_KEY", "hostinger-test-key")
|
||||
|
||||
model, provider, api_key, api_base = get_llm_provider(
|
||||
model="hostinger_router/claude-sonnet-5",
|
||||
custom_llm_provider=None,
|
||||
api_base=None,
|
||||
api_key=None,
|
||||
)
|
||||
|
||||
assert model == "claude-sonnet-5"
|
||||
assert provider == "hostinger_router"
|
||||
assert api_key == "hostinger-test-key"
|
||||
assert api_base == "https://router.hostinger.com/v1"
|
||||
|
||||
|
||||
def test_hostinger_router_provider_keeps_explicit_credentials(monkeypatch: pytest.MonkeyPatch):
|
||||
from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
|
||||
|
||||
monkeypatch.setenv("HOSTINGER_ROUTER_API_KEY", "hostinger-env-key")
|
||||
|
||||
_, provider, api_key, api_base = get_llm_provider(
|
||||
model="hostinger_router/claude-sonnet-5",
|
||||
custom_llm_provider=None,
|
||||
api_base="https://router.internal.example/v1",
|
||||
api_key="hostinger-explicit-key",
|
||||
)
|
||||
|
||||
assert provider == "hostinger_router"
|
||||
assert api_key == "hostinger-explicit-key"
|
||||
assert api_base == "https://router.internal.example/v1"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", ["gpt-5-nano", "claude-sonnet-5"])
|
||||
def test_hostinger_router_api_base_keeps_unprefixed_model_provider(model: str):
|
||||
from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
|
||||
|
||||
_, provider_without_api_base, _, _ = get_llm_provider(model=model)
|
||||
_, provider_with_hostinger_api_base, _, _ = get_llm_provider(
|
||||
model=model,
|
||||
api_base="https://router.hostinger.com/v1",
|
||||
api_key="hostinger-test-key",
|
||||
)
|
||||
|
||||
assert provider_with_hostinger_api_base == provider_without_api_base
|
||||
|
||||
|
||||
HOSTINGER_ROUTER_MODELS = tuple(sorted(name for name in litellm.model_cost if name.startswith("hostinger_router/")))
|
||||
HOSTINGER_ROUTER_CHAT_MODELS = tuple(
|
||||
name for name in HOSTINGER_ROUTER_MODELS if litellm.get_model_info(name)["mode"] == "chat"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", HOSTINGER_ROUTER_CHAT_MODELS)
|
||||
def test_hostinger_router_model_cost_and_capabilities(model: str):
|
||||
from litellm.cost_calculator import cost_per_token
|
||||
|
||||
prompt_cost, completion_cost = cost_per_token(
|
||||
model=model,
|
||||
prompt_tokens=1_000_000,
|
||||
completion_tokens=1_000_000,
|
||||
custom_llm_provider="hostinger_router",
|
||||
)
|
||||
model_info = litellm.get_model_info(model)
|
||||
|
||||
assert prompt_cost == pytest.approx(model_info["input_cost_per_token"] * 1_000_000)
|
||||
assert completion_cost == pytest.approx(model_info["output_cost_per_token"] * 1_000_000)
|
||||
assert model_info["output_cost_per_token"] > 0
|
||||
assert model_info["max_tokens"] == model_info["max_output_tokens"] <= model_info["max_input_tokens"]
|
||||
assert model_info["litellm_provider"] == "hostinger_router"
|
||||
assert model_info["mode"] == "chat"
|
||||
if model_info.get("supports_prompt_caching"):
|
||||
assert 0 < model_info["cache_read_input_token_cost"] < model_info["input_cost_per_token"]
|
||||
capability_checks: Final = (
|
||||
("supports_function_calling", litellm.supports_function_calling),
|
||||
("supports_reasoning", litellm.supports_reasoning),
|
||||
("supports_response_schema", litellm.supports_response_schema),
|
||||
("supports_vision", litellm.supports_vision),
|
||||
)
|
||||
assert {name: check(model) for name, check in capability_checks} == {
|
||||
name: model_info.get(name) is True for name, _ in capability_checks
|
||||
}
|
||||
|
||||
|
||||
def test_hostinger_router_chat_completions_request():
|
||||
with respx.mock() as upstream:
|
||||
route: Final = upstream.post("https://router.hostinger.com/v1/chat/completions").respond(
|
||||
200,
|
||||
json={
|
||||
"id": "chatcmpl_hostinger",
|
||||
"object": "chat.completion",
|
||||
"created": 1_789_550_000,
|
||||
"model": "claude-sonnet-5",
|
||||
"choices": [
|
||||
{
|
||||
"index": 0,
|
||||
"message": {"role": "assistant", "content": "Hello from Hostinger Router"},
|
||||
"finish_reason": "stop",
|
||||
}
|
||||
],
|
||||
"usage": {"prompt_tokens": 4, "completion_tokens": 3, "total_tokens": 7},
|
||||
},
|
||||
)
|
||||
response: Final = litellm.completion(
|
||||
model="hostinger_router/claude-sonnet-5",
|
||||
messages=[{"role": "user", "content": "Say hello"}],
|
||||
api_key="hostinger-test-key",
|
||||
)
|
||||
|
||||
request: Final = route.calls.last.request
|
||||
body: Final = json.loads(request.content)
|
||||
assert route.call_count == 1
|
||||
assert request.headers["authorization"] == "Bearer hostinger-test-key"
|
||||
assert body["model"] == "claude-sonnet-5"
|
||||
assert response.choices[0].message.content == "Hello from Hostinger Router"
|
||||
|
||||
|
||||
def test_hostinger_router_chat_completions_streaming_request():
|
||||
chunks = [
|
||||
'data: {"id":"chatcmpl_hostinger","object":"chat.completion.chunk","created":1789550000,"model":"claude-sonnet-5","choices":[{"index":0,"delta":{"role":"assistant","content":"Hi"},"finish_reason":null}]}\n\n',
|
||||
'data: {"id":"chatcmpl_hostinger","object":"chat.completion.chunk","created":1789550000,"model":"claude-sonnet-5","choices":[{"index":0,"delta":{},"finish_reason":"stop"}],"usage":{"prompt_tokens":4,"completion_tokens":1,"total_tokens":5}}\n\n',
|
||||
"data: [DONE]\n\n",
|
||||
]
|
||||
with respx.mock() as upstream:
|
||||
route: Final = upstream.post("https://router.hostinger.com/v1/chat/completions").respond(
|
||||
200,
|
||||
headers={"content-type": "text/event-stream"},
|
||||
content="".join(chunks).encode(),
|
||||
)
|
||||
response: Final = litellm.completion(
|
||||
model="hostinger_router/claude-sonnet-5",
|
||||
messages=[{"role": "user", "content": "Say hello"}],
|
||||
stream=True,
|
||||
api_key="hostinger-test-key",
|
||||
)
|
||||
collected = [chunk for chunk in response]
|
||||
|
||||
request: Final = route.calls.last.request
|
||||
body: Final = json.loads(request.content)
|
||||
assert route.call_count == 1
|
||||
assert body["stream"] is True
|
||||
assert any(chunk.choices and chunk.choices[0].delta.content == "Hi" for chunk in collected)
|
||||
|
||||
|
||||
def test_hostinger_router_responses_request():
|
||||
with respx.mock() as upstream:
|
||||
route: Final = upstream.post("https://router.hostinger.com/v1/responses").respond(
|
||||
200,
|
||||
json={
|
||||
"id": "resp_hostinger",
|
||||
"object": "response",
|
||||
"created_at": 1_789_550_000,
|
||||
"model": "gpt-5.2",
|
||||
"status": "completed",
|
||||
"output": [
|
||||
{
|
||||
"id": "msg_hostinger",
|
||||
"type": "message",
|
||||
"role": "assistant",
|
||||
"status": "completed",
|
||||
"content": [{"type": "output_text", "text": "Hello from Hostinger Router", "annotations": []}],
|
||||
}
|
||||
],
|
||||
"usage": {"input_tokens": 4, "output_tokens": 3, "total_tokens": 7},
|
||||
},
|
||||
)
|
||||
response: Final = litellm.responses(
|
||||
model="hostinger_router/gpt-5.2",
|
||||
input="Say hello",
|
||||
api_key="hostinger-test-key",
|
||||
)
|
||||
|
||||
request: Final = route.calls.last.request
|
||||
body: Final = json.loads(request.content)
|
||||
assert route.call_count == 1
|
||||
assert str(request.url) == "https://router.hostinger.com/v1/responses"
|
||||
assert request.headers["authorization"] == "Bearer hostinger-test-key"
|
||||
assert body["model"] == "gpt-5.2"
|
||||
assert body["input"] == "Say hello"
|
||||
assert response.output[0].content[0].text == "Hello from Hostinger Router"
|
||||
|
||||
|
||||
EMBEDDING_RESPONSE: Final = {
|
||||
"object": "list",
|
||||
"data": [{"object": "embedding", "index": 0, "embedding": [0.1, 0.2, 0.3]}],
|
||||
"model": "text-embedding-3-small",
|
||||
"usage": {"prompt_tokens": 2, "total_tokens": 2},
|
||||
}
|
||||
|
||||
|
||||
def test_hostinger_router_embeddings_request():
|
||||
with respx.mock() as upstream:
|
||||
route: Final = upstream.post("https://router.hostinger.com/v1/embeddings").respond(200, json=EMBEDDING_RESPONSE)
|
||||
response: Final = litellm.embedding(
|
||||
model="hostinger_router/text-embedding-3-small",
|
||||
input=["Say hello"],
|
||||
api_key="hostinger-test-key",
|
||||
)
|
||||
|
||||
request: Final = route.calls.last.request
|
||||
body: Final = json.loads(request.content)
|
||||
assert route.call_count == 1
|
||||
assert request.headers["authorization"] == "Bearer hostinger-test-key"
|
||||
assert body["model"] == "text-embedding-3-small"
|
||||
assert response.data[0]["embedding"] == [0.1, 0.2, 0.3]
|
||||
|
||||
|
||||
def test_hostinger_router_embeddings_send_extra_headers_as_http_headers():
|
||||
with respx.mock() as upstream:
|
||||
route: Final = upstream.post("https://router.hostinger.com/v1/embeddings").respond(200, json=EMBEDDING_RESPONSE)
|
||||
litellm.embedding(
|
||||
model="hostinger_router/text-embedding-3-small",
|
||||
input=["Say hello"],
|
||||
api_key="hostinger-test-key",
|
||||
extra_headers={"X-Trace-Id": "trace-123"},
|
||||
)
|
||||
|
||||
request: Final = route.calls.last.request
|
||||
body: Final = json.loads(request.content)
|
||||
assert request.headers["x-trace-id"] == "trace-123"
|
||||
assert request.headers["authorization"] == "Bearer hostinger-test-key"
|
||||
assert request.headers["content-type"] == "application/json"
|
||||
assert set(body) == {"model", "input"}
|
||||
|
||||
|
||||
def test_hostinger_router_embeddings_extra_headers_replace_key_headers_of_any_case():
|
||||
with respx.mock() as upstream:
|
||||
route: Final = upstream.post("https://router.hostinger.com/v1/embeddings").respond(200, json=EMBEDDING_RESPONSE)
|
||||
litellm.embedding(
|
||||
model="hostinger_router/text-embedding-3-small",
|
||||
input=["Say hello"],
|
||||
api_key="hostinger-test-key",
|
||||
extra_headers={"authorization": "Bearer caller-key", "x-trace-id": "trace-1"},
|
||||
)
|
||||
|
||||
request: Final = route.calls.last.request
|
||||
assert request.headers.get_list("authorization") == ["Bearer caller-key"]
|
||||
assert request.headers.get_list("content-type") == ["application/json"]
|
||||
assert request.headers.get_list("x-trace-id") == ["trace-1"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("configured", "sent"), [(12345, "12345"), (1.5, "1.5")])
|
||||
def test_hostinger_router_embeddings_send_numeric_extra_headers_as_text(configured: int | float, sent: str):
|
||||
with respx.mock() as upstream:
|
||||
route: Final = upstream.post("https://router.hostinger.com/v1/embeddings").respond(200, json=EMBEDDING_RESPONSE)
|
||||
litellm.embedding(
|
||||
model="hostinger_router/text-embedding-3-small",
|
||||
input=["Say hello"],
|
||||
api_key="hostinger-test-key",
|
||||
extra_headers={"X-Org-Id": configured},
|
||||
)
|
||||
|
||||
assert route.calls.last.request.headers.get_list("x-org-id") == [sent]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("configured", [True, None, ["a", "b"]])
|
||||
def test_hostinger_router_embeddings_refuse_non_text_extra_headers(configured: object):
|
||||
with respx.mock(assert_all_called=False) as upstream:
|
||||
route: Final = upstream.post("https://router.hostinger.com/v1/embeddings").respond(200, json=EMBEDDING_RESPONSE)
|
||||
with pytest.raises(litellm.UnsupportedParamsError, match="X-Debug") as refused:
|
||||
litellm.embedding(
|
||||
model="hostinger_router/text-embedding-3-small",
|
||||
input=["Say hello"],
|
||||
api_key="hostinger-test-key",
|
||||
extra_headers={"X-Debug": configured},
|
||||
)
|
||||
|
||||
assert refused.value.status_code == 400
|
||||
assert route.call_count == 0
|
||||
|
||||
|
||||
@pytest.mark.parametrize("configured", [True, None, ["a", "b"]])
|
||||
def test_hostinger_router_embeddings_drop_non_text_extra_headers_under_drop_params(configured: object):
|
||||
with respx.mock() as upstream:
|
||||
route: Final = upstream.post("https://router.hostinger.com/v1/embeddings").respond(200, json=EMBEDDING_RESPONSE)
|
||||
litellm.embedding(
|
||||
model="hostinger_router/text-embedding-3-small",
|
||||
input=["Say hello"],
|
||||
api_key="hostinger-test-key",
|
||||
extra_headers={"X-Debug": configured, "X-Trace-Id": "trace-1"},
|
||||
drop_params=True,
|
||||
)
|
||||
|
||||
request: Final = route.calls.last.request
|
||||
assert "x-debug" not in request.headers
|
||||
assert request.headers.get_list("x-trace-id") == ["trace-1"]
|
||||
assert set(json.loads(request.content)) == {"model", "input"}
|
||||
|
||||
|
||||
TRANSPORT_EXTRA_HEADERS: Final = [
|
||||
("Host", "other.example"),
|
||||
("content-length", "1"),
|
||||
("Transfer-Encoding", "chunked"),
|
||||
("Connection", "close"),
|
||||
("Keep-Alive", "timeout=5"),
|
||||
("Proxy-Connection", "close"),
|
||||
("TE", "gzip"),
|
||||
("Trailer", "X-Checksum"),
|
||||
("Upgrade", "websocket"),
|
||||
("Expect", "100-continue"),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("name", "configured"), TRANSPORT_EXTRA_HEADERS)
|
||||
def test_hostinger_router_embeddings_ignore_transport_extra_headers(name: str, configured: str):
|
||||
with respx.mock() as upstream:
|
||||
route: Final = upstream.post("https://router.hostinger.com/v1/embeddings").respond(200, json=EMBEDDING_RESPONSE)
|
||||
litellm.embedding(
|
||||
model="hostinger_router/text-embedding-3-small",
|
||||
input=["Say hello"],
|
||||
api_key="hostinger-test-key",
|
||||
extra_headers={name: configured, "X-Trace-Id": "trace-1"},
|
||||
)
|
||||
|
||||
request: Final = route.calls.last.request
|
||||
assert request.headers.get_list("host") == ["router.hostinger.com"]
|
||||
assert request.headers.get_list("content-length") == [str(len(request.content))]
|
||||
assert "transfer-encoding" not in request.headers
|
||||
assert configured not in request.headers.get_list(name)
|
||||
assert request.headers.get_list("x-trace-id") == ["trace-1"]
|
||||
|
||||
|
||||
MALFORMED_HEADER_NAMES: Final = ["Host:evil.example", "X Note", "X-Caf\u00e9"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("name", MALFORMED_HEADER_NAMES)
|
||||
def test_hostinger_router_embeddings_refuse_malformed_extra_header_names(name: str):
|
||||
with respx.mock(assert_all_called=False) as upstream:
|
||||
route: Final = upstream.post("https://router.hostinger.com/v1/embeddings").respond(200, json=EMBEDDING_RESPONSE)
|
||||
with pytest.raises(litellm.UnsupportedParamsError) as refused:
|
||||
litellm.embedding(
|
||||
model="hostinger_router/text-embedding-3-small",
|
||||
input=["Say hello"],
|
||||
api_key="hostinger-test-key",
|
||||
extra_headers={name: "x"},
|
||||
)
|
||||
|
||||
assert refused.value.status_code == 400
|
||||
assert repr(name) in refused.value.message
|
||||
assert route.call_count == 0
|
||||
|
||||
|
||||
@pytest.mark.parametrize("name", MALFORMED_HEADER_NAMES)
|
||||
def test_hostinger_router_embeddings_drop_malformed_extra_header_names_under_drop_params(name: str):
|
||||
with respx.mock() as upstream:
|
||||
route: Final = upstream.post("https://router.hostinger.com/v1/embeddings").respond(200, json=EMBEDDING_RESPONSE)
|
||||
litellm.embedding(
|
||||
model="hostinger_router/text-embedding-3-small",
|
||||
input=["Say hello"],
|
||||
api_key="hostinger-test-key",
|
||||
extra_headers={name: "x", "X-Trace-Id": "trace-1"},
|
||||
drop_params=True,
|
||||
)
|
||||
|
||||
request: Final = route.calls.last.request
|
||||
assert request.headers.get_list("host") == ["router.hostinger.com"]
|
||||
assert request.headers.get_list("x-trace-id") == ["trace-1"]
|
||||
assert name.lower() not in request.headers.keys()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("configured", ["a\r\nHost: evil.example", "line\nbreak", "caf\u00e9", "a\x7fb"])
|
||||
def test_hostinger_router_embeddings_drop_extra_header_values_http_cannot_carry(configured: str):
|
||||
with respx.mock() as upstream:
|
||||
route: Final = upstream.post("https://router.hostinger.com/v1/embeddings").respond(200, json=EMBEDDING_RESPONSE)
|
||||
litellm.embedding(
|
||||
model="hostinger_router/text-embedding-3-small",
|
||||
input=["Say hello"],
|
||||
api_key="hostinger-test-key",
|
||||
extra_headers={"x-litellm-user_api_key_alias": configured, "X-Trace-Id": "trace-1"},
|
||||
)
|
||||
|
||||
request: Final = route.calls.last.request
|
||||
assert request.headers.get_list("host") == ["router.hostinger.com"]
|
||||
assert request.headers.get_list("x-trace-id") == ["trace-1"]
|
||||
assert "x-litellm-user_api_key_alias" not in request.headers
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("configured", "sent"),
|
||||
[(" x ", "x"), ("\t", ""), (" a\tb ", "a\tb"), ("env-secret\n", "env-secret"), ("\r\n x \r\n", "x")],
|
||||
)
|
||||
def test_hostinger_router_embeddings_trim_whitespace_around_extra_header_values(configured: str, sent: str):
|
||||
with respx.mock() as upstream:
|
||||
route: Final = upstream.post("https://router.hostinger.com/v1/embeddings").respond(200, json=EMBEDDING_RESPONSE)
|
||||
litellm.embedding(
|
||||
model="hostinger_router/text-embedding-3-small",
|
||||
input=["Say hello"],
|
||||
api_key="hostinger-test-key",
|
||||
extra_headers={"X-Rig": configured},
|
||||
)
|
||||
|
||||
assert route.calls.last.request.headers.get_list("x-rig") == [sent]
|
||||
|
||||
|
||||
def test_hostinger_router_embeddings_extra_headers_stay_with_their_request():
|
||||
router: Final = litellm.Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "shared",
|
||||
"litellm_params": {
|
||||
"model": "hostinger_router/text-embedding-3-small",
|
||||
"api_key": "hostinger-test-key",
|
||||
"headers": {"X-Org-Id": "org-1"},
|
||||
},
|
||||
}
|
||||
]
|
||||
)
|
||||
|
||||
with respx.mock() as upstream:
|
||||
route: Final = upstream.post("https://router.hostinger.com/v1/embeddings").respond(200, json=EMBEDDING_RESPONSE)
|
||||
router.embedding(
|
||||
model="shared", input=["first"], extra_headers={"Authorization": "Bearer caller-a", "X-Leak": "a"}
|
||||
)
|
||||
router.embedding(model="shared", input=["second"])
|
||||
|
||||
first: Final = route.calls[0].request
|
||||
second: Final = route.calls[1].request
|
||||
assert first.headers.get_list("x-leak") == ["a"]
|
||||
assert second.headers.get_list("authorization") == ["Bearer hostinger-test-key"]
|
||||
assert "x-leak" not in second.headers
|
||||
assert second.headers.get_list("x-org-id") == ["org-1"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("param", ["headers", "extra_headers"])
|
||||
def test_hostinger_router_embeddings_send_headers_given_as_name_value_pairs(param: str):
|
||||
with respx.mock() as upstream:
|
||||
route: Final = upstream.post("https://router.hostinger.com/v1/embeddings").respond(200, json=EMBEDDING_RESPONSE)
|
||||
litellm.embedding(
|
||||
model="hostinger_router/text-embedding-3-small",
|
||||
input=["Say hello"],
|
||||
api_key="hostinger-test-key",
|
||||
**{param: [["X-Trace-Id", "trace-1"]]},
|
||||
)
|
||||
|
||||
request: Final = route.calls.last.request
|
||||
assert request.headers.get_list("x-trace-id") == ["trace-1"]
|
||||
assert request.headers.get_list("authorization") == ["Bearer hostinger-test-key"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("param", ["headers", "extra_headers"])
|
||||
def test_hostinger_router_embeddings_refuse_headers_that_are_not_an_object(param: str):
|
||||
with respx.mock(assert_all_called=False) as upstream:
|
||||
route: Final = upstream.post("https://router.hostinger.com/v1/embeddings").respond(200, json=EMBEDDING_RESPONSE)
|
||||
with pytest.raises(litellm.UnsupportedParamsError, match=rf"\b{param} must be a JSON object") as refused:
|
||||
litellm.embedding(
|
||||
model="hostinger_router/text-embedding-3-small",
|
||||
input=["Say hello"],
|
||||
api_key="hostinger-test-key",
|
||||
**{param: "X-Trace-Id: trace-1"},
|
||||
)
|
||||
|
||||
assert refused.value.status_code == 400
|
||||
assert route.call_count == 0
|
||||
|
||||
|
||||
@pytest.mark.parametrize("param", ["headers", "extra_headers"])
|
||||
def test_hostinger_router_embeddings_drop_headers_that_are_not_an_object_under_drop_params(param: str):
|
||||
with respx.mock() as upstream:
|
||||
route: Final = upstream.post("https://router.hostinger.com/v1/embeddings").respond(200, json=EMBEDDING_RESPONSE)
|
||||
litellm.embedding(
|
||||
model="hostinger_router/text-embedding-3-small",
|
||||
input=["Say hello"],
|
||||
api_key="hostinger-test-key",
|
||||
drop_params=True,
|
||||
**{param: "X-Trace-Id: trace-1"},
|
||||
)
|
||||
|
||||
request: Final = route.calls.last.request
|
||||
assert "x-trace-id" not in request.headers
|
||||
assert request.headers.get_list("authorization") == ["Bearer hostinger-test-key"]
|
||||
assert set(json.loads(request.content)) == {"model", "input"}
|
||||
|
||||
|
||||
def test_hostinger_router_embeddings_never_borrow_another_providers_key(monkeypatch: pytest.MonkeyPatch):
|
||||
monkeypatch.delenv("HOSTINGER_ROUTER_API_KEY", raising=False)
|
||||
monkeypatch.setenv("OPENAI_LIKE_API_KEY", "other-provider-key")
|
||||
monkeypatch.setenv("OPENAI_API_KEY", "other-provider-key")
|
||||
|
||||
with respx.mock(assert_all_called=False) as upstream:
|
||||
route: Final = upstream.post("https://router.hostinger.com/v1/embeddings").respond(200, json=EMBEDDING_RESPONSE)
|
||||
with pytest.raises(litellm.BadRequestError, match="Missing API Key"):
|
||||
litellm.embedding(model="hostinger_router/text-embedding-3-small", input=["Say hello"])
|
||||
|
||||
assert route.call_count == 0
|
||||
|
||||
|
||||
def test_json_provider_without_embeddings_endpoint_refuses_embeddings():
|
||||
from litellm.llms.openai_like.json_loader import JSONProviderRegistry
|
||||
|
||||
slug: Final = next(
|
||||
slug
|
||||
for slug in JSONProviderRegistry.list_providers()
|
||||
if not JSONProviderRegistry.supports_embeddings(slug) and slug not in litellm.provider_list
|
||||
)
|
||||
|
||||
with respx.mock(assert_all_called=False) as upstream:
|
||||
route: Final = upstream.route().respond(200, json=EMBEDDING_RESPONSE)
|
||||
with pytest.raises(litellm.BadRequestError):
|
||||
litellm.embedding(model=f"{slug}/text-embedding-3-small", input=["Say hello"], api_key="test-key")
|
||||
|
||||
assert route.call_count == 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_hostinger_router_anthropic_messages_request(monkeypatch: pytest.MonkeyPatch):
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
monkeypatch.setattr(litellm, "in_memory_llm_clients_cache", LLMClientCache())
|
||||
with respx.mock() as upstream:
|
||||
route: Final = upstream.post("https://router.hostinger.com/v1/messages").respond(
|
||||
200,
|
||||
json={
|
||||
"id": "msg_hostinger",
|
||||
"type": "message",
|
||||
"role": "assistant",
|
||||
"model": "claude-sonnet-5",
|
||||
"content": [{"type": "text", "text": "Hello from Hostinger Router"}],
|
||||
"stop_reason": "end_turn",
|
||||
"stop_sequence": None,
|
||||
"usage": {"input_tokens": 4, "output_tokens": 3},
|
||||
},
|
||||
)
|
||||
response: Final = await litellm.anthropic.messages.acreate(
|
||||
model="hostinger_router/claude-sonnet-5",
|
||||
messages=[{"role": "user", "content": "Say hello"}],
|
||||
max_tokens=32,
|
||||
api_key="hostinger-test-key",
|
||||
)
|
||||
|
||||
request: Final = route.calls.last.request
|
||||
body: Final = json.loads(request.content)
|
||||
assert route.call_count == 1
|
||||
assert str(request.url) == "https://router.hostinger.com/v1/messages"
|
||||
assert request.headers["authorization"] == "Bearer hostinger-test-key"
|
||||
assert request.headers["anthropic-version"] == "2023-06-01"
|
||||
assert body["model"] == "claude-sonnet-5"
|
||||
assert body["messages"] == [{"role": "user", "content": "Say hello"}]
|
||||
assert response["content"][0]["text"] == "Hello from Hostinger Router"
|
||||
|
||||
|
||||
CHAT_COMPLETION_RESPONSE: Final = {
|
||||
"id": "chatcmpl_hostinger",
|
||||
"object": "chat.completion",
|
||||
"created": 1_789_550_000,
|
||||
"model": "gpt-5.2",
|
||||
"choices": [
|
||||
{
|
||||
"index": 0,
|
||||
"message": {"role": "assistant", "content": "Hello from Hostinger Router"},
|
||||
"finish_reason": "stop",
|
||||
}
|
||||
],
|
||||
"usage": {"prompt_tokens": 4, "completion_tokens": 3, "total_tokens": 7},
|
||||
}
|
||||
UNSUPPORTED_ENDPOINT: Final = {"error": {"message": "Model does not support this endpoint."}}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("model", ["gpt-5.2", "gpt-5-nano"])
|
||||
async def test_hostinger_router_messages_bridge_models_without_native_messages(
|
||||
model: str, monkeypatch: pytest.MonkeyPatch
|
||||
):
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
monkeypatch.setattr(litellm, "in_memory_llm_clients_cache", LLMClientCache())
|
||||
with respx.mock(assert_all_called=False) as upstream:
|
||||
native: Final = upstream.post("https://router.hostinger.com/v1/messages").respond(
|
||||
403, json=UNSUPPORTED_ENDPOINT
|
||||
)
|
||||
chat: Final = upstream.post("https://router.hostinger.com/v1/chat/completions").respond(
|
||||
200, json=CHAT_COMPLETION_RESPONSE
|
||||
)
|
||||
response: Final = await litellm.anthropic.messages.acreate(
|
||||
model=f"hostinger_router/{model}",
|
||||
messages=[{"role": "user", "content": "Say hello"}],
|
||||
max_tokens=32,
|
||||
api_key="hostinger-test-key",
|
||||
)
|
||||
|
||||
assert native.call_count == 0
|
||||
assert chat.call_count == 1
|
||||
assert json.loads(chat.calls.last.request.content)["model"] == model
|
||||
assert response["content"][0]["text"] == "Hello from Hostinger Router"
|
||||
|
||||
|
||||
def test_hostinger_router_responses_bridge_models_without_native_responses():
|
||||
with respx.mock(assert_all_called=False) as upstream:
|
||||
native: Final = upstream.post("https://router.hostinger.com/v1/responses").respond(
|
||||
403, json=UNSUPPORTED_ENDPOINT
|
||||
)
|
||||
chat: Final = upstream.post("https://router.hostinger.com/v1/chat/completions").respond(
|
||||
200, json=CHAT_COMPLETION_RESPONSE
|
||||
)
|
||||
response: Final = litellm.responses(
|
||||
model="hostinger_router/claude-sonnet-5",
|
||||
input="Say hello",
|
||||
api_key="hostinger-test-key",
|
||||
)
|
||||
|
||||
assert native.call_count == 0
|
||||
assert chat.call_count == 1
|
||||
assert json.loads(chat.calls.last.request.content)["model"] == "claude-sonnet-5"
|
||||
assert response.output[0].content[0].text == "Hello from Hostinger Router"
|
||||
|
||||
|
||||
ANTHROPIC_MESSAGE_RESPONSE: Final = {
|
||||
"id": "msg_hostinger",
|
||||
"type": "message",
|
||||
"role": "assistant",
|
||||
"model": "claude-haiku-4-5",
|
||||
"content": [{"type": "text", "text": "Hello from Hostinger Router"}],
|
||||
"stop_reason": "end_turn",
|
||||
"stop_sequence": None,
|
||||
"usage": {"input_tokens": 4, "output_tokens": 3},
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"sibling_model_info",
|
||||
[
|
||||
{"supported_endpoints": ["/v1/chat/completions", "/v1/messages", "/v1/responses"]},
|
||||
{"supported_endpoints": "/v1/messages"},
|
||||
],
|
||||
)
|
||||
async def test_hostinger_router_endpoint_opt_in_stays_with_its_own_deployment(
|
||||
sibling_model_info: dict[str, object], monkeypatch: pytest.MonkeyPatch
|
||||
):
|
||||
monkeypatch.setattr(litellm, "model_cost", copy.deepcopy(litellm.model_cost))
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
monkeypatch.setattr(litellm, "in_memory_llm_clients_cache", LLMClientCache())
|
||||
deployment_params: Final = {"model": "hostinger_router/gpt-5-nano", "api_key": "hostinger-test-key"}
|
||||
router: Final = litellm.Router(
|
||||
model_list=[
|
||||
{"model_name": "opted-in", "litellm_params": deployment_params, "model_info": sibling_model_info},
|
||||
{"model_name": "plain", "litellm_params": deployment_params},
|
||||
]
|
||||
)
|
||||
|
||||
with respx.mock(assert_all_called=False) as upstream:
|
||||
native_messages: Final = upstream.post("https://router.hostinger.com/v1/messages").respond(
|
||||
403, json=UNSUPPORTED_ENDPOINT
|
||||
)
|
||||
native_responses: Final = upstream.post("https://router.hostinger.com/v1/responses").respond(
|
||||
403, json=UNSUPPORTED_ENDPOINT
|
||||
)
|
||||
chat: Final = upstream.post("https://router.hostinger.com/v1/chat/completions").respond(
|
||||
200, json=CHAT_COMPLETION_RESPONSE
|
||||
)
|
||||
messages_response: Final = await router.aanthropic_messages(
|
||||
model="plain", messages=[{"role": "user", "content": "Say hello"}], max_tokens=32
|
||||
)
|
||||
responses_response: Final = await router.aresponses(model="plain", input="Say hello")
|
||||
|
||||
assert native_messages.call_count == 0
|
||||
assert native_responses.call_count == 0
|
||||
assert chat.call_count == 2
|
||||
assert messages_response["content"][0]["text"] == "Hello from Hostinger Router"
|
||||
assert responses_response.output[0].content[0].text == "Hello from Hostinger Router"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_hostinger_router_deployment_cache_ttl_opt_in_keeps_one_hour_ttl(monkeypatch: pytest.MonkeyPatch):
|
||||
monkeypatch.setattr(litellm, "model_cost", copy.deepcopy(litellm.model_cost))
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
monkeypatch.setattr(litellm, "in_memory_llm_clients_cache", LLMClientCache())
|
||||
router: Final = litellm.Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "opted-in",
|
||||
"litellm_params": {
|
||||
"model": "hostinger_router/claude-haiku-4-5",
|
||||
"api_base": "https://router.hostinger.com/v1",
|
||||
"api_key": "hostinger-test-key",
|
||||
},
|
||||
"model_info": {"supported_endpoints": ["/v1/messages"], "cache_control_ttl": True},
|
||||
}
|
||||
]
|
||||
)
|
||||
one_hour_cache: Final = {"type": "ephemeral", "ttl": "1h"}
|
||||
|
||||
with respx.mock() as upstream:
|
||||
native: Final = upstream.post("https://router.hostinger.com/v1/messages").respond(
|
||||
200, json=ANTHROPIC_MESSAGE_RESPONSE
|
||||
)
|
||||
await router.aanthropic_messages(
|
||||
model="opted-in",
|
||||
messages=[
|
||||
{"role": "user", "content": [{"type": "text", "text": "Say hello", "cache_control": one_hour_cache}]}
|
||||
],
|
||||
max_tokens=32,
|
||||
)
|
||||
|
||||
body: Final = json.loads(native.calls.last.request.content)
|
||||
assert body["messages"][0]["content"][0]["cache_control"] == one_hour_cache
|
||||
|
|
@ -1,5 +1,8 @@
|
|||
|
||||
import copy
|
||||
import json
|
||||
from collections.abc import Iterator
|
||||
from itertools import chain
|
||||
from typing import Final
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
|
|
@ -874,6 +877,58 @@ def test_openrouter_streaming_reasoning_is_preserved() -> None:
|
|||
assert assembled.choices[0].message.reasoning_content == "Consider the evidence. Apply the rule."
|
||||
|
||||
|
||||
def _five_message_level_cache_hints() -> list[dict[str, object]]:
|
||||
return [
|
||||
{"role": "system", "content": "system prompt", "cache_control": {"type": "ephemeral"}},
|
||||
{"role": "user", "content": "turn 1", "cache_control": {"type": "ephemeral"}},
|
||||
{"role": "assistant", "content": "answer 1", "cache_control": {"type": "ephemeral"}},
|
||||
{"role": "user", "content": "turn 2", "cache_control": {"type": "ephemeral"}},
|
||||
{"role": "assistant", "content": "answer 2"},
|
||||
{"role": "user", "content": "turn 3", "cache_control": {"type": "ephemeral"}},
|
||||
]
|
||||
|
||||
|
||||
def _cached_texts(node: object) -> Iterator[str]:
|
||||
if isinstance(node, dict):
|
||||
yield from ((node["text"],) if "cache_control" in node and "text" in node else ())
|
||||
yield from chain.from_iterable(map(_cached_texts, node.values()))
|
||||
elif isinstance(node, list):
|
||||
yield from chain.from_iterable(map(_cached_texts, node))
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("drop_params", "expected_cached_texts"),
|
||||
[
|
||||
(True, ["answer 1", "turn 1", "turn 2", "turn 3"]),
|
||||
("true", ["answer 1", "turn 1", "turn 2", "turn 3"]),
|
||||
(False, ["answer 1", "system prompt", "turn 1", "turn 2", "turn 3"]),
|
||||
],
|
||||
ids=[
|
||||
"drop_params_keeps_the_newest_four",
|
||||
"string_drop_params_keeps_the_newest_four",
|
||||
"without_drop_params_all_five_pass_through",
|
||||
],
|
||||
)
|
||||
def test_openrouter_claude_request_over_the_cache_control_cap_drops_the_oldest_hints_only_under_drop_params(
|
||||
drop_params: bool | str, expected_cached_texts: list[str], monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "drop_params", False)
|
||||
messages: Final = _five_message_level_cache_hints()
|
||||
messages_before: Final = copy.deepcopy(messages)
|
||||
|
||||
body: Final = OpenrouterConfig().transform_request(
|
||||
model="anthropic/claude-haiku-5.5",
|
||||
messages=messages,
|
||||
optional_params={},
|
||||
litellm_params={"drop_params": drop_params, "custom_llm_provider": "openrouter"},
|
||||
headers={},
|
||||
)
|
||||
|
||||
assert sorted(_cached_texts(body["messages"])) == expected_cached_texts
|
||||
assert json.dumps(body).count('"cache_control"') == len(expected_cached_texts)
|
||||
assert messages == messages_before
|
||||
|
||||
|
||||
def _transform_openrouter_response(body: dict[str, object]) -> litellm.ModelResponse:
|
||||
return OpenrouterConfig().transform_response(
|
||||
model="openrouter/openai/gpt-4o-mini",
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
import json
|
||||
from typing import Final, Literal
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
|
|
@ -2507,3 +2508,46 @@ def test_check_and_create_cache_clamps_negative_creation_tokens() -> None:
|
|||
|
||||
assert cache_name == "cachedContents/created"
|
||||
assert logging_obj.vertex_ai_cache_creation_input_tokens == 0
|
||||
|
||||
|
||||
def _recording_gemini_client(calls: list[httpx.Request]) -> HTTPHandler:
|
||||
def route(request: httpx.Request) -> httpx.Response:
|
||||
calls.append(request)
|
||||
if request.url.path.endswith("/cachedContents"):
|
||||
return httpx.Response(200, json={} if request.method == "GET" else _created_cache_body(3000))
|
||||
return httpx.Response(
|
||||
200,
|
||||
json={
|
||||
"candidates": [{"content": {"role": "model", "parts": [{"text": "OK"}]}, "finishReason": "STOP"}],
|
||||
"usageMetadata": {"promptTokenCount": 3000, "candidatesTokenCount": 1, "totalTokenCount": 3001},
|
||||
},
|
||||
)
|
||||
|
||||
return HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(route)))
|
||||
|
||||
|
||||
def test_a_plain_system_prompt_goes_into_the_gemini_cache_instead_of_beside_it(local_model_cost_map) -> None:
|
||||
calls: Final[list[httpx.Request]] = []
|
||||
litellm.completion(
|
||||
model="gemini/gemini-flash-latest",
|
||||
messages=[
|
||||
{"role": "system", "content": "You are terse."},
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "plain system prefix " * 3000, "cache_control": {"type": "ephemeral"}},
|
||||
{"type": "text", "text": "Reply with OK only."},
|
||||
],
|
||||
},
|
||||
],
|
||||
api_key="test-key",
|
||||
client=_recording_gemini_client(calls),
|
||||
)
|
||||
|
||||
created: Final = next(
|
||||
json.loads(call.content) for call in calls if call.method == "POST" and call.url.path.endswith("/cachedContents")
|
||||
)
|
||||
generated: Final = next(json.loads(call.content) for call in calls if call.url.path.endswith(":generateContent"))
|
||||
assert created["system_instruction"] == {"parts": [{"text": "You are terse."}]}
|
||||
assert generated["cachedContent"] == "cachedContents/created"
|
||||
assert "system_instruction" not in generated
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
import asyncio
|
||||
import copy
|
||||
import importlib
|
||||
import json
|
||||
from typing import Final, Optional, cast
|
||||
|
|
@ -10,6 +11,7 @@ import pytest
|
|||
import litellm
|
||||
from litellm import completion, embedding
|
||||
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
|
||||
from litellm.llms.watsonx.chat.transformation import IBMWatsonXChatConfig
|
||||
from tests._vcr_conftest_common import install_live_call_probe, record_vcr_outcome
|
||||
|
||||
|
||||
|
|
@ -542,3 +544,43 @@ def test_watsonx_deployment_space_id_embedding(monkeypatch, watsonx_embedding_ca
|
|||
|
||||
# nor space_id or project_id is required by wx.ai API when inferencing deployment
|
||||
assert "project_id" not in json_data and "space_id" not in json_data
|
||||
|
||||
|
||||
def _strict_weather_tool() -> dict[str, object]:
|
||||
return {
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "get_weather",
|
||||
"strict": True,
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {"city": {"type": "string"}},
|
||||
"required": ["city"],
|
||||
"additionalProperties": False,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def test_watsonx_chat_tools_are_rewritten_without_touching_the_callers_tool() -> None:
|
||||
tools: Final = [_strict_weather_tool()]
|
||||
tools_before: Final = copy.deepcopy(tools)
|
||||
|
||||
optional_params: Final = IBMWatsonXChatConfig().map_openai_params(
|
||||
non_default_params={"tools": tools}, optional_params={}, model="ibm/granite-4-h-small", drop_params=False
|
||||
)
|
||||
|
||||
assert optional_params["tools"] == [
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "get_weather",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {"city": {"type": "string"}},
|
||||
"required": ["city"],
|
||||
},
|
||||
},
|
||||
}
|
||||
]
|
||||
assert tools == tools_before
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
import copy
|
||||
import json
|
||||
import os
|
||||
from typing import Final
|
||||
|
|
@ -581,3 +582,44 @@ def test_xai_grok_4_frequency_penalty_not_supported(model):
|
|||
"""
|
||||
supported_params = XAIChatConfig().get_supported_openai_params(model=model)
|
||||
assert "frequency_penalty" not in supported_params
|
||||
|
||||
|
||||
def _strict_weather_tool() -> dict[str, object]:
|
||||
return {
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "get_weather",
|
||||
"strict": True,
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {"city": {"type": "string"}},
|
||||
"required": ["city"],
|
||||
"additionalProperties": False,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def test_xai_tools_are_rewritten_without_touching_the_callers_tool() -> None:
|
||||
tools: Final = [_strict_weather_tool()]
|
||||
tools_before: Final = copy.deepcopy(tools)
|
||||
|
||||
optional_params: Final = XAIChatConfig().map_openai_params(
|
||||
non_default_params={"tools": tools}, optional_params={}, model="grok-4.3", drop_params=False
|
||||
)
|
||||
|
||||
assert optional_params["tools"] == [
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "get_weather",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {"city": {"type": "string"}},
|
||||
"required": ["city"],
|
||||
"additionalProperties": False,
|
||||
},
|
||||
},
|
||||
}
|
||||
]
|
||||
assert tools == tools_before
|
||||
|
|
|
|||
|
|
@ -31,7 +31,7 @@ from starlette.datastructures import UploadFile as StarletteUploadFile
|
|||
import litellm
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
from litellm.caching.dual_cache import DualCache
|
||||
from litellm.constants import DEFAULT_REQUEST_TIMEOUT_SECONDS
|
||||
from litellm.constants import ANTHROPIC_COMPATIBLE_MAX_USER_ID_LENGTH, ANTHROPIC_MAX_USER_ID_LENGTH, DEFAULT_REQUEST_TIMEOUT_SECONDS
|
||||
from litellm.integrations.custom_guardrail import CustomGuardrail
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.integrations.prometheus import PrometheusLogger
|
||||
|
|
@ -6621,6 +6621,272 @@ async def test_pass_through_body_stays_byte_identical_outside_the_opt_in(
|
|||
assert "x-litellm-call-id" in response.headers
|
||||
|
||||
|
||||
def _echo_upstream_body(upstream_request: httpx.Request) -> httpx.Response:
|
||||
return httpx.Response(
|
||||
200, content=upstream_request.content, headers={"content-type": "application/json"}, request=upstream_request
|
||||
)
|
||||
|
||||
|
||||
async def _relayed_body(response: Response) -> bytes:
|
||||
if isinstance(response, StreamingResponse):
|
||||
return b"".join([chunk async for chunk in response.body_iterator])
|
||||
return bytes(response.body)
|
||||
|
||||
|
||||
async def _relay_through_pass_through(
|
||||
client_body: Mapping[str, object],
|
||||
target: str,
|
||||
guardrails_config: dict[str, object] | None,
|
||||
custom_body: dict[str, object] | None = None,
|
||||
) -> Mapping[str, object]:
|
||||
async def receive() -> Mapping[str, object]:
|
||||
return {"type": "http.request", "body": json.dumps(dict(client_body)).encode()}
|
||||
|
||||
request: Final = Request(
|
||||
{
|
||||
"type": "http",
|
||||
"method": "POST",
|
||||
"path": "/my-claude-route",
|
||||
"query_string": b"",
|
||||
"headers": [(b"content-type", b"application/json")],
|
||||
},
|
||||
receive,
|
||||
)
|
||||
fake_client, cleanup = _inject_fake_passthrough_client(httpx.MockTransport(_echo_upstream_body), timeout=None)
|
||||
try:
|
||||
with ExitStack() as stack:
|
||||
_enter_relay_logging_mocks(stack, {})
|
||||
stack.enter_context(
|
||||
patch("litellm.proxy.proxy_server.proxy_logging_obj", ProxyLogging(user_api_key_cache=DualCache()))
|
||||
)
|
||||
response: Final = await pass_through_request(
|
||||
request=request,
|
||||
target=target,
|
||||
custom_headers={},
|
||||
user_api_key_dict=UserAPIKeyAuth(api_key="sk-relay-test"),
|
||||
custom_body=custom_body,
|
||||
guardrails_config=guardrails_config,
|
||||
)
|
||||
relayed: Final = await _relayed_body(response)
|
||||
finally:
|
||||
cleanup()
|
||||
await fake_client.aclose()
|
||||
|
||||
assert response.status_code == 200
|
||||
return MappingProxyType(json.loads(relayed))
|
||||
|
||||
|
||||
_MESSAGES_BODY: Final = MappingProxyType(
|
||||
{"model": "claude-sonnet-4-5", "max_tokens": 16, "messages": [{"role": "user", "content": "hi"}]}
|
||||
)
|
||||
_ANTHROPIC_MESSAGES: Final = "https://api.anthropic.com/v1/messages"
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def no_configured_anthropic_base(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.delenv("ANTHROPIC_API_BASE", raising=False)
|
||||
monkeypatch.delenv("ANTHROPIC_BASE_URL", raising=False)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.usefixtures("no_configured_anthropic_base")
|
||||
@pytest.mark.parametrize("stream", [False, True], ids=["non-streaming", "streaming"])
|
||||
@pytest.mark.parametrize(
|
||||
"client_metadata, target, forwarded_metadata",
|
||||
[
|
||||
({"user_id": "user-1"}, _ANTHROPIC_MESSAGES, {"user_id": "user-1"}),
|
||||
({"user_id": "user-1"}, f"{_ANTHROPIC_MESSAGES}/", {"user_id": "user-1"}),
|
||||
({"user_id": "user-1"}, f"{_ANTHROPIC_MESSAGES}?beta=true", {"user_id": "user-1"}),
|
||||
(
|
||||
{"user_id": "user-1", "tags": ["a"], "trace_id": "t", "generation_name": "g", "foo": "bar"},
|
||||
_ANTHROPIC_MESSAGES,
|
||||
{"user_id": "user-1"},
|
||||
),
|
||||
({"user_id": "1234567890"}, _ANTHROPIC_MESSAGES, {"user_id": "1234567890"}),
|
||||
({"tags": ["a"], "foo": "bar"}, _ANTHROPIC_MESSAGES, None),
|
||||
({"user_id": "a@example.com"}, _ANTHROPIC_MESSAGES, None),
|
||||
({"user_id": "x" * (ANTHROPIC_MAX_USER_ID_LENGTH + 1)}, _ANTHROPIC_MESSAGES, None),
|
||||
({"user_id": 42}, _ANTHROPIC_MESSAGES, None),
|
||||
({"user_id": "line one\nline two"}, _ANTHROPIC_MESSAGES, None),
|
||||
({"user_id": "a\x1f@example.com"}, _ANTHROPIC_MESSAGES, None),
|
||||
({"user_id": "user-1"}, f"{_ANTHROPIC_MESSAGES}/count_tokens", None),
|
||||
({"user_id": "user-1"}, f"{_ANTHROPIC_MESSAGES}/batches", None),
|
||||
({"user_id": "user-1"}, "https://api.anthropic.com/v1/models", None),
|
||||
({"user_id": "user-1"}, "https://other-vendor.test/anthropic/v1/messages", None),
|
||||
({"user_id": "user-1"}, "https://other-vendor.test/v1/responses", None),
|
||||
({"user_id": "user-1"}, "https://other-vendor.test/v1/chat/completions", None),
|
||||
],
|
||||
ids=[
|
||||
"messages",
|
||||
"messages-trailing-slash",
|
||||
"messages-with-query",
|
||||
"user-id-beside-litellm-and-unknown-keys",
|
||||
"digits-user-id",
|
||||
"no-user-id",
|
||||
"email-user-id",
|
||||
"over-long-user-id",
|
||||
"integer-user-id",
|
||||
"newline-user-id",
|
||||
"control-character-before-at-sign",
|
||||
"count-tokens",
|
||||
"batches",
|
||||
"other-anthropic-path",
|
||||
"other-vendor-messages",
|
||||
"other-vendor-responses",
|
||||
"other-vendor-chat-completions",
|
||||
],
|
||||
)
|
||||
async def test_pass_through_forwards_only_an_acceptable_user_id_to_anthropic_messages(
|
||||
stream: bool, client_metadata: object, target: str, forwarded_metadata: Mapping[str, object] | None
|
||||
) -> None:
|
||||
client_body: Final = MappingProxyType(
|
||||
{**_MESSAGES_BODY, "metadata": client_metadata, **({"stream": True} if stream else {})}
|
||||
)
|
||||
body_without_metadata: Final = {k: v for k, v in client_body.items() if k != "metadata"}
|
||||
|
||||
relayed: Final = await _relay_through_pass_through(client_body, target, guardrails_config=None)
|
||||
|
||||
assert relayed == (
|
||||
body_without_metadata if forwarded_metadata is None else {**body_without_metadata, "metadata": forwarded_metadata}
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.usefixtures("no_configured_anthropic_base")
|
||||
@pytest.mark.parametrize("base_env", ["ANTHROPIC_API_BASE", "ANTHROPIC_BASE_URL"])
|
||||
@pytest.mark.parametrize(
|
||||
"target, forwarded",
|
||||
[
|
||||
("http://gateway.test:8080/anthropic/v1/messages", True),
|
||||
("http://gateway.test:9090/v1/messages", False),
|
||||
("http://other-gateway.test:8080/v1/messages", False),
|
||||
],
|
||||
ids=["configured-upstream", "other-port", "other-host"],
|
||||
)
|
||||
async def test_pass_through_treats_the_configured_anthropic_base_as_anthropic(
|
||||
monkeypatch: pytest.MonkeyPatch, base_env: str, target: str, forwarded: bool
|
||||
) -> None:
|
||||
monkeypatch.setenv(base_env, "http://gateway.test:8080/anthropic")
|
||||
client_body: Final = MappingProxyType({**_MESSAGES_BODY, "metadata": {"user_id": "user-1"}})
|
||||
|
||||
relayed: Final = await _relay_through_pass_through(client_body, target, guardrails_config=None)
|
||||
|
||||
assert relayed == (client_body if forwarded else _MESSAGES_BODY)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.usefixtures("no_configured_anthropic_base")
|
||||
@pytest.mark.parametrize(
|
||||
"user_id_length, target, forwarded",
|
||||
[
|
||||
(ANTHROPIC_COMPATIBLE_MAX_USER_ID_LENGTH, "http://gateway.test/v1/messages", True),
|
||||
(ANTHROPIC_COMPATIBLE_MAX_USER_ID_LENGTH + 1, "http://gateway.test/v1/messages", False),
|
||||
(ANTHROPIC_MAX_USER_ID_LENGTH, _ANTHROPIC_MESSAGES, True),
|
||||
],
|
||||
ids=["at-the-configured-base-limit", "over-the-configured-base-limit", "direct-anthropic-keeps-its-limit"],
|
||||
)
|
||||
async def test_pass_through_caps_the_user_id_for_a_configured_anthropic_compatible_base(
|
||||
monkeypatch: pytest.MonkeyPatch, user_id_length: int, target: str, forwarded: bool
|
||||
) -> None:
|
||||
monkeypatch.setenv("ANTHROPIC_API_BASE", "http://gateway.test")
|
||||
client_body: Final = MappingProxyType({**_MESSAGES_BODY, "metadata": {"user_id": "u" * user_id_length}})
|
||||
|
||||
relayed: Final = await _relay_through_pass_through(client_body, target, guardrails_config=None)
|
||||
|
||||
assert relayed == (client_body if forwarded else _MESSAGES_BODY)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.usefixtures("no_configured_anthropic_base")
|
||||
@pytest.mark.parametrize(
|
||||
"default_on, event_hook, guardrails_config, client_fields, stream_scope, forwarded",
|
||||
[
|
||||
(False, GuardrailEventHooks.pre_call, None, {}, None, True),
|
||||
(False, GuardrailEventHooks.pre_call, {"unregistered-guard": {}}, {}, None, True),
|
||||
(True, GuardrailEventHooks.post_call, None, {}, None, True),
|
||||
(True, GuardrailEventHooks.pre_call, None, {}, None, False),
|
||||
(False, GuardrailEventHooks.pre_call, {"pii-guard": {}}, {}, None, False),
|
||||
(False, GuardrailEventHooks.pre_call, None, {"litellm_metadata": {"guardrails": ["pii-guard"]}}, None, False),
|
||||
(False, GuardrailEventHooks.pre_call, {"pii-guard": {}}, {"litellm_metadata": {"guardrails": []}}, None, False),
|
||||
(True, GuardrailEventHooks.pre_call, None, {"stream": True}, "streaming", False),
|
||||
(True, GuardrailEventHooks.pre_call, None, {}, "streaming", True),
|
||||
(True, GuardrailEventHooks.pre_call, None, {"stream": True}, "non_streaming", True),
|
||||
(True, GuardrailEventHooks.pre_call, None, {}, "non_streaming", False),
|
||||
],
|
||||
ids=[
|
||||
"guardrail-not-requested",
|
||||
"guardrail-not-registered",
|
||||
"default-on-post-call-guardrail",
|
||||
"default-on-pre-call-guardrail",
|
||||
"guardrail-on-the-route",
|
||||
"guardrail-in-litellm-metadata",
|
||||
"route-guardrail-beside-an-empty-client-list",
|
||||
"streaming-guardrail-on-a-stream",
|
||||
"streaming-guardrail-off-a-stream",
|
||||
"non-streaming-guardrail-on-a-stream",
|
||||
"non-streaming-guardrail-off-a-stream",
|
||||
],
|
||||
)
|
||||
async def test_pass_through_keeps_the_user_id_from_routes_a_guardrail_scans(
|
||||
default_on: bool,
|
||||
event_hook: GuardrailEventHooks,
|
||||
guardrails_config: dict[str, object] | None,
|
||||
client_fields: Mapping[str, object],
|
||||
stream_scope: str | None,
|
||||
forwarded: bool,
|
||||
) -> None:
|
||||
guardrail: Final = CustomGuardrail(
|
||||
guardrail_name="pii-guard", default_on=default_on, event_hook=event_hook, stream_scope=stream_scope
|
||||
)
|
||||
client_body: Final = MappingProxyType({**_MESSAGES_BODY, "metadata": {"user_id": "user-1"}, **client_fields})
|
||||
body_without_client_fields: Final = {k: v for k, v in client_body.items() if k not in ("metadata", "litellm_metadata")}
|
||||
|
||||
with patch.object(litellm, "callbacks", [guardrail]):
|
||||
relayed: Final = await _relay_through_pass_through(
|
||||
client_body, _ANTHROPIC_MESSAGES, guardrails_config=guardrails_config
|
||||
)
|
||||
|
||||
assert relayed == (
|
||||
{**body_without_client_fields, "metadata": {"user_id": "user-1"}} if forwarded else body_without_client_fields
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.usefixtures("no_configured_anthropic_base")
|
||||
async def test_pass_through_warns_without_the_value_when_it_drops_a_refused_user_id(
|
||||
caplog: pytest.LogCaptureFixture,
|
||||
) -> None:
|
||||
client_body: Final = MappingProxyType({**_MESSAGES_BODY, "metadata": {"user_id": "person@example.com"}})
|
||||
|
||||
with caplog.at_level(logging.WARNING, logger=verbose_proxy_logger.name):
|
||||
await _relay_through_pass_through(client_body, _ANTHROPIC_MESSAGES, guardrails_config=None)
|
||||
|
||||
assert "Not forwarding metadata.user_id" in caplog.text
|
||||
assert "person@example.com" not in caplog.text
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.usefixtures("no_configured_anthropic_base")
|
||||
async def test_pass_through_drops_metadata_that_is_not_an_object() -> None:
|
||||
client_body: Final = MappingProxyType({**_MESSAGES_BODY, "metadata": "user-1"})
|
||||
|
||||
relayed: Final = await _relay_through_pass_through(client_body, _ANTHROPIC_MESSAGES, guardrails_config=None)
|
||||
|
||||
assert relayed == _MESSAGES_BODY
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.usefixtures("no_configured_anthropic_base")
|
||||
async def test_pass_through_custom_body_replaces_the_client_user_id() -> None:
|
||||
client_body: Final = MappingProxyType({**_MESSAGES_BODY, "metadata": {"user_id": "user-1"}})
|
||||
|
||||
relayed: Final = await _relay_through_pass_through(
|
||||
client_body, _ANTHROPIC_MESSAGES, guardrails_config=None, custom_body=dict(_MESSAGES_BODY)
|
||||
)
|
||||
|
||||
assert relayed == _MESSAGES_BODY
|
||||
|
||||
|
||||
_PARTIAL_RELAY_WARNING_MARKER = "ended before upstream body was fully relayed"
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -996,3 +996,21 @@ async def test_guardrails_only_and_skip_guardrails_are_mutually_exclusive(
|
|||
guardrails_only=True,
|
||||
skip_guardrails=True,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"request_data, expected",
|
||||
[
|
||||
({"metadata": {"_guardrail_pipelines": ["p1"]}}, True),
|
||||
({"metadata": {"user_id": "u"}, "litellm_metadata": {"_guardrail_pipelines": ["p1"]}}, True),
|
||||
({"metadata": {"_guardrail_pipelines": 5}}, True),
|
||||
({"metadata": {"user_id": "u"}}, False),
|
||||
],
|
||||
ids=["pipeline-in-metadata", "pipeline-in-litellm-metadata", "non-list-pipeline-value", "no-pipeline"],
|
||||
)
|
||||
def test_request_has_pre_call_guardrails_reads_pipelines_from_both_metadata_buckets(
|
||||
proxy_logging, monkeypatch, request_data, expected
|
||||
):
|
||||
monkeypatch.setattr(litellm, "callbacks", [])
|
||||
|
||||
assert proxy_logging.request_has_pre_call_guardrails(request_data) is expected
|
||||
|
|
|
|||
|
|
@ -6314,6 +6314,17 @@ def test_marks_past_the_provider_cap_keep_the_newest_under_drop_params(drop_para
|
|||
assert messages[1] == {"role": "user", "content": "turn 0"}
|
||||
|
||||
|
||||
def test_marks_past_the_provider_cap_inside_one_message_keep_the_newest_under_drop_params():
|
||||
parts: Final = tuple(
|
||||
{"type": "input_text", "text": f"part {index}", "cache_control": _EPHEMERAL} for index in range(1, 6)
|
||||
)
|
||||
|
||||
messages: Final = _bridged_request({"role": "user", "content": list(parts)}, drop_params=True)
|
||||
|
||||
content: Final = TypeAdapter(list[dict[str, object]]).validate_python(messages[-1]["content"])
|
||||
assert ["cache_control" in part for part in content] == [False, True, True, True, True]
|
||||
|
||||
|
||||
def test_global_drop_params_keeps_the_newest_marks(monkeypatch: pytest.MonkeyPatch):
|
||||
monkeypatch.setattr(litellm, "drop_params", True)
|
||||
|
||||
|
|
|
|||
16
ui/litellm-dashboard/package-lock.json
generated
16
ui/litellm-dashboard/package-lock.json
generated
|
|
@ -2037,6 +2037,13 @@
|
|||
"url": "https://opencollective.com/libvips"
|
||||
}
|
||||
},
|
||||
"node_modules/@isaacs/nonrecursive-glob-match": {
|
||||
"version": "1.0.2",
|
||||
"resolved": "https://registry.npmjs.org/@isaacs/nonrecursive-glob-match/-/nonrecursive-glob-match-1.0.2.tgz",
|
||||
"integrity": "sha512-GJPHIgPPtFl+nLhFHyF0zZoFcOFLsZWbHFFzo+Zr+PWDXJbl9hHxb2w+9SiF2fHmLgKaXIpUzQo3SbBBZ9WObw==",
|
||||
"dev": true,
|
||||
"license": "BlueOak-1.0.0"
|
||||
},
|
||||
"node_modules/@jridgewell/gen-mapping": {
|
||||
"version": "0.3.13",
|
||||
"resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.3.13.tgz",
|
||||
|
|
@ -9657,13 +9664,14 @@
|
|||
}
|
||||
},
|
||||
"node_modules/minimatch": {
|
||||
"version": "10.2.4",
|
||||
"resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.2.4.tgz",
|
||||
"integrity": "sha512-oRjTw/97aTBN0RHbYCdtF1MQfvusSIBQM0IZEgzl6426+8jSC0nF1a/GmnVLpfB9yyr6g6FTqWqiZVbxrtaCIg==",
|
||||
"version": "11.0.2",
|
||||
"resolved": "https://registry.npmjs.org/minimatch/-/minimatch-11.0.2.tgz",
|
||||
"integrity": "sha512-pWTgz5CN3tjqyZNSK2tePlSmu9IwM41q8yn8Usdqqwe9TIHh+9dEpLLjF4uApsfPEAZY71qtSiA2B2APEr+1Yw==",
|
||||
"dev": true,
|
||||
"license": "BlueOak-1.0.0",
|
||||
"dependencies": {
|
||||
"brace-expansion": "^5.0.2"
|
||||
"@isaacs/nonrecursive-glob-match": "^1.0.2",
|
||||
"brace-expansion": "^5.0.12"
|
||||
},
|
||||
"engines": {
|
||||
"node": "18 || 20 || >=22"
|
||||
|
|
|
|||
|
|
@ -117,7 +117,7 @@
|
|||
"js-yaml": "4.3.2",
|
||||
"brace-expansion": "5.0.12",
|
||||
"glob": "13.0.0",
|
||||
"minimatch": "10.2.4",
|
||||
"minimatch": "11.0.2",
|
||||
"ws": "8.21.0",
|
||||
"braces": "3.0.3",
|
||||
"axios": "1.13.6",
|
||||
|
|
|
|||
BIN
ui/litellm-dashboard/public/assets/logos/hostinger.png
Normal file
BIN
ui/litellm-dashboard/public/assets/logos/hostinger.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 16 KiB |
|
|
@ -30,6 +30,7 @@ import googleLogo from "../../public/assets/logos/google.svg";
|
|||
import groqLogo from "../../public/assets/logos/groq.svg";
|
||||
import huggingfaceLogo from "../../public/assets/logos/huggingface.svg";
|
||||
import hyperbolicLogo from "../../public/assets/logos/hyperbolic.svg";
|
||||
import hostingerLogo from "../../public/assets/logos/hostinger.png";
|
||||
import infinityLogo from "../../public/assets/logos/infinity.png";
|
||||
import jinaLogo from "../../public/assets/logos/jina.png";
|
||||
import lambdaLogo from "../../public/assets/logos/lambda.svg";
|
||||
|
|
@ -131,6 +132,7 @@ export enum Providers {
|
|||
Hosted_Vllm = "Hosted vLLM",
|
||||
HUGGINGFACE = "Huggingface",
|
||||
HYPERBOLIC = "Hyperbolic",
|
||||
HOSTINGER_ROUTER = "Hostinger Router",
|
||||
Infinity = "Infinity",
|
||||
JinaAI = "Jina AI",
|
||||
OPENCODE_GO = "OpenCode Go",
|
||||
|
|
@ -263,6 +265,7 @@ export const provider_map: Record<string, string> = {
|
|||
Hosted_Vllm: "hosted_vllm",
|
||||
HUGGINGFACE: "huggingface",
|
||||
HYPERBOLIC: "hyperbolic",
|
||||
HOSTINGER_ROUTER: "hostinger_router",
|
||||
Infinity: "infinity",
|
||||
JinaAI: "jina_ai",
|
||||
OPENCODE_GO: "opencode_go",
|
||||
|
|
@ -389,6 +392,7 @@ export const providerLogoMap: Partial<Record<Providers, string>> = {
|
|||
[Providers.Hosted_Vllm]: vllmLogo.src,
|
||||
[Providers.HUGGINGFACE]: huggingfaceLogo.src,
|
||||
[Providers.HYPERBOLIC]: hyperbolicLogo.src,
|
||||
[Providers.HOSTINGER_ROUTER]: hostingerLogo.src,
|
||||
[Providers.Infinity]: infinityLogo.src,
|
||||
[Providers.JinaAI]: jinaLogo.src,
|
||||
[Providers.OPENCODE_GO]: opencodeLogo.src,
|
||||
|
|
@ -493,6 +497,7 @@ const providerPlaceholderMap: Partial<Record<Providers, string>> = {
|
|||
[Providers.EDENAI]: "edenai/openai/gpt-mini-latest",
|
||||
[Providers.FalAI]: "fal_ai/fal-ai/flux-pro/v1.1-ultra",
|
||||
[Providers.Google_AI_Studio]: "gemini-pro",
|
||||
[Providers.HOSTINGER_ROUTER]: "hostinger_router/claude-sonnet-5",
|
||||
[Providers.JinaAI]: "jina_ai/",
|
||||
[Providers.META]: "meta/muse-spark-1.1",
|
||||
[Providers.MICROSOFT_365_COPILOT]: "microsoft_365_copilot/chat",
|
||||
|
|
|
|||
13
uv.lock
generated
13
uv.lock
generated
|
|
@ -16,6 +16,7 @@ exclude-newer-span = "P3D"
|
|||
|
||||
[options.exclude-newer-package]
|
||||
brotli = "2026-10-22T14:01:00Z"
|
||||
pypdf = "2026-10-28T17:01:00Z"
|
||||
|
||||
[manifest]
|
||||
members = [
|
||||
|
|
@ -8372,11 +8373,11 @@ wheels = [
|
|||
|
||||
[[package]]
|
||||
name = "pyasn1"
|
||||
version = "0.6.4"
|
||||
version = "0.6.5"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/a4/9a/23310166d960def5897e91fe20e5b724601b02a22e84ba1f94232c0b7f67/pyasn1-0.6.4.tar.gz", hash = "sha256:9c447d8431c947fe4c8febc4ed9e760bc29011a5b01e5c74b67025bd9fb8ce81", size = 151262, upload-time = "2026-07-09T01:12:33.988Z" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/88/7e/3c0da41061a2021cac7cd524fd57092adbe5b515e8bf9005702e366802e2/pyasn1-0.6.5.tar.gz", hash = "sha256:6b11616a6bcf3a40e635f082bb232fa57b5e0dd7f517bd7ddd69ec3794508a47", size = 155152, upload-time = "2026-10-27T06:32:49.125Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/9a/3b/6163796d69c3977d1e4287bea4a6979161cbbdd170ebb430511e8e1999ce/pyasn1-0.6.4-py3-none-any.whl", hash = "sha256:deda9277cfd454080ec40b207fb6df82206a3a2688735233cdcd8d3d565f088b", size = 84410, upload-time = "2026-07-09T01:12:32.92Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/19/1f/3eefd5c84c35cae44549ce1c7f7e58f7945b34f5a6bce07a135438522073/pyasn1-0.6.5-py3-none-any.whl", hash = "sha256:4017b0fce125096b8573b39c989fdb1a4b19cf761e683398184550fea0e66587", size = 84807, upload-time = "2026-10-27T06:32:48.045Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -8869,14 +8870,14 @@ wheels = [
|
|||
|
||||
[[package]]
|
||||
name = "pypdf"
|
||||
version = "6.21.2"
|
||||
version = "6.21.4"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "typing-extensions", version = "4.15.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/06/28/081f1cbcca7042441dafc8de00f48ba60853f89e5bff92c5f26477346d81/pypdf-6.21.2.tar.gz", hash = "sha256:0a3e1c5c886a1be03646c16a04827cc166c2a0575c971b26a4bf89686551c2d4", size = 7144571, upload-time = "2026-10-21T16:44:55.272Z" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/1e/e1/ec8f4d155c997c45917153a5521e60d6ee6389d68abdcbc8e44e564caffc/pypdf-6.21.4.tar.gz", hash = "sha256:c8817e3f888d47a087872ded69812c1ef30cacb871d6e053e0c00bbaf0bc2163", size = 7109255, upload-time = "2026-10-28T17:00:54.81Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/1b/4d/c9d2d67f204eff463237a1a8f5e25a6923b7b1043982b91fcaf104c8ef60/pypdf-6.21.2-py3-none-any.whl", hash = "sha256:1d44d1a3a200f25062adff0085b02040e4646431ac2ccac22e42679e91466084", size = 407766, upload-time = "2026-10-21T16:44:53.626Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/3f/4b/77e8ad4ac42c9d93600f40541a50e9feb3a61d2bbee2d76f659346072051/pypdf-6.21.4-py3-none-any.whl", hash = "sha256:33e4fb41ca352810eb4e9838aa0d82d6888a5f8bf682f43e99cce5faa666255c", size = 410430, upload-time = "2026-10-28T17:00:53.146Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
|
|||
16
vscode-extension/package-lock.json
generated
16
vscode-extension/package-lock.json
generated
|
|
@ -644,6 +644,13 @@
|
|||
"node": ">=18"
|
||||
}
|
||||
},
|
||||
"node_modules/@isaacs/nonrecursive-glob-match": {
|
||||
"version": "1.0.2",
|
||||
"resolved": "https://registry.npmjs.org/@isaacs/nonrecursive-glob-match/-/nonrecursive-glob-match-1.0.2.tgz",
|
||||
"integrity": "sha512-GJPHIgPPtFl+nLhFHyF0zZoFcOFLsZWbHFFzo+Zr+PWDXJbl9hHxb2w+9SiF2fHmLgKaXIpUzQo3SbBBZ9WObw==",
|
||||
"dev": true,
|
||||
"license": "BlueOak-1.0.0"
|
||||
},
|
||||
"node_modules/@jridgewell/sourcemap-codec": {
|
||||
"version": "1.6.0",
|
||||
"resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.6.0.tgz",
|
||||
|
|
@ -2700,13 +2707,14 @@
|
|||
}
|
||||
},
|
||||
"node_modules/minimatch": {
|
||||
"version": "10.2.6",
|
||||
"resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.2.6.tgz",
|
||||
"integrity": "sha512-vpLQEs+VLCr1nU0BXS07maYoFwlDAH0gngQuuttxIwutDFEMHq2blX+8vpgxDdK3J1PwjCJiep77OitTZ4Ll1A==",
|
||||
"version": "11.0.2",
|
||||
"resolved": "https://registry.npmjs.org/minimatch/-/minimatch-11.0.2.tgz",
|
||||
"integrity": "sha512-pWTgz5CN3tjqyZNSK2tePlSmu9IwM41q8yn8Usdqqwe9TIHh+9dEpLLjF4uApsfPEAZY71qtSiA2B2APEr+1Yw==",
|
||||
"dev": true,
|
||||
"license": "BlueOak-1.0.0",
|
||||
"dependencies": {
|
||||
"brace-expansion": "^5.0.8"
|
||||
"@isaacs/nonrecursive-glob-match": "^1.0.2",
|
||||
"brace-expansion": "^5.0.12"
|
||||
},
|
||||
"engines": {
|
||||
"node": "18 || 20 || >=22"
|
||||
|
|
|
|||
|
|
@ -83,5 +83,8 @@
|
|||
"esbuild": "^0.28.2",
|
||||
"typescript": "^5.9.3",
|
||||
"vitest": "^4.1.12"
|
||||
},
|
||||
"overrides": {
|
||||
"minimatch": "11.0.2"
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue