diff --git a/.github/workflows/test-linting.yml b/.github/workflows/test-linting.yml index 89ca60ad9d1..bd7343ea222 100644 --- a/.github/workflows/test-linting.yml +++ b/.github/workflows/test-linting.yml @@ -376,10 +376,10 @@ jobs: exit 0 fi files="$(git diff --name-only "$base" HEAD)" - if grep -Eq '^(litellm/(proxy|types)/|ui/litellm-dashboard/(src/lib/http/schema\.d\.ts|scripts/gen-api-types\.mjs|package(-lock)?\.json)$|\.github/workflows/test-linting\.yml$)' <<< "$files"; then + if grep -Eq '^(litellm/|enterprise/|(pyproject\.toml|uv\.lock)$|ui/litellm-dashboard/(src/lib/http/schema\.d\.ts|scripts/gen-api-types\.mjs|package(-lock)?\.json)$|\.github/workflows/test-linting\.yml$)' <<< "$files"; then echo "relevant=true" >> "$GITHUB_OUTPUT" else - echo "No proxy, types or generator changes in this pull request, nothing to verify." + echo "No litellm, enterprise, dependency or generator changes in this pull request, nothing to verify." echo "relevant=false" >> "$GITHUB_OUTPUT" fi diff --git a/.github/workflows/test-terraform-provider.yml b/.github/workflows/test-terraform-provider.yml index ff9db13bd25..6aeb66ed37c 100644 --- a/.github/workflows/test-terraform-provider.yml +++ b/.github/workflows/test-terraform-provider.yml @@ -12,6 +12,7 @@ on: paths: - "terraform/provider/**" - "litellm/proxy/**" + - "enterprise/**" - ".github/workflows/test-terraform-provider.yml" permissions: diff --git a/litellm/__init__.py b/litellm/__init__.py index 5e8f4246cef..a1bda4aa9f7 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -685,6 +685,7 @@ moonshot_models: Set = set() publicai_models: Set = set() darkbloom_models: Set = set() openinfer_models: Final[Set[str]] = set() +kyma_models: Final[set[str]] = set() # mutable-ok: populated when the shared model catalog loads streamlake_models: Final[Set[str]] = set() zerogpu_models: Final[Set[str]] = set() v0_models: Set = set() @@ -981,6 +982,8 @@ def _populate_provider_model_sets(model_cost_map: Mapping[str, Mapping[str, obje darkbloom_models.add(key) elif value.get("litellm_provider") == "openinfer": openinfer_models.add(key) + elif value.get("litellm_provider") == "kyma": + kyma_models.add(key) elif value.get("litellm_provider") == "streamlake": streamlake_models.add(key) elif value.get("litellm_provider") == "zerogpu": @@ -1159,6 +1162,7 @@ model_list = list( | publicai_models | darkbloom_models | openinfer_models + | kyma_models | streamlake_models | zerogpu_models | v0_models @@ -1290,6 +1294,7 @@ def _build_models_by_provider() -> dict: "publicai": publicai_models, "darkbloom": darkbloom_models, "openinfer": openinfer_models, + "kyma": kyma_models, "streamlake": streamlake_models, "zerogpu": zerogpu_models, "v0": v0_models, diff --git a/litellm/_internal_context.py b/litellm/_internal_context.py index 244efc8047b..c2243b90fbf 100644 --- a/litellm/_internal_context.py +++ b/litellm/_internal_context.py @@ -36,6 +36,8 @@ REDIS_FAMILIES_METADATA_KEY: Final = "families" _service_caller: Final[ContextVar[str | None]] = ContextVar("service_caller", default=None) +_mirrored_call: Final[ContextVar[bool]] = ContextVar("mirrored_call", default=False) + @contextmanager def internal_sub_call() -> Generator[None]: @@ -123,6 +125,30 @@ def current_service_caller() -> str | None: return _service_caller.get() +@contextmanager +def mirrored_call() -> Generator[None]: + """A silent_model mirror of a request: every litellm call inside, its retries and fallbacks included.""" + token: Final = _mirrored_call.set(True) + try: + yield + finally: + _restore(_mirrored_call, token) + + +def in_mirrored_call() -> bool: + return _mirrored_call.get() + + +@contextmanager +def outside_mirrored_call() -> Generator[None]: + """Logging a mirrored call: the calls its callbacks start are not mirrors themselves.""" + token: Final = _mirrored_call.set(False) + try: + yield + finally: + _restore(_mirrored_call, token) + + @contextmanager def pinned_billing_time(moment: datetime) -> Generator[None]: """Price every rate lookup inside this block at ``moment`` rather than at each one's own clock read.""" diff --git a/litellm/anthropic_beta_gated_fields.py b/litellm/anthropic_beta_gated_fields.py new file mode 100644 index 00000000000..fc36699bc93 --- /dev/null +++ b/litellm/anthropic_beta_gated_fields.py @@ -0,0 +1,325 @@ +"""Body fields that an ``anthropic-beta`` value gates. + +``anthropic_beta_headers_config.json`` says which beta each provider leg +carries. When a leg drops a beta, the request fields that beta unlocks +would make the provider 400. Under ``drop_params`` the same request degrades on +that leg by dropping those fields too; without it the request goes out as sent +and the provider answers its own 400, like any unsupported param. Every helper +here is copy-on-write: the caller's containers are returned untouched when +nothing is stripped, and a shallow copy is returned otherwise, so a fallback +leg still sends the full request. +""" + +from collections.abc import Mapping, Sequence, Set +from dataclasses import dataclass +from types import MappingProxyType +from typing import Final + +from pydantic import TypeAdapter, ValidationError + +from litellm._logging import verbose_logger +from litellm.litellm_core_utils.core_helpers import drops_unsupported_params +from litellm.types.llms.anthropic import ( + ANTHROPIC_BETA_HEADER_VALUES, + ANTHROPIC_MID_CONVERSATION_OUTPUT_CONFIG_BETA_HEADER, + ANTHROPIC_MID_CONVERSATION_TOOL_CHANGES_BETA_HEADER, + ANTHROPIC_THINKING_BINDING_CONTROLS_BETA_HEADER, + ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER, +) + +ANTHROPIC_MID_CONVERSATION_SYSTEM_CLEAR_AT_BETA_HEADER: Final = "mid-conversation-system-clear-at-2026-08-21" +ANTHROPIC_INLINE_TOOLS_BETA_HEADER: Final = "inline-tools-2026-09-15" + +THINKING_BLOCK_BINDING: Final = "thinking.block_binding" +THINKING_DISPLAY_UPDATES: Final = "thinking.display=updates" +SYSTEM_CLEAR_AT: Final = "messages[].system.clear_at" +SYSTEM_OUTPUT_CONFIG: Final = "messages[].system.output_config" +SYSTEM_TOOL_CHANGES: Final = "messages[].system.content[].tool_addition|tool_removal" +SYSTEM_MESSAGE_EMPTIED: Final = "messages[].system (emptied by the strip)" + +_TOOL_CHANGE_BLOCK_TYPES: Final = frozenset({"tool_addition", "tool_removal"}) + +BETA_GATED_FIELDS: Final[Mapping[str, tuple[str, ...]]] = MappingProxyType( + { + ANTHROPIC_THINKING_BINDING_CONTROLS_BETA_HEADER: (THINKING_BLOCK_BINDING,), + ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER: (THINKING_DISPLAY_UPDATES,), + ANTHROPIC_MID_CONVERSATION_SYSTEM_CLEAR_AT_BETA_HEADER: (SYSTEM_CLEAR_AT,), + ANTHROPIC_MID_CONVERSATION_OUTPUT_CONFIG_BETA_HEADER: (SYSTEM_OUTPUT_CONFIG,), + ANTHROPIC_BETA_HEADER_VALUES.PER_TURN_CONTROL_2026_07_01.value: (SYSTEM_OUTPUT_CONFIG,), + ANTHROPIC_MID_CONVERSATION_TOOL_CHANGES_BETA_HEADER: (SYSTEM_TOOL_CHANGES,), + ANTHROPIC_INLINE_TOOLS_BETA_HEADER: (SYSTEM_TOOL_CHANGES,), + } +) + + +def _betas_unlocking(field: str) -> frozenset[str]: + return frozenset(beta for beta, fields in BETA_GATED_FIELDS.items() if field in fields) + + +_FIELD_BETAS: Final[Mapping[str, frozenset[str]]] = MappingProxyType( + { + field: _betas_unlocking(field) for fields in BETA_GATED_FIELDS.values() for field in fields + } # comprehension-ok: flattening the beta-to-fields map into a field-to-betas index +) + + +def field_is_gated(field: str, active_betas: Set[str]) -> bool: + """True when no active beta unlocks ``field``, so the field must be stripped.""" + return _FIELD_BETAS[field].isdisjoint(active_betas) + + +_MAPPING_ADAPTER: Final = TypeAdapter(Mapping[str, object]) +_SEQUENCE_ADAPTER: Final = TypeAdapter(tuple[object, ...]) +_HEADERS_ADAPTER: Final = TypeAdapter(Mapping[str, str]) + + +def _as_mapping(value: object) -> Mapping[str, object] | None: + try: + return _MAPPING_ADAPTER.validate_python(value, strict=True) + except ValidationError: + return None + + +def _as_sequence(value: object) -> tuple[object, ...] | None: + try: + return _SEQUENCE_ADAPTER.validate_python(value) + except ValidationError: + return None + + +def _is_tool_change_block(block: object) -> bool: + mapping: Final = _as_mapping(block) + block_type: Final = mapping.get("type") if mapping is not None else None + return isinstance(block_type, str) and block_type in _TOOL_CHANGE_BLOCK_TYPES + + +def _header_beta_values(headers: Mapping[str, str]) -> frozenset[str]: + joined: Final = ",".join(value for key, value in headers.items() if key.lower() == "anthropic-beta") + return frozenset(piece.strip() for piece in joined.split(",") if piece.strip()) + + +def _validated_headers(headers: object) -> Mapping[str, str]: + try: + return _HEADERS_ADAPTER.validate_python(headers) + except ValidationError: + return MappingProxyType({}) + + +def active_beta_values(headers: object, body_betas: object = ()) -> frozenset[str]: + """The ``anthropic-beta`` values a request will send: the comma-joined header (key matched + case-insensitively) plus the body's ``anthropic_beta`` list on legs that carry betas in the body.""" + body_values: Final = _as_sequence(body_betas) or () + body_betas_as_strings: Final = frozenset(beta for beta in body_values if isinstance(beta, str)) + return _header_beta_values(_validated_headers(headers)) | body_betas_as_strings + + +@dataclass(frozen=True, slots=True) +class StrippedThinking: + thinking: object + stripped: tuple[str, ...] + + +@dataclass(frozen=True, slots=True) +class StrippedMessages: + messages: Sequence[object] + stripped: tuple[str, ...] + + +@dataclass(frozen=True, slots=True) +class StrippedRequest: + request: Mapping[str, object] + stripped: tuple[str, ...] + + +def strip_beta_gated_thinking(thinking: object, active_betas: Set[str], *, drop_params: object) -> StrippedThinking: + """Drop ``thinking`` sub-fields whose beta is absent. Returns the same object when nothing is gated.""" + mapping: Final = _as_mapping(thinking) + if mapping is None or not drops_unsupported_params(drop_params): + return StrippedThinking(thinking=thinking, stripped=()) + binding_gated: Final = "block_binding" in mapping and field_is_gated(THINKING_BLOCK_BINDING, active_betas) + display_gated: Final = mapping.get("display") == "updates" and field_is_gated( + THINKING_DISPLAY_UPDATES, active_betas + ) + if not binding_gated and not display_gated: + return StrippedThinking(thinking=thinking, stripped=()) + gated_keys: Final = frozenset( + {*(("block_binding",) if binding_gated else ()), *(("display",) if display_gated else ())} + ) + stripped: Final = ( + *((THINKING_BLOCK_BINDING,) if binding_gated else ()), + *((THINKING_DISPLAY_UPDATES,) if display_gated else ()), + ) + return StrippedThinking( + thinking={key: value for key, value in mapping.items() if key not in gated_keys}, + stripped=stripped, + ) + + +def _strip_beta_gated_system_message(message: Mapping[str, object], active_betas: Set[str]) -> StrippedMessages: + """One mid-conversation ``role: system`` message. ``messages`` holds the rewritten message, or nothing + when the strip left it with neither content nor ``output_config``, since such a system turn is itself a 400.""" + clear_at_gated: Final = "clear_at" in message and field_is_gated(SYSTEM_CLEAR_AT, active_betas) + output_config_gated: Final = "output_config" in message and field_is_gated(SYSTEM_OUTPUT_CONFIG, active_betas) + content: Final = message.get("content") + blocks: Final = _as_sequence(content) if not isinstance(content, str) else None + tool_changes_gated: Final = ( + blocks is not None + and any(_is_tool_change_block(block) for block in blocks) + and field_is_gated(SYSTEM_TOOL_CHANGES, active_betas) + ) + if not clear_at_gated and not output_config_gated and not tool_changes_gated: + return StrippedMessages(messages=(message,), stripped=()) + kept_content: Final[object] = ( + [block for block in blocks if not _is_tool_change_block(block)] + if tool_changes_gated and blocks is not None + else content + ) + gated_keys: Final = frozenset( + {*(("clear_at",) if clear_at_gated else ()), *(("output_config",) if output_config_gated else ())} + ) + rewritten: Final = { + key: (kept_content if key == "content" else value) for key, value in message.items() if key not in gated_keys + } + stripped: Final = ( + *((SYSTEM_CLEAR_AT,) if clear_at_gated else ()), + *((SYSTEM_OUTPUT_CONFIG,) if output_config_gated else ()), + *((SYSTEM_TOOL_CHANGES,) if tool_changes_gated else ()), + ) + if kept_content in (None, "", []) and "output_config" not in rewritten: + return StrippedMessages(messages=(), stripped=(*stripped, SYSTEM_MESSAGE_EMPTIED)) + return StrippedMessages(messages=(rewritten,), stripped=stripped) + + +def _strip_beta_gated_message(message: object, active_betas: Set[str]) -> StrippedMessages: + mapping: Final = _as_mapping(message) + if mapping is None or mapping.get("role") != "system": + return StrippedMessages(messages=(message,), stripped=()) + return _strip_beta_gated_system_message(mapping, active_betas) + + +def _ends_on_assistant(messages: Sequence[object]) -> bool: + last: Final = _as_mapping(messages[-1]) if messages else None + return last is not None and last.get("role") == "assistant" + + +def strip_beta_gated_system_messages( + messages: Sequence[object], active_betas: Set[str], *, drop_params: object +) -> StrippedMessages: + """Drop the gated fields of every mid-conversation ``role: system`` message. Returns the same + sequence when nothing is gated, or when dropping an emptied system turn would leave the conversation + ending on an assistant turn, which the model would read as a prefill to continue.""" + if not drops_unsupported_params(drop_params): + return StrippedMessages(messages=messages, stripped=()) + per_message: Final = tuple(_strip_beta_gated_message(message, active_betas) for message in messages) + stripped: Final = tuple( + name for result in per_message for name in result.stripped + ) # comprehension-ok: flattening per-message strip reports + if not stripped: + return StrippedMessages(messages=messages, stripped=()) + kept: Final = [ + message for result in per_message for message in result.messages + ] # comprehension-ok: flattening per-message results + if _ends_on_assistant(kept) and not _ends_on_assistant(messages): + return StrippedMessages(messages=messages, stripped=()) + return StrippedMessages(messages=kept, stripped=stripped) + + +def strip_beta_gated_request_fields( + request: Mapping[str, object], active_betas: Set[str], *, drop_params: object +) -> StrippedRequest: + """Apply both strips to an Anthropic ``/v1/messages`` shaped body. Returns the same mapping when + nothing is gated, otherwise a shallow copy with ``thinking`` and ``messages`` replaced.""" + thinking: Final = strip_beta_gated_thinking(request.get("thinking"), active_betas, drop_params=drop_params) + messages: Final = request.get("messages") + message_sequence: Final = _as_sequence(messages) if not isinstance(messages, str) else None + system_messages: Final = ( + strip_beta_gated_system_messages(message_sequence, active_betas, drop_params=drop_params) + if message_sequence is not None + else StrippedMessages(messages=(), stripped=()) + ) + stripped: Final = (*thinking.stripped, *system_messages.stripped) + if not stripped: + return StrippedRequest(request=request, stripped=()) + rewritten: Final = { + **request, + **({"thinking": thinking.thinking} if thinking.stripped else {}), + **({"messages": system_messages.messages} if system_messages.stripped else {}), + } + return StrippedRequest(request=rewritten, stripped=stripped) + + +@dataclass(frozen=True, slots=True) +class StrippedNativeRequest: + thinking: object + thinking_stripped: bool + messages: Sequence[object] + messages_stripped: bool + stripped: tuple[str, ...] + + +def unstripped_native_request(optional_params: object, messages: object) -> StrippedNativeRequest: + """The identity result for legs that strip later, once their final ``anthropic_beta`` list exists.""" + params: Final = _as_mapping(optional_params) or MappingProxyType({}) + message_sequence: Final = _as_sequence(messages) if not isinstance(messages, str) else None + return StrippedNativeRequest( + thinking=params.get("thinking"), + thinking_stripped=False, + messages=message_sequence if message_sequence is not None else (), + messages_stripped=False, + stripped=(), + ) + + +def strip_beta_gated_native_request( + headers: object, optional_params: object, messages: object, *, body_betas: object = (), drop_params: object +) -> StrippedNativeRequest: + """The ``/v1/messages`` seam, fed the final (already filtered) headers, the final body ``anthropic_beta`` + list on legs that carry betas in the body, plus the body parts it may rewrite.""" + if not drops_unsupported_params(drop_params): + return unstripped_native_request(optional_params, messages) + active_betas: Final = active_beta_values(headers, body_betas) + params: Final = _as_mapping(optional_params) or MappingProxyType({}) + thinking: Final = strip_beta_gated_thinking(params.get("thinking"), active_betas, drop_params=drop_params) + message_sequence: Final = _as_sequence(messages) if not isinstance(messages, str) else None + system_messages: Final = ( + strip_beta_gated_system_messages(message_sequence, active_betas, drop_params=drop_params) + if message_sequence is not None + else StrippedMessages(messages=(), stripped=()) + ) + return StrippedNativeRequest( + thinking=thinking.thinking, + thinking_stripped=bool(thinking.stripped), + messages=system_messages.messages, + messages_stripped=bool(system_messages.stripped), + stripped=(*thinking.stripped, *system_messages.stripped), + ) + + +def log_beta_gated_strips(stripped: Sequence[str], model: str | None, provider: str) -> None: + if not stripped: + return + described: Final = ", ".join( + f"{field} (unlocked by anthropic-beta {' or '.join(sorted(_FIELD_BETAS[field]))})" + if field in _FIELD_BETAS + else field + for field in sorted(set(stripped)) + ) + verbose_logger.warning( + "Dropped %s for model=%s provider=%s because drop_params is on and the provider would reject them on " + "this request, either since this provider leg does not send that anthropic-beta (see " + "anthropic_beta_headers_config.json) or since the model turns mid-conversation system messages into user " + "messages.", + described, + model, + provider, + ) + + +def strip_beta_gated_request_body( + request: Mapping[str, object], headers: object, *, drop_params: object, model: str | None, provider: str +) -> Mapping[str, object]: + gated: Final = strip_beta_gated_request_fields( + request, active_beta_values(headers, request.get("anthropic_beta") or ()), drop_params=drop_params + ) + log_beta_gated_strips(gated.stripped, model=model, provider=provider) + return gated.request diff --git a/litellm/anthropic_beta_headers_config.json b/litellm/anthropic_beta_headers_config.json index d68de6c9d4d..cee5ce15469 100644 --- a/litellm/anthropic_beta_headers_config.json +++ b/litellm/anthropic_beta_headers_config.json @@ -248,6 +248,7 @@ "text_editor_20241022": null, "text_editor_20250124": null, "thinking-binding-controls-2026-08-01": "thinking-binding-controls-2026-08-01", + "thinking-display-updates-2026-08-18": "thinking-display-updates-2026-08-18", "token-efficient-tools-2025-02-19": "token-efficient-tools-2025-02-19", "web-fetch-2025-09-10": null, "web-search-2025-03-05": "web-search-2025-03-05" diff --git a/litellm/anthropic_beta_headers_manager.py b/litellm/anthropic_beta_headers_manager.py index fec1d8e4465..07730a0fbf8 100644 --- a/litellm/anthropic_beta_headers_manager.py +++ b/litellm/anthropic_beta_headers_manager.py @@ -241,6 +241,13 @@ def get_provider_name(provider: str) -> str: return aliases.get(provider, provider) +def has_beta_headers_config(provider: str) -> bool: + config: Final = _bundled_beta_headers_config() + aliases: Final = config.get("provider_aliases", {}) + canonical_provider: Final = (aliases.get(provider) if isinstance(aliases, Mapping) else None) or provider + return canonical_provider in config + + def filter_and_transform_beta_headers( beta_headers: list[str], provider: str, @@ -397,7 +404,7 @@ def update_request_with_filtered_beta( headers: dict, request_data: dict, provider: str, -) -> tuple[dict, dict]: +) -> tuple[dict[str, str], dict[str, object]]: """ Update both headers and request body beta fields based on provider support. Modifies both dicts in place and returns them. diff --git a/litellm/caching/caching.py b/litellm/caching/caching.py index 5f5b8210096..1ced13445de 100644 --- a/litellm/caching/caching.py +++ b/litellm/caching/caching.py @@ -455,17 +455,33 @@ class Cache: 3. Else use the `model` passed in kwargs, plus `model_id` when set (e.g. a SageMaker inference component) """ litellm_params: Final[dict] = kwargs.get("litellm_params", {}) or {} - metadata_sources: Final[tuple[dict, ...]] = ( - kwargs.get("metadata") or {}, - kwargs.get("litellm_metadata") or {}, - litellm_params.get("metadata") or {}, - litellm_params.get("litellm_metadata") or {}, + metadata_sources: Final = cast( # cast-ok: each source is the untyped request's metadata dict or empty + "tuple[dict[str, object], dict[str, object], dict[str, object], dict[str, object]]", + ( + kwargs.get("metadata") or {}, + kwargs.get("litellm_metadata") or {}, + litellm_params.get("metadata") or {}, + litellm_params.get("litellm_metadata") or {}, + ), ) - model_group: Final[str | None] = next( - (source["model_group"] for source in metadata_sources if source.get("model_group")), None + metadata, litellm_metadata, params_metadata, params_litellm_metadata = metadata_sources + call_model_info: Final[object] = kwargs.get("model_info") + routed_source: Final = next( + ( + source + for source in (litellm_metadata, metadata, params_litellm_metadata, params_metadata) + if call_model_info and source.get("model_group") and source.get("model_info") == call_model_info + ), + None, ) + model_group: Final = ( + str(routed_source["model_group"]) + if routed_source is not None + else next((str(source["model_group"]) for source in metadata_sources if source.get("model_group")), None) + ) + caching_group_sources: Final = (routed_source,) if routed_source is not None else metadata_sources caching_group: Final = next( - (group for source in metadata_sources if (group := self._get_caching_group(source, model_group))), None + (group for source in caching_group_sources if (group := self._get_caching_group(source, model_group))), None ) model_id_suffix: Final = ( f" model_id: {kwargs['model_id']}" if "model_id" in kwargs and kwargs["model_id"] is not None else "" diff --git a/litellm/caching/qdrant_semantic_cache.py b/litellm/caching/qdrant_semantic_cache.py index 924506dfa45..560a5ba06b8 100644 --- a/litellm/caching/qdrant_semantic_cache.py +++ b/litellm/caching/qdrant_semantic_cache.py @@ -13,7 +13,7 @@ import asyncio import json import os from types import MappingProxyType -from typing import TYPE_CHECKING, Any, Final, Protocol, cast +from typing import TYPE_CHECKING, Any, Final, Protocol from pydantic import TypeAdapter @@ -292,10 +292,7 @@ class QdrantSemanticCache(BaseCache): prompt: Final = get_semantic_cache_prompt_from_messages(messages) # create an embedding for prompt - embedding_response: Final = cast( - EmbeddingResponse, - self._get_embedding(prompt, metadata=kwargs.get("metadata")), - ) + embedding_response: Final = self._get_embedding(prompt, metadata=kwargs.get("metadata")) # get the embedding embedding: Final = _EMBEDDING_VECTOR.validate_python(embedding_response["data"][0]["embedding"]) @@ -331,10 +328,7 @@ class QdrantSemanticCache(BaseCache): prompt: Final = get_semantic_cache_prompt_from_messages(messages) # convert to embedding - embedding_response: Final = cast( - EmbeddingResponse, - self._get_embedding(prompt, metadata=kwargs.get("metadata")), - ) + embedding_response: Final = self._get_embedding(prompt, metadata=kwargs.get("metadata")) # get the embedding embedding: Final = _EMBEDDING_VECTOR.validate_python(embedding_response["data"][0]["embedding"]) diff --git a/litellm/caching/redis_semantic_cache.py b/litellm/caching/redis_semantic_cache.py index b892e4c3a1d..7268511fe7e 100644 --- a/litellm/caching/redis_semantic_cache.py +++ b/litellm/caching/redis_semantic_cache.py @@ -14,7 +14,9 @@ import asyncio import json import os from collections.abc import Callable, Mapping -from typing import TYPE_CHECKING, Any, Final, cast +from typing import TYPE_CHECKING, Final, cast + +from pydantic import ConfigDict, TypeAdapter import litellm from litellm._logging import print_verbose, verbose_logger @@ -38,6 +40,11 @@ if TYPE_CHECKING: from litellm.router import Router +_CACHE_RESPONSE_ADAPTER: Final[TypeAdapter[str | bytes | None]] = TypeAdapter( + str | bytes | None, config=ConfigDict(strict=True, hide_input_in_errors=True) +) + + class RedisSemanticCache(BaseCache): """ Redis-backed semantic cache for LLM responses. @@ -380,7 +387,7 @@ class RedisSemanticCache(BaseCache): ) return embedding_response["data"][0]["embedding"] - def _get_cache_logic(self, cached_response: Any) -> object: + def _get_cache_logic(self, cached_response: str | bytes | None) -> object: """ Process the cached response to prepare it for use. @@ -391,24 +398,23 @@ class RedisSemanticCache(BaseCache): The processed cache response, or None if input was None """ if cached_response is None: - return cached_response + return None # Convert bytes to string if needed - if isinstance(cached_response, bytes): - cached_response = cached_response.decode("utf-8") + response_text: Final = ( + cached_response.decode("utf-8") if isinstance(cached_response, bytes) else cached_response + ) # Convert string representation to Python object try: - cached_response = json.loads(cached_response) + return json.loads(response_text) except json.JSONDecodeError: try: - cached_response = ast.literal_eval(cached_response) + return ast.literal_eval(response_text) except (ValueError, SyntaxError) as e: print_verbose(f"Error parsing cached response: {e}") return None - return cached_response - def set_cache(self, key: str, value: object, **kwargs) -> None: """ Store a value in the semantic cache. @@ -494,7 +500,7 @@ class RedisSemanticCache(BaseCache): similarity: Final = 1 - vector_distance cached_prompt: Final = cache_hit["prompt"] - cached_response: Final = cache_hit["response"] + cached_response: Final = _CACHE_RESPONSE_ADAPTER.validate_python(cache_hit["response"]) # update kwargs["metadata"] with similarity, don't rewrite the original metadata kwargs.setdefault("metadata", {})["semantic-similarity"] = similarity @@ -645,7 +651,7 @@ class RedisSemanticCache(BaseCache): similarity: Final = 1 - vector_distance cached_prompt: Final = cache_hit["prompt"] - cached_response: Final = cache_hit["response"] + cached_response: Final = _CACHE_RESPONSE_ADAPTER.validate_python(cache_hit["response"]) # update kwargs["metadata"] with similarity, don't rewrite the original metadata kwargs.setdefault("metadata", {})["semantic-similarity"] = similarity diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index b5dc0ce13de..a4a9a5bee39 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -22,7 +22,7 @@ from litellm.constants import ( OPENAI_LIVE_TERMINAL_RESPONSE_EVENT_TYPES, REALTIME_METERED_USAGE_KEY, ) -from litellm.litellm_core_utils.core_helpers import as_str_mapping +from litellm.litellm_core_utils.core_helpers import as_str_mapping, billed_usage_cost from litellm.litellm_core_utils.cost_discount import resolve_cost_discount from litellm.litellm_core_utils.decisions_pricing import decisions_pricing_fields, is_decisions_call_type from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR @@ -50,6 +50,7 @@ from litellm.litellm_core_utils.llm_cost_calc.utils import ( parse_prompt_tokens_details, select_cost_metric_for_model, ) +from litellm.litellm_core_utils.llm_response_utils.response_metadata import response_ms_since_logging_start from litellm.llms.anthropic.cost_calculation import ( cost_per_token as anthropic_cost_per_token, ) @@ -1396,9 +1397,9 @@ def _apply_cost_margin( def _store_cost_breakdown_in_logging_obj( litellm_logging_obj: LitellmLoggingObject | None, - prompt_tokens_cost_usd_dollar: float, - completion_tokens_cost_usd_dollar: float, - cost_for_built_in_tools_cost_usd_dollar: float, + prompt_tokens_cost_usd_dollar: float | None, + completion_tokens_cost_usd_dollar: float | None, + cost_for_built_in_tools_cost_usd_dollar: float | None, total_cost_usd_dollar: float, additional_costs: dict | None = None, original_cost: float | None = None, @@ -1500,9 +1501,45 @@ def _response_time_ms_for_cost( return float(stamped) if total_time: return total_time - if litellm_logging_obj is not None: - return litellm_logging_obj.get_response_ms() - return 0.0 + if litellm_logging_obj is None: + return 0.0 + reported_ms: Final = response_ms_since_logging_start(litellm_logging_obj) + if reported_ms is not None and litellm_logging_obj.stream is not True: + return reported_ms + return litellm_logging_obj.get_response_ms() + + +def _adjusted_provider_billed_cost( + reported_cost: float, + custom_llm_provider: str | None, + model: str | None, + region_name: str | None, + litellm_logging_obj: LitellmLoggingObject | None, + service_tier: str | None, + data_residency: str | None, +) -> float: + discounted_cost, discount_percent, discount_amount = _apply_cost_discount( + base_cost=reported_cost, custom_llm_provider=custom_llm_provider, model=model, region_name=region_name + ) + adjusted_cost, margin_percent, margin_fixed_amount, margin_total_amount = _apply_cost_margin( + base_cost=discounted_cost, custom_llm_provider=custom_llm_provider + ) + _store_cost_breakdown_in_logging_obj( + litellm_logging_obj=litellm_logging_obj, + prompt_tokens_cost_usd_dollar=None, + completion_tokens_cost_usd_dollar=None, + cost_for_built_in_tools_cost_usd_dollar=None, + total_cost_usd_dollar=adjusted_cost, + original_cost=reported_cost, + discount_percent=discount_percent, + discount_amount=discount_amount, + margin_percent=margin_percent, + margin_fixed_amount=margin_fixed_amount, + margin_total_amount=margin_total_amount, + service_tier=service_tier, + data_residency=data_residency, + ) + return adjusted_cost def completion_cost( @@ -1616,6 +1653,9 @@ def completion_cost( ) explicit_pricing: Final = custom_pricing is True or base_model is not None + caller_set_prices: Final = ( + explicit_pricing or custom_cost_per_token is not None or custom_cost_per_second is not None + ) custom_priced_request_model: Final = model if custom_pricing is True else None configured_cost_per_query: Final = ( _configured_search_cost_per_query(litellm_logging_obj) if custom_pricing else None @@ -1666,6 +1706,7 @@ def completion_cost( raw_candidates if custom_pricing is True else _priced_candidates_before_unpriced_rules(raw_candidates) ) + priced_response: Final[object] = completion_response for idx, (candidate_model, candidate_pricing_provider) in enumerate(potential_model_names): model = candidate_model try: @@ -1984,6 +2025,19 @@ def completion_cost( total_time=total_time, ) model_for_discount = custom_priced_request_model or model + if ( + not caller_set_prices + and (reported_cost := billed_usage_cost(priced_response, custom_llm_provider)) is not None + ): + return _adjusted_provider_billed_cost( + reported_cost=reported_cost, + custom_llm_provider=custom_llm_provider, + model=model_for_discount, + region_name=region_name if isinstance(region_name, str) else None, + litellm_logging_obj=litellm_logging_obj, + service_tier=service_tier, + data_residency=data_residency, + ) # Calculate cost based on prompt_tokens, completion_tokens if ( "togethercomputer" in model or "together_ai" in model or custom_llm_provider == "together_ai" diff --git a/litellm/images/main.py b/litellm/images/main.py index 6e77cbb7046..b3247ff938b 100644 --- a/litellm/images/main.py +++ b/litellm/images/main.py @@ -641,6 +641,8 @@ def image_generation( timeout=timeout, client=custom_client, ) + elif custom_llm_provider == LlmProviders.KYMA.value: + raise LiteLLMUnknownProvider(model=model, custom_llm_provider=custom_llm_provider) return model_response except Exception as e: diff --git a/litellm/integrations/custom_logger.py b/litellm/integrations/custom_logger.py index a4de1be1842..0579eb15baf 100644 --- a/litellm/integrations/custom_logger.py +++ b/litellm/integrations/custom_logger.py @@ -513,7 +513,7 @@ class CustomLogger: # https://docs.litellm.ai/docs/observability/custom_callbac data: dict, user_api_key_dict: UserAPIKeyAuth, call_type: CallTypesLiteral, - ) -> Any: + ) -> object: pass async def async_post_call_streaming_hook( diff --git a/litellm/integrations/datadog/datadog_cost_management.py b/litellm/integrations/datadog/datadog_cost_management.py index 1a81fa019dc..d4b75fc0bce 100644 --- a/litellm/integrations/datadog/datadog_cost_management.py +++ b/litellm/integrations/datadog/datadog_cost_management.py @@ -205,11 +205,12 @@ class DatadogCostManagementLogger(CustomBatchLogger): # see _RESERVED_TAG_KEYS for the rationale. if self.cost_tag_keys: allow: Final = set(self.cost_tag_keys) + client_allow: Final = allow - {"is_silent_experiment"} for rt in log.get("request_tags") or []: if not isinstance(rt, str) or ":" not in rt: continue k, _, v = rt.partition(":") - if k in allow and v: + if k in client_allow and v: self._set_custom_tag(tags, k, v) for k, v in metadata.items(): if k in allow and v is not None and not isinstance(v, (dict, list)): @@ -218,7 +219,7 @@ class DatadogCostManagementLogger(CustomBatchLogger): nested = metadata.get(nested_key) if isinstance(nested, dict): for k, v in nested.items(): - if k in allow and v is not None and not isinstance(v, (dict, list)): + if k in client_allow and v is not None and not isinstance(v, (dict, list)): self._set_custom_tag(tags, k, str(v)) return tags diff --git a/litellm/integrations/langfuse/langfuse.py b/litellm/integrations/langfuse/langfuse.py index b8c6e91d0d7..bd68ffe8cd6 100644 --- a/litellm/integrations/langfuse/langfuse.py +++ b/litellm/integrations/langfuse/langfuse.py @@ -833,6 +833,12 @@ class LangFuseLogger: ("vertex_location", vertex_location, bool(vertex_location)), ("aws_region_name", aws_region_name, bool(aws_region_name)), ("cache_hit", kwargs.get("cache_hit") or False, "cache_hit" in kwargs), + ( + "is_silent_experiment", + "is_silent_experiment" in allowlisted_metadata + and allowlisted_metadata["is_silent_experiment"] is True, + "is_silent_experiment" in allowlisted_metadata, + ), ) enrichments: Final[Mapping[str, object]] = { key: value for key, value, include in candidate_enrichments if include diff --git a/litellm/integrations/opentelemetry_utils/base_otel_llm_obs_attributes.py b/litellm/integrations/opentelemetry_utils/base_otel_llm_obs_attributes.py index 32cf93b7937..97a970c8966 100644 --- a/litellm/integrations/opentelemetry_utils/base_otel_llm_obs_attributes.py +++ b/litellm/integrations/opentelemetry_utils/base_otel_llm_obs_attributes.py @@ -15,7 +15,7 @@ class BaseLLMObsOTELAttributes(ABC): pass -def cast_as_primitive_value_type(value) -> str | bool | int | float: +def cast_as_primitive_value_type(value: object) -> str | bool | int | float: """ Converts a value to an OTEL-supported primitive for Arize/Phoenix observability. """ diff --git a/litellm/integrations/prometheus.py b/litellm/integrations/prometheus.py index d15956c0e95..e62b99f8b0f 100644 --- a/litellm/integrations/prometheus.py +++ b/litellm/integrations/prometheus.py @@ -5148,6 +5148,12 @@ def _get_combined_custom_metadata_from_standard_logging_payload( **(requester_metadata if isinstance(requester_metadata, dict) else {}), **(user_api_key_auth_metadata if isinstance(user_api_key_auth_metadata, dict) else {}), **(spend_logs_metadata if isinstance(spend_logs_metadata, dict) else {}), + "is_silent_experiment": ( + "true" + if "is_silent_experiment" in standard_logging_metadata + and standard_logging_metadata["is_silent_experiment"] is True + else None + ), } diff --git a/litellm/interactions/litellm_responses_transformation/handler.py b/litellm/interactions/litellm_responses_transformation/handler.py index d84f0b5ffbd..f95b5611831 100644 --- a/litellm/interactions/litellm_responses_transformation/handler.py +++ b/litellm/interactions/litellm_responses_transformation/handler.py @@ -103,6 +103,7 @@ class LiteLLMResponsesInteractionsHandler: return LiteLLMResponsesInteractionsConfig.transform_responses_response_to_interactions_response( responses_response=responses_api_response, model=model, + custom_llm_provider=custom_llm_provider, ) async def async_interactions_api_handler( @@ -138,4 +139,5 @@ class LiteLLMResponsesInteractionsHandler: return LiteLLMResponsesInteractionsConfig.transform_responses_response_to_interactions_response( responses_response=responses_api_response, model=model, + custom_llm_provider=responses_request.get("custom_llm_provider"), ) diff --git a/litellm/interactions/litellm_responses_transformation/transformation.py b/litellm/interactions/litellm_responses_transformation/transformation.py index ade33831cdc..2e9b62dcfe2 100644 --- a/litellm/interactions/litellm_responses_transformation/transformation.py +++ b/litellm/interactions/litellm_responses_transformation/transformation.py @@ -12,6 +12,7 @@ from typing import Final, cast from pydantic import BaseModel +from litellm.litellm_core_utils.core_helpers import billed_usage_cost from litellm.types.interactions import ( InteractionInput, InteractionsAPIOptionalRequestParams, @@ -190,6 +191,7 @@ class LiteLLMResponsesInteractionsConfig: def transform_responses_response_to_interactions_response( responses_response: ResponsesAPIResponse, model: str | None = None, + custom_llm_provider: str | None = None, ) -> InteractionsAPIResponse: """ Transform a Responses API response to an Interactions API response. @@ -266,9 +268,11 @@ class LiteLLMResponsesInteractionsConfig: # (total_input_tokens, total_output_tokens) usage: Final[object] = getattr(responses_response, "usage", None) if usage: + reported_cost: Final = billed_usage_cost(responses_response, custom_llm_provider) interactions_response_dict["usage"] = { "total_input_tokens": getattr(usage, "input_tokens", 0), "total_output_tokens": getattr(usage, "output_tokens", 0), + **({} if reported_cost is None else {"cost": reported_cost}), } # Add updated (same as created for now) diff --git a/litellm/litellm_core_utils/core_helpers.py b/litellm/litellm_core_utils/core_helpers.py index 8a53f0cbb42..d940baf8b56 100644 --- a/litellm/litellm_core_utils/core_helpers.py +++ b/litellm/litellm_core_utils/core_helpers.py @@ -2,6 +2,7 @@ ## Helper utilities import copy import logging +import math import re from collections.abc import Collection, Iterable, Iterator, Mapping from types import MappingProxyType @@ -1016,6 +1017,16 @@ def chunk_usage(chunk: object) -> object: return getattr(chunk, "usage", None) +USAGE_COST_BILLED_PROVIDERS: Final = frozenset({"kyma"}) + + +def billed_usage_cost(response: object, custom_llm_provider: str | None) -> float | None: + if custom_llm_provider not in USAGE_COST_BILLED_PROVIDERS: + return None + cost: Final = provider_reported_usage_cost(_usage_field(response, "usage"), custom_llm_provider) + return cost if cost is not None and math.isfinite(cost) and cost >= 0 else None + + _MESSAGES_REPORTED_COST_PROVIDERS: Final = frozenset({"opencode_go"}) diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 77c4136efda..c0e988412a1 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -23,7 +23,7 @@ from httpx import Response, Timeout from pydantic import BaseModel, JsonValue, TypeAdapter import litellm -from litellm._internal_context import post_response_phase +from litellm._internal_context import in_mirrored_call, outside_mirrored_call, post_response_phase from litellm._logging import ( is_debugging_on, redact_string, @@ -162,7 +162,7 @@ from litellm.types.llms.openai import ( from litellm.types.mcp import MCPPostCallResponseObject from litellm.types.prompts.init_prompts import PromptSpec from litellm.types.rerank import RerankResponse -from litellm.types.router import RouterRateLimitError +from litellm.types.router import RouterRateLimitError, SilentExperimentFlag from litellm.types.utils import ( DEPLOYMENT_SCOPED_PRICING_FIELDS, AgenticLoopParams, @@ -315,7 +315,7 @@ else: _in_memory_loggers: Final[list[CustomLogger]] = [] _STANDARD_LOGGING_METADATA_RESOLVED_KEYS: Final[frozenset[str]] = frozenset( - ("used_client_oauth_token", "usage_object", KUBERNETES_POD_ROUTING_KEY) + ("used_client_oauth_token", "usage_object", KUBERNETES_POD_ROUTING_KEY, "is_silent_experiment") ) _STANDARD_LOGGING_METADATA_KEYS: Final[frozenset[str]] = ( frozenset(StandardLoggingMetadata.__annotations__.keys()) - _STANDARD_LOGGING_METADATA_RESOLVED_KEYS @@ -809,6 +809,7 @@ class Logging(LiteLLMLoggingBaseClass): litellm_params = scrub_sensitive_keys_in_metadata(litellm_params) self.litellm_params = litellm_params + self.is_silent_experiment: Final = in_mirrored_call() # Initialize cost breakdown field self.cost_breakdown: CostBreakdown | None = None @@ -1941,10 +1942,10 @@ class Logging(LiteLLMLoggingBaseClass): def set_cost_breakdown( self, - input_cost: float, - output_cost: float, + input_cost: float | None, + output_cost: float | None, total_cost: float, - cost_for_built_in_tools_cost_usd_dollar: float, + cost_for_built_in_tools_cost_usd_dollar: float | None, additional_costs: dict | None = None, original_cost: float | None = None, discount_percent: float | None = None, @@ -1983,14 +1984,17 @@ class Logging(LiteLLMLoggingBaseClass): self.billed_token_rates = billed_token_rates self.cost_breakdown = CostBreakdown( - input_cost=input_cost, - output_cost=output_cost, total_cost=total_cost, - tool_usage_cost=cost_for_built_in_tools_cost_usd_dollar, service_tier=service_tier, data_residency=data_residency, vertex_location=vertex_location, ) + if input_cost is not None: + self.cost_breakdown["input_cost"] = input_cost + if output_cost is not None: + self.cost_breakdown["output_cost"] = output_cost + if cost_for_built_in_tools_cost_usd_dollar is not None: + self.cost_breakdown["tool_usage_cost"] = cost_for_built_in_tools_cost_usd_dollar if cache_read_cost is not None and cache_read_cost > 0: self.cost_breakdown["cache_read_cost"] = cache_read_cost if cache_creation_cost is not None and cache_creation_cost > 0: @@ -3253,7 +3257,7 @@ class Logging(LiteLLMLoggingBaseClass): """Restores trace_id/session_id contextvars once this attempt's own success logging (including any nested calls its callbacks trigger) is fully done.""" try: - with post_response_phase(): + with post_response_phase(), outside_mirrored_call(): return self._success_handler_body( result=result, start_time=start_time, @@ -3753,7 +3757,7 @@ class Logging(LiteLLMLoggingBaseClass): """Restores trace_id/session_id contextvars once this attempt's own success logging (including any nested calls its callbacks trigger) is fully done.""" try: - with post_response_phase(): + with post_response_phase(), outside_mirrored_call(): return await self._async_success_handler_body( result=result, start_time=start_time, end_time=end_time, cache_hit=cache_hit, **kwargs ) @@ -4270,13 +4274,14 @@ class Logging(LiteLLMLoggingBaseClass): ``for_async_request`` marks the sync run of a failure whose async run also logs it, so a CustomLogger that gets ``async_log_failure_event`` from that async run skips its sync hook.""" try: - return self._failure_handler_body( - exception=exception, - traceback_exception=traceback_exception, - start_time=start_time, - end_time=end_time, - for_async_request=for_async_request, - ) + with outside_mirrored_call(): + return self._failure_handler_body( + exception=exception, + traceback_exception=traceback_exception, + start_time=start_time, + end_time=end_time, + for_async_request=for_async_request, + ) finally: self._restore_correlation_context() @@ -4469,12 +4474,13 @@ class Logging(LiteLLMLoggingBaseClass): """Restores trace_id/session_id contextvars once this attempt's own failure logging (including any nested calls its callbacks trigger) is fully done.""" try: - return await self._async_failure_handler_body( - exception=exception, - traceback_exception=traceback_exception, - start_time=start_time, - end_time=end_time, - ) + with outside_mirrored_call(): + return await self._async_failure_handler_body( + exception=exception, + traceback_exception=traceback_exception, + start_time=start_time, + end_time=end_time, + ) finally: self._restore_correlation_context() @@ -6563,6 +6569,7 @@ class StandardLoggingPayloadSetup: start_time: dt_object | None = None, response_id: str | None = None, custom_llm_provider: str | None = None, + is_silent_experiment: bool = False, ) -> StandardLoggingMetadata: """ Clean and filter the metadata dictionary to include only the specified keys in StandardLoggingMetadata. @@ -6636,6 +6643,7 @@ class StandardLoggingPayloadSetup: proxy_stamped_used_client_oauth_token(metadata, litellm_params), custom_llm_provider, ), + **(SilentExperimentFlag(is_silent_experiment=True) if is_silent_experiment else SilentExperimentFlag()), ) if isinstance(metadata, dict): for key in metadata.keys() & _STANDARD_LOGGING_METADATA_KEYS: @@ -7436,6 +7444,7 @@ def get_standard_logging_object_payload( start_time=start_time, response_id=id, custom_llm_provider=selected_provider if isinstance(selected_provider, str) else None, + is_silent_experiment=logging_obj.is_silent_experiment is True, ) _request_body: Final = proxy_server_request.get("body", {}) end_user_id: Final = clean_metadata["user_api_key_end_user_id"] or _request_body.get( @@ -7720,9 +7729,8 @@ def get_standard_logging_metadata( ) if isinstance(metadata, dict): # Update the clean_metadata with values from input metadata that match StandardLoggingMetadata fields - for key in StandardLoggingMetadata.__annotations__: - if key in metadata: - clean_metadata[key] = metadata[key] + for key in (metadata.keys() & StandardLoggingMetadata.__annotations__.keys()) - {"is_silent_experiment"}: + clean_metadata[key] = metadata[key] if metadata.get("user_api_key") is not None: if is_valid_sha256_hash(str(metadata.get("user_api_key"))): diff --git a/litellm/litellm_core_utils/llm_response_utils/response_metadata.py b/litellm/litellm_core_utils/llm_response_utils/response_metadata.py index ace57a64aae..051a8fdf256 100644 --- a/litellm/litellm_core_utils/llm_response_utils/response_metadata.py +++ b/litellm/litellm_core_utils/llm_response_utils/response_metadata.py @@ -27,6 +27,15 @@ def _timing_window_start( return start_time, False +def response_ms_since_logging_start(logging_obj: LiteLLMLoggingObject) -> float | None: + reported_ms: Final = logging_obj.response_timing_metrics.get("_response_ms") + if reported_ms is None: + return None + logging_start: Final = logging_obj.start_time + window_start: Final = _timing_window_start(logging_start, logging_obj)[0] + return reported_ms - max(0.0, logging_start.timestamp() - window_start.timestamp()) * 1000 + + def _union_duration_ms(windows: object, lower: float, upper: float) -> float | None: if not isinstance(windows, (list, tuple)): return None diff --git a/litellm/litellm_core_utils/streaming_chunk_builder_utils.py b/litellm/litellm_core_utils/streaming_chunk_builder_utils.py index 719f75546b9..7c91ffdff76 100644 --- a/litellm/litellm_core_utils/streaming_chunk_builder_utils.py +++ b/litellm/litellm_core_utils/streaming_chunk_builder_utils.py @@ -12,7 +12,11 @@ from typing_extensions import ReadOnly, Required import litellm from litellm._logging import verbose_logger -from litellm.litellm_core_utils.core_helpers import chunk_usage, provider_reported_usage_cost +from litellm.litellm_core_utils.core_helpers import ( + USAGE_COST_BILLED_PROVIDERS, + chunk_usage, + provider_reported_usage_cost, +) from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR, set_hidden_params from litellm.litellm_core_utils.llm_cost_calc.guardrail_cost import BEDROCK_TRACE_FIELDS, object_items, object_mapping from litellm.litellm_core_utils.prompt_templates.common_utils import ( @@ -1352,7 +1356,7 @@ def stream_chunk_builder_text_completion( return TextCompletionResponse(**response) -_CALCULATOR_PRICED_REPORTED_COST_PROVIDERS: Final = frozenset({LlmProviders.XAI.value}) +_CALCULATOR_PRICED_REPORTED_COST_PROVIDERS: Final = frozenset({LlmProviders.XAI.value}) | USAGE_COST_BILLED_PROVIDERS def _logged_provider(logging_obj: Optional["Logging"]) -> object: diff --git a/litellm/llms/anthropic/chat/guardrail_translation/handler.py b/litellm/llms/anthropic/chat/guardrail_translation/handler.py index d371aab4581..31b0f06251d 100644 --- a/litellm/llms/anthropic/chat/guardrail_translation/handler.py +++ b/litellm/llms/anthropic/chat/guardrail_translation/handler.py @@ -432,27 +432,30 @@ class AnthropicMessagesHandler(BaseTranslation): message, _ = serialize_http_exception_detail(exc.detail) return tuple(anthropic_sse_error_frames(message)) - def _standalone_block_chunks(self, exc: "ModifyResponseException") -> list[bytes]: + def build_block_response(self, exc: "ModifyResponseException") -> "AnthropicMessagesResponse": import uuid - from litellm.llms.anthropic.messages.native.fake_stream_iterator import ( - FakeAnthropicMessagesStreamIterator, - ) from litellm.llms.base_llm.guardrail_translation.utils import ( blocked_response_usage, ) from litellm.types.utils import AnthropicMessagesResponse - block_response: Final = AnthropicMessagesResponse( + return AnthropicMessagesResponse( id=f"msg_{uuid.uuid4()}", type="message", role="assistant", content=[{"type": "text", "text": exc.message}], model=exc.model, stop_reason="end_turn", - usage=blocked_response_usage(getattr(exc, "original_response", None)), + usage=blocked_response_usage(exc.original_response), ) - return list(FakeAnthropicMessagesStreamIterator(response=block_response)) + + def _standalone_block_chunks(self, exc: "ModifyResponseException") -> list[bytes]: + from litellm.llms.anthropic.messages.native.fake_stream_iterator import ( + FakeAnthropicMessagesStreamIterator, + ) + + return list(FakeAnthropicMessagesStreamIterator(response=self.build_block_response(exc))) def _block_continuation_chunks( self, exc: "ModifyResponseException", responses_so_far: Sequence[object] diff --git a/litellm/llms/anthropic/chat/handler.py b/litellm/llms/anthropic/chat/handler.py index 8a82ed98991..e1436473426 100644 --- a/litellm/llms/anthropic/chat/handler.py +++ b/litellm/llms/anthropic/chat/handler.py @@ -17,6 +17,7 @@ import litellm import litellm.litellm_core_utils import litellm.types import litellm.types.utils +from litellm.anthropic_beta_gated_fields import strip_beta_gated_request_body from litellm.anthropic_beta_headers_manager import ( update_request_with_filtered_beta, ) @@ -506,11 +507,20 @@ class AnthropicChatCompletion(BaseLLM): The pair stays mutable because the streaming path rewrites it in place (`data["stream"] = True`) before sending. """ - request_headers, data = update_request_with_filtered_beta( + request_headers, filtered_data = update_request_with_filtered_beta( headers=headers, request_data=request_data, provider=custom_llm_provider, ) + data: Final = dict( + strip_beta_gated_request_body( + filtered_data, + request_headers, + drop_params=litellm_params.get("drop_params"), # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # litellm_params is the handler's bare dict + model=model, + provider=custom_llm_provider, + ) + ) logging_obj.pre_call( input=messages, api_key=api_key, diff --git a/litellm/llms/anthropic/common_utils.py b/litellm/llms/anthropic/common_utils.py index bb88a9476b9..4c63f0e0f1d 100644 --- a/litellm/llms/anthropic/common_utils.py +++ b/litellm/llms/anthropic/common_utils.py @@ -51,6 +51,7 @@ from litellm.types.llms.anthropic import ( ANTHROPIC_OAUTH_BETA_HEADER, ANTHROPIC_OAUTH_TOKEN_PREFIX, ANTHROPIC_SERVER_TOOL_USE_ID_PREFIX, + ANTHROPIC_THINKING_BINDING_CONTROLS_BETA_HEADER, ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER, ANTHROPIC_TOOL_SEARCH_TOOL_TYPES, AllAnthropicToolsValues, @@ -600,6 +601,20 @@ class AnthropicModelInfo(BaseLLMModelInfo): return False return thinking.get("type") in ("adaptive", "enabled") and thinking.get("display") == "updates" + def thinking_betas(self, thinking: AnthropicThinkingParam | None) -> tuple[str, ...]: + return ( + *( + (ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER,) + if self.is_thinking_display_updates_used(thinking) + else () + ), + *( + (ANTHROPIC_THINKING_BINDING_CONTROLS_BETA_HEADER,) + if isinstance(thinking, dict) and "block_binding" in thinking + else () + ), + ) + def is_mid_conversation_tool_change_used(self, messages: Sequence[object]) -> bool: for message in messages: if message_field(message, "role") != "system": @@ -1197,7 +1212,7 @@ class AnthropicModelInfo(BaseLLMModelInfo): *, custom_llm_provider: str, is_mid_conversation_output_config_used: bool = False, - is_thinking_display_updates_used: bool = False, + thinking: AnthropicThinkingParam | None = None, is_mid_conversation_tool_change_used: bool = False, ) -> list[str]: """ @@ -1234,13 +1249,10 @@ class AnthropicModelInfo(BaseLLMModelInfo): if is_mid_conversation_output_config_used: betas.append(ANTHROPIC_MID_CONVERSATION_OUTPUT_CONFIG_BETA_HEADER) - thinking_display_betas: Final = ( - (ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER,) if is_thinking_display_updates_used else () - ) tool_change_betas: Final = ( (ANTHROPIC_MID_CONVERSATION_TOOL_CHANGES_BETA_HEADER,) if is_mid_conversation_tool_change_used else () ) - return list(set(betas).union(thinking_display_betas, tool_change_betas)) + return list(set(betas).union(self.thinking_betas(thinking), tool_change_betas)) @staticmethod def _make_api_key_auth_header( @@ -1302,7 +1314,7 @@ class AnthropicModelInfo(BaseLLMModelInfo): api_base: str | None = None, use_bearer_for_custom_base: bool = False, is_mid_conversation_output_config_used: bool = False, - is_thinking_display_updates_used: bool = False, + thinking: AnthropicThinkingParam | None = None, is_mid_conversation_tool_change_used: bool = False, wif_minted: bool = False, ) -> dict: @@ -1363,7 +1375,7 @@ class AnthropicModelInfo(BaseLLMModelInfo): betas.update(user_anthropic_beta_headers) all_betas: Final = betas.union( - (ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER,) if is_thinking_display_updates_used else (), + self.thinking_betas(thinking), (ANTHROPIC_MID_CONVERSATION_TOOL_CHANGES_BETA_HEADER,) if is_mid_conversation_tool_change_used else (), ) @@ -1450,7 +1462,7 @@ class AnthropicModelInfo(BaseLLMModelInfo): auth_token=auth_token, file_id_used=file_id_used, is_mid_conversation_output_config_used=is_mid_conversation_output_config_used, - is_thinking_display_updates_used=self.is_thinking_display_updates_used(optional_params.get("thinking")), + thinking=optional_params.get("thinking"), is_mid_conversation_tool_change_used=self.is_mid_conversation_tool_change_used(messages), web_search_tool_used=web_search_tool_used, is_vertex_request=optional_params.get("is_vertex_request", False), diff --git a/litellm/llms/anthropic/messages/adapters/transformation.py b/litellm/llms/anthropic/messages/adapters/transformation.py index 18b33fa6085..f5b8d1b80c1 100644 --- a/litellm/llms/anthropic/messages/adapters/transformation.py +++ b/litellm/llms/anthropic/messages/adapters/transformation.py @@ -204,7 +204,7 @@ from litellm.types.llms.openai import ( ChatCompletionUserMessage, ToolMessageContentPart, ) -from litellm.types.utils import Choices, ModelResponse, StreamingChoices, Usage +from litellm.types.utils import Choices, LlmProviders, ModelResponse, StreamingChoices, Usage from litellm.utils import supports_mid_conversation_system from .streaming_iterator import AnthropicSSEStream, AnthropicStreamWrapper @@ -364,6 +364,9 @@ class AnthropicAdapter: _BlockT: Final = TypeVar("_BlockT", bound=Mapping[str, object]) +_CLAUDE_OVER_REASONING_EFFORT_PROVIDERS: Final = frozenset({LlmProviders.KYMA.value}) + + class LiteLLMAnthropicMessagesAdapter: def __init__(self): pass @@ -1135,7 +1138,8 @@ class LiteLLMAnthropicMessagesAdapter: """Translate Anthropic thinking to either thinking or reasoning_effort. A Claude-family target keeps ``thinking`` verbatim, since every bridged provider serving one - speaks that param. Carrying its adaptive effort tier alongside takes two different params, + speaks that param, except Kyma, which serves Claude over the OpenAI dialect and takes only + ``reasoning_effort``, so its Claude models get the non-Claude mapping. Carrying its adaptive effort tier alongside takes two different params, because the two are not interchangeable at the provider mapping below. Bedrock takes ``output_config`` directly, which attaches the tier and leaves ``thinking`` @@ -1166,7 +1170,9 @@ class LiteLLMAnthropicMessagesAdapter: is_bedrock_target: Final = model.startswith(("bedrock/", "converse/", "invoke/")) or self.is_bedrock_arn_model( model ) - is_claude_target: Final = self.is_anthropic_claude_model(model) or self.is_bedrock_arn_model(model) + is_claude_target: Final = ( + self.is_anthropic_claude_model(model) or self.is_bedrock_arn_model(model) + ) and custom_llm_provider not in _CLAUDE_OVER_REASONING_EFFORT_PROVIDERS output_config: Final = anthropic_message_request.get("output_config") if is_claude_target: diff --git a/litellm/llms/anthropic/messages/native/transformation.py b/litellm/llms/anthropic/messages/native/transformation.py index dc8e2ef2cf9..557e2039054 100644 --- a/litellm/llms/anthropic/messages/native/transformation.py +++ b/litellm/llms/anthropic/messages/native/transformation.py @@ -2,10 +2,21 @@ from collections.abc import AsyncIterator, Mapping, Sequence from typing import ClassVar, Final import httpx +from pydantic import TypeAdapter import litellm from litellm._logging import verbose_logger -from litellm.anthropic_beta_headers_manager import get_unsupported_headers, is_beta_header_supported_by_bundled_config +from litellm.anthropic_beta_gated_fields import ( + log_beta_gated_strips, + strip_beta_gated_native_request, + strip_beta_gated_system_messages, + unstripped_native_request, +) +from litellm.anthropic_beta_headers_manager import ( + get_unsupported_headers, + has_beta_headers_config, + is_beta_header_supported_by_bundled_config, +) from litellm.constants import ANTHROPIC_BILLING_METADATA_PREFIX from litellm.exceptions import AuthenticationError from litellm.litellm_core_utils.core_helpers import drops_unsupported_params @@ -20,7 +31,6 @@ from litellm.types.llms.anthropic import ( ANTHROPIC_ADVISOR_TOOL_TYPE, ANTHROPIC_BETA_HEADER_VALUES, ANTHROPIC_MID_CONVERSATION_TOOL_CHANGES_BETA_HEADER, - ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER, AnthropicMessagesRequest, ) from litellm.types.llms.anthropic_messages.anthropic_response import ( @@ -51,6 +61,8 @@ DEFAULT_ANTHROPIC_API_VERSION: Final = "2023-06-01" _CALLER_CREDENTIAL_HEADERS: Final = frozenset({"x-api-key", "authorization"}) +_MESSAGE_MAPPINGS: Final = TypeAdapter(tuple[Mapping[str, object], ...]) + def _carries_caller_credential(headers: Mapping[str, str]) -> bool: """Whether the caller sent their own Anthropic credential, in which case this passthrough @@ -85,6 +97,9 @@ def _ensure_stop_sequence(response: AnthropicMessagesResponse) -> AnthropicMessa return response +_MESSAGES_ADAPTER: Final = TypeAdapter(list[dict[str, object]]) + + class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): _workload_identity_eligible: ClassVar[bool] = True _report_reasoning_drops: ClassVar[bool] = True @@ -230,7 +245,18 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): else: return system_param - def _normalize_system_role_messages(self, anthropic_messages_request: dict, model: str) -> None: + def _strip_beta_gated_system_turns( + self, messages: Sequence[Mapping[str, object]], model: str, drop_params: object + ) -> Sequence[Mapping[str, object]]: + if not has_beta_headers_config(self._resolved_provider): + return messages + gated: Final = strip_beta_gated_system_messages(messages, frozenset(), drop_params=drop_params) + log_beta_gated_strips(gated.stripped, model=model, provider=self._resolved_provider) + return _MESSAGE_MAPPINGS.validate_python(gated.messages) if gated.stripped else messages + + def _normalize_system_role_messages( + self, anthropic_messages_request: dict, model: str, drop_params: object + ) -> None: """Normalize ``role: "system"`` entries in ``messages`` per the Anthropic ``/v1/messages`` contract, which the first-party API, Bedrock Invoke, Vertex, and Azure Foundry all enforce identically. @@ -278,7 +304,15 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): custom_llm_provider=self.custom_llm_provider, key="supports_mid_conversation_system", ) - else list(convert_mid_conversation_system_turns(messages[leading_count:])) + else list( + convert_mid_conversation_system_turns( + self._strip_beta_gated_system_turns( + messages[leading_count:], # pyright: ignore[reportUnknownArgumentType] # messages comes from the bare request dict + model=model, + drop_params=drop_params, + ) + ) + ) ) if hoisted or remaining != messages: anthropic_messages_request["messages"] = remaining @@ -337,7 +371,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): model=model, ), ) - return self._finalize_messages_headers(headers, optional_params, messages), api_base + return self.finalize_messages_headers(headers, optional_params, messages), api_base async def avalidate_anthropic_messages_environment( self, @@ -377,7 +411,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): model=model, ), ) - return self._finalize_messages_headers(oauth_headers, optional_params, messages), api_base + return self.finalize_messages_headers(oauth_headers, optional_params, messages), api_base def _require_auth_header(self, auth_header: Mapping[str, str] | None, model: str) -> Mapping[str, str]: if auth_header is None: @@ -411,7 +445,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): return config_allows_workload_identity(self) - def _finalize_messages_headers( + def finalize_messages_headers( self, headers: dict, # mutable-ok: out-param optional_params: dict, # mutable-ok: out-param @@ -620,6 +654,12 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): if thinking_enabled or effort_enabled: optional_params.pop("temperature", None) + def strips_beta_gated_fields_from_final_headers(self) -> bool: + """False on legs whose final ``anthropic_beta`` list is only known after this transform (Bedrock Invoke), + which strip beta-gated fields themselves once that list exists, and on providers with no row in + ``anthropic_beta_headers_config.json``, whose filter drops every beta and so cannot say which fields fail.""" + return has_beta_headers_config(self._resolved_provider) + def transform_anthropic_messages_request( self, model: str, @@ -723,18 +763,37 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): if not _has_advisor: messages = strip_advisor_blocks_from_messages(messages) + gated: Final = ( + strip_beta_gated_native_request( + headers, # pyright: ignore[reportUnknownArgumentType] # the base signature types this as a bare dict + anthropic_messages_optional_request_params, # pyright: ignore[reportUnknownArgumentType] # the base signature types this as a bare dict + messages, # pyright: ignore[reportUnknownArgumentType] # the base signature types this as a bare list + drop_params=getattr(litellm_params, "drop_params", None), + ) + if self.strips_beta_gated_fields_from_final_headers() + else unstripped_native_request(anthropic_messages_optional_request_params, messages) # pyright: ignore[reportUnknownArgumentType] # the base signature types these as bare dicts + ) + log_beta_gated_strips(gated.stripped, model=model, provider=self._resolved_provider) anthropic_messages_request: Final[AnthropicMessagesRequest] = AnthropicMessagesRequest( messages=strip_encrypted_reasoning_blocks_from_anthropic_messages( - drop_non_string_tool_result_texts(messages, getattr(litellm_params, "drop_params", None)) + drop_non_string_tool_result_texts( + _MESSAGES_ADAPTER.validate_python(gated.messages) if gated.messages_stripped else messages, + getattr(litellm_params, "drop_params", None), + ) ), max_tokens=max_tokens, model=model, **anthropic_messages_optional_request_params, ) + request: Final = ( + {**dict(anthropic_messages_request), "thinking": gated.thinking} + if gated.thinking_stripped + else dict(anthropic_messages_request) # pyright: ignore[reportUnknownVariableType] # built from the base signature's bare dict + ) if not isinstance(tools, list): - return dict(anthropic_messages_request) + return request return { - **anthropic_messages_request, + **request, "tools": self.tools_for_upstream( OBJECT_LIST_ADAPTER.validate_python(tools), model, litellm_params.drop_params ), @@ -869,17 +928,13 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): if AnthropicModelInfo().is_tool_search_used(tools): beta_values.add(get_tool_search_beta_header(custom_llm_provider)) - thinking_display_betas: Final = ( - (ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER,) - if AnthropicModelInfo().is_thinking_display_updates_used(optional_params.get("thinking")) - else () - ) + thinking_betas: Final = AnthropicModelInfo().thinking_betas(optional_params.get("thinking")) tool_change_betas: Final = ( (ANTHROPIC_MID_CONVERSATION_TOOL_CHANGES_BETA_HEADER,) if AnthropicModelInfo().is_mid_conversation_tool_change_used(messages) else () ) - all_beta_values: Final = beta_values.union(thinking_display_betas, tool_change_betas) + all_beta_values: Final = beta_values.union(thinking_betas, tool_change_betas) dangerous_tool_use: Final = ANTHROPIC_BETA_HEADER_VALUES.DANGEROUS_TOOL_USE_2026_09_03.value forwarded_beta_values: Final = ( # pyright: ignore[reportUnknownVariableType] # derived from the bare beta set all_beta_values - {dangerous_tool_use} diff --git a/litellm/llms/anthropic/prompt_cache_prediction.py b/litellm/llms/anthropic/prompt_cache_prediction.py index 022260b86f7..2da071ebdd2 100644 --- a/litellm/llms/anthropic/prompt_cache_prediction.py +++ b/litellm/llms/anthropic/prompt_cache_prediction.py @@ -665,12 +665,13 @@ def prepare_native_baseline_body(request: Mapping[str, object], model: str) -> M TypeAdapter(tuple[str, ...]).validate_python(owned.get("additional_drop_params") or ()), dict(options), ) - body: Final = _BaselineAnthropicMessagesConfig().transform_anthropic_messages_request( + config: Final = _BaselineAnthropicMessagesConfig() + body: Final = config.transform_anthropic_messages_request( model=resolved_model, messages=strip_provider_specific_fields_from_anthropic_messages(messages), anthropic_messages_optional_request_params=filtered, litellm_params=GenericLiteLLMParams(), - headers={}, + headers=config.finalize_messages_headers({}, dict(filtered), list(messages)), # pyright: ignore[reportUnknownMemberType] # the shared helper takes and returns bare dicts ) return MappingProxyType(_JSON_OBJECT.validate_python(body)) except Exception: # noqa: BLE001 # an unsupported hypothetical request is unavailable, never an inference failure diff --git a/litellm/llms/azure_ai/anthropic/handler.py b/litellm/llms/azure_ai/anthropic/handler.py index ca4dad5ee3e..1a2003e6262 100644 --- a/litellm/llms/azure_ai/anthropic/handler.py +++ b/litellm/llms/azure_ai/anthropic/handler.py @@ -4,11 +4,12 @@ Azure Anthropic handler - reuses AnthropicChatCompletion logic with Azure authen import copy import json -from collections.abc import Callable +from collections.abc import Callable, Mapping from typing import Final import httpx +from litellm.anthropic_beta_gated_fields import strip_beta_gated_request_body from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.llms.anthropic.chat.handler import AnthropicChatCompletion from litellm.llms.anthropic.chat.transformation import JsonModeRequest @@ -75,13 +76,22 @@ class AzureAnthropicChatCompletion(AnthropicChatCompletion): litellm_params=litellm_params, ) - data: Final = config.transform_request( + request_body: Final[Mapping[str, object]] = config.transform_request( model=model, messages=messages, optional_params=optional_params, litellm_params=litellm_params, headers=headers, ) + data: Final = dict( + strip_beta_gated_request_body( + request_body, + headers, # pyright: ignore[reportUnknownArgumentType] # validate_environment returns a bare dict + drop_params=litellm_params.get("drop_params"), # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # litellm_params is the handler's bare dict + model=model, + provider=custom_llm_provider, + ) + ) ## LOGGING logging_obj.pre_call( diff --git a/litellm/llms/azure_ai/anthropic/messages_transformation.py b/litellm/llms/azure_ai/anthropic/messages_transformation.py index 6b2da96de23..1f9ded827b6 100644 --- a/litellm/llms/azure_ai/anthropic/messages_transformation.py +++ b/litellm/llms/azure_ai/anthropic/messages_transformation.py @@ -145,6 +145,8 @@ class AzureAnthropicMessagesConfig(AnthropicMessagesConfig): litellm_params=litellm_params, headers=headers, ) - self._normalize_system_role_messages(anthropic_messages_request, model=model) + self._normalize_system_role_messages( + anthropic_messages_request, model=model, drop_params=getattr(litellm_params, "drop_params", None) + ) self._remove_scope_from_cache_control(anthropic_messages_request) return anthropic_messages_request diff --git a/litellm/llms/azure_ai/anthropic/transformation.py b/litellm/llms/azure_ai/anthropic/transformation.py index c75213c3e2c..6516827359d 100644 --- a/litellm/llms/azure_ai/anthropic/transformation.py +++ b/litellm/llms/azure_ai/anthropic/transformation.py @@ -107,9 +107,11 @@ class AzureAnthropicConfig(AnthropicConfig): is_vertex_request=optional_params.get("is_vertex_request", False), user_anthropic_beta_headers=user_anthropic_beta_headers, mcp_server_used=mcp_server_used, + thinking=optional_params.get("thinking"), # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # optional_params is the base config's bare dict ) # Merge headers - Azure auth (api-key or Authorization) takes precedence - headers = {**anthropic_headers, **headers} + merged_beta_header: Final = {k: v for k, v in anthropic_headers.items() if k == "anthropic-beta"} + headers = {**anthropic_headers, **headers, **merged_beta_header} # Ensure anthropic-version header is set if "anthropic-version" not in headers: diff --git a/litellm/llms/azure_ai/passthrough/transformation.py b/litellm/llms/azure_ai/passthrough/transformation.py index 35b7642d7de..2e5514b3975 100644 --- a/litellm/llms/azure_ai/passthrough/transformation.py +++ b/litellm/llms/azure_ai/passthrough/transformation.py @@ -5,8 +5,9 @@ from types import MappingProxyType from typing import TYPE_CHECKING, Final import httpx -from pydantic import TypeAdapter, ValidationError +from pydantic import ConfigDict, TypeAdapter, ValidationError +from litellm._lazy_imports import get_default_encoding from litellm._logging import verbose_logger from litellm.llms.azure_ai.common_utils import ( AzureFoundryModelInfo, @@ -22,10 +23,15 @@ from litellm.llms.base_llm.passthrough.transformation import ( relayed_body, strip_leading_model_segment, ) +from litellm.llms.cohere.embed.transformation import ( + CohereEmbeddingRowShape, + CohereImageTokenBilling, + cohere_embedding_response, +) from litellm.rust_bridge.ocr.entrypoints import NATIVE_OCR_PASSTHROUGH_RESPONSE, NativeOcrPassthroughResponse from litellm.types.llms.openai import AllMessageValues from litellm.types.rerank import RerankResponse -from litellm.types.utils import CallTypes, ImageResponse, StandardPassThroughResponseObject +from litellm.types.utils import CallTypes, EmbeddingResponse, ImageResponse, StandardPassThroughResponseObject if TYPE_CHECKING: from httpx import URL, Response @@ -35,6 +41,7 @@ if TYPE_CHECKING: EMPTY_QUERY: Final[Mapping[str, object]] = MappingProxyType({}) +ECHOED_TEXTS: Final = TypeAdapter(tuple[str, ...], config=ConfigDict(hide_input_in_errors=True)) def api_version_from(litellm_params: Mapping[str, object]) -> str | None: @@ -85,9 +92,29 @@ def relay_query_params( return MappingProxyType({**(request_query_params or EMPTY_QUERY), "api-version": api_version}) +def echoed_texts(body: Mapping[str, object]) -> tuple[str, ...]: + try: + return ECHOED_TEXTS.validate_python(body.get("texts")) + except ValidationError: + return () + + +def cohere_embed_relay_response(body: Mapping[str, object]) -> EmbeddingResponse: + return cohere_embedding_response( + body, + None, + get_default_encoding(), + echoed_texts(body), + CohereImageTokenBilling.FOLDED_INTO_INPUT_TOKENS, + CohereEmbeddingRowShape.TYPED, + ) + + FOUNDRY_RELAY_SHAPES: Final = ( RelayShape("/rerank", CallTypes.arerank, RerankResponse.model_validate), RelayShape("/providers/blackforestlabs/v1/flux-2-pro", CallTypes.aimage_generation, ImageResponse.model_validate), + RelayShape("/providers/cohere/v1/embed", CallTypes.aembedding, cohere_embed_relay_response), + RelayShape("/providers/cohere/v2/embed", CallTypes.aembedding, cohere_embed_relay_response), ) diff --git a/litellm/llms/bedrock/chat/converse_transformation.py b/litellm/llms/bedrock/chat/converse_transformation.py index 03f845cf2a3..1cd161ca79a 100644 --- a/litellm/llms/bedrock/chat/converse_transformation.py +++ b/litellm/llms/bedrock/chat/converse_transformation.py @@ -17,6 +17,8 @@ from typing_extensions import ReadOnly, TypedDict import litellm from litellm._logging import verbose_logger +from litellm.anthropic_beta_gated_fields import active_beta_values, log_beta_gated_strips, strip_beta_gated_thinking +from litellm.anthropic_beta_headers_manager import filter_and_transform_beta_headers from litellm.constants import ( BEDROCK_MIN_THINKING_BUDGET_TOKENS, RESPONSE_FORMAT_TOOL_NAME, @@ -1828,7 +1830,16 @@ class AmazonConverseConfig(BaseConfig): filtered_tools.append(tool) base_model: Final = BedrockModelInfo.get_base_model(model) - client_beta_list: Final = get_anthropic_beta_from_headers(headers or {}) + header_beta_list: Final = get_anthropic_beta_from_headers(headers or {}) + client_beta_list: Final = header_beta_list or filter_and_transform_beta_headers( + sorted( + active_beta_values( + {}, + additional_request_params.get("anthropic_beta"), # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # additional_request_params is a bare dict + ) + ), + provider="bedrock_converse", + ) eager_beta: Final = ( (ANTHROPIC_FINE_GRAINED_TOOL_STREAMING_BETA_HEADER,) if base_model.startswith("anthropic") @@ -1836,7 +1847,15 @@ class AmazonConverseConfig(BaseConfig): and ANTHROPIC_FINE_GRAINED_TOOL_STREAMING_BETA_HEADER not in client_beta_list else () ) - anthropic_beta_list: Final = [*client_beta_list, *eager_beta] + thinking_betas: Final = AnthropicModelInfo().thinking_betas( + additional_request_params.get("thinking"), # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # additional_request_params is a bare dict + ) + missing_thinking_betas: Final = ( + tuple(beta for beta in thinking_betas if beta not in client_beta_list) + if base_model.startswith("anthropic") + else () + ) + anthropic_beta_list: Final = [*client_beta_list, *eager_beta, *missing_thinking_betas] # Only separate tools if computer use tools are actually present if filtered_tools and self.is_computer_use_tool_used(filtered_tools, model): @@ -1906,7 +1925,8 @@ class AmazonConverseConfig(BaseConfig): else: computer_use_header = "computer-use-2024-10-22" - anthropic_beta_list.append(computer_use_header) + if computer_use_header not in anthropic_beta_list: + anthropic_beta_list.append(computer_use_header) # Transform computer use tools to proper Bedrock format transformed_computer_tools: Final = self._transform_computer_use_tools(computer_use_tools) additional_request_params["tools"] = transformed_computer_tools @@ -2031,6 +2051,16 @@ class AmazonConverseConfig(BaseConfig): bedrock_tools, anthropic_beta_list = self._process_tools_and_beta( original_tools, model, headers, additional_request_params, litellm_params, deployment_model ) + requested_thinking: Final = additional_request_params.get("thinking") # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # _prepare_request_params hands back a bare dict + body_betas: Final = additional_request_params.get("anthropic_beta") # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # same bare dict + gated_thinking: Final = strip_beta_gated_thinking( + requested_thinking, # pyright: ignore[reportUnknownArgumentType] # same bare dict + active_beta_values({}, body_betas) | frozenset(anthropic_beta_list), # pyright: ignore[reportUnknownArgumentType] # _process_tools_and_beta hands back a bare list + drop_params=drop_params, + ) + if gated_thinking.stripped: + log_beta_gated_strips(gated_thinking.stripped, model=model, provider="bedrock_converse") + additional_request_params["thinking"] = gated_thinking.thinking # Append cachePoint to tools if cache_control_injection_points has tool_config cache_injection_points: Final = additional_request_params.pop("cache_control_injection_points", None) diff --git a/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude_transformation.py b/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude_transformation.py index eb6ed5c75e3..e32df0bbdbb 100644 --- a/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude_transformation.py +++ b/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude_transformation.py @@ -4,6 +4,7 @@ from typing import TYPE_CHECKING, Any, Final import httpx from pydantic import TypeAdapter +from litellm.anthropic_beta_gated_fields import strip_beta_gated_request_body from litellm.anthropic_beta_headers_manager import filter_and_transform_beta_headers from litellm.litellm_core_utils.llm_cost_calc.guardrail_cost import object_mapping from litellm.litellm_core_utils.prompt_templates.factory import ( @@ -210,7 +211,15 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig): if beta_list: _anthropic_request["anthropic_beta"] = beta_list - return _anthropic_request + return dict( + strip_beta_gated_request_body( + _anthropic_request, # pyright: ignore[reportUnknownArgumentType] # built by the base signature's bare dict transform + {}, + drop_params=litellm_params.get("drop_params"), # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # litellm_params is the base signature's bare dict + model=model, + provider="bedrock_invoke", + ) + ) @property def uses_async_transform_request(self) -> bool: @@ -294,7 +303,6 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig): programmatic_tool_calling_used: Final = self.is_programmatic_tool_calling_used(tools) input_examples_used: Final = self.is_input_examples_used(tools) is_mid_conversation_output_config_used: Final = self.is_mid_conversation_output_config_used(messages) - is_thinking_display_updates_used: Final = self.is_thinking_display_updates_used(thinking) client_beta_set: Final = set(get_anthropic_beta_from_headers(headers)) user_beta_set: Final = ( @@ -312,7 +320,7 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig): mcp_server_used=self.is_mcp_server_used(optional_params.get("mcp_servers")), custom_llm_provider="bedrock", is_mid_conversation_output_config_used=is_mid_conversation_output_config_used, - is_thinking_display_updates_used=is_thinking_display_updates_used, + thinking=thinking, ) beta_set.update(auto_betas) diff --git a/litellm/llms/bedrock/claude_platform/transformation.py b/litellm/llms/bedrock/claude_platform/transformation.py index 0194f1c7e45..a5ecc37d394 100644 --- a/litellm/llms/bedrock/claude_platform/transformation.py +++ b/litellm/llms/bedrock/claude_platform/transformation.py @@ -72,6 +72,7 @@ class BedrockClaudePlatformConfig(BedrockClaudePlatformMixin, AnthropicConfig): ), code_execution_tool_used=self.is_code_execution_tool_used(tools=optional_params.get("tools")), container_with_skills_used=self.is_container_with_skills_used(optional_params=optional_params), + thinking=optional_params.get("thinking"), # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # optional_params is the base config's bare dict ) anthropic_headers["anthropic-workspace-id"] = workspace_id return {**headers, **anthropic_headers} diff --git a/litellm/llms/bedrock/embed/embedding.py b/litellm/llms/bedrock/embed/embedding.py index 4d27d94664f..2f0ed4d0e82 100644 --- a/litellm/llms/bedrock/embed/embedding.py +++ b/litellm/llms/bedrock/embed/embedding.py @@ -9,9 +9,11 @@ from collections.abc import Callable, Mapping from typing import TYPE_CHECKING, Final, get_args, overload import httpx +from pydantic import TypeAdapter import litellm from litellm.constants import BEDROCK_EMBEDDING_PROVIDERS_LITERAL +from litellm.litellm_core_utils.hidden_params import set_hidden_param from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.cohere.embed.handler import embedding as cohere_embedding from litellm.llms.custom_httpx.http_handler import ( @@ -36,7 +38,7 @@ from ..base_aws_llm import ( pop_aws_auth_params, run_aws_signing, ) -from ..common_utils import BedrockError +from ..common_utils import BedrockError, split_bedrock_region_path from .amazon_nova_transformation import AmazonNovaEmbeddingConfig from .amazon_titan_g1_transformation import AmazonTitanG1Config from .amazon_titan_multimodal_transformation import ( @@ -64,12 +66,21 @@ def _sign_get_request( return request.prepare() +_OPTIONAL_STR: Final[TypeAdapter[str | None]] = TypeAdapter(str | None) + + +def _without_region_path(model: str) -> str: + region, model_id = split_bedrock_region_path(model) + return model if region is None else model_id + + class BedrockEmbedding(BaseAWSLLM): @overload def _load_credentials( self, optional_params: dict, # mutable-ok: the implementation pops the aws_* keys out of the caller's dict in place bearer_token: None = None, + region_from_model: str | None = None, ) -> tuple[Credentials, str]: ... @overload @@ -77,15 +88,18 @@ class BedrockEmbedding(BaseAWSLLM): self, optional_params: dict, # mutable-ok: the implementation pops the aws_* keys out of the caller's dict in place bearer_token: str, + region_from_model: str | None = None, ) -> tuple[None, str]: ... def _load_credentials( self, optional_params: dict, bearer_token: str | None = None, + region_from_model: str | None = None, ) -> tuple[Credentials | None, str]: auth_params: Final = pop_aws_auth_params(optional_params) - aws_region_name = optional_params.pop("aws_region_name", None) + explicit_region_name: Final = optional_params.pop("aws_region_name", None) + aws_region_name = region_from_model if explicit_region_name is None else explicit_region_name ### SET REGION NAME ### if aws_region_name is None: @@ -184,14 +198,17 @@ class BedrockEmbedding(BaseAWSLLM): provider: BEDROCK_EMBEDDING_PROVIDERS_LITERAL, is_async_invoke: bool | None = False, batch_data: list[dict] | None = None, + aws_region_name: str | None = None, ) -> EmbeddingResponse | None: """ Transforms the response from the Bedrock embedding provider to the OpenAI format. """ returned_response: EmbeddingResponse | None = None + bedrock_model: Final = _without_region_path(model) + is_async_job: Final = bool(is_async_invoke) and len(response_list) == 1 and "invocationArn" in response_list[0] # Handle async invoke responses (single response with invocationArn) - if is_async_invoke and len(response_list) == 1 and "invocationArn" in response_list[0]: + if is_async_job: if provider == "twelvelabs": returned_response = TwelveLabsMarengoEmbeddingConfig().transform_async_invoke_response( response=response_list[0], model=model @@ -227,15 +244,15 @@ class BedrockEmbedding(BaseAWSLLM): ) else: # Handle regular invoke responses - if model == "amazon.titan-embed-image-v1": + if bedrock_model == "amazon.titan-embed-image-v1": returned_response = AmazonTitanMultimodalEmbeddingG1Config().transform_response( response_list=response_list, model=model, batch_data=batch_data ) - elif model == "amazon.titan-embed-text-v1": + elif bedrock_model == "amazon.titan-embed-text-v1": returned_response = AmazonTitanG1Config().transform_response(response_list=response_list, model=model) - elif model == "amazon.titan-embed-text-v2:0": + elif bedrock_model == "amazon.titan-embed-text-v2:0": returned_response = AmazonTitanV2Config().transform_response(response_list=response_list, model=model) - elif model == "amazon.titan-embed-g1-text-02": + elif bedrock_model == "amazon.titan-embed-g1-text-02": returned_response = AmazonTitanG1Config().transform_response(response_list=response_list, model=model) elif provider == "twelvelabs": returned_response = TwelveLabsMarengoEmbeddingConfig().transform_response( @@ -251,6 +268,8 @@ class BedrockEmbedding(BaseAWSLLM): ########################################################## if returned_response is None: raise Exception(f"Unable to map model response to known provider format. model={model}") + if aws_region_name is not None and not is_async_job: + set_hidden_param(returned_response, "region_name", aws_region_name) return returned_response def _single_func_embeddings( @@ -319,6 +338,7 @@ class BedrockEmbedding(BaseAWSLLM): provider=provider, is_async_invoke=is_async_invoke, batch_data=batch_data, + aws_region_name=aws_region_name, ) async def _async_single_func_embeddings( @@ -390,6 +410,7 @@ class BedrockEmbedding(BaseAWSLLM): provider=provider, is_async_invoke=is_async_invoke, batch_data=batch_data, + aws_region_name=aws_region_name, ) def embeddings( @@ -409,26 +430,29 @@ class BedrockEmbedding(BaseAWSLLM): litellm_params: dict, api_key: str | None = None, ) -> EmbeddingResponse: + model_id_param: Final = _OPTIONAL_STR.validate_python(optional_params.pop("model_id", None)) or None + has_async_invoke: Final = "async_invoke/" in model + model_without_async_route: Final = model.replace("async_invoke/", "", 1) + path_region: Final = split_bedrock_region_path(model_without_async_route)[0] if model_id_param is None else None + bedrock_model: Final = _without_region_path(model_without_async_route) credentials, aws_region_name = self._load_credentials( - optional_params, bearer_token=bedrock_bearer_token(api_key) + optional_params, + bearer_token=bedrock_bearer_token(api_key), + region_from_model=self._get_aws_region_from_model_arn(model_id_param) or path_region, ) ### TRANSFORMATION ### - unencoded_model_id: Final = optional_params.pop("model_id", None) or model # default to model if not passed + unencoded_model_id: Final = model_id_param or bedrock_model modelId: Final = urllib.parse.quote(unencoded_model_id, safe="") aws_region_name = self._get_aws_region_name( optional_params={"aws_region_name": aws_region_name}, - model=model, + model=bedrock_model, model_id=unencoded_model_id, ) - # Check async invoke needs to be used - has_async_invoke: Final = "async_invoke/" in model - if has_async_invoke: - model = model.replace("async_invoke/", "", 1) - provider: Final = self.get_bedrock_embedding_provider(model) + provider: Final = self.get_bedrock_embedding_provider(bedrock_model) if provider is None: raise Exception( - f"Unable to determine bedrock embedding provider for model: {model}. " + f"Unable to determine bedrock embedding provider for model: {bedrock_model}. " f"Supported providers: {list(get_args(BEDROCK_EMBEDDING_PROVIDERS_LITERAL))}" ) inference_params = copy.deepcopy(optional_params) @@ -441,9 +465,9 @@ class BedrockEmbedding(BaseAWSLLM): batch_data: list | None = None if provider == "cohere": data = BedrockCohereEmbeddingConfig().transform_request( - model=model, input=input, inference_params=inference_params + model=bedrock_model, input=input, inference_params=inference_params ) - elif provider == "amazon" and model in [ + elif provider == "amazon" and bedrock_model in [ "amazon.titan-embed-image-v1", "amazon.titan-embed-text-v1", "amazon.titan-embed-text-v2:0", @@ -451,28 +475,28 @@ class BedrockEmbedding(BaseAWSLLM): ]: batch_data = [] for i in input: - if model == "amazon.titan-embed-image-v1": + if bedrock_model == "amazon.titan-embed-image-v1": transformed_request: AmazonEmbeddingRequest = ( AmazonTitanMultimodalEmbeddingG1Config().transform_request( input=i, inference_params=inference_params ) ) - elif model == "amazon.titan-embed-text-v1": + elif bedrock_model == "amazon.titan-embed-text-v1": transformed_request = AmazonTitanG1Config().transform_request( input=i, inference_params=inference_params ) - elif model == "amazon.titan-embed-text-v2:0": + elif bedrock_model == "amazon.titan-embed-text-v2:0": transformed_request = AmazonTitanV2Config().transform_request( input=i, inference_params=inference_params ) - elif model == "amazon.titan-embed-g1-text-02": + elif bedrock_model == "amazon.titan-embed-g1-text-02": transformed_request = AmazonTitanG1Config().transform_request( input=i, inference_params=inference_params ) else: raise Exception( "Unmapped model. Received={}. Expected={}".format( - model, + bedrock_model, [ "amazon.titan-embed-image-v1", "amazon.titan-embed-text-v1", @@ -485,7 +509,7 @@ class BedrockEmbedding(BaseAWSLLM): elif provider == "twelvelabs": batch_data = [] for i in input: - twelvelabs_request = TwelveLabsMarengoEmbeddingConfig(model=model).transform_request( + twelvelabs_request = TwelveLabsMarengoEmbeddingConfig(model=bedrock_model).transform_request( input=i, inference_params=inference_params, async_invoke_route=has_async_invoke, @@ -527,7 +551,7 @@ class BedrockEmbedding(BaseAWSLLM): extra_headers=extra_headers, endpoint_url=endpoint_url, aws_region_name=aws_region_name, - model=model, + model=model_without_async_route, logging_obj=logging_obj, api_key=api_key, provider=provider, @@ -541,7 +565,7 @@ class BedrockEmbedding(BaseAWSLLM): extra_headers=extra_headers, endpoint_url=endpoint_url, aws_region_name=aws_region_name, - model=model, + model=model_without_async_route, logging_obj=logging_obj, api_key=api_key, provider=provider, @@ -570,8 +594,9 @@ class BedrockEmbedding(BaseAWSLLM): ## ROUTING ## # Convert CaseInsensitiveDict to regular dict for httpx compatibility headers_for_request: Final = dict(prepped.headers) if hasattr(prepped, "headers") else {} + set_hidden_param(model_response, "region_name", aws_region_name) return cohere_embedding( - model=model, + model=model_without_async_route, input=input, model_response=model_response, logging_obj=logging_obj, diff --git a/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude_transformation.py b/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude_transformation.py index 60ce73abd82..e5d6a168d57 100644 --- a/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude_transformation.py +++ b/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude_transformation.py @@ -7,6 +7,7 @@ from pydantic import TypeAdapter import litellm from litellm._logging import verbose_logger +from litellm.anthropic_beta_gated_fields import log_beta_gated_strips, strip_beta_gated_native_request from litellm.anthropic_beta_headers_manager import filter_and_transform_beta_headers from litellm.constants import ( BEDROCK_MIN_THINKING_BUDGET_TOKENS, @@ -549,6 +550,9 @@ class AmazonAnthropicClaudeMessagesConfig( "edits": retained_edits, } + def strips_beta_gated_fields_from_final_headers(self) -> bool: + return False + def _get_bedrock_invoke_anthropic_beta_headers( self, model: str, @@ -584,9 +588,7 @@ class AmazonAnthropicClaudeMessagesConfig( ), custom_llm_provider="bedrock", is_mid_conversation_output_config_used=is_mid_conversation_output_config_used, - is_thinking_display_updates_used=anthropic_model_info.is_thinking_display_updates_used( - anthropic_messages_request.get("thinking") - ), + thinking=anthropic_messages_request.get("thinking"), is_mid_conversation_tool_change_used=anthropic_model_info.is_mid_conversation_tool_change_used( outgoing_messages_typed ), @@ -757,7 +759,9 @@ class AmazonAnthropicClaudeMessagesConfig( ), "display": "updates", } - self._normalize_system_role_messages(anthropic_messages_request, model=model) + self._normalize_system_role_messages( + anthropic_messages_request, model=model, drop_params=getattr(litellm_params, "drop_params", None) + ) ######################################################### ############## BEDROCK Invoke SPECIFIC TRANSFORMATION ### ######################################################### @@ -825,6 +829,20 @@ class AmazonAnthropicClaudeMessagesConfig( if filtered_betas: anthropic_messages_request["anthropic_beta"] = filtered_betas + final_request: Final = OBJECT_MAPPING_ADAPTER.validate_python(anthropic_messages_request) # pyright: ignore[reportUnknownArgumentType] # built from the base signature's bare dict + gated: Final = strip_beta_gated_native_request( + {}, + final_request, + final_request.get("messages"), + body_betas=filtered_betas, + drop_params=getattr(litellm_params, "drop_params", None), + ) + log_beta_gated_strips(gated.stripped, model=model, provider="bedrock_invoke") + if gated.thinking_stripped: + anthropic_messages_request["thinking"] = gated.thinking + if gated.messages_stripped: + anthropic_messages_request["messages"] = list(gated.messages) + outbound_tools: Final = tools_without_eager_input_streaming(anthropic_messages_request) if outbound_tools is not None: anthropic_messages_request["tools"] = outbound_tools diff --git a/litellm/llms/cohere/embed/transformation.py b/litellm/llms/cohere/embed/transformation.py index 590164ffa0f..1477e45997b 100644 --- a/litellm/llms/cohere/embed/transformation.py +++ b/litellm/llms/cohere/embed/transformation.py @@ -10,11 +10,13 @@ Convers Docs - https://docs.cohere.com/v2/reference/embed """ -from collections.abc import Iterable, Mapping, Sized -from typing import Final, Protocol, cast +from collections.abc import Iterable, Iterator, Mapping, Sequence, Sized +from enum import Enum +from itertools import chain, starmap +from typing import Final, Protocol, TypeAlias, cast import httpx -from pydantic import ConfigDict, TypeAdapter +from pydantic import BaseModel, ConfigDict, Field, TypeAdapter import litellm from litellm import COHERE_DEFAULT_EMBEDDING_INPUT_TYPE @@ -32,14 +34,117 @@ from litellm.utils import is_base64_encoded from ..common_utils import CohereError _JSON_OBJECT: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(hide_input_in_errors=True)) -_EMBEDDINGS_BY_TYPE: Final = TypeAdapter(Mapping[str, Iterable[object]], config=ConfigDict(hide_input_in_errors=True)) _DECODED_JSON: Final = TypeAdapter(object) class _SupportsEncode(Protocol): """Tokenizer handle: the embedding usage path only encodes text to measure its token length.""" - def encode(self, text: str, /) -> Sized: ... + def encode_ordinary(self, text: str, /) -> Sized: ... + + +class CohereImageTokenBilling(Enum): + SEPARATE_FROM_INPUT_TOKENS = "separate_from_input_tokens" + FOLDED_INTO_INPUT_TOKENS = "folded_into_input_tokens" + + +class CohereEmbeddingRowShape(Enum): + OPENAI = "openai" + TYPED = "typed" + + +class _CohereBilledUnits(BaseModel): + model_config = ConfigDict(frozen=True, hide_input_in_errors=True) + + input_tokens: int | None = None + image_tokens: int | None = None + images: int | None = None + + +class _CohereEmbedMeta(BaseModel): + model_config = ConfigDict(frozen=True, hide_input_in_errors=True) + + billed_units: _CohereBilledUnits = Field(default_factory=_CohereBilledUnits) + + +_CohereEmbeddings: TypeAlias = Mapping[str, Sequence[object]] | Sequence[object] + + +class _CohereEmbedBody(BaseModel): + model_config = ConfigDict(frozen=True, hide_input_in_errors=True) + + embeddings: _CohereEmbeddings + meta: _CohereEmbedMeta = Field(default_factory=_CohereEmbedMeta) + + +def _embedding_rows(vectors: Sequence[object]) -> Iterator[Mapping[str, object]]: + return ({"object": "embedding", "index": index, "embedding": vector} for index, vector in enumerate(vectors)) + + +def _typed_embedding_rows(embedding_type: str, vectors: Sequence[object]) -> Iterator[Mapping[str, object]]: + return ({**row, "type": embedding_type} for row in _embedding_rows(vectors)) + + +def _response_rows(embeddings: _CohereEmbeddings, row_shape: CohereEmbeddingRowShape) -> Iterable[Mapping[str, object]]: + if not isinstance(embeddings, Mapping): + return _embedding_rows(embeddings) + match row_shape: + case CohereEmbeddingRowShape.OPENAI: + return chain.from_iterable(map(_embedding_rows, embeddings.values())) + case CohereEmbeddingRowShape.TYPED: + return chain.from_iterable(starmap(_typed_embedding_rows, embeddings.items())) + + +def _prompt_usage(prompt_tokens: int, prompt_tokens_details: PromptTokensDetailsWrapper | None) -> Usage: + return Usage( + prompt_tokens=prompt_tokens, + completion_tokens=0, + total_tokens=prompt_tokens, + prompt_tokens_details=prompt_tokens_details, + ) + + +def _billed_usage(billed_units: _CohereBilledUnits, image_billing: CohereImageTokenBilling) -> Usage: + match image_billing: + case CohereImageTokenBilling.SEPARATE_FROM_INPUT_TOKENS: + image_units: Final = (billed_units.image_tokens or 0) + (billed_units.images or 0) + return _prompt_usage( + (billed_units.input_tokens or 0) + image_units, + PromptTokensDetailsWrapper( + text_tokens=billed_units.input_tokens, + image_tokens=image_units or None, + image_count=billed_units.images, + ), + ) + case CohereImageTokenBilling.FOLDED_INTO_INPUT_TOKENS: + return _prompt_usage(billed_units.input_tokens or 0, None) + + +def _cohere_embed_usage( + billed_units: _CohereBilledUnits, + input_texts: Iterable[str], + encoding: _SupportsEncode, + image_billing: CohereImageTokenBilling, +) -> Usage: + if billed_units.input_tokens is None and billed_units.image_tokens is None and billed_units.images is None: + return _prompt_usage(sum(len(encoding.encode_ordinary(text)) for text in input_texts), None) + return _billed_usage(billed_units, image_billing) + + +def cohere_embedding_response( + body: Mapping[str, object], + model: str | None, + encoding: _SupportsEncode, + input_texts: Iterable[str], + image_billing: CohereImageTokenBilling, + row_shape: CohereEmbeddingRowShape, +) -> EmbeddingResponse: + cohere_body: Final = _CohereEmbedBody.model_validate(body) + return EmbeddingResponse( + model=model, + data=list(_response_rows(cohere_body.embeddings, row_shape)), + usage=_cohere_embed_usage(cohere_body.meta.billed_units, input_texts, encoding, image_billing), + ) class CohereEmbeddingConfig(BaseEmbeddingConfig): @@ -145,44 +250,16 @@ class CohereEmbeddingConfig(BaseEmbeddingConfig): ), ) - def _calculate_usage(self, input: list[str], encoding: _SupportsEncode, meta: dict) -> Usage: - input_tokens = 0 - - text_tokens: Final[int | None] = meta.get("billed_units", {}).get("input_tokens") - - image_tokens: Final[int | None] = meta.get("billed_units", {}).get("images") - - prompt_tokens_details: PromptTokensDetailsWrapper | None = None - if image_tokens is None and text_tokens is None: - for text in input: - input_tokens += len(encoding.encode(text)) - else: - prompt_tokens_details = PromptTokensDetailsWrapper( - image_tokens=image_tokens, - text_tokens=text_tokens, - ) - if image_tokens: - input_tokens += image_tokens - if text_tokens: - input_tokens += text_tokens - - return Usage( - prompt_tokens=input_tokens, - completion_tokens=0, - total_tokens=input_tokens, - prompt_tokens_details=prompt_tokens_details, - ) - - def _transform_response( + def transform_native_embed_response( self, response: httpx.Response, api_key: str | None, logging_obj: LiteLLMLoggingObj, - data: dict | CohereEmbeddingRequest, - model_response: EmbeddingResponse, + data: Mapping[str, object] | CohereEmbeddingRequest, model: str, encoding: _SupportsEncode, - input: list, + input: Sequence[str], + row_shape: CohereEmbeddingRowShape, ) -> EmbeddingResponse: response_json: Final = _DECODED_JSON.validate_python(response.json()) ## LOGGING @@ -192,38 +269,15 @@ class CohereEmbeddingConfig(BaseEmbeddingConfig): additional_args={"complete_input_dict": data}, original_response=response_json, ) - """ - response - { - 'object': "list", - 'data': [ - - ] - 'model', - 'usage' - } - """ - payload: Final = _JSON_OBJECT.validate_python(response_json) - embeddings: Final = _EMBEDDINGS_BY_TYPE.validate_python(payload["embeddings"]) - output_data: Final = [] - for k, embedding_list in embeddings.items(): - for idx, embedding in enumerate(embedding_list): - output_data.append({"object": "embedding", "index": idx, "embedding": embedding}) - model_response.object = "list" - model_response.data = output_data - model_response.model = model - input_tokens = 0 - for text in input: - input_tokens += len(encoding.encode(text)) - - setattr( - model_response, - "usage", - self._calculate_usage(input, encoding, dict(_JSON_OBJECT.validate_python(payload.get("meta", {})))), + return cohere_embedding_response( + _JSON_OBJECT.validate_python(response_json), + model, + encoding, + input, + CohereImageTokenBilling.SEPARATE_FROM_INPUT_TOKENS, + row_shape, ) - return model_response - def transform_embedding_response( self, model: str, @@ -235,15 +289,15 @@ class CohereEmbeddingConfig(BaseEmbeddingConfig): optional_params: dict, litellm_params: dict, ) -> EmbeddingResponse: - return self._transform_response( + return self.transform_native_embed_response( response=raw_response, api_key=api_key, logging_obj=logging_obj, data=request_data, - model_response=model_response, model=model, encoding=litellm.encoding, input=logging_obj.model_call_details["input"], + row_shape=CohereEmbeddingRowShape.OPENAI, ) def get_error_class(self, error_message: str, status_code: int, headers: dict | httpx.Headers) -> BaseLLMException: diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index 9e125db4211..4c0f84a003d 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -2581,6 +2581,7 @@ class BaseLLMHTTPHandler: "preset_cache_key": None, "stream_response": {}, "model_info": kwargs.get("model_info"), + **litellm_params.per_second_pricing(), **vertex_location_params, **anthropic_messages_optional_request_params, }, diff --git a/litellm/llms/databricks/chat/transformation.py b/litellm/llms/databricks/chat/transformation.py index 48118c23ed7..60081c38073 100644 --- a/litellm/llms/databricks/chat/transformation.py +++ b/litellm/llms/databricks/chat/transformation.py @@ -10,6 +10,7 @@ from typing import TYPE_CHECKING, Any, Final, Literal, cast, overload import httpx from pydantic import BaseModel, ConfigDict, TypeAdapter +from litellm.anthropic_beta_gated_fields import log_beta_gated_strips, strip_beta_gated_thinking from litellm.constants import RESPONSE_FORMAT_TOOL_NAME from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import ( handle_invalid_parallel_tool_calls, @@ -514,6 +515,14 @@ class DatabricksConfig(DatabricksBase, OpenAILikeChatConfig, AnthropicConfig): self.update_optional_params_with_thinking_tokens( non_default_params=non_default_params, optional_params=mapped_params ) + gated_thinking: Final = strip_beta_gated_thinking( + mapped_params.get("thinking"), # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # mapped_params is the base config's bare dict + frozenset(), + drop_params=drop_params, + ) + if gated_thinking.stripped: + log_beta_gated_strips(gated_thinking.stripped, model=model, provider="databricks") + mapped_params["thinking"] = gated_thinking.thinking return mapped_params diff --git a/litellm/llms/databricks/messages/transformation.py b/litellm/llms/databricks/messages/transformation.py index e22ad85b5f9..90b63d860d3 100644 --- a/litellm/llms/databricks/messages/transformation.py +++ b/litellm/llms/databricks/messages/transformation.py @@ -144,7 +144,7 @@ class DatabricksAnthropicMessagesConfig(DatabricksBase, AnthropicMessagesConfig) ("content-type" if key.lower() == "content-type" else key): value for key, value in {**auth_headers, **client_anthropic_headers}.items() } - finalized_headers: Final[dict[str, str]] = self._finalize_messages_headers( # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # the shared helper returns a bare dict # mutable-ok: becomes the outbound headers + finalized_headers: Final[dict[str, str]] = self.finalize_messages_headers( # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # the shared helper returns a bare dict # mutable-ok: becomes the outbound headers headers=normalized_headers, optional_params=optional_params, messages=messages, @@ -191,6 +191,8 @@ class DatabricksAnthropicMessagesConfig(DatabricksBase, AnthropicMessagesConfig) litellm_params=litellm_params, headers=headers, ) - self._normalize_system_role_messages(anthropic_messages_request, model=model) # pyright: ignore[reportUnknownMemberType] # the shared helper takes a bare dict + self._normalize_system_role_messages( # pyright: ignore[reportUnknownMemberType] # the shared helper takes a bare dict + anthropic_messages_request, model=model, drop_params=getattr(litellm_params, "drop_params", None) + ) self._remove_scope_from_cache_control(anthropic_messages_request) # pyright: ignore[reportUnknownMemberType] # the shared helper takes a bare dict return anthropic_messages_request diff --git a/litellm/llms/openai/common_utils.py b/litellm/llms/openai/common_utils.py index c4925cb2701..259c9ef1991 100644 --- a/litellm/llms/openai/common_utils.py +++ b/litellm/llms/openai/common_utils.py @@ -105,6 +105,19 @@ def is_openai_backed_api_base(api_base: str) -> bool: return hostname is not None and (hostname == _OPENAI_API_HOST or hostname.endswith(f".{_OPENAI_API_HOST}")) +_OPENAI_AGENTS_API_RESOURCES: Final = frozenset({"agents", "vaults"}) + + +def openai_api_resource(path: str) -> str | None: + segments: Final = tuple(segment for segment in path.split("/") if segment) + resource_segments: Final = segments[1:] if segments[:1] == ("v1",) else segments + return next(iter(resource_segments), None) + + +def is_openai_agents_api_path(path: str) -> bool: + return openai_api_resource(path) in _OPENAI_AGENTS_API_RESOURCES + + class OpenAIError(BaseLLMException): def __init__( self, diff --git a/litellm/llms/openai_like/providers.json b/litellm/llms/openai_like/providers.json index fe88ac03e04..ad47b334ed4 100644 --- a/litellm/llms/openai_like/providers.json +++ b/litellm/llms/openai_like/providers.json @@ -270,6 +270,13 @@ "/v1/responses" ] }, + "kyma": { + "base_url": "https://kymaapi.com/v1", + "api_key_env": "KYMA_API_KEY", + "provider_info": { + "supports_function_calling": true + } + }, "streamlake": { "base_url": "https://vanchin.streamlake.ai/api/gateway/v1/endpoints", "api_key_env": "STREAMLAKE_API_KEY", diff --git a/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/messages/transformation.py b/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/messages/transformation.py index 1f0ee728cd1..bfa94396ea7 100644 --- a/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/messages/transformation.py +++ b/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/messages/transformation.py @@ -211,6 +211,12 @@ class VertexAIPartnerModelsAnthropicMessagesConfig(AnthropicMessagesConfig, Vert if messages_carry_output_config(messages): beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.PER_TURN_CONTROL_2026_07_01.value) + beta_values.update( + anthropic_model_info.thinking_betas( + optional_params.get("thinking"), # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # optional_params is the base config's bare dict + ) + ) + if beta_values: request_headers["anthropic-beta"] = ",".join(beta_values) @@ -245,7 +251,9 @@ class VertexAIPartnerModelsAnthropicMessagesConfig(AnthropicMessagesConfig, Vert headers=headers, ) - self._normalize_system_role_messages(anthropic_messages_request, model=model) + self._normalize_system_role_messages( + anthropic_messages_request, model=model, drop_params=getattr(litellm_params, "drop_params", None) + ) self._remove_scope_from_cache_control(anthropic_messages_request) diff --git a/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/transformation.py b/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/transformation.py index 363e5596cb9..79fd9f9e8b1 100644 --- a/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/transformation.py +++ b/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/transformation.py @@ -145,6 +145,7 @@ class VertexAIAnthropicConfig(AnthropicConfig): file_id_used=self.is_file_id_used(messages), mcp_server_used=self.is_mcp_server_used(optional_params.get("mcp_servers")), custom_llm_provider="vertex_ai", + thinking=optional_params.get("thinking"), # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # optional_params is the base config's bare dict ) beta_set: Final = set(auto_betas) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index aa00ea73f8c..4fad4857c60 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -78341,6 +78341,1254 @@ "supports_xhigh_reasoning_effort": true, "use_openai_responses_path": true }, + "kyma/claude-fable-5": { + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "input_cost_per_token": 1.35e-05, + "output_cost_per_token": 6.75e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true, + "supports_adaptive_thinking": true, + "thinking_always_on": true + }, + "kyma/claude-fable-5.1": { + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "input_cost_per_token": 1.35e-05, + "output_cost_per_token": 6.75e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true, + "supports_adaptive_thinking": true, + "thinking_always_on": true + }, + "kyma/claude-haiku-4-5": { + "max_input_tokens": 200000, + "max_output_tokens": 64000, + "input_cost_per_token": 1.35e-06, + "cache_read_input_token_cost": 1.35e-07, + "output_cost_per_token": 6.75e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false, + "prompt_cache_min_tokens": 4096 + }, + "kyma/claude-haiku-5.5": { + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "input_cost_per_token": 1.35e-07, + "cache_read_input_token_cost": 1.35e-08, + "output_cost_per_token": 6.75e-07, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true, + "supports_adaptive_thinking": true, + "thinking_always_on": false, + "prompt_cache_min_tokens": 512 + }, + "kyma/claude-opus-4-7": { + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "input_cost_per_token": 6.75e-06, + "cache_read_input_token_cost": 6.75e-07, + "output_cost_per_token": 3.375e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false, + "supports_adaptive_thinking": true, + "prompt_cache_min_tokens": 2048 + }, + "kyma/claude-opus-5": { + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "input_cost_per_token": 6.75e-06, + "cache_read_input_token_cost": 6.75e-07, + "output_cost_per_token": 3.375e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true, + "supports_adaptive_thinking": true, + "prompt_cache_min_tokens": 512 + }, + "kyma/claude-opus-5.5": { + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "input_cost_per_token": 5.4e-06, + "cache_read_input_token_cost": 2.7e-07, + "output_cost_per_token": 2.7e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true, + "supports_adaptive_thinking": true, + "thinking_always_on": true, + "prompt_cache_min_tokens": 512 + }, + "kyma/claude-sonnet-4-6": { + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "input_cost_per_token": 4.05e-06, + "cache_read_input_token_cost": 4.05e-07, + "output_cost_per_token": 2.025e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false, + "supports_adaptive_thinking": true, + "prompt_cache_min_tokens": 1024 + }, + "kyma/claude-sonnet-5": { + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "input_cost_per_token": 2.7e-06, + "cache_read_input_token_cost": 2.7e-07, + "output_cost_per_token": 1.35e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true, + "supports_adaptive_thinking": true, + "prompt_cache_min_tokens": 1024 + }, + "kyma/claude-sonnet-5.5": { + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "input_cost_per_token": 2.7e-06, + "cache_read_input_token_cost": 1.35e-07, + "output_cost_per_token": 1.35e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true, + "supports_adaptive_thinking": true, + "thinking_always_on": true, + "prompt_cache_min_tokens": 512 + }, + "kyma/deepseek-r1": { + "max_input_tokens": 64000, + "max_output_tokens": 32768, + "input_cost_per_token": 7.425e-07, + "output_cost_per_token": 2.957e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/deepseek-v3": { + "max_input_tokens": 160000, + "max_output_tokens": 8192, + "input_cost_per_token": 5.18e-07, + "output_cost_per_token": 7.571e-07, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/deepseek-v4-flash": { + "max_input_tokens": 1000000, + "max_output_tokens": 65536, + "input_cost_per_token": 1.23e-07, + "cache_read_input_token_cost": 2.43e-08, + "output_cost_per_token": 2.458e-07, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/deepseek-v4-flash-vision-exp": { + "max_input_tokens": 1048576, + "max_output_tokens": 384000, + "input_cost_per_token": 3.892e-07, + "output_cost_per_token": 1.168e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/deepseek-v4.1-flash": { + "max_input_tokens": 1048576, + "max_output_tokens": 131072, + "input_cost_per_token": 2.7e-07, + "output_cost_per_token": 8.1e-07, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/gemini-3-flash": { + "max_input_tokens": 1048576, + "max_output_tokens": 8192, + "input_cost_per_token": 6.75e-07, + "cache_read_input_token_cost": 6.75e-08, + "output_cost_per_token": 4.05e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/gemini-3.1-pro": { + "max_input_tokens": 1048576, + "max_output_tokens": 65536, + "input_cost_per_token": 2.7e-06, + "cache_read_input_token_cost": 2.7e-07, + "output_cost_per_token": 1.62e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/gemini-3.5-flash": { + "max_input_tokens": 1048576, + "max_output_tokens": 8192, + "input_cost_per_token": 2.025e-06, + "cache_read_input_token_cost": 2.025e-07, + "output_cost_per_token": 1.215e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/gemini-3.5-flash-lite": { + "max_input_tokens": 1048576, + "max_output_tokens": 8192, + "input_cost_per_token": 4.05e-07, + "cache_read_input_token_cost": 4.05e-08, + "output_cost_per_token": 3.375e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/gemini-3.6-flash": { + "max_input_tokens": 1048576, + "max_output_tokens": 8192, + "input_cost_per_token": 1.013e-06, + "cache_read_input_token_cost": 1.0125e-07, + "output_cost_per_token": 5.063e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": false, + "supports_pdf_input": false + }, + "kyma/gemini-3.7-flash": { + "max_input_tokens": 1048576, + "max_output_tokens": 65536, + "input_cost_per_token": 2.026e-06, + "cache_read_input_token_cost": 1.0125e-07, + "output_cost_per_token": 1.0126e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/gemini-3.8-flash": { + "max_input_tokens": 1048576, + "max_output_tokens": 65536, + "input_cost_per_token": 2.026e-06, + "output_cost_per_token": 1.0126e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/gemini-4-argon": { + "max_input_tokens": 1048576, + "max_output_tokens": 65536, + "input_cost_per_token": 5.4e-06, + "cache_read_input_token_cost": 2.7e-07, + "output_cost_per_token": 2.7e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/gemma-4-31b": { + "max_input_tokens": 128000, + "max_output_tokens": 8192, + "input_cost_per_token": 1.642e-07, + "cache_read_input_token_cost": 1.642e-07, + "output_cost_per_token": 4.689e-07, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/glm-4.5-air": { + "max_input_tokens": 131072, + "max_output_tokens": 8192, + "input_cost_per_token": 1.797e-07, + "cache_read_input_token_cost": 3.375e-08, + "output_cost_per_token": 1.175e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/glm-4.7-flash": { + "max_input_tokens": 203000, + "max_output_tokens": 65536, + "input_cost_per_token": 8.1e-08, + "cache_read_input_token_cost": 1.35e-08, + "output_cost_per_token": 5.4e-07, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/glm-5.1": { + "max_input_tokens": 203000, + "max_output_tokens": 65536, + "input_cost_per_token": 1.89e-06, + "cache_read_input_token_cost": 2.7675e-07, + "output_cost_per_token": 5.94e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/glm-5.2": { + "max_input_tokens": 1000000, + "max_output_tokens": 131072, + "input_cost_per_token": 9.029e-07, + "cache_read_input_token_cost": 2.9835e-07, + "output_cost_per_token": 3.083e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/glm-5.3": { + "max_input_tokens": 1310720, + "max_output_tokens": 131072, + "input_cost_per_token": 1.178e-06, + "output_cost_per_token": 3.702e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/glm-5.3-flash": { + "max_input_tokens": 1310720, + "max_output_tokens": 65536, + "input_cost_per_token": 1.72e-07, + "output_cost_per_token": 5.733e-07, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/gpt-5.6-luna": { + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "input_cost_per_token": 2.944e-07, + "cache_read_input_token_cost": 2.7e-08, + "output_cost_per_token": 1.767e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/gpt-5.6-luna-pro": { + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "input_cost_per_token": 2.703e-07, + "cache_read_input_token_cost": 2.7e-08, + "output_cost_per_token": 1.621e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/gpt-5.6-sol": { + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "input_cost_per_token": 5.4e-06, + "cache_read_input_token_cost": 6.75e-07, + "output_cost_per_token": 2.7e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/gpt-5.6-sol-pro": { + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "input_cost_per_token": 5.4e-06, + "cache_read_input_token_cost": 6.75e-07, + "output_cost_per_token": 2.7e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/gpt-5.6-terra": { + "max_input_tokens": 1050000, + "max_output_tokens": 8192, + "input_cost_per_token": 2.7e-06, + "cache_read_input_token_cost": 2.7e-07, + "output_cost_per_token": 1.62e-05, + "input_cost_per_token_above_272k_tokens": 5.4e-06, + "output_cost_per_token_above_272k_tokens": 2.43e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/gpt-5.6-terra-pro": { + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "input_cost_per_token": 2.7e-06, + "cache_read_input_token_cost": 2.7e-07, + "output_cost_per_token": 1.62e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/gpt-6-astra": { + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "input_cost_per_token": 1.35e-05, + "cache_read_input_token_cost": 1.35e-06, + "output_cost_per_token": 6.75e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/gpt-6-luna": { + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "input_cost_per_token": 1.639e-07, + "cache_read_input_token_cost": 1.35e-08, + "output_cost_per_token": 8.195e-07, + "input_cost_per_token_above_272k_tokens": 2.7e-07, + "output_cost_per_token_above_272k_tokens": 1.0125e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/gpt-6.1-sol": { + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "input_cost_per_token": 2.7e-06, + "cache_read_input_token_cost": 1.35e-07, + "output_cost_per_token": 1.35e-05, + "input_cost_per_token_above_272k_tokens": 5.4e-06, + "output_cost_per_token_above_272k_tokens": 2.025e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/gpt-oss-120b": { + "max_input_tokens": 128000, + "max_output_tokens": 8192, + "input_cost_per_token": 4.73e-08, + "output_cost_per_token": 2.303e-07, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/grok-4.20": { + "max_input_tokens": 2000000, + "max_output_tokens": 8192, + "input_cost_per_token": 1.763e-06, + "output_cost_per_token": 3.528e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/grok-4.20-multi-agent": { + "max_input_tokens": 2000000, + "max_output_tokens": 8192, + "input_cost_per_token": 1.767e-06, + "output_cost_per_token": 3.534e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": false, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/grok-4.3": { + "max_input_tokens": 1000000, + "max_output_tokens": 32768, + "input_cost_per_token": 1.92e-06, + "cache_read_input_token_cost": 2.7e-07, + "output_cost_per_token": 3.838e-06, + "input_cost_per_token_above_200k_tokens": 3.375e-06, + "output_cost_per_token_above_200k_tokens": 6.75e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/grok-4.5": { + "max_input_tokens": 500000, + "max_output_tokens": 8192, + "input_cost_per_token": 2.73e-06, + "output_cost_per_token": 8.189e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/grok-4.6": { + "max_input_tokens": 500000, + "max_output_tokens": 8192, + "input_cost_per_token": 2.758e-06, + "output_cost_per_token": 8.274e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/grok-4.7": { + "max_input_tokens": 500000, + "max_output_tokens": 450000, + "input_cost_per_token": 2.16e-06, + "cache_read_input_token_cost": 5.4e-07, + "output_cost_per_token": 6.48e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/grok-build": { + "max_input_tokens": 256000, + "max_output_tokens": 32768, + "input_cost_per_token": 1.389e-06, + "cache_read_input_token_cost": 2.7e-07, + "output_cost_per_token": 2.777e-06, + "input_cost_per_token_above_200k_tokens": 2.7e-06, + "output_cost_per_token_above_200k_tokens": 5.4e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/hy3": { + "max_input_tokens": 262144, + "max_output_tokens": 8192, + "input_cost_per_token": 1.594e-07, + "output_cost_per_token": 6.576e-07, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/hy4-preview": { + "max_input_tokens": 1048576, + "max_output_tokens": 64000, + "input_cost_per_token": 1.1259e-06, + "output_cost_per_token": 3.3764e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/kimi-k2.6": { + "max_input_tokens": 262144, + "max_output_tokens": 32768, + "input_cost_per_token": 5.909e-07, + "output_cost_per_token": 2.758e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/kimi-k2.7-code": { + "max_input_tokens": 262144, + "max_output_tokens": 16384, + "input_cost_per_token": 1.009e-06, + "output_cost_per_token": 4.774e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/kimi-k3": { + "max_input_tokens": 1048576, + "max_output_tokens": 32768, + "input_cost_per_token": 3.052e-06, + "cache_read_input_token_cost": 4.05e-07, + "output_cost_per_token": 1.5258e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/kimi-k3.1-pro": { + "max_input_tokens": 1048576, + "max_output_tokens": 943718, + "input_cost_per_token": 2.7e-06, + "cache_read_input_token_cost": 2.7e-07, + "output_cost_per_token": 1.35e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/llama-3.3-70b": { + "max_input_tokens": 128000, + "max_output_tokens": 8192, + "input_cost_per_token": 1.35e-07, + "output_cost_per_token": 4.32e-07, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": true, + "supports_reasoning": false, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/llama-4-maverick": { + "max_input_tokens": 1048576, + "max_output_tokens": 16384, + "input_cost_per_token": 3.344e-07, + "output_cost_per_token": 1.337e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": false, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/mimo-v2.5": { + "max_input_tokens": 1050000, + "max_output_tokens": 16384, + "input_cost_per_token": 1.859e-07, + "output_cost_per_token": 3.718e-07, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/mimo-v2.6-flash": { + "max_input_tokens": 1050000, + "max_output_tokens": 131072, + "input_cost_per_token": 1.89e-07, + "output_cost_per_token": 3.78e-07, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/mimo-v2.6-pro": { + "max_input_tokens": 1050000, + "max_output_tokens": 131072, + "input_cost_per_token": 5.805e-07, + "output_cost_per_token": 1.1745e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/minimax-m2.5": { + "max_input_tokens": 196608, + "max_output_tokens": 32768, + "input_cost_per_token": 4.188e-07, + "output_cost_per_token": 1.675e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/minimax-m2.7": { + "max_input_tokens": 204800, + "max_output_tokens": 32768, + "input_cost_per_token": 4.05e-07, + "output_cost_per_token": 1.62e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/minimax-m3": { + "max_input_tokens": 1048576, + "max_output_tokens": 32768, + "input_cost_per_token": 3.956e-07, + "cache_read_input_token_cost": 7.56e-08, + "output_cost_per_token": 1.582e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/minimax-m3.1-flash": { + "max_input_tokens": 1048576, + "max_output_tokens": 524288, + "input_cost_per_token": 2.025e-07, + "cache_read_input_token_cost": 2.025e-08, + "output_cost_per_token": 8.1e-07, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/muse-glimmer-30b": { + "max_input_tokens": 131072, + "max_output_tokens": 8192, + "input_cost_per_token": 4.554e-07, + "output_cost_per_token": 1.821e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/muse-spark-1.1": { + "max_input_tokens": 1048576, + "max_output_tokens": 8192, + "input_cost_per_token": 1.542e-06, + "output_cost_per_token": 5.243e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/muse-spark-1.2": { + "max_input_tokens": 1048576, + "max_output_tokens": 8192, + "input_cost_per_token": 1.773e-06, + "output_cost_per_token": 6.025e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/muse-spark-1.3": { + "max_input_tokens": 1048576, + "max_output_tokens": 8192, + "input_cost_per_token": 1.688e-06, + "output_cost_per_token": 5.738e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/muse-spark-open": { + "max_input_tokens": 1048576, + "max_output_tokens": 943718, + "input_cost_per_token": 1.688e-06, + "output_cost_per_token": 5.738e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/muse-wonder": { + "max_input_tokens": 1048576, + "max_output_tokens": 943718, + "input_cost_per_token": 5.4e-06, + "output_cost_per_token": 2.7e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/nemotron-3-ultra-550b": { + "max_input_tokens": 1000000, + "max_output_tokens": 32768, + "input_cost_per_token": 6.75e-07, + "cache_read_input_token_cost": 1.35e-07, + "output_cost_per_token": 3.375e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/nemotron-3.5-super-vl": { + "max_input_tokens": 1048576, + "max_output_tokens": 235929, + "input_cost_per_token": 8.1e-08, + "cache_read_input_token_cost": 1.35e-08, + "output_cost_per_token": 4.725e-07, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/qwen-3-32b": { + "max_input_tokens": 32768, + "max_output_tokens": 8192, + "input_cost_per_token": 1.08e-07, + "output_cost_per_token": 3.78e-07, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/qwen-3-coder": { + "max_input_tokens": 131072, + "max_output_tokens": 32768, + "input_cost_per_token": 3.546e-07, + "cache_read_input_token_cost": 1.35e-07, + "output_cost_per_token": 1.612e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": true, + "supports_reasoning": false, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/qwen-3.6-plus": { + "max_input_tokens": 1000000, + "max_output_tokens": 65536, + "input_cost_per_token": 4.388e-07, + "output_cost_per_token": 2.633e-06, + "input_cost_per_token_above_256k_tokens": 1.755e-06, + "output_cost_per_token_above_256k_tokens": 5.265e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/qwen-3.7-max": { + "max_input_tokens": 1000000, + "max_output_tokens": 32768, + "input_cost_per_token": 2.071e-06, + "cache_read_input_token_cost": 1.6875e-07, + "output_cost_per_token": 6.207e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/qwen-3.7-plus": { + "max_input_tokens": 1000000, + "max_output_tokens": 32768, + "input_cost_per_token": 4.325e-07, + "cache_read_input_token_cost": 8.64e-08, + "output_cost_per_token": 1.731e-06, + "input_cost_per_token_above_256k_tokens": 1.296e-06, + "output_cost_per_token_above_256k_tokens": 5.184e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/qwen-3.8-max": { + "max_input_tokens": 1000000, + "max_output_tokens": 131072, + "input_cost_per_token": 2.228e-06, + "cache_read_input_token_cost": 2.781e-07, + "output_cost_per_token": 6.684e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/qwen3.7-flash": { + "max_input_tokens": 1000000, + "max_output_tokens": 8192, + "input_cost_per_token": 4.35e-08, + "cache_read_input_token_cost": 8.1e-09, + "output_cost_per_token": 1.886e-07, + "input_cost_per_token_above_32k_tokens": 1.35e-07, + "output_cost_per_token_above_32k_tokens": 5.4e-07, + "input_cost_per_token_above_256k_tokens": 2.7e-07, + "output_cost_per_token_above_256k_tokens": 1.08e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/qwen3.8-27b": { + "max_input_tokens": 1000000, + "max_output_tokens": 131072, + "input_cost_per_token": 3.826e-07, + "output_cost_per_token": 2.732e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/qwen3.8-flash": { + "max_input_tokens": 1000000, + "max_output_tokens": 131072, + "input_cost_per_token": 2.045e-07, + "output_cost_per_token": 6.41e-07, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/sonar": { + "max_input_tokens": 127072, + "max_output_tokens": 4096, + "input_cost_per_token": 1.35e-06, + "output_cost_per_token": 1.35e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": false, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": false, + "supports_response_schema": false, + "supports_pdf_input": false + }, + "kyma/sonar-pro": { + "max_input_tokens": 200000, + "max_output_tokens": 8000, + "input_cost_per_token": 4.05e-06, + "output_cost_per_token": 2.025e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": false, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": false, + "supports_response_schema": false, + "supports_pdf_input": false + }, + "kyma/step-3.7-flash": { + "max_input_tokens": 256000, + "max_output_tokens": 8192, + "input_cost_per_token": 2.229e-07, + "cache_read_input_token_cost": 5.4e-08, + "output_cost_per_token": 1.282e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/step-5-preview": { + "max_input_tokens": 1048576, + "max_output_tokens": 64000, + "input_cost_per_token": 1.35e-06, + "cache_read_input_token_cost": 1.35e-07, + "output_cost_per_token": 3.645e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, "databricks/databricks-claude-haiku-5-5": { "cache_creation_input_token_cost": 1.2502e-07, "cache_creation_input_token_cost_above_100k_tokens": 6.2503e-07, diff --git a/litellm/provider_endpoints_support_backup.json b/litellm/provider_endpoints_support_backup.json index 7b0234e28d7..4188866de08 100644 --- a/litellm/provider_endpoints_support_backup.json +++ b/litellm/provider_endpoints_support_backup.json @@ -5327,6 +5327,32 @@ "generateContent": false } }, + "kyma": { + "display_name": "Kyma API (`kyma`)", + "url": "https://docs.litellm.ai/docs/providers/kyma", + "endpoints": { + "chat_completions": true, + "responses": true, + "text_completion": true, + "embeddings": false, + "image_generations": false, + "image_edits": false, + "audio_transcriptions": false, + "audio_speech": false, + "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": true, + "interactions": true, + "systemone": false, + "a2a": true, + "generateContent": true + } + }, "bigmodel": { "display_name": "BigModel (Zhipu AI) (`bigmodel`)", "url": "https://docs.bigmodel.cn/cn/guide/develop/responses/introduction", diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index dd873ae49dd..4047fa3f0ad 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -3748,6 +3748,7 @@ class SpendLogsMetadata(TypedDict): internal_call_origin: InternalCallOrigin | None billed_retry_attempt: ReadOnly[NotRequired[bool | None]] litellm_roi_estimator: ReadOnly[NotRequired[bool | None]] + is_silent_experiment: ReadOnly[NotRequired[bool | None]] guardrail_information: list[StandardLoggingGuardrailInformation] | None eval_information: Any | None status: StandardLoggingPayloadStatus diff --git a/litellm/proxy/anthropic_endpoints/endpoints.py b/litellm/proxy/anthropic_endpoints/endpoints.py index 1d88e133cf9..1a2e610d5c4 100644 --- a/litellm/proxy/anthropic_endpoints/endpoints.py +++ b/litellm/proxy/anthropic_endpoints/endpoints.py @@ -2,7 +2,7 @@ Unified /v1/messages endpoint - (Anthropic Spec) """ -from collections.abc import Mapping +from collections.abc import AsyncGenerator, Mapping from types import MappingProxyType from typing import Final @@ -19,12 +19,10 @@ from litellm.anthropic_interface.exceptions import ( ) from litellm.integrations.custom_guardrail import ModifyResponseException from litellm.litellm_core_utils.tool_fields import first_validation_error +from litellm.llms.anthropic.chat.guardrail_translation.handler import AnthropicMessagesHandler from litellm.llms.anthropic.messages.context_management import ( AnthropicContextManagementError, ) -from litellm.llms.base_llm.guardrail_translation.utils import ( - blocked_response_usage as _blocked_response_usage, -) from litellm.proxy._types import * from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.common_request_processing import ( @@ -44,6 +42,7 @@ from litellm.proxy.common_utils.http_parsing_utils import ( # noqa: F401 # leg from litellm.proxy.common_utils.openai_error_payload import ( LITELLM_CALL_ID_HEADER, error_status_code, + litellm_call_id_headers, openai_error_param, openai_error_type, with_litellm_call_id, @@ -58,6 +57,8 @@ from litellm.router import Router router: Final = APIRouter() +_NO_HEADERS: Final[Mapping[str, str]] = MappingProxyType({}) + def _with_provider_specific_fields(exc: ProxyException, detail: AnthropicErrorDetail) -> AnthropicErrorDetail: if not exc.provider_specific_fields: @@ -229,44 +230,22 @@ async def anthropic_response( request_data=_data, ) - # Create Anthropic-formatted response with violation message - import uuid + block_headers: Final = litellm_call_id_headers(base_llm_response_processor.litellm_call_id) or _NO_HEADERS + block_handler: Final = AnthropicMessagesHandler() - from litellm.types.utils import AnthropicMessagesResponse + if data.get("stream") is True: + block_chunks: Final = block_handler.build_block_sse_chunks(e) - # Report the blocked LLM response's real token usage (carried on the - # exception) instead of discarding it; zero for pre-call blocks. - _usage: Final = _blocked_response_usage(e.original_response) - - _anthropic_response: Final = AnthropicMessagesResponse( - id=f"msg_{uuid.uuid4()}", - type="message", - role="assistant", - content=[{"type": "text", "text": e.message}], - model=e.model, - stop_reason="end_turn", - usage=_usage, - ) - - if data.get("stream", None) is not None and data["stream"] is True: - # For streaming, use the standard SSE data generator - async def _passthrough_stream_generator(): - yield _anthropic_response - - selected_data_generator: Final = ProxyBaseLLMRequestProcessing.async_sse_data_generator( - response=_passthrough_stream_generator(), - user_api_key_dict=user_api_key_dict, - request_data=_data, - proxy_logging_obj=proxy_logging_obj, - ) + async def _blocked_stream() -> AsyncGenerator[str, None]: + for chunk in block_chunks: + yield chunk.decode() return await create_response( - generator=selected_data_generator, - media_type="text/event-stream", - headers={}, + generator=_blocked_stream(), media_type="text/event-stream", headers=block_headers ) - return _anthropic_response + fastapi_response.headers.update(block_headers) + return block_handler.build_block_response(e) except AnthropicContextManagementError as e: if e.status_code >= 500: # Server-side polyfill failures hit the failure hook for spend/alert diff --git a/litellm/proxy/health_check_utils/shared_health_check_manager.py b/litellm/proxy/health_check_utils/shared_health_check_manager.py index 62c973196b9..d689ab2a537 100644 --- a/litellm/proxy/health_check_utils/shared_health_check_manager.py +++ b/litellm/proxy/health_check_utils/shared_health_check_manager.py @@ -2,7 +2,10 @@ import asyncio import json import time from collections.abc import Mapping, Sequence -from typing import TYPE_CHECKING, Any, Final +from typing import TYPE_CHECKING, Final + +from pydantic import ConfigDict, TypeAdapter, with_config +from typing_extensions import NotRequired, ReadOnly, TypedDict from litellm._internal_context import with_service_target from litellm._logging import verbose_proxy_logger @@ -19,6 +22,17 @@ if TYPE_CHECKING: from litellm.router import Router +@with_config(ConfigDict(extra="allow", strict=True)) +class _CachedHealthCheckResults(TypedDict): + healthy_endpoints: NotRequired[ReadOnly[list[Mapping[str, object]]]] + unhealthy_endpoints: NotRequired[ReadOnly[list[Mapping[str, object]]]] + timestamp: NotRequired[ReadOnly[int | float]] + checked_by: NotRequired[ReadOnly[object]] + + +_CACHED_HEALTH_CHECK_RESULTS: Final = TypeAdapter(_CachedHealthCheckResults) + + class SharedHealthCheckManager: """ Manager for coordinating health checks across multiple pods using Redis. @@ -109,7 +123,7 @@ class SharedHealthCheckManager: verbose_proxy_logger.error("Error releasing health check lock: %s", str(e)) @with_service_target(HEALTH_CHECKS_TARGET) - async def get_cached_health_check_results(self) -> dict[str, Any] | None: + async def get_cached_health_check_results(self) -> _CachedHealthCheckResults | None: """ Get cached health check results from Redis. @@ -127,10 +141,9 @@ class SharedHealthCheckManager: return None # Parse the cached data - if isinstance(cached_data, str): - cached_results = json.loads(cached_data) - else: - cached_results = cached_data + cached_results: Final = _CACHED_HEALTH_CHECK_RESULTS.validate_python( + json.loads(cached_data) if isinstance(cached_data, str) else cached_data + ) # Check if the cache is still valid cache_timestamp: Final = cached_results.get("timestamp", 0) diff --git a/litellm/proxy/management_endpoints/fallback_management_endpoints.py b/litellm/proxy/management_endpoints/fallback_management_endpoints.py index 6a9a30a7689..ec8ee5ca213 100644 --- a/litellm/proxy/management_endpoints/fallback_management_endpoints.py +++ b/litellm/proxy/management_endpoints/fallback_management_endpoints.py @@ -20,7 +20,7 @@ from litellm._logging import verbose_proxy_logger from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.auth.model_checks import get_all_fallbacks from litellm.proxy.auth.user_api_key_auth import user_api_key_auth -from litellm.proxy.utils import PrismaClient, evict_config_param, invalidate_config_param +from litellm.proxy.utils import PrismaClient, invalidate_config_param if TYPE_CHECKING: from fastapi import APIRouter, Depends, HTTPException, status @@ -48,22 +48,50 @@ FallbackRule = dict[str, list[str]] StoredFallback = Annotated[FallbackRule | dict[str, object] | str, Field(union_mode="left_to_right")] _STORED_FALLBACKS: Final = TypeAdapter(list[StoredFallback]) _JSON_RULES: Final = TypeAdapter(list[JsonValue]) +_STORED_ROUTER_SETTINGS: Final = TypeAdapter(dict[str, JsonValue]) def _rule_covers(entry: StoredFallback, model: str) -> bool: return isinstance(entry, dict) and model in entry -async def _router_settings_fresh_from_db(proxy_config: "ProxyConfig") -> dict[str, object]: - await evict_config_param(ROUTER_SETTINGS_PARAM) - config: Final = await proxy_config.get_config() - return config.get(ROUTER_SETTINGS_PARAM, {}) +async def _reload_config_file_ownership(proxy_config: "ProxyConfig") -> None: + await proxy_config.get_config() -def _reject_config_owned_rules(proxy_config: "ProxyConfig", fallback_key: str, rules: Sequence[StoredFallback]) -> None: +async def _stored_router_settings(prisma_client: PrismaClient) -> Mapping[str, JsonValue]: + try: + row: Final = await ConfigRepository(prisma_client, use_writer=True).get_param(ROUTER_SETTINGS_PARAM) + except json.JSONDecodeError: + return {} + if row is None or not isinstance(row.param_value, Mapping): + return {} + return _STORED_ROUTER_SETTINGS.validate_python(row.param_value) + + +def _editable_rules( + proxy_config: "ProxyConfig", stored_settings: Mapping[str, JsonValue], fallback_key: str +) -> list[StoredFallback]: + file_settings: Final = proxy_config.router_settings + if not file_settings.owned_by_config(fallback_key): + return _STORED_FALLBACKS.validate_python(stored_settings.get(fallback_key) or []) + file_rules: Final = file_settings.get(fallback_key) + return _STORED_FALLBACKS.validate_python(file_rules if isinstance(file_rules, list) else []) + + +async def _write_rules( + prisma_client: PrismaClient, + proxy_config: "ProxyConfig", + stored_settings: Mapping[str, JsonValue], + fallback_key: str, + rules: Sequence[StoredFallback], +) -> None: proxy_config.reject_config_owned_writes( section_name=ROUTER_SETTINGS_PARAM, changed_keys={fallback_key: _JSON_RULES.validate_python(rules)} ) + if proxy_config.router_settings.owned_by_config(fallback_key): + return + await _persist_router_settings(prisma_client, {**stored_settings, fallback_key: rules}) async def _persist_router_settings(prisma_client: PrismaClient, router_settings: Mapping[str, object]) -> None: @@ -162,7 +190,8 @@ async def create_fallback( }, ) - router_settings: Final = await _router_settings_fresh_from_db(proxy_config) + await _reload_config_file_ownership(proxy_config) + router_settings: Final = await _stored_router_settings(prisma_client) # Get the appropriate fallback list based on type fallback_key = "fallbacks" @@ -172,7 +201,7 @@ async def create_fallback( fallback_key = "content_policy_fallbacks" # Get existing fallbacks - existing_fallbacks: Final = _STORED_FALLBACKS.validate_python(router_settings.get(fallback_key) or []) + existing_fallbacks: Final = _editable_rules(proxy_config, router_settings, fallback_key) # Update or add the fallback configuration fallback_updated = False @@ -187,12 +216,7 @@ async def create_fallback( # Add new fallback existing_fallbacks.append({data.model: data.fallback_models}) - _reject_config_owned_rules(proxy_config, fallback_key, existing_fallbacks) - - # Update router settings - router_settings[fallback_key] = existing_fallbacks - - await _persist_router_settings(prisma_client, router_settings) + await _write_rules(prisma_client, proxy_config, router_settings, fallback_key, existing_fallbacks) # Update the in-memory router configuration setattr(llm_router, fallback_key, existing_fallbacks) @@ -320,7 +344,8 @@ async def delete_fallback( }, ) - router_settings: Final = await _router_settings_fresh_from_db(proxy_config) + await _reload_config_file_ownership(proxy_config) + router_settings: Final = await _stored_router_settings(prisma_client) # Get the appropriate fallback list based on type fallback_key = "fallbacks" @@ -330,7 +355,7 @@ async def delete_fallback( fallback_key = "content_policy_fallbacks" # Get existing fallbacks - existing_fallbacks: Final = _STORED_FALLBACKS.validate_python(router_settings.get(fallback_key) or []) + existing_fallbacks: Final = _editable_rules(proxy_config, router_settings, fallback_key) updated_fallbacks: Final = [rule for rule in existing_fallbacks if not _rule_covers(rule, model)] @@ -340,12 +365,7 @@ async def delete_fallback( detail={"error": f"No {fallback_type} fallbacks configured for model '{model}'"}, ) - _reject_config_owned_rules(proxy_config, fallback_key, updated_fallbacks) - - # Update router settings - router_settings[fallback_key] = updated_fallbacks - - await _persist_router_settings(prisma_client, router_settings) + await _write_rules(prisma_client, proxy_config, router_settings, fallback_key, updated_fallbacks) # Update the in-memory router configuration setattr(llm_router, fallback_key, updated_fallbacks) diff --git a/litellm/proxy/mcp_server/oauth_issuer_stamp_backfill.py b/litellm/proxy/mcp_server/oauth_issuer_stamp_backfill.py index c417187396f..6c6be219a92 100644 --- a/litellm/proxy/mcp_server/oauth_issuer_stamp_backfill.py +++ b/litellm/proxy/mcp_server/oauth_issuer_stamp_backfill.py @@ -43,6 +43,7 @@ from litellm._logging import verbose_proxy_logger from litellm.proxy.db.db_span import db_span from litellm.proxy.mcp_server.oauth_utils import canonicalize_url_identity from litellm.proxy.utils import PrismaClient +from litellm.repositories.table_repositories import MCPServerRepository # The actor the removed discovery write-back stamped rows with. _DISCOVERY_ACTOR: Final = "mcp_oauth_discovery" @@ -106,7 +107,7 @@ def _is_stamped_issuer_row(row: _MCPServerRow) -> bool: async def backfill_discovery_stamped_issuers(prisma_client: PrismaClient) -> int: """Clear gateway-written issuer stamps, returning the number of rows healed.""" - candidate_rows: Final[list[_MCPServerRow]] = await prisma_client.db.litellm_mcpservertable.find_many( + candidate_rows: Final = await MCPServerRepository(prisma_client).table.find_many( where={ "updated_by": _DISCOVERY_ACTOR, "auth_type": {"in": list(_AUTH_TYPES_WITH_ISSUER_ANCHORING)}, diff --git a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py index 972ec18d487..f96fab75bd1 100644 --- a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py @@ -68,6 +68,7 @@ from litellm.llms.deepgram.common_utils import ( from litellm.llms.fal_ai.cost_calculator import fal_ai_passthrough_cost, fal_ai_queue_base from litellm.llms.nvidia_nim.passthrough.transformation import nvidia_nim_model_group_in_path from litellm.llms.openai.common_utils import OpenAIError as LiteLLMOpenAIError +from litellm.llms.openai.common_utils import is_openai_agents_api_path, openai_api_resource from litellm.llms.openai.workload_identity import get_workload_identity_bearer_token_for_api_base from litellm.llms.oss_decision import OssDecisionProvider, oss_connection, validate_oss_request from litellm.llms.vertex_ai.vertex_llm_base import VertexBase @@ -3530,6 +3531,10 @@ async def deepgram_listen_websocket_route( ) +_OPENAI_BETA_HEADER: Final = "openai-beta" +_CLIENT_OPENAI_BETA_RESOURCES: Final = frozenset({"agents", "vaults", "chatkit"}) + + class BaseOpenAIPassThroughHandler: @staticmethod async def base_openai_pass_through_handler( @@ -3540,7 +3545,7 @@ class BaseOpenAIPassThroughHandler: base_target_url: str, api_key: str | None, custom_llm_provider: litellm.LlmProviders, - extra_headers: dict | None = None, + extra_headers: Mapping[str, str] | None = None, ): encoded_endpoint = httpx.URL(endpoint).path # Ensure endpoint starts with '/' for proper URL construction @@ -3565,7 +3570,11 @@ class BaseOpenAIPassThroughHandler: endpoint=endpoint, target=str(updated_url), custom_headers=BaseOpenAIPassThroughHandler._assemble_headers( - api_key=api_key, request=request, extra_headers=extra_headers + api_key=api_key, + endpoint=endpoint, + request=request, + custom_llm_provider=custom_llm_provider, + extra_headers=extra_headers, ), is_streaming_request=is_streaming_request, custom_llm_provider=( @@ -3587,27 +3596,41 @@ class BaseOpenAIPassThroughHandler: _base_openai_pass_through_handler = base_openai_pass_through_handler @staticmethod - def _append_openai_beta_header(headers: dict, request: Request) -> dict: - """ - Appends the OpenAI-Beta header to the headers if the request is an OpenAI Assistants API request - """ - if RouteChecks.is_assistants_api_request(request) is True and "OpenAI-Beta" not in headers: - headers["OpenAI-Beta"] = "assistants=v2" - return headers + def _openai_beta(endpoint: str, request: Request, custom_llm_provider: litellm.LlmProviders) -> str | None: + if ( + custom_llm_provider == litellm.LlmProviders.OPENAI + and openai_api_resource(endpoint) in _CLIENT_OPENAI_BETA_RESOURCES + ): + return request.headers.get(_OPENAI_BETA_HEADER) or ( + "agents=v1" if is_openai_agents_api_path(endpoint) else None + ) + return "assistants=v2" if RouteChecks.is_assistants_api_request(request) else None @staticmethod - def _assemble_headers(api_key: str | None, request: Request, extra_headers: dict | None = None) -> dict: - base_headers = {} - if api_key is not None: - base_headers = { - "authorization": f"Bearer {api_key}", - "api-key": f"{api_key}", - } - if extra_headers is not None: - base_headers.update(extra_headers) + def _append_openai_beta_header( + headers: Mapping[str, str], endpoint: str, request: Request, custom_llm_provider: litellm.LlmProviders + ) -> dict[str, str]: + if any(name.lower() == _OPENAI_BETA_HEADER for name in headers): + return {**headers} + openai_beta: Final = BaseOpenAIPassThroughHandler._openai_beta(endpoint, request, custom_llm_provider) + return {**headers} if openai_beta is None else {**headers, _OPENAI_BETA_HEADER: openai_beta} + + @staticmethod + def _assemble_headers( + api_key: str | None, + endpoint: str, + request: Request, + custom_llm_provider: litellm.LlmProviders, + extra_headers: Mapping[str, str] | None = None, + ) -> dict[str, str]: + api_key_headers: Final[Mapping[str, str]] = ( + {} if api_key is None else {"authorization": f"Bearer {api_key}", "api-key": api_key} + ) return BaseOpenAIPassThroughHandler._append_openai_beta_header( - headers=base_headers, + headers={**api_key_headers, **(extra_headers or {})}, + endpoint=endpoint, request=request, + custom_llm_provider=custom_llm_provider, ) diff --git a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/cohere_passthrough_logging_handler.py b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/cohere_passthrough_logging_handler.py index a11b0be3983..4eeb9b26174 100644 --- a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/cohere_passthrough_logging_handler.py +++ b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/cohere_passthrough_logging_handler.py @@ -1,10 +1,12 @@ from datetime import datetime from typing import Final +from urllib.parse import urlparse import httpx import litellm from litellm import stream_chunk_builder +from litellm._logging import verbose_proxy_logger from litellm.litellm_core_utils.hidden_params import set_hidden_param from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.litellm_core_utils.litellm_logging import ( @@ -16,7 +18,7 @@ from litellm.llms.cohere.chat.v2_transformation import CohereV2ChatConfig from litellm.llms.cohere.common_utils import ( ModelResponseIterator as CohereModelResponseIterator, ) -from litellm.llms.cohere.embed.v1_transformation import CohereEmbeddingConfig +from litellm.llms.cohere.embed.transformation import CohereEmbeddingConfig, CohereEmbeddingRowShape from litellm.proxy._types import PassThroughEndpointLoggingTypedDict from litellm.types.passthrough_endpoints.pass_through_endpoints import ( PassthroughStandardLoggingPayload, @@ -29,6 +31,12 @@ from litellm.types.utils import ( from .base_passthrough_logging_handler import BasePassthroughLoggingHandler +COHERE_EMBED_PATHS: Final = ("/v1/embed", "/v2/embed") + + +def is_cohere_embed_route(url_route: str) -> bool: + return urlparse(url_route).path.rstrip("/").endswith(COHERE_EMBED_PATHS) + class CoherePassthroughLoggingHandler(BasePassthroughLoggingHandler): @property @@ -82,28 +90,24 @@ class CoherePassthroughLoggingHandler(BasePassthroughLoggingHandler): """ Handle Cohere passthrough logging with route detection and cost tracking. """ - # Check if this is an embed endpoint - if "/v1/embed" in url_route and "/v1/embeddings" not in url_route: + if is_cohere_embed_route(url_route): model: Final = request_body.get("model", response_body.get("model", "")) try: - cohere_embed_config: Final = CohereEmbeddingConfig() - litellm_model_response = litellm.EmbeddingResponse() handler_instance: Final = CoherePassthroughLoggingHandler() input_texts = request_body.get("texts", []) if not input_texts: input_texts = request_body.get("input", []) - # Transform the response - litellm_model_response = cohere_embed_config.transform_response( + litellm_model_response: Final = CohereEmbeddingConfig().transform_native_embed_response( response=httpx_response, api_key="", logging_obj=logging_obj, data=request_body, - model_response=litellm_model_response, model=model, encoding=litellm.encoding, input=input_texts, + row_shape=CohereEmbeddingRowShape.TYPED, ) # Calculate cost using LiteLLM's cost calculator @@ -153,20 +157,11 @@ class CoherePassthroughLoggingHandler(BasePassthroughLoggingHandler): "result": litellm_model_response, "kwargs": kwargs, } - except Exception: - # For other routes (e.g., /v2/chat), fall back to chat handler - return super().passthrough_chat_handler( - httpx_response=httpx_response, - response_body=response_body, - logging_obj=logging_obj, - url_route=url_route, - result=result, - start_time=start_time, - end_time=end_time, - cache_hit=cache_hit, - request_body=request_body, - **kwargs, + except Exception as e: + verbose_proxy_logger.warning( + "Could not price the Cohere passthrough embed response, logging it without a cost: %s", e ) + return {"result": None, "kwargs": kwargs} # For non-embed routes (e.g., /v2/chat), fall back to chat handler return super().passthrough_chat_handler( diff --git a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/openai_passthrough_logging_handler.py b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/openai_passthrough_logging_handler.py index c410150df70..b98dd42e3de 100644 --- a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/openai_passthrough_logging_handler.py +++ b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/openai_passthrough_logging_handler.py @@ -10,7 +10,7 @@ from typing import Final, Literal, TypeAlias from urllib.parse import urlparse import httpx -from pydantic import TypeAdapter +from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError import litellm from litellm._logging import verbose_proxy_logger @@ -21,6 +21,8 @@ from litellm.litellm_core_utils.litellm_logging import ( get_standard_logging_object_payload, ) from litellm.litellm_core_utils.token_counter import high_detail_image_token_upper_bound +from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator +from litellm.llms.openai.common_utils import is_openai_agents_api_path from litellm.llms.openai.openai import OpenAIConfig from litellm.llms.openai.openai import OpenAIConfig as OpenAIConfigType from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig @@ -38,7 +40,13 @@ from litellm.types.passthrough_endpoints.pass_through_endpoints import ( EndpointType, PassthroughStandardLoggingPayload, ) -from litellm.types.utils import EmbeddingResponse, ImageResponse, LlmProviders, PassthroughCallTypes +from litellm.types.utils import ( + EmbeddingResponse, + ImageResponse, + LlmProviders, + PassthroughCallTypes, + StandardPassThroughResponseObject, +) from litellm.utils import ModelResponse, TextCompletionResponse, convert_to_model_response_object # Hostnames that route to OpenAI-compatible APIs. @@ -121,6 +129,52 @@ def _responses_sub_route(path: str) -> tuple[str, ...] | None: return None if anchor is None else tuple(segments[anchor + 1 :]) +_STR_MAPPING: Final = TypeAdapter(Mapping[str, object]) + + +class _AgentConfig(BaseModel): + model_config = ConfigDict(frozen=True) + + model: str + + +class _AgentsApiBody(BaseModel): + model_config = ConfigDict(frozen=True) + + agent: _AgentConfig + + +class _AgentsApiSessionEvent(BaseModel): + model_config = ConfigDict(frozen=True) + + session: _AgentsApiBody + + +_AGENTS_API_BODY: Final = TypeAdapter(_AgentsApiBody | _AgentConfig) + + +def _agent_model(body: Mapping[str, object]) -> str | None: + try: + parsed: Final = _AGENTS_API_BODY.validate_python(body) + except ValidationError: + return None + return parsed.agent.model if isinstance(parsed, _AgentsApiBody) else parsed.model + + +def _session_event_model(chunk: str) -> str | None: + event: Final = BaseModelResponseIterator.string_to_dict_parser(str_line=chunk) + if event is None: + return None + try: + return _AgentsApiSessionEvent.model_validate(event).session.agent.model + except ValidationError: + return None + + +def _streamed_agent_model(all_chunks: Sequence[str]) -> str | None: + return next((model for chunk in all_chunks if (model := _session_event_model(chunk)) is not None), None) + + def _is_remote_high_detail_image(part: object) -> bool: if not isinstance(part, Mapping) or part.get("type") != "image_url": return False @@ -220,14 +274,44 @@ class OpenAIPassthroughLoggingHandler(BasePassthroughLoggingHandler): return "billed" @staticmethod - def _label_unbilled_responses_call( + def is_openai_agents_route(url_route: str) -> bool: + return is_openai_compatible_url(url_route) and is_openai_agents_api_path(urlparse(url_route).path) + + @staticmethod + def _unbilled_call_model( + url_route: str, request_body: Mapping[str, object], response_body: Mapping[str, object] + ) -> str | None: + if OpenAIPassthroughLoggingHandler.is_openai_agents_route(url_route): + return _agent_model(response_body) or _agent_model(request_body) + response_model: Final = response_body.get("model") + if OpenAIPassthroughLoggingHandler.is_openai_responses_route(url_route) and isinstance(response_model, str): + return response_model + return None + + @staticmethod + def _label_model(logging_obj: LiteLLMLoggingObj, model: str | None) -> None: + if model is not None: + logging_obj.model_call_details["model"] = model # rebind-ok: spend rows read the model from here + + @staticmethod + def _label_unbilled_call( logging_obj: LiteLLMLoggingObj, url_route: str, + request_body: Mapping[str, object], response_body: Mapping[str, object], ) -> None: - model: Final = response_body.get("model") - if OpenAIPassthroughLoggingHandler.is_openai_responses_route(url_route) and isinstance(model, str): - logging_obj.model_call_details["model"] = model # rebind-ok: spend rows read the model from here + OpenAIPassthroughLoggingHandler._label_model( + logging_obj, OpenAIPassthroughLoggingHandler._unbilled_call_model(url_route, request_body, response_body) + ) + + @staticmethod + def _log_agents_stream( + litellm_logging_obj: LiteLLMLoggingObj, request_body: Mapping[str, object], all_chunks: Sequence[str] + ) -> PassThroughEndpointLoggingTypedDict: + OpenAIPassthroughLoggingHandler._label_model( + litellm_logging_obj, _streamed_agent_model(all_chunks) or _agent_model(request_body) + ) + return {"result": StandardPassThroughResponseObject(response="\n".join(all_chunks)), "kwargs": {}} @staticmethod def is_openai_embeddings_route(url_route: str) -> bool: @@ -397,7 +481,12 @@ class OpenAIPassthroughLoggingHandler(BasePassthroughLoggingHandler): is_responses: Final = responses_billing != "unbilled" if not (is_chat_completions or is_embeddings or is_image_generation or is_image_editing or is_responses): - OpenAIPassthroughLoggingHandler._label_unbilled_responses_call(logging_obj, url_route, response_body) + OpenAIPassthroughLoggingHandler._label_unbilled_call( + logging_obj, + url_route, + request_body=_STR_MAPPING.validate_python(request_body), + response_body=_STR_MAPPING.validate_python(response_body), + ) return { "result": None, "kwargs": kwargs, @@ -676,6 +765,10 @@ class OpenAIPassthroughLoggingHandler(BasePassthroughLoggingHandler): """ Handle logging for collected OpenAI streaming chunks with cost tracking. """ + if OpenAIPassthroughLoggingHandler.is_openai_agents_route(url_route): + return OpenAIPassthroughLoggingHandler._log_agents_stream( + litellm_logging_obj, _STR_MAPPING.validate_python(request_body), all_chunks + ) try: is_responses: Final = OpenAIPassthroughLoggingHandler.is_openai_responses_route(url_route) terminal_response: Final = ( diff --git a/litellm/proxy/pass_through_endpoints/success_handler.py b/litellm/proxy/pass_through_endpoints/success_handler.py index 9c6c3111209..27e31c8f74d 100644 --- a/litellm/proxy/pass_through_endpoints/success_handler.py +++ b/litellm/proxy/pass_through_endpoints/success_handler.py @@ -22,6 +22,7 @@ from .llm_provider_handlers.assembly_passthrough_logging_handler import ( ) from .llm_provider_handlers.cohere_passthrough_logging_handler import ( CoherePassthroughLoggingHandler, + is_cohere_embed_route, ) from .llm_provider_handlers.cursor_passthrough_logging_handler import ( CursorPassthroughLoggingHandler, @@ -91,7 +92,7 @@ class PassThroughEndpointLogging: self.TRACKED_ANTHROPIC_ROUTES = ["/messages", "/v1/messages/batches"] # Cohere - self.TRACKED_COHERE_ROUTES = ["/v2/chat", "/v1/embed"] + self.TRACKED_COHERE_ROUTES = ["/v2/chat"] self.assemblyai_passthrough_logging_handler = AssemblyAIPassthroughLoggingHandler() # Langfuse @@ -536,13 +537,7 @@ class PassThroughEndpointLogging: return False def is_cohere_route(self, url_route: str) -> bool: - for route in self.TRACKED_COHERE_ROUTES: - if route not in url_route: - continue - if route == "/v1/embed" and "/v1/embeddings" in url_route: - continue - return True - return False + return is_cohere_embed_route(url_route) or any(route in url_route for route in self.TRACKED_COHERE_ROUTES) def is_assemblyai_route(self, url_route: str): parsed_url: Final = urlparse(url_route) @@ -645,6 +640,7 @@ class PassThroughEndpointLogging: or OpenAIPassthroughLoggingHandler.is_openai_image_generation_route(url_route) or OpenAIPassthroughLoggingHandler.is_openai_image_editing_route(url_route) or OpenAIPassthroughLoggingHandler.is_openai_responses_route(url_route) + or OpenAIPassthroughLoggingHandler.is_openai_agents_route(url_route) ) def _set_cost_per_request( diff --git a/litellm/proxy/plugin_routes.py b/litellm/proxy/plugin_routes.py index 1dfe5cbc6bb..cfbcfefe9e8 100644 --- a/litellm/proxy/plugin_routes.py +++ b/litellm/proxy/plugin_routes.py @@ -31,6 +31,8 @@ from typing import Final from cryptography.fernet import Fernet, InvalidToken from fastapi import APIRouter, Depends, HTTPException, Request, Response +from pydantic import ConfigDict, TypeAdapter, with_config +from typing_extensions import NotRequired, ReadOnly, TypedDict from litellm.llms.custom_httpx.http_handler import get_async_httpx_client from litellm.proxy._types import PluginConfig, SpecialHeaders, UserAPIKeyAuth @@ -39,6 +41,18 @@ from litellm.types.llms.custom_http import httpxSpecialProvider router: Final = APIRouter() + +@with_config(ConfigDict(extra="allow", strict=True)) +class PluginSessionClaim(TypedDict): + plugin: NotRequired[ReadOnly[object]] + user_id: NotRequired[ReadOnly[str]] + user_role: NotRequired[ReadOnly[str]] + exp: NotRequired[ReadOnly[int | float | str]] + + +_PLUGIN_SESSION_CLAIM: Final = TypeAdapter(PluginSessionClaim) + + # Hop-by-hop headers (RFC 7230) and the litellm session cookie — never forwarded # to a plugin backend. Credential headers are added on top per-request from the # canonical SpecialHeaders set so the plugin only ever authenticates via its own @@ -155,7 +169,7 @@ def issue_plugin_session_claim(plugin_name: str, user_id: str | None, user_role: return _plugin_fernet(plugin_name).encrypt(json.dumps(claim).encode()).decode() -def verify_plugin_session_claim(plugin_name: str, ciphertext: str) -> dict: +def verify_plugin_session_claim(plugin_name: str, ciphertext: str) -> PluginSessionClaim: """Verify and decode a plugin session claim. Raises ValueError if the HMAC is invalid, the audience is wrong, or @@ -163,7 +177,7 @@ def verify_plugin_session_claim(plugin_name: str, ciphertext: str) -> dict: """ try: raw: Final = _plugin_fernet(plugin_name).decrypt(ciphertext.encode(), ttl=_CLAIM_TTL_SECONDS) - claim: Final = json.loads(raw) + claim: Final = _PLUGIN_SESSION_CLAIM.validate_python(json.loads(raw)) except (InvalidToken, Exception) as exc: raise ValueError("Invalid, tampered, or expired plugin session claim") from exc diff --git a/litellm/proxy/public_endpoints/provider_create_fields.json b/litellm/proxy/public_endpoints/provider_create_fields.json index fd17079b15f..4237517508b 100644 --- a/litellm/proxy/public_endpoints/provider_create_fields.json +++ b/litellm/proxy/public_endpoints/provider_create_fields.json @@ -2023,6 +2023,34 @@ ], "default_model_placeholder": "jina_ai/" }, + { + "provider": "KYMA", + "provider_display_name": "Kyma API", + "litellm_provider": "kyma", + "credential_fields": [ + { + "key": "api_base", + "label": "API Base", + "placeholder": "https://kymaapi.com/v1", + "tooltip": null, + "required": false, + "field_type": "text", + "options": null, + "default_value": null + }, + { + "key": "api_key", + "label": "API Key", + "placeholder": null, + "tooltip": null, + "required": true, + "field_type": "password", + "options": null, + "default_value": null + } + ], + "default_model_placeholder": "kyma/qwen3.7-flash" + }, { "provider": "Laya", "provider_display_name": "Laya", diff --git a/litellm/proxy/spend_tracking/spend_tracking_utils.py b/litellm/proxy/spend_tracking/spend_tracking_utils.py index 94b079163e9..833ee71ae9c 100644 --- a/litellm/proxy/spend_tracking/spend_tracking_utils.py +++ b/litellm/proxy/spend_tracking/spend_tracking_utils.py @@ -80,7 +80,7 @@ from litellm.types.passthrough_endpoints.pass_through_endpoints import ( SETTLES_BACKGROUND_RESPONSE_KWARG, BackgroundResponseSettlement, ) -from litellm.types.router import DeploymentTypedDict, LiteLLM_Params +from litellm.types.router import DeploymentTypedDict, LiteLLM_Params, SilentExperimentFlag from litellm.types.utils import ( BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN, PROMPT_CARRYING_GUARDRAIL_FIELDS, @@ -194,6 +194,7 @@ _STAMPED_METADATA_KEYS: Final = frozenset( "used_client_oauth_token", KUBERNETES_POD_ROUTING_KEY, "litellm_roi_estimator", + "is_silent_experiment", BILLED_RETRY_ATTEMPT_METADATA_KEY, "user_api_key_user_email", ) @@ -227,6 +228,7 @@ def _get_spend_logs_metadata( azure_spillover: AzureSpillover | None = None, used_client_oauth_token: bool | None = None, kubernetes_pod_routing: StandardLoggingKubernetesPodRouting | None = None, + is_silent_experiment: bool = False, ) -> SpendLogsMetadata: if metadata is None: return SpendLogsMetadata( @@ -294,6 +296,7 @@ def _get_spend_logs_metadata( used_client_oauth_token=used_client_oauth_token, kubernetes_pod_routing=kubernetes_pod_routing, litellm_roi_estimator=metadata.get("litellm_roi_estimator") is True, + **(SilentExperimentFlag(is_silent_experiment=True) if is_silent_experiment else SilentExperimentFlag()), billed_retry_attempt=billed_retry_attempt, user_api_key_user_email=_stored_user_email(metadata), ) @@ -738,6 +741,11 @@ def get_logging_payload( # clean up litellm metadata clean_metadata = _get_spend_logs_metadata( persisted_metadata, + is_silent_experiment=( + standard_logging_payload is not None + and "is_silent_experiment" in standard_logging_payload["metadata"] + and standard_logging_payload["metadata"]["is_silent_experiment"] + ), applied_guardrails=( standard_logging_payload["metadata"].get("applied_guardrails", None) if standard_logging_payload is not None diff --git a/litellm/responses/main.py b/litellm/responses/main.py index 11e1602e9f8..98356e9a5d4 100644 --- a/litellm/responses/main.py +++ b/litellm/responses/main.py @@ -93,7 +93,6 @@ from litellm.types.utils import ( AgenticLoopParams, CallTypes, CostPerToken, - CustomPricingLiteLLMParams, all_litellm_params, ) from litellm.utils import ( @@ -566,12 +565,6 @@ async def aresponses_api_with_mcp( return response -def _per_call_custom_pricing(litellm_params: GenericLiteLLMParams) -> Mapping[str, float]: - return MappingProxyType( - litellm_params.model_dump(include=set(CustomPricingLiteLLMParams.model_fields), exclude_none=True) - ) - - class _DeploymentModelInfo(BaseModel): model_config = ConfigDict(frozen=True) id: object = None @@ -1800,7 +1793,7 @@ def responses( optional_params=dict(responses_api_request_params), litellm_params={ **responses_api_request_params, - **_per_call_custom_pricing(litellm_params), + **litellm_params.custom_pricing_overrides(), "aresponses": _is_async, "litellm_call_id": litellm_call_id, "model_info": kwargs.get("model_info"), diff --git a/litellm/router.py b/litellm/router.py index 3809a5d6555..ff31f48c49e 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -75,7 +75,7 @@ from typing_extensions import assert_never, overload import litellm import litellm.litellm_core_utils.exception_mapping_utils from litellm import get_secret_str -from litellm._internal_context import service_target, with_service_target +from litellm._internal_context import in_mirrored_call, mirrored_call, service_target, with_service_target from litellm._logging import verbose_router_logger from litellm._uuid import uuid from litellm.caching.caching import ( @@ -754,7 +754,7 @@ def _every_deployment_refused( def _silent_experiment_targets(silent_model: object) -> tuple[str, ...]: - if silent_model is None: + if silent_model is None or in_mirrored_call(): return () try: targets: Final = _SILENT_MODEL_ADAPTER.validate_python(silent_model) @@ -3372,16 +3372,17 @@ class Router: remaining_kwargs: Final = MappingProxyType( {key: value for key, value in silent_kwargs.items() if key != "stream"} ) - response: Final = await self.acompletion( - model=silent_model, - messages=cast(list[AllMessageValues], messages), - stream=bool(silent_kwargs.get("stream", False)), - **remaining_kwargs, - ) - if not isinstance(response, CustomStreamWrapper): - return - async for _ in response: - pass + with mirrored_call(): + response: Final = await self.acompletion( + model=silent_model, + messages=cast(list[AllMessageValues], messages), + stream=bool(silent_kwargs.get("stream", False)), + **remaining_kwargs, + ) + if not isinstance(response, CustomStreamWrapper): + return + async for _ in response: + pass def _silent_experiment_completion(self, silent_model: str, messages: Sequence[Mapping[str, str]], **kwargs): """ diff --git a/litellm/types/integrations/custom_logger.py b/litellm/types/integrations/custom_logger.py index 13542f22f88..77e75dea030 100644 --- a/litellm/types/integrations/custom_logger.py +++ b/litellm/types/integrations/custom_logger.py @@ -97,9 +97,9 @@ class AgenticLoopRequestPatch(LiteLLMBaseModel): model: str | None = None messages: list[dict[str, Any]] | None = None - tools: list[dict[str, Any]] | None = None + tools: list[dict[str, object]] | None = None max_tokens: int | None = None - optional_params: dict[str, Any] = Field(default_factory=dict) + optional_params: dict[str, object] = Field(default_factory=dict) kwargs: dict[str, Any] = Field(default_factory=dict) diff --git a/litellm/types/integrations/datadog_llm_obs.py b/litellm/types/integrations/datadog_llm_obs.py index f4fcabf53ed..f2042cb7967 100644 --- a/litellm/types/integrations/datadog_llm_obs.py +++ b/litellm/types/integrations/datadog_llm_obs.py @@ -5,7 +5,7 @@ API Reference: https://docs.datadoghq.com/llm_observability/setup/api/?tab=examp """ from collections.abc import Sequence -from typing import Any, Literal +from typing import Literal from typing_extensions import ReadOnly, TypedDict @@ -16,7 +16,7 @@ class ToolCall(TypedDict, total=False): """A tool call on a message, as LLM Obs names its fields.""" name: ReadOnly[str] - arguments: ReadOnly[dict[str, Any] | str] # parsed object, or the raw string when it will not parse to one + arguments: ReadOnly[dict[str, object] | str] # parsed object, or the raw string when it will not parse to one tool_id: ReadOnly[str] type: ReadOnly[str] @@ -35,7 +35,7 @@ class ToolDefinition(TypedDict, total=False): name: ReadOnly[str] description: ReadOnly[str] - schema: ReadOnly[dict[str, Any]] + schema: ReadOnly[dict[str, object]] class Message(TypedDict, total=False): @@ -49,13 +49,15 @@ class Message(TypedDict, total=False): class InputMeta(TypedDict): - messages: Sequence[ - Message | dict[str, Any] # changed to fit with tool calls + messages: ReadOnly[ + Sequence[ + Message | dict[str, object] # changed to fit with tool calls + ] ] # Relevant Issue: https://github.com/BerriAI/litellm/issues/9494 class OutputMeta(TypedDict): - messages: Sequence[Any] + messages: ReadOnly[Sequence[object]] class DDLLMObsError(TypedDict, total=False): @@ -71,7 +73,7 @@ class Meta(TypedDict, total=False): kind: Literal["llm", "tool", "task", "embedding", "retrieval"] input: InputMeta # The span's input information. output: OutputMeta # The span's output information. - metadata: dict[str, Any] + metadata: ReadOnly[dict[str, object]] error: DDLLMObsError | None # Error information on the span tool_definitions: ReadOnly[Sequence[ToolDefinition]] # The tools offered to the model on this request diff --git a/litellm/types/llms/anthropic.py b/litellm/types/llms/anthropic.py index de83e6ea66e..50c4a2340a3 100644 --- a/litellm/types/llms/anthropic.py +++ b/litellm/types/llms/anthropic.py @@ -832,6 +832,7 @@ ANTHROPIC_EFFORT_BETA_HEADER: Final = "effort-2025-11-24" ANTHROPIC_MID_CONVERSATION_OUTPUT_CONFIG_BETA_HEADER: Final = "mid-conversation-output-config-2026-07-01" ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER: Final = "thinking-display-updates-2026-08-18" +ANTHROPIC_THINKING_BINDING_CONTROLS_BETA_HEADER: Final = "thinking-binding-controls-2026-08-01" ANTHROPIC_MID_CONVERSATION_TOOL_CHANGES_BETA_HEADER: Final = "mid-conversation-tool-changes-2026-07-01" ANTHROPIC_FINE_GRAINED_TOOL_STREAMING_BETA_HEADER: Final = "fine-grained-tool-streaming-2025-05-14" diff --git a/litellm/types/llms/openai.py b/litellm/types/llms/openai.py index 6ce51abcd8a..d9d86cf54b3 100644 --- a/litellm/types/llms/openai.py +++ b/litellm/types/llms/openai.py @@ -1600,7 +1600,7 @@ class ResponsesAPIResponse(BaseLiteLLMOpenAIResponseObject): @field_validator("reasoning", mode="before") @classmethod - def validate_reasoning_to_dict(cls, value: Any) -> dict[str, Any] | None: + def validate_reasoning_to_dict(cls, value: Any) -> builtins.object: """Accept API reasoning dict (including effort 'none'/'xhigh'); always store as dict.""" if value is None: return None diff --git a/litellm/types/llms/vertex_ai.py b/litellm/types/llms/vertex_ai.py index 7cb5432d009..a938f6a7b1c 100644 --- a/litellm/types/llms/vertex_ai.py +++ b/litellm/types/llms/vertex_ai.py @@ -1,6 +1,6 @@ from collections.abc import Sequence from enum import Enum -from typing import Any, Final, Literal, Protocol +from typing import Final, Literal, Protocol from typing_extensions import ( ReadOnly, @@ -140,7 +140,7 @@ class Schema(TypedDict, total=False): title: str description: str nullable: bool - default: Any + default: ReadOnly[object] items: "Schema" minItems: str maxItems: str @@ -155,7 +155,7 @@ class Schema(TypedDict, total=False): minLength: str maxLength: str pattern: str - example: Any + example: ReadOnly[object] anyOf: list["Schema"] @@ -794,9 +794,9 @@ class VertexVideoGenerationResponse(TypedDict, total=False): name: str done: bool - response: dict[str, Any] - metadata: dict[str, Any] - error: dict[str, Any] + response: ReadOnly[dict[str, object]] + metadata: ReadOnly[dict[str, object]] + error: ReadOnly[dict[str, object]] VERTEX_CREDENTIALS_TYPES = str | dict[str, str] diff --git a/litellm/types/proxy/guardrails/guardrail_hooks/generic_guardrail_api.py b/litellm/types/proxy/guardrails/guardrail_hooks/generic_guardrail_api.py index d38e54986b9..4e4983c53f0 100644 --- a/litellm/types/proxy/guardrails/guardrail_hooks/generic_guardrail_api.py +++ b/litellm/types/proxy/guardrails/guardrail_hooks/generic_guardrail_api.py @@ -48,7 +48,7 @@ class GenericGuardrailAPIMetadata(TypedDict, total=False): class GenericGuardrailAPIOptionalParams(LiteLLMBaseModel): """Optional parameters for the Generic Guardrail API""" - additional_provider_specific_params: dict[str, Any] | None = Field( + additional_provider_specific_params: dict[str, object] | None = Field( default=None, description="Additional provider-specific parameters to send with the guardrail request", ) @@ -161,7 +161,7 @@ class GenericGuardrailAPIRequest(LiteLLMBaseModel): default=None, description="LiteLLM library version running this proxy.", ) - additional_provider_specific_params: dict[str, Any] | None = None + additional_provider_specific_params: dict[str, object] | None = None tool_calls: list[ChatCompletionToolCallChunk] | list[ChatCompletionMessageToolCall] | None = None model: str | None = None # the model being used for the LLM call diff --git a/litellm/types/proxy/guardrails/guardrail_hooks/ibm/ibm_detector.py b/litellm/types/proxy/guardrails/guardrail_hooks/ibm/ibm_detector.py index b58d1bff8a2..b04be71dc9c 100644 --- a/litellm/types/proxy/guardrails/guardrail_hooks/ibm/ibm_detector.py +++ b/litellm/types/proxy/guardrails/guardrail_hooks/ibm/ibm_detector.py @@ -1,7 +1,5 @@ -from typing import Any - from pydantic import Field -from typing_extensions import TypedDict +from typing_extensions import ReadOnly, TypedDict from litellm.types.llms.base import LiteLLMBaseModel @@ -14,41 +12,41 @@ from .base import IBMGuardrailsBaseConfigModel class IBMDetectorRequestBodyDetectorServer(TypedDict): """Request body for calling IBM Detector Server directly""" - contents: list[str] - detector_params: dict[str, Any] + contents: ReadOnly[list[str]] + detector_params: ReadOnly[dict[str, object]] class IBMDetectorRequestBodyOrchestrator(TypedDict): """Request body for calling IBM Detector via FMS Guardrails Orchestrator""" - content: str - detectors: dict[str, dict[str, Any]] + content: ReadOnly[str] + detectors: ReadOnly[dict[str, dict[str, object]]] class IBMDetectorDetection(TypedDict, total=False): """Individual detection from IBM Detector""" - start: int - end: int - text: str - detection: str - detection_type: str - score: float - evidences: list[Any] - metadata: dict[str, Any] - detector_id: str | None # Only present in orchestrator response + start: ReadOnly[int] + end: ReadOnly[int] + text: ReadOnly[str] + detection: ReadOnly[str] + detection_type: ReadOnly[str] + score: ReadOnly[float] + evidences: ReadOnly[list[object]] + metadata: ReadOnly[dict[str, object]] + detector_id: ReadOnly[str | None] # Only present in orchestrator response class IBMDetectorResponseDetectorServer(TypedDict): """Response from IBM Detector Server (returns list of lists)""" - detections: list[list[IBMDetectorDetection]] + detections: ReadOnly[list[list[IBMDetectorDetection]]] class IBMDetectorResponseOrchestrator(TypedDict): """Response from IBM FMS Guardrails Orchestrator""" - detections: list[IBMDetectorDetection] + detections: ReadOnly[list[IBMDetectorDetection]] # Pydantic Config Models @@ -57,12 +55,12 @@ class IBMDetectorResponseOrchestrator(TypedDict): class IBMDetectorOptionalParams(LiteLLMBaseModel): """Optional parameters for IBM Detector guardrail""" - detector_params: dict[str, Any] | None = Field( + detector_params: dict[str, object] | None = Field( default_factory=lambda: {}, description="Dictionary of arguments to pass to the detector.", ) - extra_headers: dict[str, Any] | None = Field( + extra_headers: dict[str, object] | None = Field( default_factory=lambda: {}, description="Dictionary of extra headers to pass to the detector.", ) diff --git a/litellm/types/proxy/management_endpoints/key_management_endpoints.py b/litellm/types/proxy/management_endpoints/key_management_endpoints.py index f4348391710..d4f41b538e6 100644 --- a/litellm/types/proxy/management_endpoints/key_management_endpoints.py +++ b/litellm/types/proxy/management_endpoints/key_management_endpoints.py @@ -1,5 +1,5 @@ from datetime import datetime -from typing import Any, Final, Literal, TypeAlias +from typing import Final, Literal, TypeAlias from pydantic import ConfigDict, model_validator from typing_extensions import ReadOnly, TypedDict @@ -50,14 +50,14 @@ class SuccessfulKeyUpdate(LiteLLMBaseModel): """Successfully updated key with its updated information""" key: str - key_info: dict[str, Any] + key_info: dict[str, object] class FailedKeyUpdate(LiteLLMBaseModel): """Failed key update with reason""" key: str - key_info: dict[str, Any] | None = None + key_info: dict[str, object] | None = None failed_reason: str @@ -78,14 +78,14 @@ class KeyUpdateFields(LiteLLMBaseModel): max_budget: float | None = None budget_id: str | None = None budget_duration: str | None = None - budget_limits: list[Any] | None = None - model_max_budget: dict[str, Any] | None = None + budget_limits: list[object] | None = None + model_max_budget: dict[str, object] | None = None # Rate limits tpm_limit: int | None = None rpm_limit: int | None = None - model_tpm_limit: dict[str, Any] | None = None - model_rpm_limit: dict[str, Any] | None = None + model_tpm_limit: dict[str, object] | None = None + model_rpm_limit: dict[str, object] | None = None max_parallel_requests: int | None = None rpm_limit_type: Literal["guaranteed_throughput", "best_effort_throughput", "dynamic"] | None = None tpm_limit_type: Literal["guaranteed_throughput", "best_effort_throughput", "dynamic"] | None = None @@ -99,7 +99,7 @@ class KeyUpdateFields(LiteLLMBaseModel): # Operational metadata tags: list[str] | None = None - metadata: dict[str, Any] | None = None + metadata: dict[str, object] | None = None @model_validator(mode="after") def validate_temp_budget(self) -> "KeyUpdateFields": diff --git a/litellm/types/router.py b/litellm/types/router.py index 60c9e596588..db8be9e91a3 100644 --- a/litellm/types/router.py +++ b/litellm/types/router.py @@ -67,6 +67,10 @@ from .utils import ( AllowedModelRegion: TypeAlias = Literal["eu", "us"] +class SilentExperimentFlag(TypedDict, total=False): + is_silent_experiment: ReadOnly[bool] + + class FallbackSkipReason(str, enum.Enum): NOT_AUTHORIZED = "not_authorized" OVER_BUDGET = "over_budget" diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 9c3329d6387..e9984a35ad1 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -3374,6 +3374,7 @@ class StandardLoggingMetadata(StandardLoggingUserAPIKeyMetadata): team_alias: str | None team_id: str | None used_client_oauth_token: ReadOnly[bool | None] + is_silent_experiment: ReadOnly[NotRequired[bool]] class AzureSpillover(TypedDict): @@ -4130,6 +4131,18 @@ class CustomPricingLiteLLMParams(MirroredPricingParams): """ return {k: v for k, v in model_info.items() if k not in cls.model_fields} + def custom_pricing_overrides(self) -> Mapping[str, object]: + return MappingProxyType( + self.model_dump(include=set(CustomPricingLiteLLMParams.model_fields), exclude_none=True) + ) + + def per_second_pricing(self) -> Mapping[str, float]: + return MappingProxyType( + self.model_dump( + include={"cost_per_second", "input_cost_per_second", "output_cost_per_second"}, exclude_none=True + ) + ) + DEPLOYMENT_SCOPED_PRICING_FIELDS: Final[frozenset[str]] = frozenset({"off_peak_pricing"}) @@ -4599,6 +4612,7 @@ class LlmProviders(str, Enum): CURSOR = "cursor" BEDROCK_MANTLE = "bedrock_mantle" GDC = "gdc" + KYMA = "kyma" # Create a set of all provider values for quick lookup diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index aa00ea73f8c..4fad4857c60 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -78341,6 +78341,1254 @@ "supports_xhigh_reasoning_effort": true, "use_openai_responses_path": true }, + "kyma/claude-fable-5": { + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "input_cost_per_token": 1.35e-05, + "output_cost_per_token": 6.75e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true, + "supports_adaptive_thinking": true, + "thinking_always_on": true + }, + "kyma/claude-fable-5.1": { + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "input_cost_per_token": 1.35e-05, + "output_cost_per_token": 6.75e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true, + "supports_adaptive_thinking": true, + "thinking_always_on": true + }, + "kyma/claude-haiku-4-5": { + "max_input_tokens": 200000, + "max_output_tokens": 64000, + "input_cost_per_token": 1.35e-06, + "cache_read_input_token_cost": 1.35e-07, + "output_cost_per_token": 6.75e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false, + "prompt_cache_min_tokens": 4096 + }, + "kyma/claude-haiku-5.5": { + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "input_cost_per_token": 1.35e-07, + "cache_read_input_token_cost": 1.35e-08, + "output_cost_per_token": 6.75e-07, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true, + "supports_adaptive_thinking": true, + "thinking_always_on": false, + "prompt_cache_min_tokens": 512 + }, + "kyma/claude-opus-4-7": { + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "input_cost_per_token": 6.75e-06, + "cache_read_input_token_cost": 6.75e-07, + "output_cost_per_token": 3.375e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false, + "supports_adaptive_thinking": true, + "prompt_cache_min_tokens": 2048 + }, + "kyma/claude-opus-5": { + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "input_cost_per_token": 6.75e-06, + "cache_read_input_token_cost": 6.75e-07, + "output_cost_per_token": 3.375e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true, + "supports_adaptive_thinking": true, + "prompt_cache_min_tokens": 512 + }, + "kyma/claude-opus-5.5": { + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "input_cost_per_token": 5.4e-06, + "cache_read_input_token_cost": 2.7e-07, + "output_cost_per_token": 2.7e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true, + "supports_adaptive_thinking": true, + "thinking_always_on": true, + "prompt_cache_min_tokens": 512 + }, + "kyma/claude-sonnet-4-6": { + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "input_cost_per_token": 4.05e-06, + "cache_read_input_token_cost": 4.05e-07, + "output_cost_per_token": 2.025e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false, + "supports_adaptive_thinking": true, + "prompt_cache_min_tokens": 1024 + }, + "kyma/claude-sonnet-5": { + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "input_cost_per_token": 2.7e-06, + "cache_read_input_token_cost": 2.7e-07, + "output_cost_per_token": 1.35e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true, + "supports_adaptive_thinking": true, + "prompt_cache_min_tokens": 1024 + }, + "kyma/claude-sonnet-5.5": { + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "input_cost_per_token": 2.7e-06, + "cache_read_input_token_cost": 1.35e-07, + "output_cost_per_token": 1.35e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true, + "supports_adaptive_thinking": true, + "thinking_always_on": true, + "prompt_cache_min_tokens": 512 + }, + "kyma/deepseek-r1": { + "max_input_tokens": 64000, + "max_output_tokens": 32768, + "input_cost_per_token": 7.425e-07, + "output_cost_per_token": 2.957e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/deepseek-v3": { + "max_input_tokens": 160000, + "max_output_tokens": 8192, + "input_cost_per_token": 5.18e-07, + "output_cost_per_token": 7.571e-07, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/deepseek-v4-flash": { + "max_input_tokens": 1000000, + "max_output_tokens": 65536, + "input_cost_per_token": 1.23e-07, + "cache_read_input_token_cost": 2.43e-08, + "output_cost_per_token": 2.458e-07, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/deepseek-v4-flash-vision-exp": { + "max_input_tokens": 1048576, + "max_output_tokens": 384000, + "input_cost_per_token": 3.892e-07, + "output_cost_per_token": 1.168e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/deepseek-v4.1-flash": { + "max_input_tokens": 1048576, + "max_output_tokens": 131072, + "input_cost_per_token": 2.7e-07, + "output_cost_per_token": 8.1e-07, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/gemini-3-flash": { + "max_input_tokens": 1048576, + "max_output_tokens": 8192, + "input_cost_per_token": 6.75e-07, + "cache_read_input_token_cost": 6.75e-08, + "output_cost_per_token": 4.05e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/gemini-3.1-pro": { + "max_input_tokens": 1048576, + "max_output_tokens": 65536, + "input_cost_per_token": 2.7e-06, + "cache_read_input_token_cost": 2.7e-07, + "output_cost_per_token": 1.62e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/gemini-3.5-flash": { + "max_input_tokens": 1048576, + "max_output_tokens": 8192, + "input_cost_per_token": 2.025e-06, + "cache_read_input_token_cost": 2.025e-07, + "output_cost_per_token": 1.215e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/gemini-3.5-flash-lite": { + "max_input_tokens": 1048576, + "max_output_tokens": 8192, + "input_cost_per_token": 4.05e-07, + "cache_read_input_token_cost": 4.05e-08, + "output_cost_per_token": 3.375e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/gemini-3.6-flash": { + "max_input_tokens": 1048576, + "max_output_tokens": 8192, + "input_cost_per_token": 1.013e-06, + "cache_read_input_token_cost": 1.0125e-07, + "output_cost_per_token": 5.063e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": false, + "supports_pdf_input": false + }, + "kyma/gemini-3.7-flash": { + "max_input_tokens": 1048576, + "max_output_tokens": 65536, + "input_cost_per_token": 2.026e-06, + "cache_read_input_token_cost": 1.0125e-07, + "output_cost_per_token": 1.0126e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/gemini-3.8-flash": { + "max_input_tokens": 1048576, + "max_output_tokens": 65536, + "input_cost_per_token": 2.026e-06, + "output_cost_per_token": 1.0126e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/gemini-4-argon": { + "max_input_tokens": 1048576, + "max_output_tokens": 65536, + "input_cost_per_token": 5.4e-06, + "cache_read_input_token_cost": 2.7e-07, + "output_cost_per_token": 2.7e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/gemma-4-31b": { + "max_input_tokens": 128000, + "max_output_tokens": 8192, + "input_cost_per_token": 1.642e-07, + "cache_read_input_token_cost": 1.642e-07, + "output_cost_per_token": 4.689e-07, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/glm-4.5-air": { + "max_input_tokens": 131072, + "max_output_tokens": 8192, + "input_cost_per_token": 1.797e-07, + "cache_read_input_token_cost": 3.375e-08, + "output_cost_per_token": 1.175e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/glm-4.7-flash": { + "max_input_tokens": 203000, + "max_output_tokens": 65536, + "input_cost_per_token": 8.1e-08, + "cache_read_input_token_cost": 1.35e-08, + "output_cost_per_token": 5.4e-07, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/glm-5.1": { + "max_input_tokens": 203000, + "max_output_tokens": 65536, + "input_cost_per_token": 1.89e-06, + "cache_read_input_token_cost": 2.7675e-07, + "output_cost_per_token": 5.94e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/glm-5.2": { + "max_input_tokens": 1000000, + "max_output_tokens": 131072, + "input_cost_per_token": 9.029e-07, + "cache_read_input_token_cost": 2.9835e-07, + "output_cost_per_token": 3.083e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/glm-5.3": { + "max_input_tokens": 1310720, + "max_output_tokens": 131072, + "input_cost_per_token": 1.178e-06, + "output_cost_per_token": 3.702e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/glm-5.3-flash": { + "max_input_tokens": 1310720, + "max_output_tokens": 65536, + "input_cost_per_token": 1.72e-07, + "output_cost_per_token": 5.733e-07, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/gpt-5.6-luna": { + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "input_cost_per_token": 2.944e-07, + "cache_read_input_token_cost": 2.7e-08, + "output_cost_per_token": 1.767e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/gpt-5.6-luna-pro": { + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "input_cost_per_token": 2.703e-07, + "cache_read_input_token_cost": 2.7e-08, + "output_cost_per_token": 1.621e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/gpt-5.6-sol": { + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "input_cost_per_token": 5.4e-06, + "cache_read_input_token_cost": 6.75e-07, + "output_cost_per_token": 2.7e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/gpt-5.6-sol-pro": { + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "input_cost_per_token": 5.4e-06, + "cache_read_input_token_cost": 6.75e-07, + "output_cost_per_token": 2.7e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/gpt-5.6-terra": { + "max_input_tokens": 1050000, + "max_output_tokens": 8192, + "input_cost_per_token": 2.7e-06, + "cache_read_input_token_cost": 2.7e-07, + "output_cost_per_token": 1.62e-05, + "input_cost_per_token_above_272k_tokens": 5.4e-06, + "output_cost_per_token_above_272k_tokens": 2.43e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/gpt-5.6-terra-pro": { + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "input_cost_per_token": 2.7e-06, + "cache_read_input_token_cost": 2.7e-07, + "output_cost_per_token": 1.62e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/gpt-6-astra": { + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "input_cost_per_token": 1.35e-05, + "cache_read_input_token_cost": 1.35e-06, + "output_cost_per_token": 6.75e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/gpt-6-luna": { + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "input_cost_per_token": 1.639e-07, + "cache_read_input_token_cost": 1.35e-08, + "output_cost_per_token": 8.195e-07, + "input_cost_per_token_above_272k_tokens": 2.7e-07, + "output_cost_per_token_above_272k_tokens": 1.0125e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/gpt-6.1-sol": { + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "input_cost_per_token": 2.7e-06, + "cache_read_input_token_cost": 1.35e-07, + "output_cost_per_token": 1.35e-05, + "input_cost_per_token_above_272k_tokens": 5.4e-06, + "output_cost_per_token_above_272k_tokens": 2.025e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/gpt-oss-120b": { + "max_input_tokens": 128000, + "max_output_tokens": 8192, + "input_cost_per_token": 4.73e-08, + "output_cost_per_token": 2.303e-07, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/grok-4.20": { + "max_input_tokens": 2000000, + "max_output_tokens": 8192, + "input_cost_per_token": 1.763e-06, + "output_cost_per_token": 3.528e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/grok-4.20-multi-agent": { + "max_input_tokens": 2000000, + "max_output_tokens": 8192, + "input_cost_per_token": 1.767e-06, + "output_cost_per_token": 3.534e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": false, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/grok-4.3": { + "max_input_tokens": 1000000, + "max_output_tokens": 32768, + "input_cost_per_token": 1.92e-06, + "cache_read_input_token_cost": 2.7e-07, + "output_cost_per_token": 3.838e-06, + "input_cost_per_token_above_200k_tokens": 3.375e-06, + "output_cost_per_token_above_200k_tokens": 6.75e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/grok-4.5": { + "max_input_tokens": 500000, + "max_output_tokens": 8192, + "input_cost_per_token": 2.73e-06, + "output_cost_per_token": 8.189e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/grok-4.6": { + "max_input_tokens": 500000, + "max_output_tokens": 8192, + "input_cost_per_token": 2.758e-06, + "output_cost_per_token": 8.274e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/grok-4.7": { + "max_input_tokens": 500000, + "max_output_tokens": 450000, + "input_cost_per_token": 2.16e-06, + "cache_read_input_token_cost": 5.4e-07, + "output_cost_per_token": 6.48e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/grok-build": { + "max_input_tokens": 256000, + "max_output_tokens": 32768, + "input_cost_per_token": 1.389e-06, + "cache_read_input_token_cost": 2.7e-07, + "output_cost_per_token": 2.777e-06, + "input_cost_per_token_above_200k_tokens": 2.7e-06, + "output_cost_per_token_above_200k_tokens": 5.4e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/hy3": { + "max_input_tokens": 262144, + "max_output_tokens": 8192, + "input_cost_per_token": 1.594e-07, + "output_cost_per_token": 6.576e-07, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/hy4-preview": { + "max_input_tokens": 1048576, + "max_output_tokens": 64000, + "input_cost_per_token": 1.1259e-06, + "output_cost_per_token": 3.3764e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/kimi-k2.6": { + "max_input_tokens": 262144, + "max_output_tokens": 32768, + "input_cost_per_token": 5.909e-07, + "output_cost_per_token": 2.758e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/kimi-k2.7-code": { + "max_input_tokens": 262144, + "max_output_tokens": 16384, + "input_cost_per_token": 1.009e-06, + "output_cost_per_token": 4.774e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/kimi-k3": { + "max_input_tokens": 1048576, + "max_output_tokens": 32768, + "input_cost_per_token": 3.052e-06, + "cache_read_input_token_cost": 4.05e-07, + "output_cost_per_token": 1.5258e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/kimi-k3.1-pro": { + "max_input_tokens": 1048576, + "max_output_tokens": 943718, + "input_cost_per_token": 2.7e-06, + "cache_read_input_token_cost": 2.7e-07, + "output_cost_per_token": 1.35e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/llama-3.3-70b": { + "max_input_tokens": 128000, + "max_output_tokens": 8192, + "input_cost_per_token": 1.35e-07, + "output_cost_per_token": 4.32e-07, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": true, + "supports_reasoning": false, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/llama-4-maverick": { + "max_input_tokens": 1048576, + "max_output_tokens": 16384, + "input_cost_per_token": 3.344e-07, + "output_cost_per_token": 1.337e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": false, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/mimo-v2.5": { + "max_input_tokens": 1050000, + "max_output_tokens": 16384, + "input_cost_per_token": 1.859e-07, + "output_cost_per_token": 3.718e-07, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/mimo-v2.6-flash": { + "max_input_tokens": 1050000, + "max_output_tokens": 131072, + "input_cost_per_token": 1.89e-07, + "output_cost_per_token": 3.78e-07, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/mimo-v2.6-pro": { + "max_input_tokens": 1050000, + "max_output_tokens": 131072, + "input_cost_per_token": 5.805e-07, + "output_cost_per_token": 1.1745e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/minimax-m2.5": { + "max_input_tokens": 196608, + "max_output_tokens": 32768, + "input_cost_per_token": 4.188e-07, + "output_cost_per_token": 1.675e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/minimax-m2.7": { + "max_input_tokens": 204800, + "max_output_tokens": 32768, + "input_cost_per_token": 4.05e-07, + "output_cost_per_token": 1.62e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/minimax-m3": { + "max_input_tokens": 1048576, + "max_output_tokens": 32768, + "input_cost_per_token": 3.956e-07, + "cache_read_input_token_cost": 7.56e-08, + "output_cost_per_token": 1.582e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/minimax-m3.1-flash": { + "max_input_tokens": 1048576, + "max_output_tokens": 524288, + "input_cost_per_token": 2.025e-07, + "cache_read_input_token_cost": 2.025e-08, + "output_cost_per_token": 8.1e-07, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/muse-glimmer-30b": { + "max_input_tokens": 131072, + "max_output_tokens": 8192, + "input_cost_per_token": 4.554e-07, + "output_cost_per_token": 1.821e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/muse-spark-1.1": { + "max_input_tokens": 1048576, + "max_output_tokens": 8192, + "input_cost_per_token": 1.542e-06, + "output_cost_per_token": 5.243e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/muse-spark-1.2": { + "max_input_tokens": 1048576, + "max_output_tokens": 8192, + "input_cost_per_token": 1.773e-06, + "output_cost_per_token": 6.025e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/muse-spark-1.3": { + "max_input_tokens": 1048576, + "max_output_tokens": 8192, + "input_cost_per_token": 1.688e-06, + "output_cost_per_token": 5.738e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/muse-spark-open": { + "max_input_tokens": 1048576, + "max_output_tokens": 943718, + "input_cost_per_token": 1.688e-06, + "output_cost_per_token": 5.738e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/muse-wonder": { + "max_input_tokens": 1048576, + "max_output_tokens": 943718, + "input_cost_per_token": 5.4e-06, + "output_cost_per_token": 2.7e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": true + }, + "kyma/nemotron-3-ultra-550b": { + "max_input_tokens": 1000000, + "max_output_tokens": 32768, + "input_cost_per_token": 6.75e-07, + "cache_read_input_token_cost": 1.35e-07, + "output_cost_per_token": 3.375e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/nemotron-3.5-super-vl": { + "max_input_tokens": 1048576, + "max_output_tokens": 235929, + "input_cost_per_token": 8.1e-08, + "cache_read_input_token_cost": 1.35e-08, + "output_cost_per_token": 4.725e-07, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/qwen-3-32b": { + "max_input_tokens": 32768, + "max_output_tokens": 8192, + "input_cost_per_token": 1.08e-07, + "output_cost_per_token": 3.78e-07, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/qwen-3-coder": { + "max_input_tokens": 131072, + "max_output_tokens": 32768, + "input_cost_per_token": 3.546e-07, + "cache_read_input_token_cost": 1.35e-07, + "output_cost_per_token": 1.612e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": true, + "supports_reasoning": false, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/qwen-3.6-plus": { + "max_input_tokens": 1000000, + "max_output_tokens": 65536, + "input_cost_per_token": 4.388e-07, + "output_cost_per_token": 2.633e-06, + "input_cost_per_token_above_256k_tokens": 1.755e-06, + "output_cost_per_token_above_256k_tokens": 5.265e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/qwen-3.7-max": { + "max_input_tokens": 1000000, + "max_output_tokens": 32768, + "input_cost_per_token": 2.071e-06, + "cache_read_input_token_cost": 1.6875e-07, + "output_cost_per_token": 6.207e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": false, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/qwen-3.7-plus": { + "max_input_tokens": 1000000, + "max_output_tokens": 32768, + "input_cost_per_token": 4.325e-07, + "cache_read_input_token_cost": 8.64e-08, + "output_cost_per_token": 1.731e-06, + "input_cost_per_token_above_256k_tokens": 1.296e-06, + "output_cost_per_token_above_256k_tokens": 5.184e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/qwen-3.8-max": { + "max_input_tokens": 1000000, + "max_output_tokens": 131072, + "input_cost_per_token": 2.228e-06, + "cache_read_input_token_cost": 2.781e-07, + "output_cost_per_token": 6.684e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/qwen3.7-flash": { + "max_input_tokens": 1000000, + "max_output_tokens": 8192, + "input_cost_per_token": 4.35e-08, + "cache_read_input_token_cost": 8.1e-09, + "output_cost_per_token": 1.886e-07, + "input_cost_per_token_above_32k_tokens": 1.35e-07, + "output_cost_per_token_above_32k_tokens": 5.4e-07, + "input_cost_per_token_above_256k_tokens": 2.7e-07, + "output_cost_per_token_above_256k_tokens": 1.08e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/qwen3.8-27b": { + "max_input_tokens": 1000000, + "max_output_tokens": 131072, + "input_cost_per_token": 3.826e-07, + "output_cost_per_token": 2.732e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/qwen3.8-flash": { + "max_input_tokens": 1000000, + "max_output_tokens": 131072, + "input_cost_per_token": 2.045e-07, + "output_cost_per_token": 6.41e-07, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/sonar": { + "max_input_tokens": 127072, + "max_output_tokens": 4096, + "input_cost_per_token": 1.35e-06, + "output_cost_per_token": 1.35e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": false, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": false, + "supports_response_schema": false, + "supports_pdf_input": false + }, + "kyma/sonar-pro": { + "max_input_tokens": 200000, + "max_output_tokens": 8000, + "input_cost_per_token": 4.05e-06, + "output_cost_per_token": 2.025e-05, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": false, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": false, + "supports_response_schema": false, + "supports_pdf_input": false + }, + "kyma/step-3.7-flash": { + "max_input_tokens": 256000, + "max_output_tokens": 8192, + "input_cost_per_token": 2.229e-07, + "cache_read_input_token_cost": 5.4e-08, + "output_cost_per_token": 1.282e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, + "kyma/step-5-preview": { + "max_input_tokens": 1048576, + "max_output_tokens": 64000, + "input_cost_per_token": 1.35e-06, + "cache_read_input_token_cost": 1.35e-07, + "output_cost_per_token": 3.645e-06, + "litellm_provider": "kyma", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_pdf_input": false + }, "databricks/databricks-claude-haiku-5-5": { "cache_creation_input_token_cost": 1.2502e-07, "cache_creation_input_token_cost_above_100k_tokens": 6.2503e-07, diff --git a/provider_endpoints_support.json b/provider_endpoints_support.json index 7b0234e28d7..4188866de08 100644 --- a/provider_endpoints_support.json +++ b/provider_endpoints_support.json @@ -5327,6 +5327,32 @@ "generateContent": false } }, + "kyma": { + "display_name": "Kyma API (`kyma`)", + "url": "https://docs.litellm.ai/docs/providers/kyma", + "endpoints": { + "chat_completions": true, + "responses": true, + "text_completion": true, + "embeddings": false, + "image_generations": false, + "image_edits": false, + "audio_transcriptions": false, + "audio_speech": false, + "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": true, + "interactions": true, + "systemone": false, + "a2a": true, + "generateContent": true + } + }, "bigmodel": { "display_name": "BigModel (Zhipu AI) (`bigmodel`)", "url": "https://docs.bigmodel.cn/cn/guide/develop/responses/introduction", diff --git a/scripts/pre_commit_lint.sh b/scripts/pre_commit_lint.sh index 371884fb791..95b5942fa8b 100755 --- a/scripts/pre_commit_lint.sh +++ b/scripts/pre_commit_lint.sh @@ -18,7 +18,8 @@ # -> ruff over ruff-tests.toml + `make lint-test-quality` (test-linting.yml's # test-tree ruff and test-quality gate steps) # - dashboard -> prettier + eslint + lint budgets (test-linting.yml's ui job) -# - proxy/types -> regenerate the lazy OpenAPI snapshot and dashboard API types, fail on drift (test-linting.yml's ui-api-types job) +# - spec_pattern -> regenerate the lazy OpenAPI snapshot and dashboard API types, fail on drift +# (test-linting.yml's ui-api-types job) # # Each block is skipped when no matching files are in scope, so unrelated commits # stay fast. This is intentionally not auto-installed as a git hook (see @@ -98,7 +99,7 @@ existing_files() { litellm_py_pattern='^litellm/.*\.py$' e2e_py_pattern='^tests/e2e(_harness)?/.*\.py$' test_tree_pattern='^(tests/.*\.py|ruff-tests\.toml|scripts/(check_test_quality|test_quality_gate)\.py)$' -spec_pattern='^(litellm/(proxy|types)/.*|ui/litellm-dashboard/(scripts/gen-api-types\.mjs|package\.json|package-lock\.json|src/lib/http/schema\.d\.ts))$' +spec_pattern='^(litellm/.*|enterprise/.*|pyproject\.toml|uv\.lock|ui/litellm-dashboard/(scripts/gen-api-types\.mjs|package\.json|package-lock\.json|src/lib/http/schema\.d\.ts))$' ui_prettier_pattern='^ui/litellm-dashboard/.*\.(js|jsx|ts|tsx|mjs|cjs|json|css|scss|md|mdx|yml|yaml|html)$' ui_eslint_pattern='^ui/litellm-dashboard/.*\.(js|jsx|ts|tsx|mjs|cjs)$' @@ -110,9 +111,6 @@ e2e_py_files=$(scope_match "$e2e_py_pattern") test_tree_files=$(scope_match "$test_tree_pattern") # ruff format (and CI's format step) skip enterprise; the rest of make lint covers it. fmt_files=$(printf '%s\n' "$litellm_py_files" | grep -v '^litellm/enterprise/' | existing_files) -# test-linting.yml triggers on any file under litellm/proxy or litellm/types -# (Prisma schema and configs included, not just Python) plus the generator and its -# lockfiles, so match that whole trigger set rather than a Python subset. spec_files=$(scope_match "$spec_pattern") # CI's ui job runs prettier over a wider extension set than eslint; keep that # split so this flags exactly what the job would. @@ -337,7 +335,7 @@ summary_item "tests/e2e checks (basedpyright + raw HTTP client ban)" "$e2e_py_fi summary_item "test-tree lint (ruff-tests.toml + test-quality gate)" "$test_tree_files" \ "no tests/ Python files or test-tree lint inputs in scope" summary_item "dashboard lint (prettier + eslint + lint budgets)" "$ui_prettier_changed$ui_eslint_changed" "no dashboard files in scope" -summary_item "dashboard API-type sync (npm run gen:api)" "$spec_files" "no litellm/proxy, litellm/types, or generator files in scope" +summary_item "dashboard API-type sync (npm run gen:api)" "$spec_files" "no litellm/, enterprise/, dependency, or generator files in scope" if [ -z "$litellm_py_files$e2e_py_files$test_tree_files$ui_prettier_changed$ui_eslint_changed$spec_files" ]; then echo "check: NOTE - no gating lint check matches the files in scope, so nothing ran:" >&2 diff --git a/tests/integration/_support/client.py b/tests/integration/_support/client.py index b8ea11f3d2f..a759126fcd4 100644 --- a/tests/integration/_support/client.py +++ b/tests/integration/_support/client.py @@ -280,12 +280,14 @@ class Scenario: self.gateway.rows('SELECT model_id FROM "LiteLLM_ProxyModelTable" WHERE model_id = %s', (identity,)) == [] ) - def model(self, *, model_info: Mapping[str, JsonValue] | None = None, **parameters: JsonValue) -> str: - name: Final = f"integration-{uuid.uuid4().hex}" + def model( + self, *, name: str | None = None, model_info: Mapping[str, JsonValue] | None = None, **parameters: JsonValue + ) -> str: + model_name: Final = name if name is not None else f"integration-{uuid.uuid4().hex}" created: Final = self.gateway.post( "/model/new", { - "model_name": name, + "model_name": model_name, "litellm_params": { "model": "openai/gpt-4o-mini", "api_key": "integration-provider-key", @@ -298,8 +300,8 @@ class Scenario: identity: Final = string_value(object_value(created["model_info"])["id"]) self.cleanups.callback(self.delete_model, identity) if self.gateway.workers > 1: - self._await_every_worker(name, identity) - return name + self._await_every_worker(model_name, identity) + return model_name def _await_every_worker(self, name: str, identity: str) -> None: from litellm.constants import PROXY_CONFIG_RELOAD_INTERVAL_SECONDS diff --git a/tests/integration/caching/test_response_cache_model_group_key.py b/tests/integration/caching/test_response_cache_model_group_key.py new file mode 100644 index 00000000000..59565b75d9a --- /dev/null +++ b/tests/integration/caching/test_response_cache_model_group_key.py @@ -0,0 +1,192 @@ +"""The response cache keys an answer by the router's model group on every endpoint the router serves. + +The router names the model group in ``metadata`` on ``/v1/chat/completions`` and in ``litellm_metadata`` on +``/v1/responses`` and ``/v1/messages``. Every cell gives each model group its own scripted upstream scenario on one +shared provider model, fills one group's entry, and reads which scenario the identical request reached next and which +answer the caller got: an answer served from the cache repeats the id of the request that filled it. +""" + +from __future__ import annotations + +import uuid +from collections.abc import Callable, Mapping +from dataclasses import dataclass +from pathlib import Path +from typing import Final, TypeAlias + +import pytest +import yaml +from anthropic import Anthropic +from openai import OpenAI +from pydantic import JsonValue + +from tests.integration._support.client import JSON_OBJECT, Gateway, Scenario, object_value +from tests.integration._support.process import OwnedState, graceful_stop_seconds, owned_proxy_process +from tests.integration._support.upstream import ScenarioHandle +from tests.integration.caching.response_cache_case import ( + CHAT, + CHAT_MODEL, + MESSAGE, + MESSAGES_MODEL, + PROVIDER_KEY, + RESPONSE, + Upstream, + await_entry, + json_response, + prompt, + redis_store, + scripted, +) + +Ask: TypeAlias = Callable[[Gateway, str, str, Mapping[str, JsonValue]], str] + + +def _root(gateway: Gateway) -> str: + return str(gateway.client.base_url).rstrip("/") + + +def _metadata(metadata: Mapping[str, JsonValue]) -> dict[str, JsonValue] | None: + return {"metadata": dict(metadata)} if metadata else None + + +def _chat(gateway: Gateway, model: str, text: str, metadata: Mapping[str, JsonValue]) -> str: + with OpenAI(base_url=f"{_root(gateway)}/v1", api_key=gateway.key, max_retries=0) as client: + return client.chat.completions.create( + model=model, messages=[{"role": "user", "content": text}], extra_body=_metadata(metadata) + ).id + + +def _responses(gateway: Gateway, model: str, text: str, metadata: Mapping[str, JsonValue]) -> str: + with OpenAI(base_url=f"{_root(gateway)}/v1", api_key=gateway.key, max_retries=0) as client: + identity: Final = client.responses.create(model=model, input=text, extra_body=_metadata(metadata)).output[0].id + assert identity is not None + return identity + + +def _messages(gateway: Gateway, model: str, text: str, metadata: Mapping[str, JsonValue]) -> str: + with Anthropic(base_url=_root(gateway), api_key=gateway.key, max_retries=0) as client: + return client.messages.create( + model=model, + max_tokens=16, + messages=[{"role": "user", "content": text}], + extra_body=_metadata(metadata), + ).id + + +@dataclass(frozen=True, slots=True) +class Endpoint: + provider_model: str + sibling_model: str + answer: Mapping[str, JsonValue] + ask: Ask + + +@dataclass(frozen=True, slots=True) +class Group: + name: str + upstream: ScenarioHandle + + +ENDPOINTS: Final = { + "chat_completions": Endpoint(CHAT_MODEL, "openai/gpt-5.6", CHAT, _chat), + "responses": Endpoint(CHAT_MODEL, "openai/gpt-5.6", RESPONSE, _responses), + "messages": Endpoint(MESSAGES_MODEL, "anthropic/claude-sonnet-5-5", MESSAGE, _messages), +} + + +def _group(scenario: Scenario, endpoint: Endpoint, name: str | None = None) -> Group: + handle: Final = scripted(scenario, json_response(endpoint.answer)) + return Group(scenario.model(name=name, model=endpoint.provider_model, api_base=handle.api_base()), handle) + + +@pytest.mark.parametrize("endpoint", ENDPOINTS.values(), ids=ENDPOINTS.keys()) +def test_identical_requests_to_two_groups_on_one_model_each_reach_their_own_deployment( + gateway: Gateway, endpoint: Endpoint +) -> None: + upstream: Final = Upstream(gateway.upstream_url) + store: Final = redis_store() + text: Final = prompt() + with gateway.scenario() as scenario: + first: Final = _group(scenario, endpoint) + second: Final = _group(scenario, endpoint) + filled: Final = endpoint.ask(gateway, first.name, text, {}) + await_entry(store, filled) + answered: Final = endpoint.ask(gateway, second.name, text, {}) + assert answered != filled, f"{second.name} was served the answer cached for {first.name}" + assert upstream.calls(second.upstream.scenario_id) == 1 + assert endpoint.ask(gateway, first.name, text, {}) == filled + assert upstream.calls(first.upstream.scenario_id) == 1 + + +PLANTS: Final[Mapping[str, Callable[[str], Mapping[str, JsonValue]]]] = { + "model_group": lambda target: {"model_group": target}, + "caching_groups": lambda target: {"caching_groups": [target]}, +} + + +@pytest.mark.parametrize("plant", PLANTS.values(), ids=PLANTS.keys()) +@pytest.mark.parametrize("endpoint", ENDPOINTS.values(), ids=ENDPOINTS.keys()) +def test_a_caller_naming_another_group_in_metadata_never_fills_that_groups_entry( + gateway: Gateway, endpoint: Endpoint, plant: Callable[[str], Mapping[str, JsonValue]] +) -> None: + upstream: Final = Upstream(gateway.upstream_url) + store: Final = redis_store() + text: Final = prompt() + with gateway.scenario() as scenario: + caller: Final = _group(scenario, endpoint) + target: Final = _group(scenario, endpoint, name=f"{caller.name}-premium") + planted: Final = endpoint.ask(gateway, caller.name, text, plant(target.name)) + assert upstream.calls(caller.upstream.scenario_id) == 1 + await_entry(store, planted) + answered: Final = endpoint.ask(gateway, target.name, text, {}) + assert answered != planted, f"{target.name} was served the answer {caller.name} planted under its name" + assert upstream.calls(target.upstream.scenario_id) == 1 + await_entry(store, answered) + assert endpoint.ask(gateway, target.name, text, {}) == answered + + +def _caching_groups_config(directory: Path, endpoint: Endpoint, first: Group, second: Group) -> Path: + rig: Final = JSON_OBJECT.validate_python( + yaml.safe_load((Path(__file__).resolve().parents[1] / "proxy_config.yaml").read_text()) + ) + router_settings: Final = object_value(rig.get("router_settings") or {}) + config: Final = { + **rig, + "model_list": [ + { + "model_name": group.name, + "litellm_params": {"model": model, "api_key": PROVIDER_KEY, "api_base": group.upstream.api_base()}, + } + for group, model in ((first, endpoint.provider_model), (second, endpoint.sibling_model)) + ], + "router_settings": {**router_settings, "caching_groups": [[first.name, second.name]]}, + } + path: Final = directory / f"proxy_config-{uuid.uuid4().hex[:8]}.yaml" + path.write_text(yaml.safe_dump(config, sort_keys=False)) + return path + + +def _configured_group(scenario: Scenario, endpoint: Endpoint) -> Group: + return Group(f"cache-group-{uuid.uuid4().hex[:8]}", scripted(scenario, json_response(endpoint.answer))) + + +@pytest.mark.timeout(2 * graceful_stop_seconds() + 120) +@pytest.mark.parametrize("endpoint", ENDPOINTS.values(), ids=ENDPOINTS.keys()) +def test_groups_in_one_caching_group_on_different_models_share_one_entry( + gateway: Gateway, endpoint: Endpoint, tmp_path: Path +) -> None: + upstream: Final = Upstream(gateway.upstream_url) + store: Final = redis_store() + text: Final = prompt() + with gateway.scenario() as scenario: + first: Final = _configured_group(scenario, endpoint) + second: Final = _configured_group(scenario, endpoint) + config: Final = _caching_groups_config(tmp_path, endpoint, first, second) + with owned_proxy_process( + gateway, tmp_path, {}, config=config, workers=1, state=OwnedState.SHARED_WITH_WORKER + ) as owned: + filled: Final = endpoint.ask(owned.gateway, first.name, text, {}) + await_entry(store, filled) + assert endpoint.ask(owned.gateway, second.name, text, {}) == filled + assert upstream.calls(second.upstream.scenario_id) == 0 + assert upstream.calls(first.upstream.scenario_id) == 1 diff --git a/tests/integration/management/test_fallback_management_public_names.py b/tests/integration/management/test_fallback_management_public_names.py index fcbeee9dc60..6f97165fb3c 100644 --- a/tests/integration/management/test_fallback_management_public_names.py +++ b/tests/integration/management/test_fallback_management_public_names.py @@ -1,6 +1,7 @@ import uuid from collections.abc import Iterator from concurrent.futures import ThreadPoolExecutor +from contextlib import contextmanager from functools import partial from itertools import chain from pathlib import Path @@ -10,6 +11,7 @@ import httpx import psutil import pytest from pydantic import JsonValue +from redis import Redis from tests.integration._support.client import ( Gateway, @@ -27,7 +29,7 @@ from tests.integration._support.process import ( setup_only_proxy_run, worker_configuration, ) -from tests.integration._support.redis_process import owned_redis +from tests.integration._support.redis_process import OwnedRedis, owned_redis pytestmark: Final = [pytest.mark.timeout(300), pytest.mark.shares_worker_state] @@ -385,6 +387,39 @@ def test_fallback_write_survives_a_redis_outage(gateway: Gateway, tmp_path: Path cache.start() +@contextmanager +def _refusing_cache_deletes(cache: OwnedRedis) -> Iterator[None]: + with Redis(host=cache.host, port=cache.port) as client: + client.acl_setuser("default", enabled=True, commands=["-del", "-unlink"]) + try: + yield + finally: + client.acl_setuser("default", enabled=True, commands=["+del", "+unlink"]) + + +@pytest.mark.usefixtures("migrated_database") +def test_consecutive_creates_keep_every_rule_when_redis_refuses_the_cache_eviction( + gateway: Gateway, tmp_path: Path +) -> None: + with owned_redis(tmp_path) as cache: + overrides: Final = {"REDIS_HOST": cache.host, "REDIS_PORT": str(cache.port)} + with owned_proxy_process(gateway, tmp_path, overrides, database_setup=()) as owned: + with owned.gateway.scenario() as scenario: + target: Final = scenario.model(model=f"openai/x3t-{uuid.uuid4().hex}") + first: Final = scenario.model(model=f"openai/x3a-{uuid.uuid4().hex}") + second: Final = scenario.model(model=f"openai/x3b-{uuid.uuid4().hex}") + for model in (target, first, second): + _wait_until_served(owned.gateway, model) + with _refusing_cache_deletes(cache): + created: Final = tuple( + _create_fallback(owned.gateway, scenario, model, [target]) for model in (first, second) + ) + assert all(response.status_code == 200 for response in created), [r.text for r in created] + covered: Final = _covered_models(_stored_fallbacks()) + assert {first, second} <= covered, covered + assert EVICTION_WARNING in owned.log.read_text(), owned.log.read_text()[-3000:] + + @pytest.mark.usefixtures("migrated_database") def test_fallback_write_survives_a_worker_kill(gateway: Gateway, tmp_path: Path) -> None: with owned_proxy_process(gateway, tmp_path, {}, workers=2, database_setup=()) as owned: diff --git a/tests/integration/messages_endpoint/test_beta_gated_fields_wire.py b/tests/integration/messages_endpoint/test_beta_gated_fields_wire.py new file mode 100644 index 00000000000..26ad9f20897 --- /dev/null +++ b/tests/integration/messages_endpoint/test_beta_gated_fields_wire.py @@ -0,0 +1,183 @@ +import json +import re +import uuid +from collections.abc import Iterator, Mapping +from dataclasses import dataclass +from pathlib import Path +from typing import Final + +import pytest +from integration._support.client import Gateway, gateway_from_environment, list_value, object_value +from integration._support.process import owned_proxy +from integration._support.wire import Reply, Request, Wire, wire_server +from pydantic import JsonValue, TypeAdapter + +pytestmark: Final = pytest.mark.shares_worker_state + +ANTHROPIC_MODEL: Final = "beta-gated-anthropic" +FOUNDRY_MODEL: Final = "beta-gated-foundry" +FOUNDRY_DROP_PARAMS_MODEL: Final = "beta-gated-foundry-drop-params" +CLEAR_AT_BETA: Final = "mid-conversation-system-clear-at-2026-08-21" +_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) +_NONCE: Final = re.compile(rb"nonce-([0-9a-f]{32})") + + +def nonce() -> str: + return f"nonce-{uuid.uuid4().hex}" + + +def claude_reply(request: Request) -> Reply: + match: Final = _NONCE.search(request.body) + assert match is not None, request.body[:200] + identity: Final = match.group(1).decode() + return Reply( + body=json.dumps( + { + "id": f"msg_{identity}", + "type": "message", + "role": "assistant", + "model": "claude-opus-4-8", + "content": [{"type": "text", "text": f"answered {identity}"}], + "stop_reason": "end_turn", + "stop_sequence": None, + "usage": {"input_tokens": 12, "output_tokens": 3}, + } + ).encode() + ) + + +def respond(request: Request) -> Reply: + if request.method != "POST": + return Reply(body=b'{"object":"list","data":[]}') + return claude_reply(request) + + +def deployment(name: str, model: str, api_base: str, drop_params: bool = False) -> dict[str, JsonValue]: + return { + "model_name": name, + "litellm_params": { + "model": model, + "api_base": api_base, + "api_key": f"synthetic-{name}-key", + **({"drop_params": True} if drop_params else {}), + }, + } + + +def write_config(directory: Path, wire_url: str) -> Path: + path: Final = directory / f"beta-gated-{uuid.uuid4().hex}.json" + path.write_text( + json.dumps( + { + "model_list": [ + deployment(ANTHROPIC_MODEL, "anthropic/claude-opus-4-8", f"{wire_url}/anthropic"), + deployment(FOUNDRY_MODEL, "azure_ai/claude-opus-4-8", f"{wire_url}/foundry"), + deployment( + FOUNDRY_DROP_PARAMS_MODEL, "azure_ai/claude-opus-4-8", f"{wire_url}/foundry", drop_params=True + ), + ], + "general_settings": { + "master_key": "os.environ/LITELLM_MASTER_KEY", + "database_url": "os.environ/DATABASE_URL", + }, + } + ) + ) + return path + + +@dataclass(frozen=True, slots=True) +class Rig: + gateway: Gateway + wire: Wire + + def send(self, model: str, request_nonce: str) -> tuple[int, bytes]: + response: Final = self.gateway.client.post( + "/v1/messages", + json={ + "model": model, + "max_tokens": 64, + "messages": [ + {"role": "user", "content": f"hi {request_nonce}"}, + {"role": "assistant", "content": "Hello"}, + {"role": "user", "content": "how are you"}, + {"role": "system", "content": "Answer in French", "clear_at": "next_user_message"}, + ], + }, + headers={"Authorization": f"Bearer {self.gateway.key}", "anthropic-beta": CLEAR_AT_BETA}, + timeout=120, + ) + return response.status_code, response.content + + def received(self, request_nonce: str) -> Request: + requests: Final = tuple(request for request in self.wire.drain() if request_nonce.encode() in request.body) + assert len(requests) == 1, [request.target for request in requests] + return requests[0] + + +@pytest.fixture(scope="module") +def rig(tmp_path_factory: pytest.TempPathFactory) -> Iterator[Rig]: + directory: Final = tmp_path_factory.mktemp("beta-gated") + with ( + gateway_from_environment() as shared, + wire_server(respond) as wire, + owned_proxy( + shared, + directory, + {"LITELLM_LOCAL_ANTHROPIC_BETA_HEADERS": "true"}, + config=write_config(directory, wire.url), + workers=1, + ) as gateway, + ): + yield Rig(gateway, wire) + + +def reply_text(status: int, raw: bytes) -> str: + assert status == 200, raw[:500] + return "".join(str(object_value(block)["text"]) for block in list_value(_JSON_OBJECT.validate_json(raw)["content"])) + + +def beta_header(headers: Mapping[str, str]) -> str: + return ",".join(value for key, value in headers.items() if key.lower() == "anthropic-beta") + + +def system_message(body: bytes) -> dict[str, JsonValue]: + messages: Final = list_value(_JSON_OBJECT.validate_json(body)["messages"]) + system: Final = [object_value(m) for m in messages if object_value(m).get("role") == "system"] + assert len(system) == 1, messages + return system[0] + + +def test_a_drop_params_leg_whose_row_lacks_the_beta_sends_neither_the_beta_nor_clear_at(rig: Rig) -> None: + request_nonce: Final = nonce() + status, raw = rig.send(FOUNDRY_DROP_PARAMS_MODEL, request_nonce) + assert reply_text(status, raw) == f"answered {request_nonce.removeprefix('nonce-')}" + upstream: Final = rig.received(request_nonce) + assert CLEAR_AT_BETA not in beta_header(upstream.headers), upstream.headers + assert system_message(upstream.body) == {"role": "system", "content": "Answer in French"} + + +def test_a_leg_whose_row_lacks_the_beta_forwards_clear_at_as_sent_without_drop_params(rig: Rig) -> None: + request_nonce: Final = nonce() + status, raw = rig.send(FOUNDRY_MODEL, request_nonce) + assert reply_text(status, raw) == f"answered {request_nonce.removeprefix('nonce-')}" + upstream: Final = rig.received(request_nonce) + assert CLEAR_AT_BETA not in beta_header(upstream.headers), upstream.headers + assert system_message(upstream.body) == { + "role": "system", + "content": "Answer in French", + "clear_at": "next_user_message", + } + + +def test_a_leg_that_carries_the_beta_forwards_clear_at_untouched(rig: Rig) -> None: + request_nonce: Final = nonce() + status, raw = rig.send(ANTHROPIC_MODEL, request_nonce) + assert reply_text(status, raw) == f"answered {request_nonce.removeprefix('nonce-')}" + upstream: Final = rig.received(request_nonce) + assert CLEAR_AT_BETA in beta_header(upstream.headers), upstream.headers + assert system_message(upstream.body) == { + "role": "system", + "content": "Answer in French", + "clear_at": "next_user_message", + } diff --git a/tests/integration/messages_endpoint/test_guardrail_block_stream.py b/tests/integration/messages_endpoint/test_guardrail_block_stream.py new file mode 100644 index 00000000000..d487f81c757 --- /dev/null +++ b/tests/integration/messages_endpoint/test_guardrail_block_stream.py @@ -0,0 +1,129 @@ +import uuid +from collections.abc import Callable, Iterator +from dataclasses import dataclass +from pathlib import Path +from typing import Final + +import anthropic +import httpx +import pytest +import yaml +from anthropic.types import Message +from integration._support.client import JSON_OBJECT, Gateway, gateway_from_environment +from integration._support.process import graceful_stop_seconds, owned_proxy +from integration._support.wire import Reply, Request, wire_server + +pytestmark: Final = pytest.mark.timeout(2 * graceful_stop_seconds() + 120) + +_DENIAL: Final = "This request is not allowed by policy." +_CALL_ID_HEADER: Final = "x-litellm-call-id" + + +@dataclass(frozen=True, slots=True) +class _Rail: + gateway: Gateway + names: tuple[str, str] + + +@dataclass(frozen=True, slots=True) +class _Ask: + model: str + rails: tuple[str, str] + call_id: str + + +@pytest.fixture(scope="module") +def denying_rail(tmp_path_factory: pytest.TempPathFactory) -> Iterator[_Rail]: + directory: Final = tmp_path_factory.mktemp("messages-guardrail-block") + name: Final = f"messages-deny-{uuid.uuid4().hex}" + post_call_name: Final = f"messages-deny-replies-{uuid.uuid4().hex}" + stock: Final = JSON_OBJECT.validate_python(yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text())) + config: Final = directory / "messages-guardrail-block.yaml" + config.write_text( + yaml.safe_dump( + { + **stock, + "guardrails": [ + { + "guardrail_name": name, + "litellm_params": { + "guardrail": "custom_code", + "mode": "pre_call", + "default_on": False, + "custom_code": ( + f"def apply_guardrail(inputs, request_data, input_type):\n return block({_DENIAL!r})\n" + ), + }, + }, + { + "guardrail_name": post_call_name, + "litellm_params": { + "guardrail": "custom_code", + "mode": "post_call", + "default_on": False, + "custom_code": ( + "def apply_guardrail(inputs, request_data, input_type):\n" + " return block('A post-call rail scanned the block message.')\n" + ), + }, + }, + ], + } + ) + ) + with gateway_from_environment() as shared, owned_proxy(shared, directory, {}, config=config) as candidate: + yield _Rail(candidate, (name, post_call_name)) + + +def _provider_must_not_answer(request: Request) -> Reply: + return Reply(status=500, body=f"unexpected {request.method} {request.target}".encode()) + + +def _streamed(client: anthropic.Anthropic, ask: _Ask) -> tuple[Message, httpx.Headers]: + with client.messages.stream( + model=ask.model, + max_tokens=32, + messages=[{"role": "user", "content": "count from 1 to 100"}], + extra_body={"guardrails": list(ask.rails)}, + extra_headers={_CALL_ID_HEADER: ask.call_id}, + ) as stream: + return stream.get_final_message(), stream.response.headers + + +def _created(client: anthropic.Anthropic, ask: _Ask) -> tuple[Message, httpx.Headers]: + raw: Final = client.messages.with_raw_response.create( + model=ask.model, + max_tokens=32, + messages=[{"role": "user", "content": "count from 1 to 100"}], + extra_body={"guardrails": list(ask.rails)}, + extra_headers={_CALL_ID_HEADER: ask.call_id}, + ) + return raw.parse(), raw.headers + + +@pytest.mark.parametrize("send", (_streamed, _created), ids=("stream", "json")) +def test_anthropic_sdk_reads_a_pre_call_block_as_a_finished_message_past_post_call_rails( + denying_rail: _Rail, send: Callable[[anthropic.Anthropic, _Ask], tuple[Message, httpx.Headers]] +) -> None: + call_id: Final = f"messages-block-{uuid.uuid4().hex}" + with ( + wire_server(_provider_must_not_answer) as provider, + denying_rail.gateway.scenario() as scenario, + httpx.Client(timeout=30, trust_env=False) as transport, + anthropic.Anthropic( + api_key=denying_rail.gateway.key, + base_url=str(denying_rail.gateway.client.base_url).rstrip("/"), + max_retries=0, + http_client=transport, + ) as client, + ): + model: Final = scenario.model( + model="anthropic/claude-haiku-5-5", api_base=provider.url, api_key="synthetic-anthropic-key" + ) + message, headers = send(client, _Ask(model, denying_rail.names, call_id)) + sent: Final = provider.drain() + + assert [(block.type, getattr(block, "text", None)) for block in message.content] == [("text", _DENIAL)], message + assert message.stop_reason == "end_turn", message + assert headers.get(_CALL_ID_HEADER) == call_id, dict(headers) + assert sent == (), sent diff --git a/tests/integration/observability/test_guardrail_effects.py b/tests/integration/observability/test_guardrail_effects.py index 8f83287696c..537eb2bbf69 100644 --- a/tests/integration/observability/test_guardrail_effects.py +++ b/tests/integration/observability/test_guardrail_effects.py @@ -1902,11 +1902,20 @@ def test_messages_pre_call_denial_streams_message(gateway: Gateway, tmp_path: Pa ) assert response.status_code == 200, response.text assert response.headers["content-type"].startswith("text/event-stream"), response.text - lines: Final = tuple(line for line in response.text.split("\n") if line.startswith("data: ")) - assert len(lines) == 1, response.text - body: Final = json.loads(lines[0].removeprefix("data: ")) - assert body["type"] == "message" and body["role"] == "assistant", body - assert body["content"] == [{"type": "text", "text": _RESPONSES_DENIAL}], body + events: Final = tuple( + json.loads(line.removeprefix("data: ")) for line in response.text.split("\n") if line.startswith("data: ") + ) + assert [event["type"] for event in events] == [ + "message_start", + "content_block_start", + "content_block_delta", + "content_block_stop", + "message_delta", + "message_stop", + ], response.text + assert events[0]["message"]["role"] == "assistant", events[0] + assert events[2]["delta"] == {"type": "text_delta", "text": _RESPONSES_DENIAL}, events[2] + assert events[4]["delta"]["stop_reason"] == "end_turn", events[4] @pytest.mark.covers("other.observability.guardrails.responses_pre_call_denial_writes_zero_spend_row") diff --git a/tests/integration/sdk/test_router_silent_model_shadow_sdk.py b/tests/integration/sdk/test_router_silent_model_shadow_sdk.py index 821e4ddd0b7..be5998e205e 100644 --- a/tests/integration/sdk/test_router_silent_model_shadow_sdk.py +++ b/tests/integration/sdk/test_router_silent_model_shadow_sdk.py @@ -82,9 +82,9 @@ class _ShadowRecorder(CustomLogger): async def async_log_success_event( self, kwargs: Mapping[str, object], response_obj: object, start_time: object, end_time: object ) -> None: - metadata: Final = _OBJECT.validate_python(_OBJECT.validate_python(kwargs["litellm_params"])["metadata"] or {}) - if metadata.get("is_silent_experiment") is True: - self.shadow_groups.put(str(metadata.get("model_group"))) + payload: Final = _OBJECT.validate_python(kwargs["standard_logging_object"]) + if _OBJECT.validate_python(payload["metadata"]).get("is_silent_experiment") is True: + self.shadow_groups.put(str(payload["model_group"])) def drained(self) -> tuple[str, ...]: return tuple(self.shadow_groups.get_nowait() for _ in range(self.shadow_groups.qsize())) diff --git a/tests/unit/caching/test_unit_test_caching.py b/tests/unit/caching/test_unit_test_caching.py index 83132b5aa1d..f8a367798db 100644 --- a/tests/unit/caching/test_unit_test_caching.py +++ b/tests/unit/caching/test_unit_test_caching.py @@ -8,6 +8,7 @@ load_dotenv() import asyncio import hashlib import random +from typing import Final import pytest @@ -16,7 +17,7 @@ from litellm import aembedding, completion, embedding from litellm.caching.caching import Cache from unittest.mock import AsyncMock, patch, MagicMock -from litellm.caching.caching_handler import LLMCachingHandler, CachingHandlerResponse +from litellm.caching.caching_handler import LLMCachingHandler, CachingHandlerResponse, _PENDING_CACHE_WRITES from litellm.caching.caching import LiteLLMCacheType from litellm.types.utils import CallTypes from litellm.types.rerank import RerankResponse @@ -278,6 +279,107 @@ def test_get_model_param_value_reads_model_group_from_litellm_metadata(): ) +def test_get_model_param_value_prefers_the_router_group_over_a_caller_model_group(): + cache: Final = Cache() + deployment: Final = {"id": "deployment-1"} + request: Final = {"model": "openai/gpt-5.6", "input": "search this text", "model_info": deployment} + router_metadata: Final = {"model_group": "caller", "model_info": deployment} + + assert ( + cache._get_model_param_value( + {**request, "metadata": {"model_group": "target"}, "litellm_metadata": router_metadata} + ) + == "caller" + ) + assert ( + cache._get_model_param_value( + { + **request, + "litellm_params": {"metadata": {"model_group": "target"}, "litellm_metadata": router_metadata}, + } + ) + == "caller" + ) + assert ( + cache._get_model_param_value( + { + **request, + "metadata": {"model_group": "target", "model_info": deployment}, + "litellm_metadata": router_metadata, + } + ) + == "caller" + ) + assert ( + cache._get_model_param_value( + { + **request, + "metadata": router_metadata, + "litellm_metadata": {"model_group": "parent", "model_info": {"id": "parent-deployment"}}, + } + ) + == "caller" + ) + + +def test_get_model_param_value_keeps_metadata_first_outside_the_router(): + cache: Final = Cache() + + assert ( + cache._get_model_param_value( + { + "model": "openai/gpt-5.6", + "input": "search this text", + "metadata": {"model_group": "group-a"}, + "litellm_metadata": {"model_group": "group-b"}, + } + ) + == "group-a" + ) + + +@pytest.mark.asyncio +async def test_router_call_keeps_its_own_group_when_forwarded_litellm_metadata_names_another( + monkeypatch: pytest.MonkeyPatch, +): + monkeypatch.setattr(litellm, "cache", Cache(type=LiteLLMCacheType.LOCAL)) + router: Final = litellm.Router( + model_list=[ + { + "model_name": group, + "litellm_params": {"model": "openai/gpt-5.6", "api_key": "sk-test", "mock_response": f"{group} answer"}, + } + for group in ("group-a", "group-b") + ] + ) + messages: Final = [{"role": "user", "content": "which group answers this"}] + + await router.acompletion( + model="group-a", + messages=messages, + caching=True, + litellm_metadata={"model_group": "group-b", "model_info": {"id": "parent-deployment"}}, + ) + await asyncio.gather(*_PENDING_CACHE_WRITES) + reply: Final = await router.acompletion(model="group-b", messages=messages, caching=True) + + assert reply.choices[0].message.content == "group-b answer" + + +@pytest.mark.parametrize("planted", [["sales-premium"], [("sales", "sales-premium")]], ids=["string", "tuple"]) +def test_get_model_param_value_reads_caching_groups_only_beside_the_router_group(planted: list[str | tuple[str, str]]): + cache: Final = Cache() + request: Final = { + "model": "openai/gpt-5.6", + "input": "search this text", + "model_info": {"id": "sales-deployment"}, + "litellm_metadata": {"model_group": "sales", "caching_groups": None, "model_info": {"id": "sales-deployment"}}, + "metadata": {"caching_groups": planted}, + } + + assert cache._get_model_param_value(request) == "sales" + + def test_preset_cache_key(): """ Test that the preset cache key is used if it is set in kwargs["litellm_params"] diff --git a/tests/unit/integrations/datadog/test_datadog_cost_management.py b/tests/unit/integrations/datadog/test_datadog_cost_management.py index 1a50a6991da..3601db12909 100644 --- a/tests/unit/integrations/datadog/test_datadog_cost_management.py +++ b/tests/unit/integrations/datadog/test_datadog_cost_management.py @@ -1,4 +1,5 @@ import time +from typing import Final from unittest.mock import AsyncMock import pytest @@ -314,6 +315,43 @@ async def test_extract_tags_nested_metadata_allowlisted(clean_env): assert "ignored" not in tags +@pytest.mark.parametrize( + ("metadata", "request_tags", "expected_tag"), + [ + ( + {"is_silent_experiment": True, "requester_metadata": {"is_silent_experiment": False}}, + ["is_silent_experiment:false"], + "true", + ), + ( + { + "requester_metadata": {"is_silent_experiment": True}, + "spend_logs_metadata": {"is_silent_experiment": "true"}, + }, + [], + None, + ), + ({}, ["is_silent_experiment:true"], None), + ], + ids=["mirrored call", "direct call with a client flag", "direct call with a client request tag"], +) +@pytest.mark.asyncio +async def test_extract_tags_takes_the_silent_experiment_flag_only_from_the_router( + clean_env: None, metadata: dict[str, object], request_tags: list[str], expected_tag: str | None +): + logger: Final = DatadogCostManagementLogger(cost_tag_keys=["is_silent_experiment"]) + log: Final = StandardLoggingPayload( + custom_llm_provider="openai", + model="gpt-4", + response_cost=0.01, + startTime=time.time(), + metadata=metadata, + request_tags=request_tags, + ) + + assert logger._extract_tags(log).get("is_silent_experiment") == expected_tag + + @pytest.mark.asyncio async def test_extract_tags_allowlist_cannot_override_reserved_dimensions(clean_env): """ diff --git a/tests/unit/integrations/test_langfuse.py b/tests/unit/integrations/test_langfuse.py index 00d85b64566..2d1e49e32df 100644 --- a/tests/unit/integrations/test_langfuse.py +++ b/tests/unit/integrations/test_langfuse.py @@ -830,6 +830,31 @@ class TestLangfuseUsageDetails(unittest.TestCase): assert generation_metadata["litellm_response_cost"] == 0.25 assert generation_metadata["api_base"] == "https://real-api-base" + def test_caller_sent_is_silent_experiment_cannot_hide_the_mirrored_call_flag(self): + payload = self._build_standard_logging_payload(trace_id="canary-trace-id") + payload["metadata"]["is_silent_experiment"] = True + payload["metadata"]["requester_metadata"] = {"is_silent_experiment": False} + kwargs = self._build_langfuse_kwargs(payload) + metadata = self._canary_request_metadata() + self.use_real_langfuse_client() + + self.logger._log_langfuse_v2( + user_id="user-1", + metadata=metadata, + litellm_params={"metadata": metadata}, + output=None, + start_time=datetime.datetime(2024, 1, 1, 12, 0, 0), + end_time=datetime.datetime(2024, 1, 1, 12, 0, 1), + kwargs=kwargs, + optional_params={}, + input=None, + response_obj=None, + level="INFO", + litellm_call_id="canary-call-id", + ) + + assert self.exported_generation_metadata()["is_silent_experiment"] is True + def test_generation_metadata_carries_the_call_id_and_response_id(self): """ v2's generation id was ``time-_``, so a generation could diff --git a/tests/unit/integrations/test_prometheus_spend_logs_metadata.py b/tests/unit/integrations/test_prometheus_spend_logs_metadata.py index e2af6fd2daf..608195a9df8 100644 --- a/tests/unit/integrations/test_prometheus_spend_logs_metadata.py +++ b/tests/unit/integrations/test_prometheus_spend_logs_metadata.py @@ -5,6 +5,10 @@ Verifies that metadata from x-litellm-spend-logs-metadata header is available in Prometheus custom labels via combined_metadata. """ +from typing import Final + +import pytest + from litellm.integrations.prometheus import ( _get_combined_custom_metadata_from_standard_logging_payload, get_custom_labels_from_metadata, @@ -205,3 +209,34 @@ def test_project_alias_accessible_without_prefix(monkeypatch): result = get_custom_labels_from_metadata(combined) assert result == {"user_api_key_project_alias": "hotel-recommendations"} + + +@pytest.mark.parametrize( + ("standard_logging_metadata", "expected_labels"), + [ + ( + {"is_silent_experiment": True, "requester_metadata": {"is_silent_experiment": False}}, + {"metadata_is_silent_experiment": "true"}, + ), + ( + { + "requester_metadata": {"is_silent_experiment": True}, + "spend_logs_metadata": {"is_silent_experiment": "true"}, + }, + {}, + ), + ], + ids=["mirrored call", "direct call with a client flag"], +) +def test_silent_experiment_label_comes_only_from_the_router_flag( + monkeypatch: pytest.MonkeyPatch, + standard_logging_metadata: dict[str, object], + expected_labels: dict[str, str], +): + monkeypatch.setattr("litellm.custom_prometheus_metadata_labels", ["metadata.is_silent_experiment"]) + + combined: Final = _get_combined_custom_metadata_from_standard_logging_payload( + {"metadata": standard_logging_metadata} + ) + + assert get_custom_labels_from_metadata(combined) == expected_labels diff --git a/tests/unit/litellm_core_utils/test_litellm_logging.py b/tests/unit/litellm_core_utils/test_litellm_logging.py index 299088a6c4f..a6d99a5ba55 100644 --- a/tests/unit/litellm_core_utils/test_litellm_logging.py +++ b/tests/unit/litellm_core_utils/test_litellm_logging.py @@ -50,6 +50,7 @@ from litellm.litellm_core_utils.litellm_logging import ( get_standard_logging_object_payload, set_callbacks, ) +from litellm.litellm_core_utils.llm_response_utils.response_metadata import update_response_metadata from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER from litellm.llms.base_llm.ocr.transformation import OCRUsageInfo from litellm.proxy._types import UserAPIKeyAuth @@ -1118,6 +1119,115 @@ class TestZeroCostDiagnostic: litellm.model_cost.pop(served_model, None) +class TestPerSecondMessagesCost: + DEPLOYMENT_ID: Final = "per-second-priced-deployment" + MODEL_GROUP: Final = "per-second-priced-claude" + RATE: Final = 0.00042 + PER_SECOND_PRICING: Final = {"input_cost_per_second": RATE, "output_cost_per_second": RATE} + MESSAGES_RESPONSE: Final = { + "id": "msg_per-second-call", + "type": "message", + "role": "assistant", + "model": "claude-haiku-5-5", + "content": [{"type": "text", "text": "Hi there friend"}], + "stop_reason": "end_turn", + "stop_sequence": None, + "usage": {"input_tokens": 10, "output_tokens": 5}, + } + + @pytest.fixture(autouse=True) + def registered_deployment(self) -> Iterator[None]: + litellm.register_model(model_cost={self.DEPLOYMENT_ID: self.PER_SECOND_PRICING}, persist_across_reloads=False) + try: + yield + finally: + litellm.model_cost.pop(self.DEPLOYMENT_ID, None) + + def _routed_messages_call( + self, *, stream: bool, received_seconds_before_start: float | None = None + ) -> LitellmLogging: + started: Final = datetime.datetime.now() + proxy_receive_stamp: Final = ( + {} + if received_seconds_before_start is None + else { + "litellm_received_at": (started - datetime.timedelta(seconds=received_seconds_before_start)).astimezone( + datetime.timezone.utc + ) + } + ) + logging_obj: Final = LitellmLogging( + model=self.MODEL_GROUP, + messages=[{"role": "user", "content": "Say hi in three words"}], + stream=stream, + call_type="anthropic_messages", + start_time=started, + litellm_call_id="per-second-call", + function_id="fn", + kwargs={"model": self.MODEL_GROUP}, + ) + logging_obj.update_from_kwargs( + kwargs={ + "model": "anthropic/claude-haiku-5-5", + "litellm_metadata": { + "deployment": "anthropic/claude-haiku-5-5", + "model_info": {"id": self.DEPLOYMENT_ID}, + **proxy_receive_stamp, + }, + }, + model="claude-haiku-5-5", + optional_params={"max_tokens": 32}, + litellm_params={ + "preset_cache_key": None, + "stream_response": {}, + "model_info": None, + **self.PER_SECOND_PRICING, + "max_tokens": 32, + }, + custom_llm_provider="anthropic", + ) + return logging_obj + + @staticmethod + def _response_handed_back_after(logging_obj: LitellmLogging, seconds: float) -> None: + started: Final = logging_obj.start_time + update_response_metadata( + result=dict(TestPerSecondMessagesCost.MESSAGES_RESPONSE), + logging_obj=logging_obj, + model="claude-haiku-5-5", + kwargs={}, + start_time=started, + end_time=started + datetime.timedelta(seconds=seconds), + ) + + def test_messages_call_prices_the_reported_response_time_however_late_the_cost_is_read(self) -> None: + logging_obj: Final = self._routed_messages_call(stream=False) + self._response_handed_back_after(logging_obj, seconds=1.0) + result: Final = logging_obj._handle_anthropic_messages_response_logging(result=dict(self.MESSAGES_RESPONSE)) + cost_for_the_response_header: Final = logging_obj._response_cost_calculator(result=result) + logging_obj.model_call_details["end_time"] = logging_obj.start_time + datetime.timedelta(seconds=1.5) + cost_for_the_spend_log: Final = logging_obj._response_cost_calculator(result=result) + + assert cost_for_the_response_header == pytest.approx(self.RATE) + assert cost_for_the_spend_log == pytest.approx(self.RATE) + + def test_messages_call_bills_from_logging_setup_not_from_proxy_receive(self) -> None: + logging_obj: Final = self._routed_messages_call(stream=False, received_seconds_before_start=2.0) + self._response_handed_back_after(logging_obj, seconds=1.0) + result: Final = logging_obj._handle_anthropic_messages_response_logging(result=dict(self.MESSAGES_RESPONSE)) + + assert logging_obj.response_timing_metrics["_response_ms"] == pytest.approx(3000) + assert logging_obj._response_cost_calculator(result=result) == pytest.approx(self.RATE) + + def test_streamed_messages_call_bills_the_whole_stream_not_the_first_byte(self) -> None: + logging_obj: Final = self._routed_messages_call(stream=True) + self._response_handed_back_after(logging_obj, seconds=0.25) + logging_obj.model_call_details["end_time"] = logging_obj.start_time + datetime.timedelta(seconds=1.0) + result: Final = logging_obj._handle_anthropic_messages_response_logging(result=dict(self.MESSAGES_RESPONSE)) + + assert logging_obj._response_cost_calculator(result=result) == pytest.approx(self.RATE) + + class TestGetRouterModelId: """Tests for the get_router_model_id helper method.""" @@ -3723,9 +3833,9 @@ def test_get_extra_header_tags_returns_none_for_absent_or_empty_header( monkeypatch.setattr(litellm, "extra_spend_tag_headers", ["x-org-id"], raising=False) assert StandardLoggingPayloadSetup._get_extra_header_tags(proxy_server_request={"headers": {}}) is None - assert StandardLoggingPayloadSetup._get_extra_header_tags( - proxy_server_request={"headers": {"x-org-id": ""}} - ) is None + assert ( + StandardLoggingPayloadSetup._get_extra_header_tags(proxy_server_request={"headers": {"x-org-id": ""}}) is None + ) def test_get_extra_header_tags_preserves_configured_header_order( @@ -11939,7 +12049,7 @@ def test_get_additional_headers(): def all_fields_present(standard_logging_metadata: StandardLoggingMetadata): - for field in StandardLoggingMetadata.__annotations__.keys(): + for field in StandardLoggingMetadata.__required_keys__: assert field in standard_logging_metadata @@ -11975,6 +12085,26 @@ def test_get_standard_logging_metadata(metadata_key, metadata_value): assert standard_logging_metadata[metadata_key] == metadata_value +@pytest.mark.usefixtures("_vcr_outcome_gate", "drain_logging_worker", "isolate_litellm_state", "setup_and_teardown") +def test_module_level_get_standard_logging_metadata_drops_only_a_caller_sent_silent_experiment_flag(): + from litellm.litellm_core_utils.litellm_logging import get_standard_logging_metadata + + caller_metadata: Final = { + "is_silent_experiment": True, + "user_api_key_alias": "alias", + "usage_object": {"total_tokens": 3}, + "used_client_oauth_token": True, + "kubernetes_pod_routing": {"pod": "a"}, + } + + result = get_standard_logging_metadata(caller_metadata) + + assert "is_silent_experiment" not in result + assert {key: result[key] for key in caller_metadata.keys() - {"is_silent_experiment"}} == { + key: value for key, value in caller_metadata.items() if key != "is_silent_experiment" + } + + @pytest.mark.usefixtures("_vcr_outcome_gate", "drain_logging_worker", "isolate_litellm_state", "setup_and_teardown") def test_get_standard_logging_metadata_user_api_key_hash(): valid_hash = "a" * 64 # 64 character string @@ -13801,7 +13931,9 @@ def test_record_native_bedrock_guardrail_bills_each_call_its_own_trace_when_call ) logging_obj.model_call_details["litellm_params"] = {"metadata": shared_metadata} logging_obj.record_native_bedrock_guardrail({"trace": trace}) - recorded = logging_obj.model_call_details["litellm_params"]["metadata"]["standard_logging_guardrail_information"] + recorded = logging_obj.model_call_details["litellm_params"]["metadata"][ + "standard_logging_guardrail_information" + ] costs.append(guardrail_information_cost(recorded)) (first_cost, second_cost) = costs diff --git a/tests/unit/litellm_core_utils/test_text_completion_conversion.py b/tests/unit/litellm_core_utils/test_text_completion_conversion.py index 4a2f91093a3..8a913bc8058 100644 --- a/tests/unit/litellm_core_utils/test_text_completion_conversion.py +++ b/tests/unit/litellm_core_utils/test_text_completion_conversion.py @@ -1,10 +1,12 @@ import asyncio +from typing import Final from unittest.mock import MagicMock, patch import pytest import litellm from litellm import text_completion +from litellm.llms.custom_llm import CustomLLM from litellm.types.utils import ModelResponse, TextCompletionResponse, Usage from litellm.utils import LiteLLMResponseObjectHandler @@ -1298,3 +1300,30 @@ def mock_post(*args, **kwargs): "usage": {"prompt_tokens": 2, "total_tokens": 18, "completion_tokens": 16}, } return mock_response + + +def test_text_completion_reads_a_custom_provider_dict_reply(monkeypatch: pytest.MonkeyPatch) -> None: + class DictReplyLLM(CustomLLM): + def completion(self, *args: object, **kwargs: object) -> dict[str, object]: + return { + "id": "dict-reply", + "object": "chat.completion", + "created": 1, + "model": "x", + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": "hi from a dict reply"}, + "finish_reason": "stop", + } + ], + } + + monkeypatch.setattr(litellm, "custom_provider_map", [{"provider": "dictreply", "custom_handler": DictReplyLLM()}]) + monkeypatch.setattr(litellm, "provider_list", [*litellm.provider_list, "dictreply"]) + monkeypatch.setattr(litellm, "_custom_providers", [*litellm._custom_providers, "dictreply"]) + + response: Final = text_completion(model="dictreply/x", prompt="hi") + + assert response.choices[0].text == "hi from a dict reply" + assert response.choices[0].finish_reason == "stop" diff --git a/tests/unit/llms/anthropic/chat/test_anthropic_chat_handler.py b/tests/unit/llms/anthropic/chat/test_anthropic_chat_handler.py index 355d407001c..5d3fc523c0c 100644 --- a/tests/unit/llms/anthropic/chat/test_anthropic_chat_handler.py +++ b/tests/unit/llms/anthropic/chat/test_anthropic_chat_handler.py @@ -537,7 +537,11 @@ def test_streamed_thinking_blocks_accumulate_in_the_openai_sdk_and_replay_withou {"type": "content_block_delta", "index": 2, "delta": {"type": "thinking_delta", "thinking": "Step 3."}}, {"type": "content_block_delta", "index": 2, "delta": {"type": "signature_delta", "signature": "sig-b"}}, {"type": "content_block_stop", "index": 2}, - {"type": "message_delta", "delta": {"stop_reason": "end_turn", "stop_sequence": None}, "usage": {"output_tokens": 9}}, + { + "type": "message_delta", + "delta": {"stop_reason": "end_turn", "stop_sequence": None}, + "usage": {"output_tokens": 9}, + }, {"type": "message_stop"}, ) @@ -558,7 +562,11 @@ def test_streamed_thinking_blocks_accumulate_in_the_openai_sdk_and_replay_withou "thinking_blocks": message["thinking_blocks"], } replayed: Final = anthropic_messages_pt( - messages=[{"role": "user", "content": "Capital of France?"}, next_turn, {"role": "user", "content": "And Spain?"}], + messages=[ + {"role": "user", "content": "Capital of France?"}, + next_turn, + {"role": "user", "content": "And Spain?"}, + ], model="claude-haiku-4-5", llm_provider="anthropic", ) @@ -3650,3 +3658,79 @@ def test_streamed_end_turn_carries_no_stop_details(): provider_fields: Final = assembled.choices[0].message.provider_specific_fields or {} assert "stop_details" not in provider_fields assert "stop_reason" not in provider_fields + + +def _anthropic_chat_request_for_bound_thinking( + extra_headers: dict[str, str] | None, drop_params: bool | None = None +) -> httpx.Request: + captured_requests: Final[list[httpx.Request]] = [] + + def respond(request: httpx.Request) -> httpx.Response: + captured_requests.append(request) + return httpx.Response( + 200, + json={ + "id": "msg_bound", + "type": "message", + "role": "assistant", + "model": "claude-opus-4-8", + "content": [{"type": "text", "text": "Hello"}], + "stop_reason": "end_turn", + "stop_sequence": None, + "usage": {"input_tokens": 1, "output_tokens": 1}, + }, + ) + + thinking: Final = {"type": "adaptive", "block_binding": {"prefix_mismatch_behavior": "error"}} + client: Final = HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(respond))) + try: + litellm.completion( + model="anthropic/claude-opus-4-8", + messages=[{"role": "user", "content": "Hello"}], + api_key="test-key", + client=client, + thinking=thinking, + extra_headers=extra_headers, + drop_params=drop_params, + ) + finally: + client.close() + assert thinking == {"type": "adaptive", "block_binding": {"prefix_mismatch_behavior": "error"}} + return captured_requests[0] + + +def _anthropic_chat_body_for_bound_thinking( + extra_headers: dict[str, str] | None, drop_params: bool | None = None +) -> dict[str, object]: + return json.loads(_anthropic_chat_request_for_bound_thinking(extra_headers, drop_params).content) + + +def test_anthropic_chat_keeps_block_binding_when_the_final_header_carries_the_binding_beta( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("LITELLM_LOCAL_ANTHROPIC_BETA_HEADERS", "true") + body: Final = _anthropic_chat_body_for_bound_thinking( + {"anthropic-beta": "thinking-binding-controls-2026-08-01"}, drop_params=True + ) + assert body["thinking"] == {"type": "adaptive", "block_binding": {"prefix_mismatch_behavior": "error"}} + + +@pytest.mark.parametrize("drop_params", [True, False]) +def test_anthropic_chat_adds_the_binding_beta_so_block_binding_is_kept( + drop_params: bool, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv("LITELLM_LOCAL_ANTHROPIC_BETA_HEADERS", "true") + monkeypatch.setattr(litellm, "drop_params", False) + request: Final = _anthropic_chat_request_for_bound_thinking(None, drop_params=drop_params) + assert json.loads(request.content)["thinking"] == { + "type": "adaptive", + "block_binding": {"prefix_mismatch_behavior": "error"}, + } + assert "thinking-binding-controls-2026-08-01" in request.headers["anthropic-beta"].split(",") + + +def test_anthropic_chat_sends_block_binding_as_is_without_drop_params(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("LITELLM_LOCAL_ANTHROPIC_BETA_HEADERS", "true") + monkeypatch.setattr(litellm, "drop_params", False) + body: Final = _anthropic_chat_body_for_bound_thinking(None) + assert body["thinking"] == {"type": "adaptive", "block_binding": {"prefix_mismatch_behavior": "error"}} diff --git a/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_router.py b/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_router.py index 56e650d3351..1d0d70d8e60 100644 --- a/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_router.py +++ b/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_router.py @@ -5,8 +5,10 @@ import base64 import json import logging import struct +import time import uuid -from collections.abc import AsyncIterable, Mapping +from collections.abc import AsyncIterable, AsyncIterator, Mapping +from datetime import datetime from typing import Final from zlib import crc32 @@ -446,6 +448,135 @@ async def test_router_aanthropic_messages_streaming_logs_usage_model_and_cost( assert payload["completion_tokens"] == max(usage.get("output_tokens", 0) for usage in usages) == 20 +_PER_SECOND_RATE: Final = 0.5 +_UPSTREAM_PAUSE_SECONDS: Final = 0.3 + + +def _priced_router(**pricing: float) -> Router: + return Router( + model_list=[ + { + "model_name": _ALIAS, + "litellm_params": {"model": _ANTHROPIC_MODEL, "api_key": "fake-key", **pricing}, + } + ] + ) + + +def _per_second_router() -> Router: + return _priced_router(input_cost_per_second=_PER_SECOND_RATE, output_cost_per_second=_PER_SECOND_RATE) + + +def _anthropic_sse_event(event: Mapping[str, object]) -> bytes: + return f"event: {event['type']}\ndata: {json.dumps(event)}\n\n".encode() + + +async def _route_like_the_proxy(router: Router, **request: object) -> object: + logging_obj, routed_request = litellm.utils.function_setup( + original_function="aanthropic_messages", + rules_obj=litellm.utils.Rules(), + start_time=datetime.now(), + litellm_call_id=str(uuid.uuid4()), + model=_ALIAS, + max_tokens=100, + **request, + ) + return await router.aanthropic_messages(**routed_request, litellm_logging_obj=logging_obj) + + +async def _sse_pausing_after_the_first_event() -> AsyncIterator[bytes]: + yield _anthropic_sse_event(_STREAM_EVENTS[0]) + await asyncio.sleep(_UPSTREAM_PAUSE_SECONDS) + for event in _STREAM_EVENTS[1:]: + yield _anthropic_sse_event(event) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "pricing", + [ + {"input_cost_per_second": _PER_SECOND_RATE, "output_cost_per_second": _PER_SECOND_RATE}, + {"cost_per_second": _PER_SECOND_RATE}, + ], +) +async def test_router_aanthropic_messages_bills_a_per_second_deployment_for_the_call_duration( + respx_mock, monkeypatch, pricing: Mapping[str, float] +): + messages: Final = _unique_messages() + recorder: Final = _RecordingLogger(messages) + monkeypatch.setattr(litellm, "callbacks", [recorder]) + + async def slow_upstream(request: httpx.Request) -> httpx.Response: + await asyncio.sleep(_UPSTREAM_PAUSE_SECONDS) + return httpx.Response(200, json=_anthropic_body()) + + respx_mock.post(url__regex=_ANTHROPIC_URL).mock(side_effect=slow_upstream) + started: Final = time.monotonic() + await _route_like_the_proxy(_priced_router(**pricing), messages=messages) + await _wait_for_payload(recorder) + elapsed: Final = time.monotonic() - started + + assert ( + _PER_SECOND_RATE * _UPSTREAM_PAUSE_SECONDS + <= recorder.payloads[0]["response_cost"] + <= _PER_SECOND_RATE * elapsed + ) + + +@pytest.mark.asyncio +async def test_router_aanthropic_messages_bills_a_per_second_deployment_for_the_whole_stream(respx_mock, monkeypatch): + messages: Final = _unique_messages() + recorder: Final = _RecordingLogger(messages) + monkeypatch.setattr(litellm, "callbacks", [recorder]) + respx_mock.post(url__regex=_ANTHROPIC_URL).mock( + side_effect=lambda request: httpx.Response( + 200, headers={"content-type": "text/event-stream"}, content=_sse_pausing_after_the_first_event() + ) + ) + started: Final = time.monotonic() + stream: Final = await _route_like_the_proxy(_per_second_router(), messages=messages, stream=True) + await _stream_events(stream) + await _wait_for_payload(recorder) + elapsed: Final = time.monotonic() - started + + assert ( + _PER_SECOND_RATE * _UPSTREAM_PAUSE_SECONDS + <= recorder.payloads[0]["response_cost"] + <= _PER_SECOND_RATE * elapsed + ) + + +async def _wait_for_payloads(recorder: _RecordingLogger, count: int) -> None: + async def arrived() -> None: + while len(recorder.payloads) < count: + await asyncio.sleep(0.05) + + await asyncio.wait_for(arrived(), timeout=30.0) + + +@pytest.mark.asyncio +async def test_router_aanthropic_messages_bills_a_short_call_on_a_long_context_rate_like_a_deployment_without_one( + respx_mock, monkeypatch +): + messages: Final = _unique_messages() + recorder: Final = _RecordingLogger(messages) + monkeypatch.setattr(litellm, "callbacks", [recorder]) + respx_mock.post(url__regex=_ANTHROPIC_URL).mock( + side_effect=lambda request: httpx.Response(200, json=_anthropic_body()) + ) + + await _route_like_the_proxy(_priced_router(), messages=messages) + await _route_like_the_proxy( + _priced_router(input_cost_per_token_above_200k_tokens=1.0, output_cost_per_token_above_200k_tokens=1.0), + messages=messages, + ) + await _wait_for_payloads(recorder, 2) + + costs: Final = {payload["response_cost"] for payload in recorder.payloads} + assert len(costs) == 1, costs + assert costs.pop() > 0 + + _LARGE_SYSTEM_PROMPT: Final = "This is a comprehensive legal agreement between Party A and Party B. " * 100 diff --git a/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_transformation.py b/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_transformation.py index 13729ecb6cb..1c004d8c36e 100644 --- a/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_transformation.py +++ b/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_transformation.py @@ -1,3 +1,4 @@ +import logging from collections.abc import Mapping from datetime import datetime, timezone from pathlib import Path @@ -6,9 +7,11 @@ from unittest.mock import patch import httpx import pytest +import respx from pydantic import JsonValue import litellm +from litellm import anthropic_beta_headers_manager from litellm.litellm_core_utils.litellm_logging import Logging from litellm.llms.anthropic.messages.native.transformation import ( AnthropicMessagesConfig, @@ -498,3 +501,133 @@ def test_drop_params_forwards_a_non_object_message_item_as_sent_on_the_native_re assert request["messages"][1] == 7 assert request["messages"][2]["content"][0]["content"] == [{"type": "text", "text": "sunny"}] + + +BINDING_AND_DISPLAY_BETAS: Final = "thinking-binding-controls-2026-08-01,thinking-display-updates-2026-08-18" + + +def test_messages_headers_add_the_binding_beta_when_thinking_carries_block_binding() -> None: + headers: Final = AnthropicMessagesConfig()._update_headers_with_anthropic_beta( + headers={}, + optional_params={"thinking": {"type": "adaptive", "block_binding": {"prefix_mismatch_behavior": "error"}}}, + messages=[], + ) + assert "thinking-binding-controls-2026-08-01" in headers["anthropic-beta"].split(",") + + +def test_messages_headers_add_no_binding_beta_without_block_binding() -> None: + headers: Final = AnthropicMessagesConfig()._update_headers_with_anthropic_beta( + headers={}, optional_params={"thinking": {"type": "adaptive"}}, messages=[] + ) + assert "thinking-binding-controls-2026-08-01" not in headers.get("anthropic-beta", "") + + +def _bound_adaptive_request(headers: dict[str, str], drop_params: bool | None = True) -> dict[str, object]: + thinking: Final = { + "type": "adaptive", + "block_binding": {"prefix_mismatch_behavior": "error"}, + "display": "updates", + } + optional_params: Final = {"max_tokens": 64, "thinking": thinking} + request: Final = AnthropicMessagesConfig().transform_anthropic_messages_request( + model="claude-opus-4-8", + messages=[{"role": "user", "content": "hi"}], + anthropic_messages_optional_request_params=optional_params, + litellm_params=GenericLiteLLMParams(drop_params=drop_params), + headers=headers, + ) + assert thinking == { + "type": "adaptive", + "block_binding": {"prefix_mismatch_behavior": "error"}, + "display": "updates", + }, "the caller's thinking dict must survive for a fallback leg" + return request + + +def test_messages_request_keeps_beta_gated_thinking_fields_when_the_final_header_carries_the_betas() -> None: + request: Final = _bound_adaptive_request({"anthropic-beta": BINDING_AND_DISPLAY_BETAS}) + assert request["thinking"] == { + "type": "adaptive", + "block_binding": {"prefix_mismatch_behavior": "error"}, + "display": "updates", + } + + +def test_messages_request_drops_beta_gated_thinking_fields_the_final_header_does_not_carry( + caplog: pytest.LogCaptureFixture, +) -> None: + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + request: Final = _bound_adaptive_request({"anthropic-beta": "thinking-binding-controls-2026-08-01"}) + assert request["thinking"] == {"type": "adaptive", "block_binding": {"prefix_mismatch_behavior": "error"}} + assert any("thinking.display=updates" in record.getMessage() for record in caplog.records) + + +def test_messages_request_sends_beta_gated_thinking_fields_as_is_without_drop_params( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + request: Final = _bound_adaptive_request({}, drop_params=None) + assert request["thinking"] == { + "type": "adaptive", + "block_binding": {"prefix_mismatch_behavior": "error"}, + "display": "updates", + } + + +def test_messages_request_drops_clear_at_from_a_mid_conversation_system_turn_without_its_beta() -> None: + system_turn: Final = {"role": "system", "content": "be terse", "clear_at": "end_turn"} + request: Final = AnthropicMessagesConfig().transform_anthropic_messages_request( + model="claude-opus-4-8", + messages=[{"role": "user", "content": "hi"}, system_turn, {"role": "user", "content": "go"}], + anthropic_messages_optional_request_params={"max_tokens": 64}, + litellm_params=GenericLiteLLMParams(drop_params=True), + headers={"anthropic-beta": "mid-conversation-system-2026-07-01"}, + ) + assert request["messages"][1] == {"role": "system", "content": "be terse"} + assert system_turn == {"role": "system", "content": "be terse", "clear_at": "end_turn"} + + +def test_a_provider_without_a_beta_headers_row_forwards_beta_gated_fields_under_drop_params( + local_beta_headers_config: None, +) -> None: + thinking: Final = {"type": "enabled", "budget_tokens": 1024, "display": "updates"} + request: Final = MinimaxMessagesConfig().transform_anthropic_messages_request( + model="MiniMax-M2", + messages=[ + {"role": "user", "content": "hi"}, + {"role": "assistant", "content": "Hello"}, + {"role": "user", "content": "how are you"}, + {"role": "system", "content": "Answer in French", "output_config": {"effort": "low"}}, + ], + anthropic_messages_optional_request_params={"max_tokens": 64, "thinking": dict(thinking)}, + litellm_params=GenericLiteLLMParams(drop_params=True), + headers={}, + ) + assert request["thinking"] == thinking + assert request["messages"][-1]["output_config"] == {"effort": "low"} + + +def test_a_first_party_messages_request_never_fetches_the_remote_beta_headers_config( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.delenv("LITELLM_LOCAL_ANTHROPIC_BETA_HEADERS", raising=False) + monkeypatch.setattr(anthropic_beta_headers_manager, "_BETA_HEADERS_CONFIG", None) + + with respx.mock(assert_all_called=False) as router: + remote_config: Final = router.get(litellm.anthropic_beta_headers_url).mock( + return_value=httpx.Response(200, json={}) + ) + request: Final = AnthropicMessagesConfig().transform_anthropic_messages_request( + model="claude-opus-4-8", + messages=[ + {"role": "user", "content": "hi"}, + {"role": "system", "content": "be terse", "clear_at": "end_turn"}, + {"role": "user", "content": "go"}, + ], + anthropic_messages_optional_request_params={"max_tokens": 64}, + litellm_params=GenericLiteLLMParams(drop_params=True), + headers={}, + ) + + assert request["messages"][1] == {"role": "system", "content": "be terse"} + assert not remote_config.called diff --git a/tests/unit/llms/anthropic/messages/native/test_response_cache.py b/tests/unit/llms/anthropic/messages/native/test_response_cache.py index 133bcd96433..bc3b1f6ac76 100644 --- a/tests/unit/llms/anthropic/messages/native/test_response_cache.py +++ b/tests/unit/llms/anthropic/messages/native/test_response_cache.py @@ -2,6 +2,7 @@ import asyncio from typing import Any, AsyncIterator, Dict, Final, List import pytest +import pytest_asyncio import datetime @@ -67,7 +68,7 @@ async def _collect(stream: AsyncIterator[bytes]) -> List[bytes]: return [chunk async for chunk in stream] -@pytest.fixture(autouse=True) +@pytest_asyncio.fixture(autouse=True, loop_scope="function") async def _drain_logging_worker(): yield await GLOBAL_LOGGING_WORKER.flush() diff --git a/tests/unit/llms/anthropic/test_anthropic_common_utils.py b/tests/unit/llms/anthropic/test_anthropic_common_utils.py index 7a339353252..75d2bc9763d 100644 --- a/tests/unit/llms/anthropic/test_anthropic_common_utils.py +++ b/tests/unit/llms/anthropic/test_anthropic_common_utils.py @@ -4296,7 +4296,14 @@ def test_shared_legacy_thinking_translation_carries_block_binding( @pytest.mark.usefixtures("local_model_cost_map") @pytest.mark.parametrize( - ("custom_llm_provider", "model", "requested_thinking", "rejected_keys", "expected_thinking"), + ( + "custom_llm_provider", + "model", + "requested_thinking", + "rejected_keys", + "expected_thinking", + "provider_rejects_block_binding", + ), ( ( "anthropic", @@ -4304,6 +4311,7 @@ def test_shared_legacy_thinking_translation_carries_block_binding( {"type": "enabled", "budget_tokens": 2048}, {"not_a_thinking_key": True}, {"type": "adaptive"}, + False, ), ( "bedrock", @@ -4311,6 +4319,7 @@ def test_shared_legacy_thinking_translation_carries_block_binding( {"type": "enabled", "budget_tokens": 2048}, {"not_a_thinking_key": True}, {"type": "adaptive"}, + False, ), ( "bedrock", @@ -4318,6 +4327,7 @@ def test_shared_legacy_thinking_translation_carries_block_binding( {"type": "enabled", "budget_tokens": 2048}, {"not_a_thinking_key": True}, {"type": "adaptive"}, + False, ), ( "bedrock", @@ -4325,6 +4335,7 @@ def test_shared_legacy_thinking_translation_carries_block_binding( {"type": "adaptive", "display": "updates"}, {"not_a_thinking_key": True, "budget_tokens": 2048}, {"type": "adaptive", "display": "updates"}, + False, ), ( "vertex_ai", @@ -4332,6 +4343,7 @@ def test_shared_legacy_thinking_translation_carries_block_binding( {"type": "enabled", "budget_tokens": 2048}, {"not_a_thinking_key": True}, {"type": "adaptive"}, + False, ), ( "databricks", @@ -4339,6 +4351,7 @@ def test_shared_legacy_thinking_translation_carries_block_binding( {"type": "enabled", "budget_tokens": 2048}, {"not_a_thinking_key": True}, {"type": "adaptive"}, + True, ), ), ids=( @@ -4357,6 +4370,7 @@ def test_chat_thinking_drops_unknown_keys_only_under_drop_params( requested_thinking: dict[str, object], rejected_keys: dict[str, object], expected_thinking: dict[str, object], + provider_rejects_block_binding: bool, drop_params_source: str | None, monkeypatch: pytest.MonkeyPatch, ) -> None: @@ -4371,7 +4385,10 @@ def test_chat_thinking_drops_unknown_keys_only_under_drop_params( ) kept_rejected_keys: Final = {} if drop_params_source else rejected_keys - assert optional_params["thinking"] == {**expected_thinking, "block_binding": block_binding, **kept_rejected_keys} + kept_block_binding: Final = ( + {} if drop_params_source and provider_rejects_block_binding else {"block_binding": block_binding} + ) + assert optional_params["thinking"] == {**expected_thinking, **kept_block_binding, **kept_rejected_keys} @pytest.mark.usefixtures("local_model_cost_map", "local_beta_headers_config") diff --git a/tests/unit/llms/anthropic/test_anthropic_prompt_cache_prediction.py b/tests/unit/llms/anthropic/test_anthropic_prompt_cache_prediction.py index e91df2b36c3..571c344a3e6 100644 --- a/tests/unit/llms/anthropic/test_anthropic_prompt_cache_prediction.py +++ b/tests/unit/llms/anthropic/test_anthropic_prompt_cache_prediction.py @@ -14,7 +14,7 @@ import litellm from litellm.caching.dual_cache import DualCache from litellm.caching.llm_caching_handler import LLMClientCache from litellm.llms.anthropic.count_tokens import handler as count_handler -from litellm.llms.anthropic.messages.native.transformation import DEFAULT_ANTHROPIC_API_VERSION +from litellm.llms.anthropic.messages.native.transformation import DEFAULT_ANTHROPIC_API_VERSION, AnthropicMessagesConfig from litellm.llms.anthropic.prompt_cache_prediction import ( CountedPromptCachePlan, NativePredictionTarget, @@ -38,7 +38,7 @@ from litellm.proxy import proxy_server from litellm.proxy.hooks.prompt_cache_prediction import PromptCacheObserver from litellm.proxy.management_endpoints.prompt_cache_prediction import predict_arm from litellm.proxy.utils import InternalUsageCache -from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo +from litellm.types.router import Deployment, GenericLiteLLMParams, LiteLLM_Params, ModelInfo from litellm.types.utils import CacheCreationTokenDetails, ModelResponse, PromptTokensDetailsWrapper, Usage _MODEL: Final = "claude-sonnet-5" @@ -52,9 +52,12 @@ def _body() -> dict[str, JsonValue]: "model": _MODEL, "system": "Keep this context", "tools": [{"name": "lookup", "input_schema": {"type": "object"}}], - "messages": [{"role": "user", "content": [ - {"type": "text", "text": "A cacheable prefix", "cache_control": {"type": "ephemeral"}} - ]}], + "messages": [ + { + "role": "user", + "content": [{"type": "text", "text": "A cacheable prefix", "cache_control": {"type": "ephemeral"}}], + } + ], } @@ -90,7 +93,8 @@ async def test_observer_records_only_version_supported_by_token_counter(version: "httpx_response": httpx.Response(200, request=wire), "first_api_call_start_time": datetime.fromtimestamp(1000.0), "standard_logging_object": { - "status": "success", "model_id": _DEPLOYMENT, + "status": "success", + "model_id": _DEPLOYMENT, "metadata": {"user_api_key_hash": _CALLER}, }, }, @@ -106,13 +110,16 @@ async def test_observer_records_only_version_supported_by_token_counter(version: assert await lookup(cache, other_scope, prefix, now=1010.0) is None -@pytest.mark.parametrize("headers, supported", [ - ({}, True), - ({"Anthropic-Version": DEFAULT_ANTHROPIC_API_VERSION}, True), - ({"anthropic-version": "2099-01-01"}, False), - ({"Anthropic-Beta": ""}, False), - ({"anthropic-beta": "future-feature"}, False), -]) +@pytest.mark.parametrize( + "headers, supported", + [ + ({}, True), + ({"Anthropic-Version": DEFAULT_ANTHROPIC_API_VERSION}, True), + ({"anthropic-version": "2099-01-01"}, False), + ({"Anthropic-Beta": ""}, False), + ({"anthropic-beta": "future-feature"}, False), + ], +) def test_prediction_header_eligibility(headers: Mapping[str, str], supported: bool) -> None: assert supported_prediction_headers(headers) is supported @@ -152,49 +159,89 @@ async def test_provider_count_uses_same_version_and_preserves_native_input( @pytest.mark.parametrize("source", ["static", "database"]) @pytest.mark.asyncio async def test_environment_credential_matches_native_count_and_observed_scope( - source: str, monkeypatch: pytest.MonkeyPatch, + source: str, + monkeypatch: pytest.MonkeyPatch, ) -> None: monkeypatch.setenv("LIT7658_PROVIDER_KEY", _KEY) monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) monkeypatch.setattr(litellm, "in_memory_llm_clients_cache", LLMClientCache()) params: Final = { - "model": f"anthropic/{_MODEL}", "api_key": "os.environ/LIT7658_PROVIDER_KEY", + "model": f"anthropic/{_MODEL}", + "api_key": "os.environ/LIT7658_PROVIDER_KEY", "api_base": "https://api.anthropic.com", } - router: Final = litellm.Router(model_list=[{ - "model_name": "test-native", "litellm_params": dict(params), "model_info": {"id": _DEPLOYMENT}, - }] if source == "static" else [], num_retries=0) + router: Final = litellm.Router( + model_list=[ + { + "model_name": "test-native", + "litellm_params": dict(params), + "model_info": {"id": _DEPLOYMENT}, + } + ] + if source == "static" + else [], + num_retries=0, + ) if source == "database": monkeypatch.setattr(proxy_server, "llm_router", router) - assert proxy_server.ProxyConfig()._add_deployment([SimpleNamespace( - model_id=_DEPLOYMENT, model_name="test-native", model_info={}, litellm_params=dict(params), - )]) == 1 + assert ( + proxy_server.ProxyConfig()._add_deployment( + [ + SimpleNamespace( + model_id=_DEPLOYMENT, + model_name="test-native", + model_info={}, + litellm_params=dict(params), + ) + ] + ) + == 1 + ) deployment: Final = router.get_deployment(_DEPLOYMENT) assert deployment is not None target: Final = resolve_prediction_target(deployment.litellm_params) assert isinstance(target, NativePredictionTarget) body: Final = _body() with respx.mock() as upstream: - native: Final = upstream.post("https://api.anthropic.com/v1/messages").respond(200, json={ - "id": "msg_test", "type": "message", "role": "assistant", "model": _MODEL, - "content": [{"type": "text", "text": "Hello"}], "stop_reason": "end_turn", "stop_sequence": None, - "usage": {"input_tokens": 11, "output_tokens": 1, "cache_read_input_tokens": 300}, - }) + native: Final = upstream.post("https://api.anthropic.com/v1/messages").respond( + 200, + json={ + "id": "msg_test", + "type": "message", + "role": "assistant", + "model": _MODEL, + "content": [{"type": "text", "text": "Hello"}], + "stop_reason": "end_turn", + "stop_sequence": None, + "usage": {"input_tokens": 11, "output_tokens": 1, "cache_read_input_tokens": 300}, + }, + ) counter: Final = upstream.post("https://api.anthropic.com/v1/messages/count_tokens").respond( - 200, json={"input_tokens": 311}, + 200, + json={"input_tokens": 311}, ) await router.aanthropic_messages( - model="test-native", max_tokens=1, **{key: value for key, value in body.items() if key != "model"}, + model="test-native", + max_tokens=1, + **{key: value for key, value in body.items() if key != "model"}, ) assert await count_prompt_tokens(target.model, target.api_key, body) == 311 assert native.call_count == counter.call_count == 1 assert native.calls.last.request.headers["x-api-key"] == counter.calls.last.request.headers["x-api-key"] == _KEY - observed: Final = parse_observed_cache(native.calls.last.request, ModelResponse( - model=_MODEL, usage=Usage( - prompt_tokens=311, completion_tokens=1, total_tokens=312, - prompt_tokens_details=PromptTokensDetailsWrapper(cached_tokens=300), + observed: Final = parse_observed_cache( + native.calls.last.request, + ModelResponse( + model=_MODEL, + usage=Usage( + prompt_tokens=311, + completion_tokens=1, + total_tokens=312, + prompt_tokens_details=PromptTokensDetailsWrapper(cached_tokens=300), + ), ), - ), _CALLER, _DEPLOYMENT) + _CALLER, + _DEPLOYMENT, + ) assert observed is not None assert observed.scope == cache_scope(_CALLER, _DEPLOYMENT, target.api_key, target.model) @@ -202,15 +249,26 @@ async def test_environment_credential_matches_native_count_and_observed_scope( @pytest.mark.parametrize("inline_key", [None, _KEY]) @pytest.mark.asyncio async def test_named_credential_is_explicitly_unsupported_before_count( - inline_key: str | None, monkeypatch: pytest.MonkeyPatch, + inline_key: str | None, + monkeypatch: pytest.MonkeyPatch, ) -> None: - monkeypatch.setattr(litellm, "credential_list", [CredentialItem( - credential_name="test-named", credential_info={}, credential_values={"api_key": "test-named-provider-key"}, - )]) + monkeypatch.setattr( + litellm, + "credential_list", + [ + CredentialItem( + credential_name="test-named", + credential_info={}, + credential_values={"api_key": "test-named-provider-key"}, + ) + ], + ) deployment: Final = Deployment( model_name="test-native", litellm_params=LiteLLM_Params( - model=f"anthropic/{_MODEL}", api_key=inline_key, litellm_credential_name="test-named", + model=f"anthropic/{_MODEL}", + api_key=inline_key, + litellm_credential_name="test-named", ), model_info=ModelInfo(id=_DEPLOYMENT), ) @@ -234,8 +292,7 @@ def _cache_plan(body: Mapping[str, JsonValue]) -> PromptCachePlan: def _text(text: str, ttl: str | None = None) -> dict[str, JsonValue]: - return {"type": "text", "text": text, - **({"cache_control": {"type": "ephemeral", "ttl": ttl}} if ttl else {})} + return {"type": "text", "text": text, **({"cache_control": {"type": "ephemeral", "ttl": ttl}} if ttl else {})} def _prompt(*blocks: dict[str, JsonValue], role: str = "user", **options: JsonValue) -> dict[str, JsonValue]: @@ -253,16 +310,19 @@ def test_public_predictor_preserves_string_message_policy(text: str, supported: def test_cache_plan_preserves_hierarchical_prefixes_and_public_policy() -> None: body: Final = _prompt( - _text("First turn", "5m"), system=[_text("Stable instructions", "1h")], - tools=[{"name": "lookup", "input_schema": {"type": "object"}, - "cache_control": {"type": "ephemeral", "ttl": "1h"}}], + _text("First turn", "5m"), + system=[_text("Stable instructions", "1h")], + tools=[ + {"name": "lookup", "input_schema": {"type": "object"}, "cache_control": {"type": "ephemeral", "ttl": "1h"}} + ], ) plan: Final = _cache_plan(body) changed: Final = _cache_plan({**body, "system": [_text("Changed instructions", "1h")]}) assert tuple(marker.ttl_seconds for marker in plan.breakpoints) == (3600, 3600, 300) assert plan.breakpoints[0].fingerprint == changed.breakpoints[0].fingerprint - assert all(left.fingerprint != right.fingerprint for left, right - in zip(plan.breakpoints[1:], changed.breakpoints[1:])) + assert all( + left.fingerprint != right.fingerprint for left, right in zip(plan.breakpoints[1:], changed.breakpoints[1:]) + ) assert plan.breakpoints[0].prefix_body == { "tools": [{"name": "lookup", "input_schema": {"type": "object"}}], "messages": [], @@ -270,51 +330,80 @@ def test_cache_plan_preserves_hierarchical_prefixes_and_public_policy() -> None: assert parse_prompt(body) is None -@pytest.mark.parametrize("kind, added, matches", [ - ("text", 19, True), ("text", 20, False), ("tool_use", 30, True), - ("tool_result", 30, True), -]) +@pytest.mark.parametrize( + "kind, added, matches", + [ + ("text", 19, True), + ("text", 20, False), + ("tool_use", 30, True), + ("tool_result", 30, True), + ], +) def test_cache_plan_lookback_counts_native_positions( - kind: str, added: int, matches: bool, + kind: str, + added: int, + matches: bool, ) -> None: previous: Final = _cache_plan(_body()) appended: Final[list[dict[str, JsonValue]]] = [ {"type": "tool_use", "id": f"tool_{index}", "name": "lookup", "input": {}} - if kind == "tool_use" else - {"type": "tool_result", "tool_use_id": f"tool_{index}", "content": "done"} - if kind == "tool_result" else - {"type": "text", "text": f"Added {index}"} + if kind == "tool_use" + else {"type": "tool_result", "tool_use_id": f"tool_{index}", "content": "done"} + if kind == "tool_result" + else {"type": "text", "text": f"Added {index}"} for index in range(added) ] - current: Final = _cache_plan({**_body(), **_prompt( - _text("A cacheable prefix"), *appended[:-1], - {**appended[-1], "cache_control": {"type": "ephemeral"}}, - )}) - assert (previous.breakpoints[0].fingerprint - in current.breakpoints[0].lookback_fingerprints) is matches + current: Final = _cache_plan( + { + **_body(), + **_prompt( + _text("A cacheable prefix"), + *appended[:-1], + {**appended[-1], "cache_control": {"type": "ephemeral"}}, + ), + } + ) + assert (previous.breakpoints[0].fingerprint in current.breakpoints[0].lookback_fingerprints) is matches -@pytest.mark.parametrize("change, same_prefix, same_content", [ - ("tool_order", False, False), ("effort", False, False), - ("standard_speed", True, True), ("ttl", False, True), -]) +@pytest.mark.parametrize( + "change, same_prefix, same_content", + [ + ("tool_order", False, False), + ("effort", False, False), + ("standard_speed", True, True), + ("ttl", False, True), + ], +) def test_cache_plan_identity_respects_settings_and_preserves_content( - change: str, same_prefix: bool, same_content: bool, + change: str, + same_prefix: bool, + same_content: bool, ) -> None: tool_input: Final[dict[str, JsonValue]] = {"a": 1, "b": 2, "cache_control": {"ttl": "user-data"}} block: Final[dict[str, JsonValue]] = { - "type": "tool_use", "id": "tool_1", "name": "lookup", "input": tool_input, + "type": "tool_use", + "id": "tool_1", + "name": "lookup", + "input": tool_input, "cache_control": {"type": "ephemeral", "ttl": "5m"}, } changed_block: Final = ( - {**block, "input": dict(reversed(tool_input.items()))} if change == "tool_order" else - {**block, "cache_control": {"type": "ephemeral", "ttl": "1h"}} if change == "ttl" else block + {**block, "input": dict(reversed(tool_input.items()))} + if change == "tool_order" + else {**block, "cache_control": {"type": "ephemeral", "ttl": "1h"}} + if change == "ttl" + else block ) before: Final = _cache_plan(_prompt(block, role="assistant", output_config={"effort": "low"})).breakpoints[0] - after: Final = _cache_plan(_prompt( - changed_block, role="assistant", output_config={"effort": "high" if change == "effort" else "low"}, - **({"speed": "standard"} if change == "standard_speed" else {}), - )).breakpoints[0] + after: Final = _cache_plan( + _prompt( + changed_block, + role="assistant", + output_config={"effort": "high" if change == "effort" else "low"}, + **({"speed": "standard"} if change == "standard_speed" else {}), + ) + ).breakpoints[0] assert (before.fingerprint == after.fingerprint) is same_prefix assert (before.fingerprint in after.lookback_fingerprints) is same_prefix assert (before.content_fingerprint == after.content_fingerprint) is same_content @@ -325,30 +414,41 @@ def test_cache_plan_identity_respects_settings_and_preserves_content( def test_cache_plan_automatic_cache_and_thinking_use_last_cacheable_block() -> None: body: Final = _prompt( - _text("A stable answer"), {"type": "thinking", "thinking": "Thinking", "signature": "signature"}, - role="assistant", thinking={"type": "adaptive"}, cache_control={"type": "ephemeral", "ttl": "1h"}, + _text("A stable answer"), + {"type": "thinking", "thinking": "Thinking", "signature": "signature"}, + role="assistant", + thinking={"type": "adaptive"}, + cache_control={"type": "ephemeral", "ttl": "1h"}, ) plan: Final = _cache_plan(body) assert len(plan.breakpoints) == 1 assert plan.breakpoints[0].ttl_seconds == 3600 assert plan.breakpoints[0].prefix_body == { "thinking": {"type": "adaptive"}, - "messages": [{"role": "assistant", "content": [ - {"type": "text", "text": "A stable answer"}, - ]}], + "messages": [ + { + "role": "assistant", + "content": [ + {"type": "text", "text": "A stable answer"}, + ], + } + ], } assert parse_prompt(body) is None -@pytest.mark.parametrize("body, reason", [ - (_prompt({"type": "image"}), "unsupported_prompt_shape"), - ({**_body(), "unknown_native_setting": True}, "unsupported_prompt_shape"), - ({**_body(), "cache_control": {"type": "ephemeral", "ttl": "1h"}}, - "conflicting_cache_ttl"), - (_prompt(_text("five", "5m"), _text("hour", "1h")), "invalid_cache_ttl_order"), -]) +@pytest.mark.parametrize( + "body, reason", + [ + (_prompt({"type": "image"}), "unsupported_prompt_shape"), + ({**_body(), "unknown_native_setting": True}, "unsupported_prompt_shape"), + ({**_body(), "cache_control": {"type": "ephemeral", "ttl": "1h"}}, "conflicting_cache_ttl"), + (_prompt(_text("five", "5m"), _text("hour", "1h")), "invalid_cache_ttl_order"), + ], +) def test_cache_plan_unsupported_is_explicit( - body: Mapping[str, JsonValue], reason: str, + body: Mapping[str, JsonValue], + reason: str, ) -> None: result: Final = parse_cache_plan(body) assert isinstance(result, UnsupportedCachePlan) @@ -356,30 +456,46 @@ def test_cache_plan_unsupported_is_explicit( @pytest.mark.asyncio -@pytest.mark.parametrize("model, counts, reason", [ - (None, (100, 150, 200), None), - (None, (100, 201, 200), "inconsistent_prefix_token_count"), - (None, (151, 150, 200), "inconsistent_prefix_token_count"), - (None, (None, 150, 200), "token_count_unavailable"), - ("claude-opus-5", (100, 150, 200), None), - ("claude-sonnet-5", (100, 150, 200), None), - ("declared-cache-model", (100, 150, 200), None), - ("unknown-cache-model", (100, 150, 200), "unsupported_thinking_cache_semantics"), - ("claude-haiku-4-5", (100, 150, 200), "unsupported_thinking_cache_semantics"), - ("claude-sonnet-4-5", (100, 150, 200), "unsupported_thinking_cache_semantics"), -]) +@pytest.mark.parametrize( + "model, counts, reason", + [ + (None, (100, 150, 200), None), + (None, (100, 201, 200), "inconsistent_prefix_token_count"), + (None, (151, 150, 200), "inconsistent_prefix_token_count"), + (None, (None, 150, 200), "token_count_unavailable"), + ("claude-opus-5", (100, 150, 200), None), + ("claude-sonnet-5", (100, 150, 200), None), + ("declared-cache-model", (100, 150, 200), None), + ("unknown-cache-model", (100, 150, 200), "unsupported_thinking_cache_semantics"), + ("claude-haiku-4-5", (100, 150, 200), "unsupported_thinking_cache_semantics"), + ("claude-sonnet-4-5", (100, 150, 200), "unsupported_thinking_cache_semantics"), + ], +) async def test_cache_plan_count_conserves_total_and_rejects_unknown( - model: str | None, counts: tuple[int | None, int | None, int], reason: str | None, + model: str | None, + counts: tuple[int | None, int | None, int], + reason: str | None, monkeypatch: pytest.MonkeyPatch, ) -> None: - monkeypatch.setitem(litellm.model_cost, "declared-cache-model", { - "litellm_provider": "anthropic", "mode": "chat", "supports_thinking_cache_preservation": True, - }) - plan: Final = _cache_plan(_prompt( - {"type": "thinking", "thinking": "Retained thought", "signature": "signature"} - if model else _text("first", "5m"), - _text("second", "5m"), _text("uncached"), role="assistant" if model else "user", - )) + monkeypatch.setitem( + litellm.model_cost, + "declared-cache-model", + { + "litellm_provider": "anthropic", + "mode": "chat", + "supports_thinking_cache_preservation": True, + }, + ) + plan: Final = _cache_plan( + _prompt( + {"type": "thinking", "thinking": "Retained thought", "signature": "signature"} + if model + else _text("first", "5m"), + _text("second", "5m"), + _text("uncached"), + role="assistant" if model else "user", + ) + ) async def count(model: str, api_key: str, body: Mapping[str, JsonValue]) -> int | None: assert reason != "unsupported_thinking_cache_semantics", "Unverified thinking retention must skip counting" @@ -401,12 +517,15 @@ async def test_cache_plan_count_conserves_total_and_rejects_unknown( @pytest.mark.parametrize("section", ["system", "tools"]) @pytest.mark.parametrize("rejects_prefix", (False, True)) async def test_native_count_preserves_settings_and_requires_every_prefix( - section: str, rejects_prefix: bool, monkeypatch: pytest.MonkeyPatch, + section: str, + rejects_prefix: bool, + monkeypatch: pytest.MonkeyPatch, ) -> None: monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) monkeypatch.setattr(litellm, "in_memory_llm_clients_cache", LLMClientCache()) params: Final = LiteLLM_Params( - model=f"anthropic/{_MODEL}", api_key=_KEY, + model=f"anthropic/{_MODEL}", + api_key=_KEY, api_base="https://gateway.example/v1/messages", ) target: Final = resolve_baseline_prediction_target(params) @@ -415,19 +534,38 @@ async def test_native_count_preserves_settings_and_requires_every_prefix( assert not isinstance(resolve_prediction_target(params), NativePredictionTarget) body: Final = _body() marker: Final[dict[str, JsonValue]] = {"type": "ephemeral", "ttl": "1h"} - body[section] = ([_text("A cached system", "1h")] if section == "system" else [{ - "name": "lookup", "input_schema": {"type": "object"}, "cache_control": marker, - }]) - plan: Final = _cache_plan({**body, **_prompt( - _text("A later prefix", "5m"), _text("An uncached suffix"), - thinking={"type": "adaptive"}, tool_choice={"type": "auto"}, output_config={"effort": "high"}, - )}) + body[section] = ( + [_text("A cached system", "1h")] + if section == "system" + else [ + { + "name": "lookup", + "input_schema": {"type": "object"}, + "cache_control": marker, + } + ] + ) + plan: Final = _cache_plan( + { + **body, + **_prompt( + _text("A later prefix", "5m"), + _text("An uncached suffix"), + thinking={"type": "adaptive"}, + tool_choice={"type": "auto"}, + output_config={"effort": "high"}, + ), + } + ) assert len(plan.breakpoints) == 2 assert plan.breakpoints[0].prefix_body["messages"] == [] async def count(model: str, api_key: str, body: Mapping[str, JsonValue]) -> int | None: return await count_prompt_tokens( - model, api_key, {**body, "max_tokens": 100}, api_base=target.api_base, + model, + api_key, + {**body, "max_tokens": 100}, + api_base=target.api_base, ) with respx.mock(assert_all_called=False) as upstream: @@ -436,12 +574,16 @@ async def test_native_count_preserves_settings_and_requires_every_prefix( upstream.post(endpoint, json={**body, "model": _MODEL}).respond( 400 if rejects_prefix and index == 1 else 200, json={"detail": {"error": "messages parameter is required"}} - if rejects_prefix and index == 1 else {"input_tokens": tokens}, + if rejects_prefix and index == 1 + else {"input_tokens": tokens}, + ) + for index, (body, tokens) in enumerate( + ( + (plan.full_body, 6000), + (plan.breakpoints[0].prefix_body, 5000), + (plan.breakpoints[1].prefix_body, 5800), + ) ) - for index, (body, tokens) in enumerate(( - (plan.full_body, 6000), (plan.breakpoints[0].prefix_body, 5000), - (plan.breakpoints[1].prefix_body, 5800), - )) ) unexpected: Final = upstream.post(endpoint).respond(200, json={"input_tokens": 1}) result: Final = await count_cache_plan(target.model, target.api_key, plan, count) @@ -474,3 +616,32 @@ def test_native_baseline_body_drops_a_non_numeric_budget_without_logging_a_drop( assert body is not None assert body["thinking"] == {"type": "adaptive"} assert [record.getMessage() for record in caplog.records if "Dropping" in record.getMessage()] == [] + + +def test_native_baseline_body_matches_the_sent_body_when_drop_params_meets_an_auto_added_beta( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(litellm, "drop_params", True) + thinking: Final = {"type": "adaptive", "block_binding": {"prefix_mismatch_behavior": "error"}} + messages: Final = [{"role": "user", "content": "Say hi in French"}] + predicted: Final = prepare_native_baseline_body( + {"model": "claude-opus-4-8", "messages": messages, "max_tokens": 64, "thinking": thinking}, "claude-opus-4-8" + ) + config: Final = AnthropicMessagesConfig() + headers, _ = config.validate_anthropic_messages_environment( + headers={}, + model="claude-opus-4-8", + messages=[dict(message) for message in messages], + optional_params={"max_tokens": 64, "thinking": dict(thinking)}, + litellm_params={}, + api_key="sk-ant-test", + ) + sent: Final = config.transform_anthropic_messages_request( + model="claude-opus-4-8", + messages=[dict(message) for message in messages], + anthropic_messages_optional_request_params={"max_tokens": 64, "thinking": dict(thinking)}, + litellm_params=GenericLiteLLMParams(), + headers=headers, + ) + assert predicted is not None + assert predicted["thinking"] == sent["thinking"] == thinking diff --git a/tests/unit/llms/azure_ai/claude/test_azure_anthropic_handler.py b/tests/unit/llms/azure_ai/claude/test_azure_anthropic_handler.py index 052c4c422f2..d886303da8f 100644 --- a/tests/unit/llms/azure_ai/claude/test_azure_anthropic_handler.py +++ b/tests/unit/llms/azure_ai/claude/test_azure_anthropic_handler.py @@ -6,6 +6,7 @@ sys.path.insert( ) import json +from typing import Final from unittest.mock import MagicMock, patch import pytest @@ -272,3 +273,71 @@ def test_completion_surfaces_the_status_and_headers_of_a_rejected_request(): assert rejected.value.status_code == 400 assert rejected.value.litellm_response_headers["x-request-id"] == "req-1" assert "max_tokens too large" in str(rejected.value) + + +@respx.mock +@pytest.mark.parametrize("drop_params", [True, False]) +def test_completion_adds_the_binding_beta_so_block_binding_is_kept( + drop_params: bool, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + route: Final = respx.post("https://example-resource.services.ai.azure.com/anthropic/v1/messages").mock( + return_value=Response( + 200, + json={ + "id": "msg_bound", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-4-6", + "content": [{"type": "text", "text": "Hello"}], + "stop_reason": "end_turn", + "stop_sequence": None, + "usage": {"input_tokens": 1, "output_tokens": 1}, + }, + ) + ) + block_binding: Final = {"prefix_mismatch_behavior": "error"} + + litellm.completion( + model="azure_ai/claude-sonnet-4-6", + messages=[{"role": "user", "content": "hi"}], + api_base="https://example-resource.services.ai.azure.com/anthropic", + api_key="azure-test-key", + thinking={"type": "enabled", "budget_tokens": 1024, "block_binding": block_binding}, + drop_params=drop_params, + ) + + sent: Final = json.loads(route.calls.last.request.content) + assert sent["thinking"] == {"type": "enabled", "budget_tokens": 1024, "block_binding": block_binding} + assert "thinking-binding-controls-2026-08-01" in route.calls.last.request.headers["anthropic-beta"].split(",") + + +@respx.mock +def test_completion_sends_the_binding_beta_next_to_the_callers_own_betas(local_beta_headers_config: None) -> None: + route: Final = respx.post("https://example-resource.services.ai.azure.com/anthropic/v1/messages").mock( + return_value=Response( + 200, + json={ + "id": "msg_bound", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-4-6", + "content": [{"type": "text", "text": "Hello"}], + "stop_reason": "end_turn", + "stop_sequence": None, + "usage": {"input_tokens": 1, "output_tokens": 1}, + }, + ) + ) + + litellm.completion( + model="azure_ai/claude-sonnet-4-6", + messages=[{"role": "user", "content": "hi"}], + api_base="https://example-resource.services.ai.azure.com/anthropic", + api_key="azure-test-key", + thinking={"type": "enabled", "budget_tokens": 1024, "block_binding": {"prefix_mismatch_behavior": "error"}}, + extra_headers={"anthropic-beta": "context-1m-2025-08-07"}, + ) + + sent_betas: Final = set(route.calls.last.request.headers["anthropic-beta"].split(",")) + assert {"context-1m-2025-08-07", "thinking-binding-controls-2026-08-01"} <= sent_betas diff --git a/tests/unit/llms/azure_ai/passthrough/test_azure_ai_passthrough_transformation.py b/tests/unit/llms/azure_ai/passthrough/test_azure_ai_passthrough_transformation.py index 24f21751065..2d8ea27edb7 100644 --- a/tests/unit/llms/azure_ai/passthrough/test_azure_ai_passthrough_transformation.py +++ b/tests/unit/llms/azure_ai/passthrough/test_azure_ai_passthrough_transformation.py @@ -1,5 +1,7 @@ import json +from collections.abc import Mapping from datetime import datetime +from typing import Final from unittest.mock import MagicMock import httpx @@ -519,6 +521,162 @@ def test_cohere_rerank_relay_is_costed_per_search_unit(): assert logging_obj.response_cost_calculator(result=result) == pytest.approx(2 * per_query) +COHERE_EMBED_BODY: Final[Mapping[str, object]] = { + "id": "embed-1", + "texts": ["hello world"], + "embeddings": {"float": [[0.1, 0.2]]}, + "meta": {"api_version": {"version": "2"}, "billed_units": {"input_tokens": 2, "image_tokens": 0}}, + "response_type": "embeddings_by_type", +} +COHERE_V1_EMBED_META: Final[Mapping[str, object]] = { + "api_version": {"version": "1"}, + "billed_units": {"input_tokens": 2, "image_tokens": 0}, +} +TWO_VECTORS: Final = [[0.1, 0.2], [0.3, 0.4]] + + +def _cohere_embed_relay_result( + body: Mapping[str, object], native_path: str = "providers/cohere/v2/embed" +) -> tuple[object, Logging]: + return _relay_logging_result(AzureAIPassthroughConfig(), "embed-v-4-0", native_path, body) + + +def _foundry_embed_price(price_key: str) -> float: + return litellm.get_model_info("azure_ai/embed-v-4-0")[price_key] + + +def test_native_cohere_embed_relay_is_costed_per_billed_input_token() -> None: + result, logging_obj = _cohere_embed_relay_result(COHERE_EMBED_BODY) + per_token: Final = _foundry_embed_price("input_cost_per_token") + + assert isinstance(result, EmbeddingResponse) + assert logging_obj.call_type == "aembedding" + assert per_token > 0 + assert logging_obj.response_cost_calculator(result=result) == pytest.approx(2 * per_token) + + +@pytest.mark.parametrize( + "body", + [ + { + "id": "embed-1", + "texts": ["hello world", "second text"], + "embeddings": {"float": TWO_VECTORS}, + "meta": COHERE_V1_EMBED_META, + "response_type": "embeddings_by_type", + }, + { + "id": "embed-1", + "texts": ["hello world", "second text"], + "embeddings": TWO_VECTORS, + "meta": COHERE_V1_EMBED_META, + "response_type": "embeddings_floats", + }, + ], + ids=["embeddings_by_type", "embeddings_floats"], +) +def test_native_cohere_v1_embed_relay_is_costed_per_billed_input_token(body: Mapping[str, object]) -> None: + result, logging_obj = _cohere_embed_relay_result(body, "providers/cohere/v1/embed") + per_token: Final = _foundry_embed_price("input_cost_per_token") + + assert isinstance(result, EmbeddingResponse) + assert [(row["index"], row["embedding"]) for row in result.data] == list(enumerate(TWO_VECTORS)) + assert logging_obj.call_type == "aembedding" + assert per_token > 0 + assert logging_obj.response_cost_calculator(result=result) == pytest.approx(2 * per_token) + + +def test_native_cohere_embed_relay_prices_an_image_at_the_input_tokens_foundry_already_folded_it_into() -> None: + body: Final = { + **COHERE_EMBED_BODY, + "texts": [], + "images": [{"width": 64, "height": 64, "format": "image/png", "bit_depth": 8}], + "meta": {"api_version": {"version": "2"}, "billed_units": {"input_tokens": 24, "image_tokens": 6}}, + } + result, logging_obj = _cohere_embed_relay_result(body) + per_token: Final = _foundry_embed_price("input_cost_per_token") + per_image_token: Final = _foundry_embed_price("input_cost_per_image_token") + + assert isinstance(result, EmbeddingResponse) + assert per_token > 0 + assert per_image_token > 0 + assert logging_obj.response_cost_calculator(result=result) == pytest.approx(24 * per_token) + + + +def test_native_cohere_embed_relay_rows_keep_their_embedding_type() -> None: + result, _ = _cohere_embed_relay_result({**COHERE_EMBED_BODY, "embeddings": {"float": TWO_VECTORS, "int8": [[1, 2]]}}) + + assert isinstance(result, EmbeddingResponse) + assert [(row["type"], row["index"], row["embedding"]) for row in result.data] == [ + ("float", 0, TWO_VECTORS[0]), + ("float", 1, TWO_VECTORS[1]), + ("int8", 0, [1, 2]), + ] + +@pytest.mark.parametrize("texts", [None, "hello world", [1, 2], ["hello", 3]]) +def test_native_cohere_embed_relay_with_billed_units_is_costed_whatever_texts_holds(texts: object) -> None: + result, logging_obj = _cohere_embed_relay_result({**COHERE_EMBED_BODY, "texts": texts}) + + assert isinstance(result, EmbeddingResponse) + assert logging_obj.call_type == "aembedding" + assert logging_obj.response_cost_calculator(result=result) == pytest.approx( + 2 * _foundry_embed_price("input_cost_per_token") + ) + + +def test_native_cohere_embed_relay_without_texts_is_costed_per_billed_input_token() -> None: + body: Final = {key: value for key, value in COHERE_EMBED_BODY.items() if key != "texts"} + result, logging_obj = _cohere_embed_relay_result(body) + + assert isinstance(result, EmbeddingResponse) + assert logging_obj.call_type == "aembedding" + assert logging_obj.response_cost_calculator(result=result) == pytest.approx( + 2 * _foundry_embed_price("input_cost_per_token") + ) + + +@pytest.mark.parametrize( + ("texts", "counted_texts"), + [ + (["hello world", "a longer second text"], ["hello world", "a longer second text"]), + (["before <|endoftext|> after"], ["before <|endoftext|> after"]), + ("hello world", []), + (["hello", 3], []), + (None, []), + ], +) +def test_native_cohere_embed_relay_without_billed_units_is_costed_from_the_echoed_texts( + texts: object, counted_texts: list[str] +) -> None: + result, logging_obj = _cohere_embed_relay_result({**COHERE_EMBED_BODY, "texts": texts, "meta": {}}) + per_token: Final = _foundry_embed_price("input_cost_per_token") + echoed_tokens: Final = sum(len(litellm.encoding.encode(text, disallowed_special=())) for text in counted_texts) + + assert isinstance(result, EmbeddingResponse) + assert logging_obj.call_type == "aembedding" + assert result.usage.prompt_tokens == echoed_tokens + assert logging_obj.response_cost_calculator(result=result) == pytest.approx(echoed_tokens * per_token) + + +@pytest.mark.parametrize( + "body", + [ + {"message": "no embeddings here"}, + {**COHERE_EMBED_BODY, "embeddings": None}, + {**COHERE_EMBED_BODY, "embeddings": "vectors"}, + {**COHERE_EMBED_BODY, "meta": "not an object"}, + ], +) +def test_native_cohere_embed_relay_with_a_non_embed_body_keeps_the_passthrough_object_and_call_type( + body: Mapping[str, object], +) -> None: + result, logging_obj = _cohere_embed_relay_result(body) + + assert result == {"response": body} + assert logging_obj.call_type == "allm_passthrough_route" + + def test_image_generation_relay_is_costed_per_image(): result, logging_obj = _relay_logging_result( AzureAIPassthroughConfig(), "FLUX.2-pro", "openai/deployments/FLUX.2-pro/images/generations", IMAGE_BODY diff --git a/tests/unit/llms/bedrock/chat/invoke_transformations/test_bedrock_chat_invoke_transformations_anthropic_claude_transformation.py b/tests/unit/llms/bedrock/chat/invoke_transformations/test_bedrock_chat_invoke_transformations_anthropic_claude_transformation.py index 86c4b29eb53..d40abb4860b 100644 --- a/tests/unit/llms/bedrock/chat/invoke_transformations/test_bedrock_chat_invoke_transformations_anthropic_claude_transformation.py +++ b/tests/unit/llms/bedrock/chat/invoke_transformations/test_bedrock_chat_invoke_transformations_anthropic_claude_transformation.py @@ -2008,3 +2008,25 @@ def test_invoke_uses_opt_in_native_output_is_false_when_native_output_did_not_co assert not AmazonAnthropicClaudeConfig().uses_opt_in_native_output( model, optional_params, {"model_info": model_info} ) + + +@pytest.mark.usefixtures("local_model_cost_map", "local_beta_headers_config") +@pytest.mark.parametrize("drop_params", [True, False]) +def test_bedrock_chat_invoke_adds_the_binding_beta_so_block_binding_is_kept( + drop_params: bool, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + block_binding: Final = {"prefix_mismatch_behavior": "error"} + result: Final = AmazonAnthropicClaudeConfig().transform_request( + model="us.anthropic.claude-sonnet-4-6", + messages=[{"role": "user", "content": "Hello"}], + optional_params={ + "max_tokens": 2048, + "thinking": {"type": "enabled", "budget_tokens": 1024, "block_binding": block_binding}, + }, + litellm_params={"drop_params": drop_params}, + headers={}, + ) + + assert result["thinking"] == {"type": "enabled", "budget_tokens": 1024, "block_binding": block_binding} + assert "thinking-binding-controls-2026-08-01" in result["anthropic_beta"] diff --git a/tests/unit/llms/bedrock/chat/test_converse_transformation.py b/tests/unit/llms/bedrock/chat/test_converse_transformation.py index 85698bd7a47..a304ec88c4b 100644 --- a/tests/unit/llms/bedrock/chat/test_converse_transformation.py +++ b/tests/unit/llms/bedrock/chat/test_converse_transformation.py @@ -25,9 +25,9 @@ from litellm.llms.anthropic.chat.transformation import ( NATIVE_OUTPUT_MAX_OPTIONAL_PARAMETERS, NATIVE_OUTPUT_MAX_UNION_PARAMETERS, ) -from litellm.llms.bedrock.common_utils import BEDROCK_NATIVE_OUTPUT_MAX_PROPERTIES from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM from litellm.llms.bedrock.chat.converse_transformation import AmazonConverseConfig +from litellm.llms.bedrock.common_utils import BEDROCK_NATIVE_OUTPUT_MAX_PROPERTIES from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler from litellm.types.llms.bedrock import ContentBlock, ConverseTokenUsageBlock, RequestObject from litellm.types.llms.openai import AllMessageValues @@ -5356,7 +5356,6 @@ def test_recursive_output_config_format_follows_the_profile_native_key( assert ("outputConfig" in request) is forwarded - def test_sonnet_4_6_json_schema_still_uses_native_output_config(monkeypatch): monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url="")) @@ -10673,7 +10672,10 @@ def test_uses_opt_in_native_output_reads_the_base_model_of_an_application_infere _CALLER_OUTPUT_CONFIG_WITH_A_SCHEMA_OBJECT: Final = { - "textFormat": {"type": "json_schema", "structure": {"jsonSchema": {"name": "capital", "schema": {"type": "object"}}}} + "textFormat": { + "type": "json_schema", + "structure": {"jsonSchema": {"name": "capital", "schema": {"type": "object"}}}, + } } _BEDROCK_SCHEMA_TYPE_ERROR: Final = "outputConfig.textFormat.structure.jsonSchema.schema must be a string" @@ -10843,3 +10845,121 @@ def test_bedrock_deepseek_r1_drop_of_an_unadvertised_param_does_not_warn( ) assert _warnings_naming(caplog, param) == () + + +def _converse_bound_thinking_request( + headers: dict[str, str] | None, body_betas: tuple[str, ...] = (), drop_params: bool = True +) -> dict[str, object]: + thinking: Final = { + "type": "adaptive", + "block_binding": {"prefix_mismatch_behavior": "error"}, + "display": "updates", + } + data: Final = AmazonConverseConfig()._transform_request_helper( + model="anthropic.claude-opus-4-8-v1:0", + system_content_blocks=[], + optional_params={"thinking": thinking, **({"anthropic_beta": list(body_betas)} if body_betas else {})}, + messages=[{"role": "user", "content": "hi"}], + headers=headers, + drop_params=drop_params, + ) + assert thinking == { + "type": "adaptive", + "block_binding": {"prefix_mismatch_behavior": "error"}, + "display": "updates", + }, "the caller's thinking dict must survive for a fallback leg" + return data["additionalModelRequestFields"] + + +def test_converse_keeps_beta_gated_thinking_fields_the_bedrock_converse_row_carries( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("LITELLM_LOCAL_ANTHROPIC_BETA_HEADERS", "true") + fields: Final = _converse_bound_thinking_request( + {"anthropic-beta": "thinking-binding-controls-2026-08-01,thinking-display-updates-2026-08-18"} + ) + assert fields["thinking"] == { + "type": "adaptive", + "block_binding": {"prefix_mismatch_behavior": "error"}, + "display": "updates", + } + + +def test_converse_adds_the_thinking_betas_so_gated_thinking_fields_are_kept( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("LITELLM_LOCAL_ANTHROPIC_BETA_HEADERS", "true") + fields: Final = _converse_bound_thinking_request(None) + assert fields["thinking"] == { + "type": "adaptive", + "block_binding": {"prefix_mismatch_behavior": "error"}, + "display": "updates", + } + assert sorted(fields["anthropic_beta"]) == [ + "thinking-binding-controls-2026-08-01", + "thinking-display-updates-2026-08-18", + ] + + +def test_converse_keeps_the_callers_body_betas_next_to_the_added_thinking_betas( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("LITELLM_LOCAL_ANTHROPIC_BETA_HEADERS", "true") + fields: Final = _converse_bound_thinking_request(None, body_betas=("context-1m-2025-08-07",)) + assert sorted(fields["anthropic_beta"]) == [ + "context-1m-2025-08-07", + "thinking-binding-controls-2026-08-01", + "thinking-display-updates-2026-08-18", + ] + + +def test_converse_sends_beta_gated_thinking_fields_as_is_without_drop_params(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("LITELLM_LOCAL_ANTHROPIC_BETA_HEADERS", "true") + monkeypatch.setattr(litellm, "drop_params", False) + fields: Final = _converse_bound_thinking_request(None, drop_params=False) + assert fields["thinking"] == { + "type": "adaptive", + "block_binding": {"prefix_mismatch_behavior": "error"}, + "display": "updates", + } + + +def test_converse_keeps_beta_gated_thinking_fields_the_body_anthropic_beta_carries( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("LITELLM_LOCAL_ANTHROPIC_BETA_HEADERS", "true") + fields: Final = _converse_bound_thinking_request( + None, body_betas=("thinking-binding-controls-2026-08-01", "thinking-display-updates-2026-08-18") + ) + assert fields["thinking"] == { + "type": "adaptive", + "block_binding": {"prefix_mismatch_behavior": "error"}, + "display": "updates", + } + + +def test_converse_drops_body_betas_the_bedrock_converse_row_rejects_when_adding_thinking_betas( + local_beta_headers_config: None, +) -> None: + fields: Final = _converse_bound_thinking_request( + None, body_betas=("per-turn-control-2026-07-01", "context-1m-2025-08-07") + ) + assert sorted(fields["anthropic_beta"]) == [ + "context-1m-2025-08-07", + "thinking-binding-controls-2026-08-01", + "thinking-display-updates-2026-08-18", + ] + + +def test_converse_sends_only_the_header_betas_when_header_and_body_both_carry_betas( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("LITELLM_LOCAL_ANTHROPIC_BETA_HEADERS", "true") + fields: Final = _converse_bound_thinking_request( + {"anthropic-beta": "context-1m-2025-08-07"}, body_betas=("not-a-real-beta-2099",) + ) + assert sorted(fields["anthropic_beta"]) == [ + "context-1m-2025-08-07", + "thinking-binding-controls-2026-08-01", + "thinking-display-updates-2026-08-18", + ] diff --git a/tests/unit/llms/bedrock/embed/test_bedrock_embedding.py b/tests/unit/llms/bedrock/embed/test_bedrock_embedding.py index c98844fef55..889faf784bf 100644 --- a/tests/unit/llms/bedrock/embed/test_bedrock_embedding.py +++ b/tests/unit/llms/bedrock/embed/test_bedrock_embedding.py @@ -189,6 +189,290 @@ def test_bedrock_cohere_embedding_returns_the_scripted_vector(respx_mock: respx. assert response.data == [{"object": "embedding", "index": 0, "embedding": [0.3, 0.2, 0.1]}] +GOVCLOUD_WEST_RUNTIME: Final = "https://bedrock-runtime.us-gov-west-1.amazonaws.com" + + +@pytest.mark.parametrize( + ("model_id", "embed_response"), + [ + ("amazon.titan-embed-text-v2:0", titan_embedding_response), + ( + "amazon.nova-2-multimodal-embeddings-v1:0", + {"embeddings": [{"embeddingType": "TEXT", "embedding": [0.1, 0.2, 0.3]}]}, + ), + ("cohere.embed-english-v3", cohere_embedding_response), + ], +) +@pytest.mark.parametrize("sync_mode", [True, False]) +async def test_bedrock_embedding_calls_the_model_id_region_with_the_bare_model_id( + respx_mock: respx.MockRouter, + monkeypatch: pytest.MonkeyPatch, + model_id: str, + embed_response: dict[str, object], + sync_mode: bool, +) -> None: + monkeypatch.setenv("AWS_REGION_NAME", "us-east-1") + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + route = respx_mock.post(f"{GOVCLOUD_WEST_RUNTIME}/model/{model_id}/invoke").mock( + return_value=httpx.Response(200, json=embed_response) + ) + request: Final = { + "model": f"bedrock/us-gov-west-1/{model_id}", + "input": [test_input], + "api_key": "test-bearer-token", + } + + response: Final = litellm.embedding(**request) if sync_mode else await litellm.aembedding(**request) + + assert route.call_count == 1 + assert response.data[0]["embedding"] == [0.1, 0.2, 0.3] + + +def test_bedrock_embedding_prices_a_region_prefixed_model_from_its_region_row( + respx_mock: respx.MockRouter, +) -> None: + model: Final = "bedrock/us-gov-west-1/amazon.titan-embed-text-v2:0" + respx_mock.post(f"{GOVCLOUD_WEST_RUNTIME}/model/amazon.titan-embed-text-v2:0/invoke").mock( + return_value=httpx.Response(200, json=titan_embedding_response) + ) + + response: Final = litellm.embedding(model=model, input=[test_input], api_key="test-bearer-token") + + assert response._hidden_params["response_cost"] == pytest.approx( + response.usage.prompt_tokens * litellm.model_cost[model]["input_cost_per_token"] + ) + + +def test_bedrock_embedding_explicit_aws_region_name_wins_over_the_model_id_region_for_routing_and_price( + respx_mock: respx.MockRouter, +) -> None: + route = respx_mock.post( + "https://bedrock-runtime.us-gov-east-1.amazonaws.com/model/amazon.titan-embed-text-v2:0/invoke" + ).mock(return_value=httpx.Response(200, json=titan_embedding_response)) + + response: Final = litellm.embedding( + model="bedrock/us-gov-west-1/amazon.titan-embed-text-v2:0", + input=[test_input], + api_key="test-bearer-token", + aws_region_name="us-gov-east-1", + ) + + assert route.call_count == 1 + assert response._hidden_params["response_cost"] == pytest.approx( + response.usage.prompt_tokens + * litellm.model_cost["bedrock/us-gov-east-1/amazon.titan-embed-text-v2:0"]["input_cost_per_token"] + ) + + +def test_bedrock_embedding_model_id_arn_region_wins_over_the_model_path_region_and_env( + respx_mock: respx.MockRouter, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("AWS_REGION_NAME", "us-west-2") + route = respx_mock.post(url__startswith=f"{GOVCLOUD_WEST_RUNTIME}/model/").mock( + return_value=httpx.Response(200, json=titan_embedding_response) + ) + + litellm.embedding( + model="bedrock/us-east-1/amazon.titan-embed-text-v2:0", + input=[test_input], + api_key="test-bearer-token", + model_id="arn:aws-us-gov:bedrock:us-gov-west-1::foundation-model/amazon.titan-embed-text-v2:0", + ) + + assert route.call_count == 1 + + +def test_bedrock_embedding_plain_model_id_calls_the_env_region_like_chat( + respx_mock: respx.MockRouter, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("AWS_REGION_NAME", "us-west-2") + route = respx_mock.post("https://bedrock-runtime.us-west-2.amazonaws.com/model/cohere.embed-english-v3/invoke").mock( + return_value=httpx.Response(200, json=cohere_embedding_response) + ) + + litellm.embedding( + model="bedrock/us-east-1/cohere.embed-english-v3", + input=[test_input], + api_key="test-bearer-token", + model_id="cohere.embed-english-v3", + ) + + assert route.call_count == 1 + + +def test_bedrock_embedding_rejects_an_empty_aws_region_name_instead_of_using_the_path_region( + respx_mock: respx.MockRouter, +) -> None: + route = respx_mock.post(url__startswith=GOVCLOUD_WEST_RUNTIME).mock( + return_value=httpx.Response(200, json=titan_embedding_response) + ) + + with pytest.raises(litellm.APIConnectionError, match="Invalid AWS region format"): + litellm.embedding( + model="bedrock/us-gov-west-1/amazon.titan-embed-text-v2:0", + input=[test_input], + api_key="test-bearer-token", + aws_region_name="", + ) + + assert route.call_count == 0 + + +@pytest.mark.parametrize( + "model,extra_params", + [ + ("bedrock/invoke/us-gov-west-1/amazon.titan-embed-text-v2:0", {}), + ("bedrock/us-gov-west-1/amazon.titan-embed-text-v2:0", {"model_id": ""}), + ], +) +def test_bedrock_embedding_region_path_variants_call_the_path_region_with_the_bare_id_and_its_price( + respx_mock: respx.MockRouter, + monkeypatch: pytest.MonkeyPatch, + model: str, + extra_params: dict[str, str], +) -> None: + monkeypatch.setenv("AWS_REGION_NAME", "us-east-1") + route = respx_mock.post(f"{GOVCLOUD_WEST_RUNTIME}/model/amazon.titan-embed-text-v2:0/invoke").mock( + return_value=httpx.Response(200, json=titan_embedding_response) + ) + + response: Final = litellm.embedding(model=model, input=[test_input], api_key="test-bearer-token", **extra_params) + + assert route.call_count == 1 + assert response._hidden_params["response_cost"] == pytest.approx( + response.usage.prompt_tokens + * litellm.model_cost["bedrock/us-gov-west-1/amazon.titan-embed-text-v2:0"]["input_cost_per_token"] + ) + + +def test_bedrock_embedding_explicit_region_outside_the_known_list_returns_the_vector_at_the_bare_price( + respx_mock: respx.MockRouter, +) -> None: + route = respx_mock.post( + "https://bedrock-runtime.zz-unlisted-1.amazonaws.com/model/amazon.titan-embed-text-v2:0/invoke" + ).mock(return_value=httpx.Response(200, json=titan_embedding_response)) + + response: Final = litellm.embedding( + model="bedrock/us-east-1/amazon.titan-embed-text-v2:0", + input=[test_input], + api_key="test-bearer-token", + aws_region_name="zz-unlisted-1", + ) + + assert route.call_count == 1 + assert response.data[0]["embedding"] == [0.1, 0.2, 0.3] + assert response._hidden_params["response_cost"] == pytest.approx( + response.usage.prompt_tokens * litellm.model_cost["amazon.titan-embed-text-v2:0"]["input_cost_per_token"] + ) + + +@pytest.mark.parametrize("sync_mode", [True, False]) +async def test_bedrock_embedding_without_a_region_path_bills_the_row_of_the_region_it_calls( + respx_mock: respx.MockRouter, + monkeypatch: pytest.MonkeyPatch, + sync_mode: bool, +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + route = respx_mock.post(f"{GOVCLOUD_WEST_RUNTIME}/model/amazon.titan-embed-text-v2:0/invoke").mock( + return_value=httpx.Response(200, json=titan_embedding_response) + ) + request: Final = { + "model": "bedrock/amazon.titan-embed-text-v2:0", + "input": [test_input], + "api_key": "test-bearer-token", + "aws_region_name": "us-gov-west-1", + } + + response: Final = litellm.embedding(**request) if sync_mode else await litellm.aembedding(**request) + + assert route.call_count == 1 + assert response.model == "amazon.titan-embed-text-v2:0" + assert response._hidden_params["response_cost"] == pytest.approx( + response.usage.prompt_tokens + * litellm.model_cost["bedrock/us-gov-west-1/amazon.titan-embed-text-v2:0"]["input_cost_per_token"] + ) + + +def test_bedrock_cohere_embedding_bills_the_row_of_the_region_it_calls( + respx_mock: respx.MockRouter, + monkeypatch: pytest.MonkeyPatch, +) -> None: + regional_key: Final = "bedrock/us-gov-west-1/cohere.embed-english-v3" + monkeypatch.setitem( + litellm.model_cost, + regional_key, + {**litellm.model_cost["cohere.embed-english-v3"], "input_cost_per_token": 7e-07}, + ) + respx_mock.post(f"{GOVCLOUD_WEST_RUNTIME}/model/cohere.embed-english-v3/invoke").mock( + return_value=httpx.Response(200, json=cohere_embedding_response) + ) + + response: Final = litellm.embedding( + model="bedrock/cohere.embed-english-v3", + input=[test_input], + api_key="test-bearer-token", + aws_region_name="us-gov-west-1", + ) + + assert response._hidden_params["response_cost"] == pytest.approx( + response.usage.prompt_tokens * litellm.model_cost[regional_key]["input_cost_per_token"] + ) + + +def test_bedrock_embedding_with_a_govcloud_arn_model_id_bills_at_the_arn_region_row( + respx_mock: respx.MockRouter, +) -> None: + arn: Final = "arn:aws-us-gov:bedrock:us-gov-west-1::foundation-model/amazon.nova-2-multimodal-embeddings-v1:0" + route = respx_mock.post(url__startswith=f"{GOVCLOUD_WEST_RUNTIME}/model/").mock( + return_value=httpx.Response( + 200, json={"embeddings": [{"embeddingType": "TEXT", "embedding": [0.1, 0.2, 0.3]}]} + ) + ) + + response: Final = litellm.embedding( + model="bedrock/amazon.nova-2-multimodal-embeddings-v1:0", + input=[test_input], + api_key="test-bearer-token", + model_id=arn, + ) + + assert route.call_count == 1 + assert response._hidden_params["response_cost"] == pytest.approx( + response.usage.prompt_tokens + * litellm.model_cost["bedrock/us-gov-west-1/amazon.nova-2-multimodal-embeddings-v1:0"]["input_cost_per_token"] + ) + + +@pytest.mark.parametrize( + "model", + [ + "bedrock/us-gov-west-1/async_invoke/amazon.nova-2-multimodal-embeddings-v1:0", + "bedrock/async_invoke/us-gov-west-1/amazon.nova-2-multimodal-embeddings-v1:0", + ], +) +def test_bedrock_async_invoke_embedding_with_a_region_path_posts_the_bare_id_to_that_region( + respx_mock: respx.MockRouter, + model: str, +) -> None: + route = respx_mock.post(f"{GOVCLOUD_WEST_RUNTIME}/async-invoke").mock( + return_value=httpx.Response( + 200, json={"invocationArn": "arn:aws-us-gov:bedrock:us-gov-west-1:123456789012:async-invoke/abc123"} + ) + ) + + litellm.embedding( + model=model, + input=[test_input], + api_key="test-bearer-token", + output_s3_uri="s3://test-bucket/async-invoke-output/", + ) + + assert route.call_count == 1 + assert json.loads(route.calls[0].request.content)["modelId"] == "amazon.nova-2-multimodal-embeddings-v1:0" + + @pytest.mark.asyncio async def test_async_bedrock_embedding_with_bearer_token(): """Test async embedding functionality with bearer token authentication""" diff --git a/tests/unit/llms/bedrock/messages/invoke_transformations/test_anthropic_claude_transformation.py b/tests/unit/llms/bedrock/messages/invoke_transformations/test_anthropic_claude_transformation.py index c096e2126ae..74f6f3030cb 100644 --- a/tests/unit/llms/bedrock/messages/invoke_transformations/test_anthropic_claude_transformation.py +++ b/tests/unit/llms/bedrock/messages/invoke_transformations/test_anthropic_claude_transformation.py @@ -16,7 +16,7 @@ import httpx import pytest import requests_mock as requests_mock_lib import respx -from pydantic import TypeAdapter +from pydantic import BaseModel, ConfigDict, TypeAdapter import litellm from litellm.constants import ( @@ -4200,6 +4200,98 @@ def test_bedrock_messages_without_guardrail_config_sends_no_guardrail_headers() assert not any(name.startswith("X-Amzn-Bedrock-Guardrail") for name in headers) +@pytest.mark.usefixtures("local_model_cost_map", "local_beta_headers_config") +def test_bedrock_messages_keeps_mid_conversation_output_config_its_auto_added_beta_gates() -> None: + """Invoke learns its body betas after the shared transform, so the gated field it supports must survive.""" + from litellm.types.router import GenericLiteLLMParams + + messages: Final = [ + {"role": "user", "content": [{"type": "text", "text": "Hello"}]}, + {"role": "assistant", "content": [{"type": "text", "text": "Hi"}]}, + {"role": "system", "content": "Answer briefly", "output_config": {"effort": "high"}}, + {"role": "user", "content": [{"type": "text", "text": "Go on"}]}, + ] + result: Final = AmazonAnthropicClaudeMessagesConfig().transform_anthropic_messages_request( + model="anthropic.claude-opus-4-8-v1:0", + messages=messages, + anthropic_messages_optional_request_params={"max_tokens": 4096}, + litellm_params=GenericLiteLLMParams(drop_params=True), + headers={}, + ) + + assert "mid-conversation-output-config-2026-07-01" in result["anthropic_beta"] + assert {"role": "system", "content": "Answer briefly", "output_config": {"effort": "high"}} in result["messages"] + + +class _SentMessage(BaseModel): + model_config = ConfigDict(frozen=True) + role: str + content: list[dict[str, object]] + + +_SENT_MESSAGES: Final = TypeAdapter(list[_SentMessage]) + + +@pytest.mark.usefixtures("local_model_cost_map", "local_beta_headers_config") +@pytest.mark.parametrize("drop_params", [True, False]) +def test_bedrock_messages_drops_tool_changes_from_a_system_turn_an_older_model_gets_as_a_user_turn( + drop_params: bool, monkeypatch: pytest.MonkeyPatch +) -> None: + """A model without mid-conversation system support sends no tool-changes beta, so under drop_params the + converted turn must not carry the blocks that beta gates, and without it they go out as sent.""" + monkeypatch.setattr(litellm, "drop_params", False) + from litellm.types.router import GenericLiteLLMParams + + messages: Final = [ + {"role": "user", "content": [{"type": "text", "text": "Hello"}]}, + {"role": "assistant", "content": [{"type": "text", "text": "Hi"}]}, + { + "role": "system", + "content": [ + {"type": "text", "text": "A weather tool is now available"}, + {"type": "tool_addition", "tool": {"type": "tool_reference", "name": "get_weather"}}, + ], + }, + {"role": "user", "content": [{"type": "text", "text": "Go on"}]}, + ] + result: Final = AmazonAnthropicClaudeMessagesConfig().transform_anthropic_messages_request( + model="anthropic.claude-3-7-sonnet-20250219-v1:0", + messages=messages, + anthropic_messages_optional_request_params={"max_tokens": 4096}, + litellm_params=GenericLiteLLMParams(drop_params=drop_params), + headers={}, + ) + + sent: Final = _SENT_MESSAGES.validate_python(result["messages"]) + blocks: Final = [ + block for message in sent for block in message.content + ] # comprehension-ok: flattening the sent content blocks + assert "mid-conversation-tool-changes-2026-07-01" not in result.get("anthropic_beta", []) + assert any(block["type"] == "tool_addition" for block in blocks) is not drop_params + assert {"type": "text", "text": "A weather tool is now available"} in blocks + assert all(message.role != "system" for message in sent) + + +@pytest.mark.usefixtures("local_model_cost_map", "local_beta_headers_config") +@pytest.mark.parametrize("drop_params", (True, False)) +def test_bedrock_messages_sends_the_binding_beta_with_block_binding_whatever_drop_params(drop_params: bool) -> None: + from litellm.types.router import GenericLiteLLMParams + + result: Final = AmazonAnthropicClaudeMessagesConfig().transform_anthropic_messages_request( + model="us.anthropic.claude-sonnet-4-6", + messages=[{"role": "user", "content": "Reply with OK"}], + anthropic_messages_optional_request_params={ + "max_tokens": 2048, + "thinking": {"type": "enabled", "budget_tokens": 1024, "block_binding": BLOCK_BINDING}, + }, + litellm_params=GenericLiteLLMParams(drop_params=drop_params), + headers={}, + ) + + assert result["thinking"]["block_binding"] == BLOCK_BINDING + assert "thinking-binding-controls-2026-08-01" in result["anthropic_beta"] + + @dataclass(slots=True) class BedrockDownAnthropicUp: anthropic_bodies: tuple[dict[str, object], ...] = () diff --git a/tests/unit/llms/bedrock/test_claude_platform_provider.py b/tests/unit/llms/bedrock/test_claude_platform_provider.py index 2353a45139b..9142fe45752 100644 --- a/tests/unit/llms/bedrock/test_claude_platform_provider.py +++ b/tests/unit/llms/bedrock/test_claude_platform_provider.py @@ -207,6 +207,26 @@ def test_claude_platform_api_key_auth_sets_workspace_and_key_headers(): assert headers["anthropic-beta"] == "skills-2025-10-02" +def test_claude_platform_chat_sends_the_binding_beta_when_thinking_carries_block_binding(): + from litellm.llms.bedrock.claude_platform.transformation import ( + BedrockClaudePlatformConfig, + ) + + headers: Final = BedrockClaudePlatformConfig().validate_environment( + api_key="fake-platform-key", + headers={}, + model="claude-opus-5-5", + messages=[{"role": "user", "content": "hello"}], + optional_params={ + "workspace_id": "wrkspc_test", + "thinking": {"type": "adaptive", "block_binding": {"prefix_mismatch_behavior": "error"}}, + }, + litellm_params={}, + ) + + assert "thinking-binding-controls-2026-08-01" in headers["anthropic-beta"].split(",") + + def test_claude_platform_does_not_use_standard_anthropic_api_key(monkeypatch): from litellm.llms.bedrock.claude_platform.transformation import ( BedrockClaudePlatformConfig, diff --git a/tests/unit/llms/cohere/embed/test_cohere_embedding_transformation.py b/tests/unit/llms/cohere/embed/test_cohere_embedding_transformation.py index 516ea1f5221..245d30a6283 100644 --- a/tests/unit/llms/cohere/embed/test_cohere_embedding_transformation.py +++ b/tests/unit/llms/cohere/embed/test_cohere_embedding_transformation.py @@ -7,7 +7,12 @@ import respx import litellm from litellm.litellm_core_utils.litellm_logging import Logging -from litellm.llms.cohere.embed.transformation import CohereEmbeddingConfig +from litellm.llms.cohere.embed.transformation import ( + CohereEmbeddingConfig, + CohereEmbeddingRowShape, + CohereImageTokenBilling, + cohere_embedding_response, +) from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler from litellm.types.utils import CallTypes, EmbeddingResponse @@ -108,18 +113,50 @@ def test_cohere_embedding_response_counts_tokens_when_provider_usage_is_missing( assert response.usage.total_tokens == response.usage.prompt_tokens -def test_cohere_embedding_usage_includes_text_and_image_token_details(): - config: Final = CohereEmbeddingConfig() - usage: Final = config._calculate_usage( - input=["hello"], - encoding=litellm.encoding, - meta={"billed_units": {"input_tokens": 5, "images": 2}}, +def test_cohere_embedding_fallback_counts_special_token_text_as_plain_text() -> None: + text: Final = "before <|endoftext|> after" + response: Final = CohereEmbeddingConfig().transform_embedding_response( + model="embed-english-v3.0", + raw_response=httpx.Response(200, json={"embeddings": {"float": [[0.1]]}}), + model_response=EmbeddingResponse(), + logging_obj=_logging_obj([text]), + api_key="test-key", + request_data={"texts": [text]}, + optional_params={}, + litellm_params={}, ) + assert response.usage.prompt_tokens == len(litellm.encoding.encode(text, disallowed_special=())) + + +@pytest.mark.parametrize("image_key", ["images", "image_tokens"]) +def test_cohere_embedding_usage_includes_text_and_image_token_details(image_key: str) -> None: + usage: Final = cohere_embedding_response( + {"embeddings": {"float": [[0.1]]}, "meta": {"billed_units": {"input_tokens": 5, image_key: 2}}}, + "embed-english-v3.0", + litellm.encoding, + ["hello"], + CohereImageTokenBilling.SEPARATE_FROM_INPUT_TOKENS, + CohereEmbeddingRowShape.OPENAI, + ).usage + assert usage.prompt_tokens == 7 assert (usage.prompt_tokens_details.text_tokens, usage.prompt_tokens_details.image_tokens) == (5, 2) +def test_cohere_embedding_usage_folded_image_tokens_count_input_tokens_only() -> None: + usage: Final = cohere_embedding_response( + {"embeddings": {"float": [[0.1]]}, "meta": {"billed_units": {"input_tokens": 24, "image_tokens": 6}}}, + "embed-v-4-0", + litellm.encoding, + [], + CohereImageTokenBilling.FOLDED_INTO_INPUT_TOKENS, + CohereEmbeddingRowShape.OPENAI, + ).usage + + assert (usage.prompt_tokens, usage.total_tokens, usage.prompt_tokens_details) == (24, 24, None) + + def test_cohere_embedding_error_class_preserves_status_and_message(): error: Final = CohereEmbeddingConfig().get_error_class( error_message="invalid embedding request", diff --git a/tests/unit/llms/cohere/embed/test_transformation.py b/tests/unit/llms/cohere/embed/test_transformation.py index c0f355cfb57..abae30013fe 100644 --- a/tests/unit/llms/cohere/embed/test_transformation.py +++ b/tests/unit/llms/cohere/embed/test_transformation.py @@ -1,6 +1,6 @@ import datetime import json -from collections.abc import Iterator +from collections.abc import Iterator, Mapping from typing import Final import httpx @@ -70,8 +70,11 @@ def test_cohere_embedding_response_lists_every_embedding_type_in_provider_order( ("meta", "expected_counts", "expected_details"), [ ({"billed_units": {"input_tokens": 5}}, (5, 0, 5), (5, None)), + ({"billed_units": {"input_tokens": 5, "image_tokens": 0}}, (5, 0, 5), (5, None)), ({"billed_units": {"images": 3}}, (3, 0, 3), (None, 3)), ({"billed_units": {"input_tokens": 5, "images": 3}}, (8, 0, 8), (5, 3)), + ({"billed_units": {"input_tokens": 0, "image_tokens": 6}}, (6, 0, 6), (0, 6)), + ({"billed_units": {"input_tokens": 3, "image_tokens": 6}}, (9, 0, 9), (3, 6)), ({"billed_units": {"input_tokens": 0}}, (0, 0, 0), (0, None)), ( {"api_version": {"version": "2"}, "billed_units": {"input_tokens": 2, "output_tokens": 9}}, @@ -80,11 +83,15 @@ def test_cohere_embedding_response_lists_every_embedding_type_in_provider_order( ), ], ) -def test_cohere_embedding_usage_comes_from_billed_units(meta, expected_counts, expected_details): - response = _transform({"embeddings": {"float": FLOAT_VECTORS}, "meta": meta}) +def test_cohere_embedding_usage_comes_from_billed_units( + meta: Mapping[str, object], + expected_counts: tuple[int, int, int], + expected_details: tuple[int | None, int | None], +) -> None: + response: Final = _transform({"embeddings": {"float": FLOAT_VECTORS}, "meta": meta}) assert response.usage is not None - details = response.usage.prompt_tokens_details + details: Final = response.usage.prompt_tokens_details assert details is not None assert _token_counts(response) == expected_counts assert (details.text_tokens, details.image_tokens) == expected_details @@ -123,8 +130,20 @@ def test_cohere_embedding_rows_pass_through_whatever_each_type_lists(embeddings, assert [row["embedding"] for row in response.data] == expected_vectors -@pytest.mark.parametrize("embeddings", [FLOAT_VECTORS, [], None, "vectors", 3]) -def test_cohere_embeddings_that_are_not_keyed_by_type_are_rejected(embeddings): +@pytest.mark.parametrize("vectors", [FLOAT_VECTORS, []]) +def test_cohere_v1_float_embeddings_list_one_row_per_vector(vectors: list[list[float]]) -> None: + response: Final = _transform( + {"embeddings": vectors, "meta": {"billed_units": {"input_tokens": 1}}, "response_type": "embeddings_floats"} + ) + + assert response.data == [ + {"object": "embedding", "index": index, "embedding": vector} for index, vector in enumerate(vectors) + ] + assert _token_counts(response) == (1, 0, 1) + + +@pytest.mark.parametrize("embeddings", [None, "vectors", 3]) +def test_cohere_embeddings_that_are_neither_keyed_by_type_nor_a_list_are_rejected(embeddings: object) -> None: with pytest.raises(ValidationError): _transform({"embeddings": embeddings, "meta": {"billed_units": {"input_tokens": 1}}}) @@ -147,12 +166,15 @@ def test_cohere_embedding_body_that_is_not_an_object_is_rejected(payload): _transform(payload) -def test_cohere_embedding_body_without_embeddings_is_rejected(): - with pytest.raises(KeyError): +def test_cohere_embedding_body_without_embeddings_is_rejected() -> None: + with pytest.raises(ValidationError): _transform({"meta": {"billed_units": {"input_tokens": 1}}}) COHERE_V2_EMBED_URL: Final = "https://api.cohere.ai/v2/embed" +ONE_PIXEL_PNG: Final = ( + "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAIAAACQd1PeAAAADElEQVR4nGP4z8AAAAMBAQDJ/pLvAAAAAElFTkSuQmCC" +) @pytest.fixture @@ -362,13 +384,43 @@ async def test_cohere_embed_v4_with_optional_params(respx_mock: respx.MockRouter assert response.data == [{"object": "embedding", "index": 0, "embedding": vector}] +@pytest.mark.usefixtures("_cohere_httpx_transport") +@pytest.mark.respx(assert_all_called=True) +@pytest.mark.parametrize(("input_tokens", "image_tokens"), [(0, 6), (3, 6)]) +async def test_cohere_embed_v4_image_tokens_are_billed_apart_from_input_tokens( + respx_mock: respx.MockRouter, input_tokens: int, image_tokens: int +) -> None: + respx_mock.post(COHERE_V2_EMBED_URL).mock( + return_value=httpx.Response( + 200, + json={ + "id": "5807ee2e-0cda-445a-9ec8-864c60a06606", + "embeddings": {"float": [[0.125, -0.375]]}, + "texts": [], + "images": [{"width": 1, "height": 1, "format": "image/png", "bit_depth": 8}], + "meta": { + "api_version": {"version": "2"}, + "billed_units": {"input_tokens": input_tokens, "image_tokens": image_tokens}, + }, + "response_type": "embeddings_by_type", + }, + ) + ) + model_info: Final = litellm.get_model_info("cohere/embed-v4.0") + + response: Final = await _embed(True, input=[ONE_PIXEL_PNG], input_type="image") + + assert model_info["input_cost_per_image_token"] > 0 + assert response._hidden_params["response_cost"] == pytest.approx( + input_tokens * model_info["input_cost_per_token"] + image_tokens * model_info["input_cost_per_image_token"] + ) + + @pytest.mark.usefixtures("_cohere_httpx_transport") @pytest.mark.respx(assert_all_called=True) @pytest.mark.parametrize("sync_mode", [True, False]) async def test_cohere_embed_v4_image_embedding(respx_mock: respx.MockRouter, sync_mode: bool) -> None: - image: Final = ( - "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAIAAACQd1PeAAAADElEQVR4nGP4z8AAAAMBAQDJ/pLvAAAAAElFTkSuQmCC" - ) + image: Final = ONE_PIXEL_PNG route: Final = respx_mock.post(COHERE_V2_EMBED_URL).mock( return_value=httpx.Response( 200, diff --git a/tests/unit/llms/databricks/chat/test_databricks_chat_transformation.py b/tests/unit/llms/databricks/chat/test_databricks_chat_transformation.py index eb2e1d8e347..19234767b19 100644 --- a/tests/unit/llms/databricks/chat/test_databricks_chat_transformation.py +++ b/tests/unit/llms/databricks/chat/test_databricks_chat_transformation.py @@ -2465,6 +2465,41 @@ def test_databricks_claude_drops_the_openai_tool_search_tool_under_drop_params(m ] +@pytest.mark.parametrize("drop_params", [True, False]) +def test_databricks_chat_drops_beta_gated_thinking_fields_only_under_drop_params( + drop_params: bool, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + thinking: Final = { + "type": "adaptive", + "block_binding": {"prefix_mismatch_behavior": "error"}, + "display": "updates", + } + mapped: Final = DatabricksConfig().map_openai_params( + non_default_params={"thinking": thinking}, + optional_params={}, + model="databricks-claude-opus-5-5", + drop_params=drop_params, + ) + assert mapped["thinking"] == ({"type": "adaptive"} if drop_params else thinking) + assert thinking == { + "type": "adaptive", + "block_binding": {"prefix_mismatch_behavior": "error"}, + "display": "updates", + } + + +def test_databricks_chat_keeps_ungated_thinking_display_under_drop_params(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + mapped: Final = DatabricksConfig().map_openai_params( + non_default_params={"thinking": {"type": "adaptive", "display": "summarized"}}, + optional_params={}, + model="databricks-claude-opus-5-5", + drop_params=True, + ) + assert mapped["thinking"] == {"type": "adaptive", "display": "summarized"} + + _JSON_TOOL_ARGUMENT_FRAGMENTS: Final = ("", "", '{"name": ', '"Ada"', ', "met', 'a": ', "{}", ', "tags": []}') _RECORD_RESPONSE_FORMAT: Final = { "type": "json_schema", diff --git a/tests/unit/llms/databricks/messages/test_databricks_messages_transformation.py b/tests/unit/llms/databricks/messages/test_databricks_messages_transformation.py index 3c5f4fd72f2..ae54d0e7437 100644 --- a/tests/unit/llms/databricks/messages/test_databricks_messages_transformation.py +++ b/tests/unit/llms/databricks/messages/test_databricks_messages_transformation.py @@ -281,7 +281,7 @@ SERVER_TOOLS: Final = [ ] -async def _workspace_request_body(**request_params: object) -> dict[str, object]: +async def _workspace_request(max_tokens: int = 64, **request_params: object) -> httpx.Request: received: Final[list[httpx.Request]] = [] # mutable-ok: the fake workspace records what litellm sent def workspace_answers(request: httpx.Request) -> httpx.Response: @@ -303,7 +303,7 @@ async def _workspace_request_body(**request_params: object) -> dict[str, object] upstream: Final = AsyncHTTPHandler(transport=httpx.MockTransport(workspace_answers)) try: await litellm.anthropic.messages.acreate( - max_tokens=64, + max_tokens=max_tokens, messages=USER_TURN, model=f"databricks/{CLAUDE_MODEL}", api_key=DATABRICKS_API_KEY, @@ -314,7 +314,30 @@ async def _workspace_request_body(**request_params: object) -> dict[str, object] finally: await upstream.close() assert len(received) == 1 - return json.loads(received[0].content) + return received[0] + + +async def _workspace_request_body(**request_params: object) -> dict[str, object]: + return json.loads((await _workspace_request(**request_params)).content) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("drop_params", [True, False]) +async def test_bound_thinking_with_display_updates_reaches_the_workspace_with_both_betas( + drop_params: bool, monkeypatch: pytest.MonkeyPatch, local_beta_headers_config: None +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + thinking: Final = { + "type": "enabled", + "budget_tokens": 1024, + "block_binding": {"prefix_mismatch_behavior": "error"}, + "display": "updates", + } + request: Final = await _workspace_request(max_tokens=4096, thinking=thinking, drop_params=drop_params) + assert json.loads(request.content)["thinking"] == thinking + assert {"thinking-binding-controls-2026-08-01", "thinking-display-updates-2026-08-18"} <= set( + request.headers["anthropic-beta"].split(",") + ) @pytest.mark.asyncio diff --git a/tests/unit/llms/edenai/conftest.py b/tests/unit/llms/edenai/conftest.py index 5ca5728354c..e11a515e953 100644 --- a/tests/unit/llms/edenai/conftest.py +++ b/tests/unit/llms/edenai/conftest.py @@ -2,7 +2,6 @@ import asyncio import uuid import pytest -import pytest_asyncio import litellm from litellm.integrations.custom_logger import CustomLogger @@ -40,9 +39,8 @@ class SpendCapture(CustomLogger): await asyncio.wait_for(GLOBAL_LOGGING_WORKER.flush(), timeout=10.0) -@pytest_asyncio.fixture -async def spend_capture(monkeypatch) -> SpendCapture: - GLOBAL_LOGGING_WORKER.start() # rebinds the worker's queue to this test's event loop +@pytest.fixture +def spend_capture(monkeypatch) -> SpendCapture: capture = SpendCapture(call_id=f"eden-{uuid.uuid4()}") monkeypatch.setattr(litellm, "callbacks", [capture]) return capture diff --git a/tests/unit/llms/openai_like/test_kyma_provider.py b/tests/unit/llms/openai_like/test_kyma_provider.py new file mode 100644 index 00000000000..2b3740ca676 --- /dev/null +++ b/tests/unit/llms/openai_like/test_kyma_provider.py @@ -0,0 +1,465 @@ +import json +import re +from collections.abc import Callable +from datetime import datetime +from itertools import chain +from pathlib import Path +from typing import Final + +import pytest +import respx + +import litellm +from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider +from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging +from litellm.proxy.auth.model_checks import get_known_models_from_wildcard +from litellm.proxy.common_request_processing import cost_headers +from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices, Usage + +_REPO_ROOT: Final = Path(__file__).resolve().parents[4] +_PRIMARY_PRICES: Final = json.loads((_REPO_ROOT / "model_prices_and_context_window.json").read_text()) +_BACKUP_PRICES: Final = json.loads((_REPO_ROOT / "litellm" / "model_prices_and_context_window_backup.json").read_text()) +_CATALOG_MODELS: Final = tuple( + key for key, row in _PRIMARY_PRICES.items() if isinstance(row, dict) and row.get("litellm_provider") == "kyma" +) +_TIER_KEY: Final = re.compile(r"^input_cost_per_token_above_(\d+)k_tokens$") + + +def _long_prompt_tiers(model: str) -> tuple[tuple[str, int], ...]: + matches: Final = map(_TIER_KEY.match, _PRIMARY_PRICES[model]) + return tuple((model, int(match.group(1))) for match in matches if match is not None) + + +_LONG_PROMPT_TIERS: Final = tuple(chain.from_iterable(map(_long_prompt_tiers, _CATALOG_MODELS))) +_CHAT_URL: Final = "https://kymaapi.com/v1/chat/completions" +_UNLISTED_MODEL: Final = "kyma/model-added-after-this-release" + + +def _kyma_chat_body(usage: dict[str, object]) -> dict[str, object]: + return { + "id": "chatcmpl_kyma", + "object": "chat.completion", + "created": 1_792_000_000, + "model": "sonar", + "choices": [{"index": 0, "message": {"role": "assistant", "content": "pong"}, "finish_reason": "stop"}], + "usage": usage, + } + + +def test_kyma_provider_resolution(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setenv("KYMA_API_KEY", "kyma-test-key") + + model, provider, api_key, api_base = get_llm_provider( + model="kyma/qwen3.7-flash", + custom_llm_provider=None, + api_base=None, + api_key=None, + ) + + assert model == "qwen3.7-flash" + assert provider == "kyma" + assert api_key == "kyma-test-key" + assert api_base == "https://kymaapi.com/v1" + + +def test_kyma_provider_keeps_explicit_credentials(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setenv("KYMA_API_KEY", "kyma-env-key") + + _, provider, api_key, api_base = get_llm_provider( + model="kyma/qwen3.7-flash", + custom_llm_provider=None, + api_base="https://kyma.internal.example/v1", + api_key="kyma-explicit-key", + ) + + assert provider == "kyma" + assert api_key == "kyma-explicit-key" + assert api_base == "https://kyma.internal.example/v1" + + +def test_bare_model_on_kyma_base_url_keeps_openai_routing(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setenv("OPENAI_API_KEY", "openai-env-key") + + model, provider, api_key, api_base = get_llm_provider( + model="gpt-6-luna", + custom_llm_provider=None, + api_base="https://kymaapi.com/v1", + api_key=None, + ) + + assert (model, provider, api_base) == ("gpt-6-luna", "openai", "https://kymaapi.com/v1") + assert api_key is None + + +def test_kyma_is_available_in_add_model_form(): + fields_path: Final = Path(litellm.__file__).parent / "proxy" / "public_endpoints" / "provider_create_fields.json" + providers: Final = json.loads(fields_path.read_text()) + kyma: Final = next(provider for provider in providers if provider["litellm_provider"] == "kyma") + + assert kyma["default_model_placeholder"] in _CATALOG_MODELS + assert {field["key"]: field["required"] for field in kyma["credential_fields"]} == { + "api_base": False, + "api_key": True, + } + + +def test_kyma_chat_completion_request(): + with respx.mock() as upstream: + route: Final = upstream.post(_CHAT_URL).respond( + 200, + json={ + "id": "chatcmpl_kyma", + "object": "chat.completion", + "created": 1_792_000_000, + "model": "qwen3.7-flash", + "choices": [ + {"index": 0, "message": {"role": "assistant", "content": "Hello from Kyma"}, "finish_reason": "stop"} + ], + "usage": {"prompt_tokens": 4, "completion_tokens": 3, "total_tokens": 7}, + }, + ) + response: Final = litellm.completion( + model="kyma/qwen3.7-flash", + messages=[{"role": "user", "content": "Say hello"}], + api_key="kyma-test-key", + ) + + request: Final = route.calls.last.request + body: Final = json.loads(request.content) + assert route.call_count == 1 + assert str(request.url) == _CHAT_URL + assert request.headers["authorization"] == "Bearer kyma-test-key" + assert body["model"] == "qwen3.7-flash" + assert body["messages"] == [{"role": "user", "content": "Say hello"}] + assert response.choices[0].message.content == "Hello from Kyma" + + +@pytest.mark.parametrize("model", _CATALOG_MODELS) +def test_catalog_row_matches_its_backup_and_fits_its_context(model: str): + row: Final = _PRIMARY_PRICES[model] + + assert row == _BACKUP_PRICES[model] + assert row["mode"] == "chat" + assert 0 < row["max_output_tokens"] <= row["max_input_tokens"] + assert row["input_cost_per_token"] > 0 + assert row["output_cost_per_token"] > 0 + + +@pytest.mark.usefixtures("local_model_cost_map") +@pytest.mark.parametrize("model", _CATALOG_MODELS) +def test_chat_response_is_priced_at_its_catalog_row(model: str): + row: Final = _PRIMARY_PRICES[model] + response: Final = litellm.ModelResponse( + model=model.removeprefix("kyma/"), + usage=litellm.Usage(prompt_tokens=1000, completion_tokens=200, total_tokens=1200), + ) + + cost: Final = litellm.completion_cost(completion_response=response, model=model) + + assert cost == pytest.approx(1000 * row["input_cost_per_token"] + 200 * row["output_cost_per_token"]) + + +@pytest.mark.usefixtures("local_model_cost_map") +@pytest.mark.parametrize(("model", "threshold_k"), _LONG_PROMPT_TIERS) +def test_long_prompt_is_priced_at_its_tier(model: str, threshold_k: int): + row: Final = _PRIMARY_PRICES[model] + prompt_tokens: Final = threshold_k * 1000 + 1000 + response: Final = litellm.ModelResponse( + model=model.removeprefix("kyma/"), + usage=litellm.Usage(prompt_tokens=prompt_tokens, completion_tokens=200, total_tokens=prompt_tokens + 200), + ) + + cost: Final = litellm.completion_cost(completion_response=response, model=model) + + assert cost == pytest.approx( + prompt_tokens * row[f"input_cost_per_token_above_{threshold_k}k_tokens"] + + 200 * row[f"output_cost_per_token_above_{threshold_k}k_tokens"] + ) + + +@pytest.mark.parametrize("model", ("kyma/sonar", _UNLISTED_MODEL)) +def test_kyma_response_is_billed_at_the_cost_kyma_reports(model: str): + reported_cost: Final = 0.00721 + with respx.mock() as upstream: + upstream.post(_CHAT_URL).respond( + 200, json=_kyma_chat_body({"prompt_tokens": 4, "completion_tokens": 3, "total_tokens": 7, "cost": reported_cost}) + ) + response: Final = litellm.completion(model=model, messages=[{"role": "user", "content": "ping"}], api_key="k") + + assert response._hidden_params["response_cost"] == pytest.approx(reported_cost) + + +def test_deployment_pricing_overrides_the_cost_kyma_reports(): + with respx.mock() as upstream: + upstream.post(_CHAT_URL).respond( + 200, json=_kyma_chat_body({"prompt_tokens": 4, "completion_tokens": 3, "total_tokens": 7, "cost": 0.00721}) + ) + response: Final = litellm.completion( + model="kyma/sonar", + messages=[{"role": "user", "content": "ping"}], + api_key="k", + input_cost_per_token=0.001, + output_cost_per_token=0.002, + ) + + assert response._hidden_params["response_cost"] == pytest.approx(4 * 0.001 + 3 * 0.002) + + + +def _catalog_price(model: str, prompt_tokens: int, completion_tokens: int) -> float: + row: Final = _PRIMARY_PRICES[model] + return prompt_tokens * row["input_cost_per_token"] + completion_tokens * row["output_cost_per_token"] + + +@pytest.mark.usefixtures("local_model_cost_map") +@pytest.mark.parametrize("reported_cost", ("-0.5", "1e999")) +def test_unusable_cost_from_kyma_falls_back_to_the_catalog_price(reported_cost: str): + usage: Final = {"prompt_tokens": 4, "completion_tokens": 3, "total_tokens": 7, "cost": "REPORTED_COST"} + with respx.mock() as upstream: + upstream.post(_CHAT_URL).respond( + 200, + content=json.dumps(_kyma_chat_body(usage)).replace('"REPORTED_COST"', reported_cost), + headers={"content-type": "application/json"}, + ) + response: Final = litellm.completion(model="kyma/sonar", messages=[{"role": "user", "content": "ping"}], api_key="k") + + assert response._hidden_params["response_cost"] == pytest.approx(_catalog_price("kyma/sonar", 4, 3)) + + +@pytest.mark.usefixtures("local_model_cost_map") +def test_deployment_base_model_overrides_the_cost_kyma_reports(): + with respx.mock() as upstream: + upstream.post(_CHAT_URL).respond( + 200, json=_kyma_chat_body({"prompt_tokens": 4, "completion_tokens": 3, "total_tokens": 7, "cost": 0.00721}) + ) + response: Final = litellm.completion( + model="kyma/sonar", + messages=[{"role": "user", "content": "ping"}], + api_key="k", + base_model="kyma/qwen3.7-flash", + ) + + assert response._hidden_params["response_cost"] == pytest.approx(_catalog_price("kyma/qwen3.7-flash", 4, 3)) + + +def test_interaction_on_kyma_is_billed_at_the_cost_kyma_reports(): + reported_cost: Final = 0.00721 + with respx.mock() as upstream: + upstream.post(_CHAT_URL).respond( + 200, json=_kyma_chat_body({"prompt_tokens": 4, "completion_tokens": 3, "total_tokens": 7, "cost": reported_cost}) + ) + interaction: Final = litellm.interactions.create(model="kyma/qwen3.7-flash", input="ping", api_key="k") + + assert interaction._hidden_params["response_cost"] == pytest.approx(reported_cost) + + +@pytest.mark.parametrize("model", ("openrouter/openai/gpt-4o-mini", "deepseek/deepseek-chat")) +def test_interaction_on_another_provider_keeps_its_upstream_cost_out_of_the_body(model: str): + chat_usage: Final = {"prompt_tokens": 4, "completion_tokens": 3, "total_tokens": 7, "cost": 0.00721} + responses_body: Final = { + "id": "resp_upstream", + "object": "response", + "created_at": 1_792_000_000, + "status": "completed", + "model": "x", + "output": [ + { + "type": "message", + "id": "msg_upstream", + "status": "completed", + "role": "assistant", + "content": [{"type": "output_text", "text": "pong", "annotations": []}], + } + ], + "usage": {"input_tokens": 4, "output_tokens": 3, "total_tokens": 7, "cost": 0.00721}, + } + with respx.mock(assert_all_called=False) as upstream: + upstream.route(path__regex=r".*/chat/completions$").respond(200, json=_kyma_chat_body(chat_usage)) + upstream.route(path__regex=r".*/responses$").respond(200, json=responses_body) + interaction: Final = litellm.interactions.create(model=model, input="ping", api_key="k") + + assert "cost" not in interaction.model_dump(exclude_none=True)["usage"] + + +def test_sdk_completion_cost_prices_kyma_at_the_cost_kyma_reports(): + reported_cost: Final = 0.00721 + with respx.mock() as upstream: + upstream.post(_CHAT_URL).respond( + 200, json=_kyma_chat_body({"prompt_tokens": 4, "completion_tokens": 3, "total_tokens": 7, "cost": reported_cost}) + ) + response: Final = litellm.completion(model="kyma/sonar", messages=[{"role": "user", "content": "ping"}], api_key="k") + + assert litellm.completion_cost(completion_response=response) == pytest.approx(reported_cost) + + +def _kyma_sdk_response(usage: dict[str, float]) -> litellm.ModelResponse: + with respx.mock() as upstream: + upstream.post(_CHAT_URL).respond(200, json=_kyma_chat_body(usage)) + response: Final = litellm.completion(model="kyma/sonar", messages=[{"role": "user", "content": "ping"}], api_key="k") + assert isinstance(response, litellm.ModelResponse) + return response + + +_KYMA_TOKEN_USAGE: Final = {"prompt_tokens": 4, "completion_tokens": 3, "total_tokens": 7} +_KYMA_REPORTED_COST: Final = 0.00721 + + +def test_sdk_completion_cost_uses_the_callers_token_prices_over_the_cost_kyma_reports(): + caller_prices: Final = {"input_cost_per_token": 0.5, "output_cost_per_token": 2.0} + with_reported_cost: Final = _kyma_sdk_response({**_KYMA_TOKEN_USAGE, "cost": _KYMA_REPORTED_COST}) + without_reported_cost: Final = _kyma_sdk_response(_KYMA_TOKEN_USAGE) + + assert litellm.completion_cost( + completion_response=with_reported_cost, custom_cost_per_token=caller_prices + ) == pytest.approx(litellm.completion_cost(completion_response=without_reported_cost, custom_cost_per_token=caller_prices)) + assert litellm.completion_cost( + completion_response=with_reported_cost, custom_cost_per_token=caller_prices + ) != pytest.approx(_KYMA_REPORTED_COST) + + +def test_sdk_completion_cost_uses_the_callers_per_second_price_over_the_cost_kyma_reports(): + response: Final = _kyma_sdk_response({**_KYMA_TOKEN_USAGE, "cost": _KYMA_REPORTED_COST}) + + assert litellm.completion_cost(completion_response=response, custom_cost_per_second=50.0) == pytest.approx( + 50.0 * response._response_ms / 1000 + ) + + +def _with_kyma_discount_and_margin(monkeypatch: pytest.MonkeyPatch) -> float: + monkeypatch.setattr(litellm, "cost_discount_config", {"kyma": 0.5}) + monkeypatch.setattr(litellm, "cost_margin_config", {"kyma": 0.1}) + return 0.5 * 1.1 + + +def test_cost_kyma_reports_takes_the_configured_discount_and_margin(monkeypatch: pytest.MonkeyPatch): + adjustment: Final = _with_kyma_discount_and_margin(monkeypatch) + with respx.mock() as upstream: + upstream.post(_CHAT_URL).respond( + 200, json=_kyma_chat_body({"prompt_tokens": 4, "completion_tokens": 3, "total_tokens": 7, "cost": 0.008}) + ) + response: Final = litellm.completion(model="kyma/sonar", messages=[{"role": "user", "content": "ping"}], api_key="k") + + assert response._hidden_params["response_cost"] == pytest.approx(0.008 * adjustment) + + +def _kyma_stream_chunk(content: str, finish_reason: str | None = None) -> ModelResponseStream: + return ModelResponseStream( + id="chatcmpl_kyma", + created=1_792_000_000, + model="sonar", + object="chat.completion.chunk", + choices=[StreamingChoices(finish_reason=finish_reason, index=0, delta=Delta(content=content, role="assistant"))], + ) + + +def test_streamed_cost_kyma_reports_takes_the_configured_discount_and_margin(monkeypatch: pytest.MonkeyPatch): + adjustment: Final = _with_kyma_discount_and_margin(monkeypatch) + usage_chunk: Final = _kyma_stream_chunk("") + usage_chunk.usage = Usage(prompt_tokens=4, completion_tokens=3, total_tokens=7, cost=0.008) + logging_obj: Final = LiteLLMLogging( + model="sonar", + messages=[{"role": "user", "content": "ping"}], + stream=True, + call_type="completion", + start_time=datetime.now(), + litellm_call_id="kyma-call", + function_id="kyma-function", + ) + logging_obj.update_environment_variables( + model="sonar", + user=None, + optional_params={}, + litellm_params={"custom_llm_provider": "kyma"}, + custom_llm_provider="kyma", + ) + + response: Final = litellm.stream_chunk_builder( + chunks=[_kyma_stream_chunk("pong", finish_reason="stop"), usage_chunk], + messages=[{"role": "user", "content": "ping"}], + logging_obj=logging_obj, + ) + + assert logging_obj.response_cost_calculator(result=response) == pytest.approx(0.008 * adjustment) + + +def test_cost_headers_show_the_discount_and_margin_taken_from_the_cost_kyma_reports(monkeypatch: pytest.MonkeyPatch): + _with_kyma_discount_and_margin(monkeypatch) + logging_obj: Final = LiteLLMLogging( + model="sonar", + messages=[{"role": "user", "content": "ping"}], + stream=False, + call_type="completion", + start_time=datetime.now(), + litellm_call_id="kyma-call", + function_id="kyma-function", + ) + logging_obj.update_environment_variables( + model="sonar", + user=None, + optional_params={}, + litellm_params={"custom_llm_provider": "kyma"}, + custom_llm_provider="kyma", + ) + response: Final = litellm.ModelResponse( + model="sonar", usage=Usage(prompt_tokens=4, completion_tokens=3, total_tokens=7, cost=0.008) + ) + + headers: Final = cost_headers(logging_obj.response_cost_calculator(result=response), logging_obj) + + assert float(headers["x-litellm-response-cost-original"] or "nan") == pytest.approx(0.008) + assert float(headers["x-litellm-response-cost-discount-amount"] or "nan") == pytest.approx(0.004) + assert float(headers["x-litellm-response-cost-margin-amount"] or "nan") == pytest.approx(0.0004) + assert float(headers["x-litellm-response-cost"] or "nan") == pytest.approx(0.0044) + assert headers["x-litellm-response-cost-input"] is None + assert headers["x-litellm-response-cost-output"] is None + + +@pytest.mark.parametrize("thinking", ({"type": "enabled", "budget_tokens": 1024}, {"type": "adaptive"})) +def test_messages_thinking_on_a_kyma_claude_model_is_sent_as_reasoning_effort(thinking: dict[str, object]): + with respx.mock() as upstream: + route: Final = upstream.post(_CHAT_URL).respond( + 200, json=_kyma_chat_body({"prompt_tokens": 4, "completion_tokens": 3, "total_tokens": 7}) + ) + litellm.anthropic.messages.create( + model="kyma/claude-haiku-4-5", + max_tokens=2048, + messages=[{"role": "user", "content": "ping"}], + thinking=thinking, + api_key="k", + ) + + sent: Final = json.loads(route.calls.last.request.content) + assert "thinking" not in sent + assert sent["reasoning_effort"] in ("low", "medium", "high") + +def test_kyma_model_missing_from_the_catalog_keeps_its_tools(): + tools: Final = [{"type": "function", "function": {"name": "lookup", "parameters": {"type": "object"}}}] + with respx.mock() as upstream: + route: Final = upstream.post(_CHAT_URL).respond( + 200, json=_kyma_chat_body({"prompt_tokens": 4, "completion_tokens": 3, "total_tokens": 7}) + ) + litellm.completion(model=_UNLISTED_MODEL, messages=[{"role": "user", "content": "ping"}], api_key="k", tools=tools) + + assert json.loads(route.calls.last.request.content)["tools"] == tools + + +def test_kyma_wildcard_lists_every_catalog_model(): + assert sorted(get_known_models_from_wildcard("kyma/*")) == sorted(_CATALOG_MODELS) + + +def _generate_image() -> object: + return litellm.image_generation(model="kyma/flux-2", prompt="a cat", api_key="k") + + +def _transcribe() -> object: + return litellm.transcription(model="kyma/whisper-1", file=("a.wav", b"RIFF0000WAVE", "audio/wav"), api_key="k") + + +@pytest.mark.parametrize("call", (_generate_image, _transcribe)) +def test_unpriced_kyma_media_calls_are_refused_without_calling_kyma(call: Callable[[], object]): + with respx.mock(assert_all_called=False) as upstream: + kyma: Final = upstream.route(host="kymaapi.com").respond(202, json={"id": "job_kyma"}) + with pytest.raises(litellm.BadRequestError): + call() + + assert kyma.call_count == 0 diff --git a/tests/unit/llms/sail/conftest.py b/tests/unit/llms/sail/conftest.py index 2b2a6e5cae0..c1156af5b12 100644 --- a/tests/unit/llms/sail/conftest.py +++ b/tests/unit/llms/sail/conftest.py @@ -4,11 +4,9 @@ from typing import Final import httpx import pytest -import pytest_asyncio import respx import litellm -from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER from tests.unit.llms.sail.helpers import SAIL_API_BASE, SpendCapture, chat_completion_body @@ -26,9 +24,8 @@ def sail_env(local_model_cost_map: None, monkeypatch: pytest.MonkeyPatch) -> Ite litellm.in_memory_llm_clients_cache.flush_cache() -@pytest_asyncio.fixture -async def spend_capture(monkeypatch: pytest.MonkeyPatch) -> SpendCapture: - GLOBAL_LOGGING_WORKER.start() +@pytest.fixture +def spend_capture(monkeypatch: pytest.MonkeyPatch) -> SpendCapture: capture: Final = SpendCapture(call_id=f"sail-{uuid.uuid4()}") monkeypatch.setattr(litellm, "callbacks", [capture]) return capture diff --git a/tests/unit/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_messages_config.py b/tests/unit/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_messages_config.py index 5916ce9f7a1..1e7c3d52814 100644 --- a/tests/unit/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_messages_config.py +++ b/tests/unit/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_messages_config.py @@ -289,7 +289,7 @@ def test_no_safeguards_leaves_dangerous_tool_use_beta_header_out(): assert "dangerous-tool-use-2026-09-03" not in updated_headers.get("anthropic-beta", "") -def _validate_vertex_headers(client_headers, messages): +def _validate_vertex_headers(client_headers, messages, thinking: Mapping[str, object] | None = None): config = VertexAIPartnerModelsAnthropicMessagesConfig() litellm_params = { "vertex_ai_project": "test-project", @@ -305,7 +305,7 @@ def _validate_vertex_headers(client_headers, messages): headers=client_headers, model="claude-opus-5-5", messages=messages, - optional_params={"max_tokens": 64}, + optional_params={"max_tokens": 64, **({"thinking": thinking} if thinking is not None else {})}, litellm_params=litellm_params, api_base=None, ) @@ -337,6 +337,25 @@ def test_per_message_output_config_reaches_vertex_with_per_turn_control_beta(cli assert filtered["anthropic-beta"].split(",").count("per-turn-control-2026-07-01") == 1 +def test_block_binding_reaches_vertex_messages_with_its_beta_under_drop_params(local_beta_headers_config: None) -> None: + from litellm.anthropic_beta_headers_manager import update_headers_with_filtered_beta + + thinking: Final = {"type": "adaptive", "block_binding": {"prefix_mismatch_behavior": "error"}} + messages: Final = [{"role": "user", "content": "Say hi in French"}] + headers: Final = update_headers_with_filtered_beta( + headers=_validate_vertex_headers({}, messages, thinking=thinking), provider="vertex_ai" + ) + request: Final = VertexAIPartnerModelsAnthropicMessagesConfig().transform_anthropic_messages_request( + model="claude-opus-5-5", + messages=[dict(message) for message in messages], + anthropic_messages_optional_request_params={"max_tokens": 64, "thinking": dict(thinking)}, + litellm_params=GenericLiteLLMParams(drop_params=True), + headers=headers, + ) + assert "thinking-binding-controls-2026-08-01" in headers["anthropic-beta"].split(",") + assert request["thinking"] == thinking + + def test_no_per_message_output_config_leaves_per_turn_control_beta_out(): headers = _validate_vertex_headers({}, [{"role": "user", "content": "Hello"}]) @@ -913,6 +932,20 @@ class TestVertexAnthropicMidConversationSystem: {"type": "text", "text": "Cite sources."}, ] + @pytest.mark.parametrize("drop_params", (False, True)) + def test_unsupported_model_sends_a_non_object_message_on_for_the_provider_to_reject( + self, local_model_cost_map, local_beta_headers_config, drop_params + ): + messages: Final = [{"role": "user", "content": "hi"}, "oops"] + result: Final = VertexAIPartnerModelsAnthropicMessagesConfig().transform_anthropic_messages_request( + model="claude-sonnet-4-6", + messages=list(messages), + anthropic_messages_optional_request_params={"max_tokens": 64}, + litellm_params=GenericLiteLLMParams(drop_params=drop_params), + headers={}, + ) + assert result["messages"] == messages + def test_unsupported_model_converts_mid_conversation_system_in_place(self, local_model_cost_map): messages = [ {"role": "user", "content": "read the file"}, diff --git a/tests/unit/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_transformation.py b/tests/unit/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_transformation.py index 1c432ec79da..3ae208bb3a8 100644 --- a/tests/unit/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_transformation.py +++ b/tests/unit/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_transformation.py @@ -155,6 +155,25 @@ def test_vertex_ai_anthropic_web_search_header_in_completion(): ), "anthropic-beta with web-search should not be present for non-Vertex requests" +def test_vertex_ai_anthropic_chat_adds_the_betas_for_bound_thinking_with_display_updates() -> None: + thinking: Final = { + "type": "adaptive", + "block_binding": {"prefix_mismatch_behavior": "error"}, + "display": "updates", + } + result: Final = VertexAIAnthropicConfig().transform_request( + model="claude-opus-4-8", + messages=[{"role": "user", "content": "Hello"}], + optional_params={"thinking": thinking, "max_tokens": 100, "is_vertex_request": True}, + litellm_params={}, + headers={}, + ) + assert result["thinking"] == thinking + assert {"thinking-binding-controls-2026-08-01", "thinking-display-updates-2026-08-18"} <= set( + result["anthropic_beta"] + ) + + def test_vertex_ai_anthropic_context_management_compact_beta_header(): """Test that context_management with compact adds the correct beta header for Vertex AI""" config = VertexAIAnthropicConfig() diff --git a/tests/unit/proxy/anthropic_endpoints/test_endpoints.py b/tests/unit/proxy/anthropic_endpoints/test_endpoints.py index 037b0a6c29a..e3e625be975 100644 --- a/tests/unit/proxy/anthropic_endpoints/test_endpoints.py +++ b/tests/unit/proxy/anthropic_endpoints/test_endpoints.py @@ -5,18 +5,25 @@ Test for anthropic_endpoints/endpoints.py, focusing on handling dictionary objec import json import logging import unittest -from collections.abc import Callable, Mapping +from collections.abc import AsyncGenerator, AsyncIterator, Callable, Mapping from typing import Final from unittest.mock import AsyncMock, MagicMock, patch import httpx import pytest +from fastapi import Request, Response +from fastapi.responses import StreamingResponse from fastapi.testclient import TestClient +from starlette.types import Message from litellm import Router from litellm._logging import verbose_proxy_logger +from litellm.caching.dual_cache import DualCache +from litellm.integrations.custom_guardrail import CustomGuardrail +from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing -from litellm.types.utils import TokenCountResponse +from litellm.types.guardrails import GuardrailEventHooks +from litellm.types.utils import ModelResponseStream, TokenCountResponse class TestAnthropicEndpoints(unittest.TestCase): @@ -70,11 +77,76 @@ class TestAnthropicEndpoints(unittest.TestCase): assert mock_safe_dumps.call_count == 2 # Called twice, once for each dict object +_BLOCK_CALL_ID: Final = "messages-block" +_BLOCK_TEXT: Final = "blocked by guardrail" + + +class _BlockEveryRequest(CustomGuardrail): + async def async_pre_call_hook( + self, user_api_key_dict: UserAPIKeyAuth, cache: DualCache, data: dict[str, object], call_type: str + ) -> None: + self.raise_passthrough_exception(violation_message=_BLOCK_TEXT, request_data=data) + + +class _RewriteEveryStreamedReply(CustomGuardrail): + async def async_post_call_streaming_iterator_hook( + self, user_api_key_dict: UserAPIKeyAuth, response: AsyncIterator[object], request_data: dict[str, object] + ) -> AsyncGenerator[ModelResponseStream, None]: + async for _ in response: + yield ModelResponseStream(choices=[]) + + +def _messages_request(body: Mapping[str, object]) -> Request: + async def receive() -> Message: + return {"type": "http.request", "body": json.dumps(body).encode(), "more_body": False} + + return Request( + { + "type": "http", + "method": "POST", + "path": "/v1/messages", + "query_string": b"", + "headers": [(b"content-type", b"application/json"), (b"x-litellm-call-id", _BLOCK_CALL_ID.encode())], + }, + receive, + ) + + +async def _blocked_messages_reply( + body: Mapping[str, object], fastapi_response: Response, rails: tuple[CustomGuardrail, ...] +) -> tuple[object, str]: + import litellm + import litellm.proxy.anthropic_endpoints.endpoints as ep + import litellm.proxy.proxy_server as proxy_server + from litellm.proxy.utils import ProxyLogging + + blocker: Final = _BlockEveryRequest( + guardrail_name="deny-list", event_hook=GuardrailEventHooks.pre_call, default_on=True + ) + with ( + patch.object( + proxy_server, "proxy_logging_obj", new=ProxyLogging(user_api_key_cache=DualCache()) + ), # test-quality-ok: module global imported at call time; no injection seam + patch.object(litellm, "callbacks", new=[blocker, *rails]), # test-quality-ok: guardrails register through this global + ): + response = await ep.anthropic_response( + fastapi_response=fastapi_response, + request=_messages_request(body), + user_api_key_dict=UserAPIKeyAuth(), + ) + if not isinstance(response, StreamingResponse): + return response, "" + streamed: Final = b"".join( + [chunk if isinstance(chunk, bytes) else chunk.encode() async for chunk in response.body_iterator] + ).decode() + return response, streamed + + class TestBlockedResponseUsage: """Blocked responses report the blocked LLM response's real usage.""" def test_uses_original_response_usage(self): - from litellm.proxy.anthropic_endpoints.endpoints import _blocked_response_usage + from litellm.llms.base_llm.guardrail_translation.utils import blocked_response_usage as _blocked_response_usage # original_response is the AnthropicMessagesResponse the LLM produced # before the guardrail blocked it; its usage is real. @@ -85,7 +157,7 @@ class TestBlockedResponseUsage: } def test_zero_usage_when_no_original_response(self): - from litellm.proxy.anthropic_endpoints.endpoints import _blocked_response_usage + from litellm.llms.base_llm.guardrail_translation.utils import blocked_response_usage as _blocked_response_usage # Pre-call blocks never invoked the LLM -> nothing consumed. assert _blocked_response_usage(None) == { @@ -131,6 +203,49 @@ class TestBlockedResponseUsage: assert response["usage"] == {"input_tokens": 12, "output_tokens": 5} mock_logging.post_call_failure_hook.assert_awaited_once() + @pytest.mark.asyncio + @pytest.mark.parametrize( + "rails", + ((), (_RewriteEveryStreamedReply(guardrail_name="post-call-rewrite", default_on=True),)), + ids=("no_rails", "post_call_rail"), + ) + async def test_streaming_block_answers_anthropic_events_with_the_call_id( + self, rails: tuple[CustomGuardrail, ...] + ) -> None: + response, streamed = await _blocked_messages_reply( + {"model": "claude-haiku-5-5", "stream": True}, Response(), rails + ) + + frames: Final = tuple(frame for frame in streamed.split("\n\n") if frame.strip()) + events: Final = tuple( + (frame.split("\n")[0].removeprefix("event: "), json.loads(frame.split("\n")[1].removeprefix("data: "))) + for frame in frames + ) + assert isinstance(response, StreamingResponse) + assert response.media_type == "text/event-stream" + assert response.headers["x-litellm-call-id"] == _BLOCK_CALL_ID + assert [name for name, _ in events] == [ + "message_start", + "content_block_start", + "content_block_delta", + "content_block_stop", + "message_delta", + "message_stop", + ] + assert all(payload["type"] == name for name, payload in events) + assert events[2][1]["delta"] == {"type": "text_delta", "text": _BLOCK_TEXT} + assert events[4][1]["delta"]["stop_reason"] == "end_turn" + + @pytest.mark.asyncio + async def test_json_block_answers_with_the_call_id(self) -> None: + fastapi_response: Final = Response() + + response, _ = await _blocked_messages_reply({"model": "claude-haiku-5-5"}, fastapi_response, ()) + + assert fastapi_response.headers["x-litellm-call-id"] == _BLOCK_CALL_ID + assert isinstance(response, Mapping) + assert response["content"] == [{"type": "text", "text": _BLOCK_TEXT}] + class TestProxyExceptionAnthropicEnvelope: @pytest.mark.asyncio diff --git a/tests/unit/proxy/management_endpoints/policy_endpoints/test_ai_policy_suggester.py b/tests/unit/proxy/management_endpoints/policy_endpoints/test_ai_policy_suggester.py index 2041c622b3b..d287f5d00ad 100644 --- a/tests/unit/proxy/management_endpoints/policy_endpoints/test_ai_policy_suggester.py +++ b/tests/unit/proxy/management_endpoints/policy_endpoints/test_ai_policy_suggester.py @@ -3,6 +3,7 @@ Tests for AiPolicySuggester class. """ import json +from typing import Final from unittest.mock import AsyncMock, MagicMock, patch import httpx @@ -214,6 +215,26 @@ class TestAiPolicySuggester: assert result["selected_templates"] == [] assert "No templates" in result["explanation"] + @pytest.mark.asyncio + @pytest.mark.parametrize("selected_templates", [{}, ""]) + async def test_suggest_reads_an_empty_non_list_selection_as_no_templates(self, selected_templates: object) -> None: + mock_tool_call: Final = MagicMock() + mock_tool_call.function.arguments = json.dumps( + {"selected_templates": selected_templates, "explanation": "No template fits"} + ) + mock_response: Final = MagicMock() + mock_response.choices = [MagicMock()] + mock_response.choices[0].message.tool_calls = [mock_tool_call] + + with patch("litellm.acompletion", new_callable=AsyncMock, return_value=mock_response): + result: Final = await AiPolicySuggester().suggest( + templates=SAMPLE_TEMPLATES, + attack_examples=["Ignore all previous instructions"], + description="", + ) + + assert result == {"selected_templates": [], "explanation": "No template fits"} + @pytest.mark.asyncio async def test_suggest_calls_litellm_with_correct_params(self): suggester = AiPolicySuggester() diff --git a/tests/unit/proxy/pass_through_endpoints/llm_provider_handlers/test_cohere_passthrough_logging_handler.py b/tests/unit/proxy/pass_through_endpoints/llm_provider_handlers/test_cohere_passthrough_logging_handler.py index 32e87d93fdb..f2bb9eaa9a5 100644 --- a/tests/unit/proxy/pass_through_endpoints/llm_provider_handlers/test_cohere_passthrough_logging_handler.py +++ b/tests/unit/proxy/pass_through_endpoints/llm_provider_handlers/test_cohere_passthrough_logging_handler.py @@ -1,18 +1,23 @@ import json +import logging +from collections.abc import Mapping from datetime import datetime +from typing import Final from unittest.mock import MagicMock, patch import httpx import pytest - +import litellm from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.proxy.pass_through_endpoints.llm_provider_handlers.cohere_passthrough_logging_handler import ( CoherePassthroughLoggingHandler, ) +from litellm.proxy.pass_through_endpoints.success_handler import PassThroughEndpointLogging from litellm.types.passthrough_endpoints.pass_through_endpoints import ( PassthroughStandardLoggingPayload, ) +from litellm.types.utils import CallTypes class TestCoherePassthroughLoggingHandler: @@ -64,46 +69,25 @@ class TestCoherePassthroughLoggingHandler: ) @patch("litellm.completion_cost") - @patch("litellm.litellm_core_utils.litellm_logging.get_standard_logging_object_payload") - @patch("litellm.llms.cohere.embed.v1_transformation.CohereEmbeddingConfig._transform_response") - def test_cohere_embed_passthrough_cost_tracking( - self, mock_transform_response, mock_get_standard_logging, mock_completion_cost - ): + def test_cohere_embed_passthrough_cost_tracking(self, mock_completion_cost: MagicMock) -> None: """Test successful cost tracking for Cohere embed passthrough""" - # Arrange - from litellm.types.utils import EmbeddingResponse + mock_completion_cost.return_value = 3.6e-07 - # Create a mock embedding response - mock_embedding_response = EmbeddingResponse() - mock_embedding_response.data = [ - {"object": "embedding", "index": 0, "embedding": [0.1, 0.2, 0.3]}, - {"object": "embedding", "index": 1, "embedding": [0.4, 0.5, 0.6]}, - ] - mock_embedding_response.model = "embed-english-v3.0" - mock_embedding_response.object = "list" - from litellm.types.utils import Usage + mock_httpx_response: Final = self._create_mock_httpx_response() + mock_logging_obj: Final = self._create_mock_logging_obj() + passthrough_payload: Final = self._create_passthrough_logging_payload() - mock_embedding_response.usage = Usage(prompt_tokens=3, completion_tokens=0, total_tokens=3) - - mock_transform_response.return_value = mock_embedding_response - mock_completion_cost.return_value = 3.6e-07 # Expected cost for embed-v4.0 - mock_get_standard_logging.return_value = {"test": "logging_payload"} - - mock_httpx_response = self._create_mock_httpx_response() - mock_logging_obj = self._create_mock_logging_obj() - passthrough_payload = self._create_passthrough_logging_payload() - - kwargs = { + kwargs: Final = { "passthrough_logging_payload": passthrough_payload, } - request_body = { + request_body: Final = { "model": "embed-english-v3.0", "texts": ["test passthrough"], } # Act - result = self.handler.cohere_passthrough_handler( + result: Final = self.handler.cohere_passthrough_handler( httpx_response=mock_httpx_response, response_body=self.mock_cohere_embed_response, logging_obj=mock_logging_obj, @@ -126,7 +110,7 @@ class TestCoherePassthroughLoggingHandler: # Verify cost calculation was called with correct parameters mock_completion_cost.assert_called_once() - call_args = mock_completion_cost.call_args + call_args: Final = mock_completion_cost.call_args assert call_args.kwargs["model"] == "embed-english-v3.0" assert call_args.kwargs["custom_llm_provider"] == "cohere" assert call_args.kwargs["call_type"] == "aembedding" @@ -174,5 +158,167 @@ class TestCoherePassthroughLoggingHandler: assert result == {"result": None, "kwargs": {}} +COHERE_EMBED_V2_TEXT_BODY: Final[Mapping[str, object]] = { + "id": "da6e531f-83f4-4bcb-8e9a-f1a2c6e7a1b3", + "embeddings": {"float": [[0.1, 0.2], [0.3, 0.4]]}, + "texts": ["hello world", "second text"], + "meta": {"api_version": {"version": "2"}, "billed_units": {"input_tokens": 9}}, + "response_type": "embeddings_by_type", +} +COHERE_EMBED_V2_IMAGE_BODY: Final[Mapping[str, object]] = { + "id": "5807ee2e-0cda-445a-9ec8-864c60a06606", + "embeddings": {"float": [[0.125, -0.375]]}, + "texts": [], + "images": [{"width": 64, "height": 64, "format": "image/png", "bit_depth": 8}], + "meta": {"api_version": {"version": "2"}, "billed_units": {"input_tokens": 0, "image_tokens": 6}}, + "response_type": "embeddings_by_type", +} +COHERE_EMBED_V1_FLOATS_BODY: Final[Mapping[str, object]] = { + "id": "5d4e1f3a-7c1b-4f3e-9a3e-1b2c3d4e5f60", + "embeddings": [[0.1, 0.2], [0.3, 0.4]], + "texts": ["hello world", "second text"], + "meta": {"api_version": {"version": "1"}, "billed_units": {"input_tokens": 3}}, + "response_type": "embeddings_floats", +} + + +_CALL_START: Final = datetime(2026, 10, 1, 12, 0, 0) +_CALL_END: Final = datetime(2026, 10, 1, 12, 0, 1) + + +def _passthrough_logging_obj() -> LiteLLMLoggingObj: + return LiteLLMLoggingObj( + model="unknown", + messages=[{"role": "user", "content": "no-message-pass-through-endpoint"}], + stream=False, + call_type=CallTypes.pass_through.value, + start_time=_CALL_START, + litellm_call_id="cohere-embed-passthrough-call", + function_id="cohere-embed-passthrough-function", + ) + + +@pytest.mark.parametrize( + ("url_route", "body", "input_tokens", "image_tokens"), + [ + ("https://api.cohere.com/v2/embed", COHERE_EMBED_V2_TEXT_BODY, 9, 0), + ("https://api.cohere.com/v2/embed", COHERE_EMBED_V2_IMAGE_BODY, 0, 6), + ("https://api.cohere.com/v1/embed", COHERE_EMBED_V1_FLOATS_BODY, 3, 0), + ], + ids=["v2_text", "v2_image", "v1_floats"], +) +def test_cohere_embed_passthrough_is_costed_from_billed_units( + url_route: str, body: Mapping[str, object], input_tokens: int, image_tokens: int +) -> None: + request_body: Final = { + "model": "embed-v4.0", + "texts": ["hello world", "second text"], + "input_type": "search_document", + } + model_info: Final = litellm.get_model_info("cohere/embed-v4.0") + expected_cost: Final = ( + input_tokens * model_info["input_cost_per_token"] + image_tokens * model_info["input_cost_per_image_token"] + ) + + logged: Final = PassThroughEndpointLogging().normalize_llm_passthrough_logging_payload( + httpx_response=httpx.Response(200, json=body, request=httpx.Request("POST", url_route)), + response_body=dict(body), + request_body=request_body, + logging_obj=_passthrough_logging_obj(), + url_route=url_route, + result="", + start_time=_CALL_START, + end_time=_CALL_END, + cache_hit=False, + ) + + assert expected_cost > 0 + assert logged["kwargs"]["custom_llm_provider"] == "cohere" + assert logged["kwargs"]["response_cost"] == pytest.approx(expected_cost) + + + +def _normalize_cohere_embed(url_route: str, body: Mapping[str, object], model: str) -> Mapping[str, object]: + return PassThroughEndpointLogging().normalize_llm_passthrough_logging_payload( + httpx_response=httpx.Response(200, json=body, request=httpx.Request("POST", url_route)), + response_body=dict(body), + request_body={"model": model, "texts": ["hello world"], "input_type": "search_document"}, + logging_obj=_passthrough_logging_obj(), + url_route=url_route, + result="", + start_time=_CALL_START, + end_time=_CALL_END, + cache_hit=False, + ) + + +def test_cohere_v3_image_embed_passthrough_is_costed_per_billed_image() -> None: + body: Final = { + "id": "8a1b2c3d-4e5f-4a6b-8c7d-9e0f1a2b3c4d", + "embeddings": {"float": [[0.125, -0.375]]}, + "texts": [], + "images": [{"width": 64, "height": 64, "format": "image/png", "bit_depth": 8}], + "meta": {"api_version": {"version": "2"}, "billed_units": {"images": 1}}, + "response_type": "embeddings_by_type", + } + per_image: Final = litellm.get_model_info("embed-english-v3.0", custom_llm_provider="cohere")["input_cost_per_image"] + + logged: Final = _normalize_cohere_embed("https://api.cohere.com/v2/embed", body, "embed-english-v3.0") + + assert per_image is not None and per_image > 0 + assert logged["kwargs"]["response_cost"] == pytest.approx(per_image) + + +def test_cohere_embed_passthrough_rows_keep_their_embedding_type() -> None: + body: Final = { + **COHERE_EMBED_V2_TEXT_BODY, + "embeddings": {"float": [[0.1, 0.2], [0.3, 0.4]], "int8": [[1, 2], [3, 4]]}, + } + + logged: Final = _normalize_cohere_embed("https://api.cohere.com/v2/embed", body, "embed-v4.0") + + assert [(row["type"], row["index"], row["embedding"]) for row in logged["standard_logging_response_object"].data] == [ + ("float", 0, [0.1, 0.2]), + ("float", 1, [0.3, 0.4]), + ("int8", 0, [1, 2]), + ("int8", 1, [3, 4]), + ] + + +def test_unpriceable_cohere_embed_passthrough_falls_back_to_the_plain_passthrough_row( + caplog: pytest.LogCaptureFixture, +) -> None: + with caplog.at_level(logging.WARNING): + logged: Final = _normalize_cohere_embed( + "https://api.cohere.com/v2/embed", COHERE_EMBED_V2_TEXT_BODY, "embed-model-missing-from-the-cost-map" + ) + + assert logged["standard_logging_response_object"] is None + assert "response_cost" not in logged["kwargs"] + assert any( + "Could not price the Cohere passthrough embed response" in record.getMessage() + for record in caplog.records + if record.levelno == logging.WARNING + ) + + +@pytest.mark.parametrize( + ("url_route", "body"), + [ + ("https://api.cohere.com/v1/embed-jobs", {"job_id": "job-1", "meta": {"api_version": {"version": "1"}}}), + ( + "https://api.cohere.com/v1/embed-jobs/job-1", + {"job_id": "job-1", "status": "processing", "model": "embed-english-v3.0"}, + ), + ], +) +def test_cohere_embed_job_passthrough_logs_the_plain_passthrough_row( + url_route: str, body: Mapping[str, object] +) -> None: + logged: Final = _normalize_cohere_embed(url_route, body, "embed-english-v3.0") + + assert logged["standard_logging_response_object"] is None + assert "response_cost" not in logged["kwargs"] + if __name__ == "__main__": pytest.main([__file__]) diff --git a/tests/unit/proxy/pass_through_endpoints/llm_provider_handlers/test_openai_passthrough_logging_handler.py b/tests/unit/proxy/pass_through_endpoints/llm_provider_handlers/test_openai_passthrough_logging_handler.py index c9064caf929..a4f3fd0a86f 100644 --- a/tests/unit/proxy/pass_through_endpoints/llm_provider_handlers/test_openai_passthrough_logging_handler.py +++ b/tests/unit/proxy/pass_through_endpoints/llm_provider_handlers/test_openai_passthrough_logging_handler.py @@ -24,8 +24,10 @@ from litellm.proxy._types import SpendLogsPayload from litellm.proxy.pass_through_endpoints.success_handler import ( PassThroughEndpointLogging, ) +from litellm.proxy.pass_through_endpoints.streaming_handler import PassThroughStreamingHandler from litellm.proxy.spend_tracking.spend_tracking_utils import get_logging_payload from litellm.types.passthrough_endpoints.pass_through_endpoints import ( + EndpointType, PassthroughStandardLoggingPayload, ) @@ -2088,6 +2090,7 @@ COMPACTION: Final = { "output": [{"id": "cmp_082492f9", "type": "compaction", "encrypted_content": "gAAAA"}], "usage": RESPONSES_USAGE, } +AGENTS_SESSIONS_URL: Final = "https://api.openai.com/v1/agents/sessions" ASSISTANT_MESSAGE: Final = { "id": "msg_080b10ff5d740c69", "type": "message", @@ -2119,6 +2122,28 @@ def _stored_response(status: str, background: bool, usage: Mapping[str, object] } +def _agent_session(agent_model: str | None) -> dict[str, object]: + agent: Final = ( + {} + if agent_model is None + else { + "agent": {"id": "agent_6a0b0f8e", "model": agent_model, "name": None, "service_tier": "auto", "tools": []} + } + ) + return { + "id": "sess_6a0b0f8e2c1d4e5f", + "object": "agent.session", + "created_at": 1792270025, + "last_active_at": 1792270025, + "status": "in_progress", + "metadata": {}, + "required_actions": [], + "vault_ids": [], + "usage": None, + **agent, + } + + @dataclass(frozen=True, slots=True) class _LoggedCall: spend_row: SpendLogsPayload @@ -2296,20 +2321,12 @@ class TestOpenAIPassthroughResponsesSubRouteBilling: assert not dispatched["kwargs"].get("response_cost") assert [record.getMessage() for record in caplog.records if record.levelno >= logging.ERROR] == [] - async def _logged_call( - self, - method: str, - sub_route: str, - request_body: dict[str, object], - response_body: dict[str, object], - litellm_call_id: str, - ) -> _LoggedCall: - url_route: Final = f"{RESPONSES_URL}{sub_route}" - recorder: Final = _SpendRowRecorder() + @staticmethod + def _recorded_logging_obj(recorder: _SpendRowRecorder, litellm_call_id: str, stream: bool) -> LiteLLMLoggingObj: logging_obj: Final = LiteLLMLoggingObj( model="unknown", messages=[], - stream=False, + stream=stream, call_type="pass_through_endpoint", start_time=DISPATCH_TIME, litellm_call_id=litellm_call_id, @@ -2319,6 +2336,36 @@ class TestOpenAIPassthroughResponsesSubRouteBilling: logging_obj.update_environment_variables( model="unknown", user="unknown", optional_params={}, litellm_params={}, call_type="pass_through_endpoint" ) + return logging_obj + + async def _logged_stream( + self, url_route: str, request_body: dict[str, object], sse_events: tuple[bytes, ...], litellm_call_id: str + ) -> _LoggedCall: + recorder: Final = _SpendRowRecorder() + await PassThroughStreamingHandler.route_streaming_logging_to_handler( + litellm_logging_obj=self._recorded_logging_obj(recorder, litellm_call_id, stream=True), + passthrough_success_handler_obj=PassThroughEndpointLogging(), + url_route=url_route, + request_body=request_body, + endpoint_type=EndpointType.OPENAI, + start_time=DISPATCH_TIME, + raw_bytes=sse_events, + end_time=DISPATCH_TIME, + ) + return await asyncio.wait_for(recorder.call, timeout=5) + + async def _logged_call( + self, + method: str, + sub_route: str, + request_body: dict[str, object], + response_body: dict[str, object], + litellm_call_id: str, + base_url: str = RESPONSES_URL, + ) -> _LoggedCall: + url_route: Final = f"{base_url}{sub_route}" + recorder: Final = _SpendRowRecorder() + logging_obj: Final = self._recorded_logging_obj(recorder, litellm_call_id, stream=False) await PassThroughEndpointLogging().pass_through_async_success_handler( httpx_response=httpx.Response(200, json=response_body, request=httpx.Request(method, url_route)), @@ -2350,6 +2397,79 @@ class TestOpenAIPassthroughResponsesSubRouteBilling: assert spend_log_row["model"] == response_body["model"] assert not spend_log_row["spend"] + @pytest.mark.asyncio + @pytest.mark.parametrize( + ("request_agent", "response_agent_model", "logged_model"), + [ + pytest.param({"agent_id": "agent_6a0b0f8e"}, "gpt-5.4-mini", "gpt-5.4-mini", id="saved_agent"), + pytest.param({"agent": {"model": "gpt-5.4-mini"}}, None, "gpt-5.4-mini", id="request_agent_only"), + pytest.param({"agent": {"model": "gpt-5.4"}}, "gpt-5.4-mini", "gpt-5.4-mini", id="response_agent_wins"), + ], + ) + async def test_unbilled_agents_session_spend_log_names_the_agent_model( + self, request_agent: dict[str, object], response_agent_model: str | None, logged_model: str + ): + spend_log_row: Final = ( + await self._logged_call( + "POST", + "", + {"environment": {"type": "none"}, "input": "Say hi in one word", **request_agent}, + _agent_session(response_agent_model), + "2f7c9e1a-4b3d-4a6e-8c5f-1d0e9b7a3c62", + AGENTS_SESSIONS_URL, + ) + ).spend_row + + assert spend_log_row["model"] == logged_model + assert not spend_log_row["spend"] + + @pytest.mark.asyncio + async def test_unbilled_saved_agent_spend_log_names_the_agent_model(self): + saved_agent: Final = {"id": "agent_6a0b0f8e", "object": "agent", "model": "gpt-5.4-mini", "tools": []} + + spend_log_row: Final = ( + await self._logged_call( + "POST", + "", + {"model": "gpt-5.4-mini", "instructions": "Answer in one word"}, + saved_agent, + "8d2e4f6a-1b3c-4d5e-9f7a-2c4e6a8b0d13", + "https://api.openai.com/v1/agents", + ) + ).spend_row + + assert spend_log_row["model"] == saved_agent["model"] + assert not spend_log_row["spend"] + + @pytest.mark.asyncio + @pytest.mark.parametrize( + ("request_body", "session_agent_model", "logged_model"), + [ + pytest.param({}, "gpt-5.4-mini", "gpt-5.4-mini", id="session_event_names_the_model"), + pytest.param({"agent": {"model": "gpt-5.4"}}, None, "gpt-5.4", id="request_agent_fallback"), + ], + ) + async def test_streamed_agents_session_spend_log_names_the_agent_model( + self, request_body: dict[str, object], session_agent_model: str | None, logged_model: str + ): + session_event: Final = {"type": "agent.session.in_progress", "session": _agent_session(session_agent_model)} + sse_events: Final = ( + f"event: agent.session.in_progress\ndata: {json.dumps(session_event)}\n\n".encode(), + b'event: agent.turn.completed\ndata: {"type": "agent.turn.completed", "usage": null}\n\n', + ) + + spend_log_row: Final = ( + await self._logged_stream( + f"{AGENTS_SESSIONS_URL}/sess_6a0b0f8e2c1d4e5f/events", + request_body, + sse_events, + "5b7d9f1a-3c5e-4a7b-8d9f-0e2a4c6b8d35", + ) + ).spend_row + + assert spend_log_row["model"] == logged_model + assert not spend_log_row["spend"] + @pytest.mark.asyncio @pytest.mark.parametrize( ("method", "sub_route", "response_body"), diff --git a/tests/unit/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py b/tests/unit/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py index 43fe5d4e786..33ddd98610b 100644 --- a/tests/unit/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py +++ b/tests/unit/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py @@ -158,63 +158,30 @@ class TestBaseOpenAIPassThroughHandler: print(f"Base URL with trailing slash: '{base_url}' + '{path}' → '{result}'") assert str(result) == "https://api.example.com/v1/chat/completions" - def test_append_openai_beta_header(self): - print("\nTesting _append_openai_beta_header method...") + def test_assemble_headers_keeps_the_operators_openai_beta_over_the_clients(self): + client_request: Final = Request( + { + "type": "http", + "method": "POST", + "path": "/azure/v1/agents/sessions", + "headers": [(b"openai-beta", b"agents=v1")], + "query_string": b"", + } + ) - # Create mock requests with different paths - assistants_request = MagicMock(spec=Request) - assistants_request.url = MagicMock() - assistants_request.url.path = "/v1/threads/thread_123456/messages" + assembled: Final = BaseOpenAIPassThroughHandler._assemble_headers( + api_key="sk-upstream", + endpoint="v1/agents/sessions", + request=client_request, + custom_llm_provider=litellm.LlmProviders.AZURE_AI, + extra_headers={"OpenAI-Beta": "agents=v2"}, + ) - non_assistants_request = MagicMock(spec=Request) - non_assistants_request.url = MagicMock() - non_assistants_request.url.path = "/v1/chat/completions" - - headers = {"authorization": "Bearer test_key"} - - # Test with assistants API request - result = BaseOpenAIPassThroughHandler._append_openai_beta_header(headers, assistants_request) - print(f"Assistants API request: Added header: {result}") - assert result["OpenAI-Beta"] == "assistants=v2" - - # Test with non-assistants API request - headers = {"authorization": "Bearer test_key"} - result = BaseOpenAIPassThroughHandler._append_openai_beta_header(headers, non_assistants_request) - print(f"Non-assistants API request: Headers: {result}") - assert "OpenAI-Beta" not in result - - # Test with assistant in the path - assistant_request = MagicMock(spec=Request) - assistant_request.url = MagicMock() - assistant_request.url.path = "/v1/assistants/asst_123456" - - headers = {"authorization": "Bearer test_key"} - result = BaseOpenAIPassThroughHandler._append_openai_beta_header(headers, assistant_request) - print(f"Assistant API request: Added header: {result}") - assert result["OpenAI-Beta"] == "assistants=v2" - - def test_assemble_headers(self): - print("\nTesting _assemble_headers method...") - - # Mock request - mock_request = MagicMock(spec=Request) - api_key = "test_api_key" - - # Patch the _append_openai_beta_header method to avoid testing it again - with patch.object( - BaseOpenAIPassThroughHandler, - "_append_openai_beta_header", - return_value={ - "authorization": "Bearer test_api_key", - "api-key": "test_api_key", - "test-header": "value", - }, - ): - result = BaseOpenAIPassThroughHandler._assemble_headers(api_key, mock_request) - print(f"Assembled headers: {result}") - assert result["authorization"] == "Bearer test_api_key" - assert result["api-key"] == "test_api_key" - assert result["test-header"] == "value" + assert assembled == { + "authorization": "Bearer sk-upstream", + "api-key": "sk-upstream", + "OpenAI-Beta": "agents=v2", + } @patch( # test-quality-ok: patching litellm internal for unit test isolation "litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints.create_pass_through_route" @@ -4057,6 +4024,129 @@ def test_openai_passthrough_forwards_verbatim_to_openai( assert route.calls.last.request.headers["authorization"] == "Bearer sk-upstream" +@pytest.mark.parametrize( + ("method", "client_base_path", "path", "client_headers", "upstream_openai_beta"), + [ + pytest.param( + "POST", "/openai/v1", "/agents/sessions", {"OpenAI-Beta": "agents=v1"}, ["agents=v1"], id="sdk_agents" + ), + pytest.param( + "GET", + "/openai_passthrough/v1", + "/agents/sessions/sess_123/events", + {"OpenAI-Beta": "agents=v1"}, + ["agents=v1"], + id="sdk_agents_events", + ), + pytest.param("POST", "/openai/v1", "/agents/sessions", {}, ["agents=v1"], id="agents_default"), + pytest.param( + "POST", + "/openai/v1", + "/agents/sessions", + {"OpenAI-Beta": "agents=v2"}, + ["agents=v2"], + id="client_value_beats_route_default", + ), + pytest.param("POST", "/openai", "/agents/sessions", {}, ["agents=v1"], id="agents_default_without_v1"), + pytest.param("GET", "/openai_passthrough/v1", "/vaults", {}, ["agents=v1"], id="vaults_default"), + pytest.param( + "POST", + "/openai/v1", + "/chatkit/sessions", + {"OpenAI-Beta": "chatkit_beta=v1"}, + ["chatkit_beta=v1"], + id="sdk_chatkit", + ), + pytest.param( + "POST", + "/openai/v1", + "/agents/sessions", + {"OpenAI-Beta": "agents=v1", "x-pass-openai-beta": "agents=v2"}, + ["agents=v2"], + id="x_pass_wins", + ), + pytest.param("POST", "/openai/v1", "/chatkit/threads", {}, [], id="chatkit_threads_no_default"), + pytest.param("POST", "/openai/v1", "/assistants", {}, ["assistants=v2"], id="assistants_default"), + pytest.param("POST", "/openai/v1", "/threads/thread_123/messages", {}, ["assistants=v2"], id="threads_default"), + pytest.param("POST", "/openai/v1", "/chat/completions", {}, [], id="no_beta"), + pytest.param( + "POST", + "/openai_passthrough/v1", + "/responses", + {"OpenAI-Beta": "12345"}, + [], + id="client_beta_dropped_off_beta_paths", + ), + pytest.param( + "POST", + "/openai/v1", + "/chat/completions", + {"OpenAI-Beta": "assistants=v2"}, + [], + id="client_beta_dropped_on_chat_completions", + ), + pytest.param( + "POST", + "/openai/v1", + "/threads/thread_123/messages", + {"OpenAI-Beta": "assistants=v1"}, + ["assistants=v2"], + id="assistants_client_beta_replaced", + ), + ], +) +def test_openai_passthrough_sends_the_openai_beta_header_upstream( + openai_passthrough_client: TestClient, + method: str, + client_base_path: str, + path: str, + client_headers: dict[str, str], + upstream_openai_beta: list[str], +) -> None: + with respx.mock(assert_all_called=True) as upstream: + route: Final = upstream.request(method, f"https://api.openai.com/v1{path}").mock( + return_value=httpx.Response(200, json={"id": "upstream_123"}) + ) + response: Final = openai_passthrough_client.request(method, f"{client_base_path}{path}", headers=client_headers) + + assert response.status_code == 200 + assert route.calls.last.request.headers.get_list("openai-beta") == upstream_openai_beta + + +@pytest.mark.parametrize( + ("method", "path", "client_headers", "upstream_openai_beta"), + [ + pytest.param("GET", "/openai/v1/vector_stores", {"OpenAI-Beta": "assistants=v2"}, [], id="client_beta_dropped"), + pytest.param("POST", "/v1/agents/sessions", {}, [], id="no_agents_default"), + pytest.param( + "POST", + "/openai/v1/threads/thread_123/messages", + {"OpenAI-Beta": "assistants=v1"}, + ["assistants=v2"], + id="assistants_default_kept", + ), + ], +) +def test_azure_passthrough_keeps_its_own_openai_beta_rule( + openai_passthrough_client: TestClient, + monkeypatch: pytest.MonkeyPatch, + method: str, + path: str, + client_headers: dict[str, str], + upstream_openai_beta: list[str], +) -> None: + monkeypatch.setenv("AZURE_API_BASE", "https://operator.openai.azure.com") + monkeypatch.setenv("AZURE_API_KEY", "operator-key") + with respx.mock(assert_all_called=True) as upstream: + route: Final = upstream.request(method, f"https://operator.openai.azure.com{path}").mock( + return_value=httpx.Response(200, json={"id": "upstream_123"}) + ) + response: Final = openai_passthrough_client.request(method, f"/azure{path}", headers=client_headers) + + assert response.status_code == 200 + assert route.calls.last.request.headers.get_list("openai-beta") == upstream_openai_beta + + @pytest.fixture def openai_wif_env(monkeypatch: pytest.MonkeyPatch, tmp_path) -> None: from litellm.llms.openai.workload_identity import _workload_identity_auth @@ -7306,9 +7396,7 @@ async def test_bedrock_native_count_tokens_logs_the_count_against_the_router_mod respx_mock: respx.MockRouter, token_count_log_lines: Callable[[], tuple[dict[str, object], ...]], ) -> None: - result: Final = await _bedrock_native_count( - monkeypatch, respx_mock, httpx.Response(200, json={"inputTokens": 12}) - ) + result: Final = await _bedrock_native_count(monkeypatch, respx_mock, httpx.Response(200, json={"inputTokens": 12})) assert result == {"input_tokens": 12} assert len(respx_mock.calls) == 1 @@ -7390,7 +7478,6 @@ async def test_bedrock_native_count_tokens_without_a_model_logs_the_400_it_answe ) - @pytest.mark.asyncio async def test_bedrock_native_count_tokens_with_a_region_only_model_answers_the_validation_error_and_logs_it( monkeypatch: pytest.MonkeyPatch, @@ -7433,6 +7520,7 @@ async def test_bedrock_native_count_tokens_with_a_region_only_model_answers_the_ }, ) + class _AzureGroupRouter: def __init__(self, captured: list[dict]) -> None: self.captured = captured diff --git a/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py b/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py index b0504af1b5e..df36d805377 100644 --- a/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py +++ b/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py @@ -10437,6 +10437,42 @@ def test_init_kwargs_no_authenticated_identity_field_is_client_settable(mock_req survived: Final = {key: metadata.get(key) for key in expected if metadata.get(key) != expected[key]} assert survived == {}, f"client-supplied values survived for: {sorted(survived)}" + +def test_init_kwargs_client_metadata_cannot_mark_a_pass_through_call_as_a_silent_experiment( + mock_user_api_key_dict: UserAPIKeyAuth, +) -> None: + from litellm.proxy.spend_tracking.spend_tracking_utils import get_logging_payload + + started: Final = datetime(2026, 1, 15, 12, 0, 0) + result: Final = HttpPassThroughEndpointHelpers._init_kwargs_for_pass_through_endpoint( + request=Request({"type": "http", "method": "POST", "path": "/test", "headers": [], "query_string": b""}), + user_api_key_dict=mock_user_api_key_dict, + passthrough_logging_payload=PassthroughStandardLoggingPayload(url="https://test.com", request_body={}), + litellm_call_id="test-call-id", + logging_obj=LiteLLMLoggingObj( + model="test-model", + messages=[], + stream=False, + call_type="test-call-type", + start_time=started, + litellm_call_id="test-call-id", + function_id="test-function-id", + ), + _parsed_body={ + "litellm_metadata": {"is_silent_experiment": True, "client_key": "kept"}, + "metadata": {"is_silent_experiment": True}, + }, + ) + + row: Final = get_logging_payload( + kwargs={"model": "test-model", "litellm_params": result["litellm_params"]}, + response_obj=litellm.ModelResponse(id="chatcmpl-test", choices=[]), + start_time=started, + end_time=started, + ) + assert "is_silent_experiment" not in json.loads(row["metadata"]) + assert result["litellm_params"]["metadata"]["client_key"] == "kept" + @pytest.mark.asyncio async def test_direct_passthrough_keeps_mapped_baggage_tags_outside_upstream_body(): from litellm.proxy.common_utils.http_parsing_utils import read_request_body diff --git a/tests/unit/proxy/spend_tracking/test_spend_tracking_utils.py b/tests/unit/proxy/spend_tracking/test_spend_tracking_utils.py index 86d064050eb..23f9ac0e2d8 100644 --- a/tests/unit/proxy/spend_tracking/test_spend_tracking_utils.py +++ b/tests/unit/proxy/spend_tracking/test_spend_tracking_utils.py @@ -6830,7 +6830,7 @@ def test_spend_logs_payload(model_id: str | None): ) assert len(payload["request_id"]) > 0 - expected_metadata_keys: Final = SpendLogsMetadata.__annotations__.keys() + expected_metadata_keys: Final = SpendLogsMetadata.__annotations__.keys() - {"is_silent_experiment"} assert "metadata" in payload assert isinstance(payload["metadata"], str) diff --git a/tests/unit/proxy/test_fallback_management_endpoints.py b/tests/unit/proxy/test_fallback_management_endpoints.py index 530d22f02be..4a0e8cb7a9a 100644 --- a/tests/unit/proxy/test_fallback_management_endpoints.py +++ b/tests/unit/proxy/test_fallback_management_endpoints.py @@ -10,16 +10,18 @@ Tests: import copy import json -from collections.abc import AsyncIterator -from types import SimpleNamespace -from typing import Final +from collections.abc import AsyncIterator, Mapping +from types import MappingProxyType, SimpleNamespace +from typing import Final, get_args from unittest.mock import AsyncMock, MagicMock, patch import pytest from fastapi import HTTPException from pydantic import JsonValue, TypeAdapter +from redis.exceptions import NoPermissionError from litellm import Router +from litellm.proxy.config_resolvers.settings_rules import Section from litellm.proxy.management_endpoints.fallback_management_endpoints import ( FallbackCreateRequest, FallbackDeleteResponse, @@ -29,10 +31,19 @@ from litellm.proxy.management_endpoints.fallback_management_endpoints import ( get_fallback, ) from litellm.proxy.proxy_server import ProxyConfig -from litellm.proxy.utils import evict_config_param, get_config_param +from litellm.proxy.utils import evict_config_param, get_config_param, litellm_config_cache from litellm.types.router import DeploymentTypedDict, LiteLLMParamsTypedDict +@pytest.fixture(autouse=True) +async def empty_config_cache() -> AsyncIterator[None]: + for section in get_args(Section): + await evict_config_param(section) + yield + for section in get_args(Section): + await evict_config_param(section) + + class TestFallbackCreateRequest: """Test the FallbackCreateRequest validation""" @@ -123,7 +134,7 @@ PRIMARY_DEPLOYMENT_ID: Final = "team-primary-id" FALLBACK_DEPLOYMENT_ID: Final = "team-fallback-id" FallbackRules = list[dict[str, list[str]] | dict[str, object] | str] -RouterSettings = dict[str, FallbackRules | None] +RouterSettings = dict[str, FallbackRules | str | None] _ROUTER_SETTINGS_JSON: Final = TypeAdapter(dict[str, JsonValue]) @@ -168,17 +179,18 @@ class TestCreateFallbackForTeamScopedModels: @pytest.fixture def prisma_client(self) -> MagicMock: client: Final = MagicMock() + client.writer_db = client.db + client.db.litellm_config.find_unique = AsyncMock(return_value=None) client.db.litellm_config.upsert = AsyncMock() + client.get_generic_data = AsyncMock(return_value=None) return client @pytest.fixture - def proxy_config(self) -> MagicMock: - config: Final = MagicMock() - config.get_config = AsyncMock(return_value={"router_settings": {}}) - return config + def proxy_config(self) -> ProxyConfig: + return _ConfigFileProxyConfig() async def _create( - self, request: FallbackCreateRequest, router: Router, prisma_client: MagicMock, proxy_config: MagicMock + self, request: FallbackCreateRequest, router: Router, prisma_client: MagicMock, proxy_config: ProxyConfig ) -> FallbackResponse: with ( patch("litellm.proxy.proxy_server.llm_router", router), @@ -254,26 +266,43 @@ def _config_row(router_settings: RouterSettings) -> SimpleNamespace: return SimpleNamespace(param_name="router_settings", param_value=router_settings) +class _ConfigFileProxyConfig(ProxyConfig): + def __init__(self, config_file: RouterSettings | None = None) -> None: + super().__init__() + self.config_file: RouterSettings = config_file or {} + self.router_settings.load_yaml(_ROUTER_SETTINGS_JSON.validate_python(self.config_file)) + + async def _get_config_from_file(self, config_file_path: str | None = None) -> dict[str, RouterSettings]: + return {"router_settings": copy.deepcopy(self.config_file)} + + class _StoredRouterSettings: """The LiteLLM_Config router_settings row, with the proxy objects that read it through the config cache""" - def __init__(self, router_settings: RouterSettings | None, config_file: RouterSettings | None = None) -> None: + def __init__( + self, + router_settings: RouterSettings | None, + config_file: RouterSettings | None = None, + *, + lagging_replica: bool = False, + ) -> None: self.row: SimpleNamespace | None = None if router_settings is None else _config_row(router_settings) self.prisma_client: Final = MagicMock() - self.prisma_client.get_generic_data = AsyncMock(side_effect=lambda **_: self.row) + self.prisma_client.get_generic_data = AsyncMock( + side_effect=lambda value, **_: self.row if value == "router_settings" else None + ) + self.prisma_client.writer_db.litellm_config.find_unique = AsyncMock( + side_effect=lambda **_: copy.deepcopy(self.row) + ) + self.prisma_client.db.litellm_config.find_unique = AsyncMock( + side_effect=lambda **_: None if lagging_replica else copy.deepcopy(self.row) + ) self.prisma_client.db.litellm_config.upsert = AsyncMock(side_effect=self._upsert) - self.proxy_config: Final = MagicMock() if config_file is None else ProxyConfig() - if config_file is not None: - self.proxy_config.router_settings.load_yaml(_ROUTER_SETTINGS_JSON.validate_python(config_file)) - self.proxy_config.get_config = AsyncMock(side_effect=self._get_config) + self.proxy_config: Final = _ConfigFileProxyConfig(config_file) async def _upsert(self, where: dict[str, str], data: dict[str, dict[str, str]]) -> None: self.row = _config_row(json.loads(data["update"]["param_value"])) - async def _get_config(self) -> dict[str, RouterSettings]: - row: Final = await get_config_param(self.prisma_client, "router_settings") - return {"router_settings": copy.deepcopy(row.param_value) if row is not None else {}} - def written_by_another_instance(self, router_settings: RouterSettings) -> None: self.row = _config_row(router_settings) @@ -295,15 +324,23 @@ NON_STANDARD_RULES: Final = [ ] +class _RedisThatRefusesDel: + def __init__(self) -> None: + self.values: Mapping[str, object] = MappingProxyType({}) + + async def async_get_cache(self, key: str, parent_otel_span: object = None) -> object: + return self.values.get(key) + + async def async_set_cache(self, key: str, value: object, **_: object) -> None: + self.values = MappingProxyType({**self.values, key: value}) + + async def async_delete_cache(self, key: str) -> None: + raise NoPermissionError("this user has no permissions to run the 'del' command") + + class TestFallbackWritesSeeTheLatestStoredRules: """A write reads the rules the database holds now and leaves no stale copy in the config cache behind""" - @pytest.fixture(autouse=True) - async def clean_config_cache(self) -> AsyncIterator[None]: - await evict_config_param("router_settings") - yield - await evict_config_param("router_settings") - async def _create(self, request: FallbackCreateRequest, stored: _StoredRouterSettings) -> FallbackResponse: with ( patch("litellm.proxy.proxy_server.llm_router", _team_router()), @@ -322,6 +359,13 @@ class TestFallbackWritesSeeTheLatestStoredRules: ): return await delete_fallback(model, "general", MagicMock()) + async def test_a_create_reads_the_rules_from_the_writer_while_the_replica_lags(self) -> None: + stored: Final = _StoredRouterSettings({"fallbacks": [TEAM_RULE]}, lagging_replica=True) + + await self._create(FallbackCreateRequest(model="gpt-5.4-mini", fallback_models=["team-fallback"]), stored) + + assert stored.stored_fallbacks() == [TEAM_RULE, GATEWAY_RULE] + async def test_a_second_create_keeps_the_first_rule(self) -> None: stored: Final = _StoredRouterSettings(None) @@ -331,6 +375,17 @@ class TestFallbackWritesSeeTheLatestStoredRules: assert stored.stored_fallbacks() == [TEAM_RULE, GATEWAY_RULE] assert await stored.cached_fallbacks() == [TEAM_RULE, GATEWAY_RULE] + async def test_a_second_create_keeps_the_first_rule_when_redis_refuses_to_evict_the_cached_settings(self) -> None: + stored: Final = _StoredRouterSettings(None) + + with patch.object(litellm_config_cache, "redis_cache", _RedisThatRefusesDel()): + await self._create(FallbackCreateRequest(model="team-primary", fallback_models=["team-fallback"]), stored) + await self._create(FallbackCreateRequest(model="gpt-5.4-mini", fallback_models=["team-fallback"]), stored) + await self._delete("team-primary", stored) + await self._create(FallbackCreateRequest(model="team-primary", fallback_models=["team-fallback"]), stored) + + assert stored.stored_fallbacks() == [GATEWAY_RULE, TEAM_RULE] + async def test_a_create_keeps_a_rule_another_instance_stored_since_this_one_last_read(self) -> None: stored: Final = _StoredRouterSettings({}) await get_config_param(stored.prisma_client, "router_settings") @@ -380,6 +435,15 @@ class TestFallbackWritesSeeTheLatestStoredRules: assert stored.stored_fallbacks() == [GATEWAY_RULE] + @pytest.mark.parametrize("param_value", [None, ["team-primary"], 7, "not json"]) + async def test_a_create_replaces_a_stored_row_that_is_not_an_object(self, param_value: JsonValue) -> None: + stored: Final = _StoredRouterSettings(None) + stored.row = SimpleNamespace(param_name="router_settings", param_value=param_value) + + await self._create(FallbackCreateRequest(model="gpt-5.4-mini", fallback_models=["team-fallback"]), stored) + + assert stored.row.param_value == {"fallbacks": [GATEWAY_RULE]} + CONFIG_RULE: Final = {"gpt-5.4-mini": ["team-fallback"]} CONFIG_WINDOW_RULE: Final = {"team-primary": ["gpt-5.4-mini"]} @@ -438,6 +502,18 @@ class TestFallbackWritesHonourTheConfigFile: stored.prisma_client.db.litellm_config.upsert.assert_not_awaited() assert router.fallbacks == [CONFIG_RULE] + async def test_a_create_after_the_config_file_takes_over_the_key_is_refused(self, router: Router) -> None: + stored: Final = _StoredRouterSettings({"fallbacks": [TEAM_RULE]}) + stored.proxy_config.config_file = {"fallbacks": [CONFIG_RULE]} + request: Final = FallbackCreateRequest(model="team-primary", fallback_models=["gpt-5.4-mini"]) + + with pytest.raises(HTTPException) as refused: + await self._create(request, router, stored) + + assert refused.value.status_code == 400 + assert "key 'fallbacks' is set in the config file" in str(refused.value.detail) + assert stored.stored_fallbacks() == [TEAM_RULE] + async def test_a_delete_of_a_config_owned_rule_is_refused( self, router: Router, stored: _StoredRouterSettings ) -> None: @@ -452,6 +528,57 @@ class TestFallbackWritesHonourTheConfigFile: stored.prisma_client.db.litellm_config.upsert.assert_not_awaited() assert router.fallbacks == [CONFIG_RULE] + async def test_a_delete_of_a_rule_only_the_config_file_holds_is_refused(self, router: Router) -> None: + stored: Final = _StoredRouterSettings(None, config_file={"fallbacks": [CONFIG_RULE]}) + router.fallbacks = [CONFIG_RULE] + + with pytest.raises(HTTPException) as refused: + await self._delete("gpt-5.4-mini", router, stored) + + assert refused.value.status_code == 400 + assert "key 'fallbacks' is set in the config file" in str(refused.value.detail) + assert stored.row is None + assert router.fallbacks == [CONFIG_RULE] + + async def test_writes_to_a_config_file_env_reference_that_resolves_to_nothing_are_refused( + self, router: Router, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.delenv("LITELLM_TEST_UNSET_FALLBACKS", raising=False) + stored: Final = _StoredRouterSettings(None, config_file={"fallbacks": "os.environ/LITELLM_TEST_UNSET_FALLBACKS"}) + + with pytest.raises(HTTPException) as refused: + await self._create(FallbackCreateRequest(model="team-primary", fallback_models=["team-fallback"]), router, stored) + with pytest.raises(HTTPException) as not_found: + await self._delete("team-primary", router, stored) + + assert refused.value.status_code == 400 + assert "key 'fallbacks' is set in the config file" in str(refused.value.detail) + assert not_found.value.status_code == 404 + assert stored.row is None + + async def test_a_create_that_restates_one_of_several_config_rules_writes_nothing(self, router: Router) -> None: + stored: Final = _StoredRouterSettings(None, config_file={"fallbacks": [CONFIG_RULE, CONFIG_WINDOW_RULE]}) + request: Final = FallbackCreateRequest(model="gpt-5.4-mini", fallback_models=["team-fallback"]) + + response: Final = await self._create(request, router, stored) + + assert response.fallback_models == ["team-fallback"] + assert stored.row is None + assert router.fallbacks == [CONFIG_RULE, CONFIG_WINDOW_RULE] + + async def test_a_create_that_restates_a_config_rule_next_to_an_env_reference_writes_nothing( + self, router: Router, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.setenv("LITELLM_TEST_FALLBACK_MODEL", "team-fallback") + env_rule: Final = {"team-primary": "os.environ/LITELLM_TEST_FALLBACK_MODEL"} + stored: Final = _StoredRouterSettings(None, config_file={"fallbacks": [CONFIG_RULE, env_rule]}) + request: Final = FallbackCreateRequest(model="gpt-5.4-mini", fallback_models=["team-fallback"]) + + response: Final = await self._create(request, router, stored) + + assert response.fallback_models == ["team-fallback"] + assert stored.row is None + async def test_a_create_that_restates_the_config_rule_is_accepted( self, router: Router, stored: _StoredRouterSettings ) -> None: @@ -462,6 +589,20 @@ class TestFallbackWritesHonourTheConfigFile: assert response.fallback_models == ["team-fallback"] assert stored.stored_fallbacks() == [CONFIG_RULE] + async def test_a_write_stores_only_the_database_rules_never_the_config_files_router_settings( + self, router: Router + ) -> None: + stored: Final = _StoredRouterSettings( + {"fallbacks": [TEAM_RULE]}, config_file={"routing_strategy": "latency-based-routing"} + ) + + await self._create(FallbackCreateRequest(model="gpt-5.4-mini", fallback_models=["team-fallback"]), router, stored) + assert stored.row is not None + assert stored.row.param_value == {"fallbacks": [TEAM_RULE, GATEWAY_RULE]} + + await self._delete("team-primary", router, stored) + assert stored.row.param_value == {"fallbacks": [GATEWAY_RULE]} + async def test_a_sibling_list_the_config_file_does_not_set_stays_writable( self, router: Router, stored: _StoredRouterSettings ) -> None: @@ -496,16 +637,17 @@ class TestCreateFallback: def mock_prisma_client(self): """Create a mock prisma client""" client = MagicMock() + client.writer_db = client.db + client.db.litellm_config.find_unique = AsyncMock(return_value=None) client.db.litellm_config.upsert = AsyncMock() + client.get_generic_data = AsyncMock(return_value=None) client.jsonify_object = lambda x: x return client @pytest.fixture def mock_proxy_config(self): """Create a mock proxy config""" - config = MagicMock() - config.get_config = AsyncMock(return_value={"router_settings": {}}) - return config + return _ConfigFileProxyConfig() @pytest.fixture def mock_user_api_key_dict(self): @@ -803,22 +945,19 @@ class TestDeleteFallback: def mock_prisma_client(self): """Create a mock prisma client""" client = MagicMock() + client.writer_db = client.db + client.db.litellm_config.find_unique = AsyncMock( + return_value=_config_row({"fallbacks": [{"gpt-3.5-turbo": ["gpt-4", "claude-3-haiku"]}]}) + ) client.db.litellm_config.upsert = AsyncMock() + client.get_generic_data = AsyncMock(return_value=None) client.jsonify_object = lambda x: x return client @pytest.fixture def mock_proxy_config(self): """Create a mock proxy config""" - config = MagicMock() - config.get_config = AsyncMock( - return_value={ - "router_settings": { - "fallbacks": [{"gpt-3.5-turbo": ["gpt-4", "claude-3-haiku"]}] - } - } - ) - return config + return _ConfigFileProxyConfig() @pytest.fixture def mock_user_api_key_dict(self): diff --git a/tests/unit/router_strategy/test_auto_router.py b/tests/unit/router_strategy/test_auto_router.py index 6cdee407ced..de775b9246f 100644 --- a/tests/unit/router_strategy/test_auto_router.py +++ b/tests/unit/router_strategy/test_auto_router.py @@ -398,6 +398,19 @@ def _auto_router(routelayer: Any, litellm_router_instance: Any = None, **kwargs: return auto_router +@pytest.mark.parametrize("routes", [{}, ""]) +def test_should_load_no_routes_from_an_empty_non_list_routes_value(routes: object) -> None: + auto_router: Final = AutoRouter( + model_name="my-auto-router", + auto_router_config=json.dumps({"routes": routes}), + default_model="fallback-model", + embedding_model="text-embedding-3-small", + litellm_router_instance=StubEmbeddingRouter(), + ) + + assert auto_router.loaded_routes == [] + + class TestAutoRouterAlwaysResolvesARoutableModel: """The hook returns the alias's default model instead of failing or leaking the alias downstream.""" diff --git a/tests/unit/router_strategy/test_budget_limiter_hotpath.py b/tests/unit/router_strategy/test_budget_limiter_hotpath.py index 10704a29347..c79ff6325d2 100644 --- a/tests/unit/router_strategy/test_budget_limiter_hotpath.py +++ b/tests/unit/router_strategy/test_budget_limiter_hotpath.py @@ -2,6 +2,7 @@ import asyncio import contextlib import gc import logging +import threading from types import SimpleNamespace from typing import Final from unittest.mock import AsyncMock, MagicMock @@ -301,6 +302,27 @@ def test_router_add_deployment_registers_deployment_budget(disable_budget_sync, assert config.max_budget == 0.000000000001 +def _leave_a_task_pending_on_a_closed_loop() -> None: + def park_a_task_then_close_the_loop() -> None: + loop: Final = asyncio.new_event_loop() + + async def park() -> None: + await loop.create_future() + + loop.create_task(park()) + loop.call_soon(loop.stop) + loop.run_forever() + loop.close() + + parker: Final = threading.Thread(target=park_a_task_then_close_the_loop) + parker.start() + parker.join() + + +def _non_asyncio_log_lines(caplog: pytest.LogCaptureFixture) -> list[str]: + return [record.getMessage() for record in caplog.records if record.name != "asyncio"] + + @pytest.mark.asyncio async def test_sync_refused_by_the_open_circuit_breaker_is_quiet_and_leaks_no_task(disable_budget_sync, caplog): """The budget sync runs every second, so an open breaker must not add an error line or an unretrieved task exception per cycle.""" @@ -322,11 +344,12 @@ async def test_sync_refused_by_the_open_circuit_breaker_is_quiet_and_leaks_no_ta with caplog.at_level(logging.ERROR): await limiter._sync_in_memory_spend_with_redis() await asyncio.sleep(0) + _leave_a_task_pending_on_a_closed_loop() gc.collect() finally: loop.set_exception_handler(None) - assert caplog.records == [] + assert _non_asyncio_log_lines(caplog) == [] unretrieved.assert_not_called() assert limiter.redis_increment_operation_queue == [ {"key": "provider_spend:openai:1d", "increment_value": 0.5, "ttl": 60} @@ -382,11 +405,12 @@ async def test_push_task_failure_is_logged_once_and_not_leaked(disable_budget_sy with caplog.at_level(logging.ERROR): await limiter._push_in_memory_increments_to_redis() await asyncio.sleep(0) + _leave_a_task_pending_on_a_closed_loop() gc.collect() finally: loop.set_exception_handler(None) - assert [record.getMessage() for record in caplog.records if record.name != "asyncio"] == [ + assert _non_asyncio_log_lines(caplog) == [ "Error syncing in-memory cache with Redis: Error 61 connecting to 127.0.0.1:6379" ] unretrieved.assert_not_called() diff --git a/tests/unit/test_anthropic_beta_gated_fields.py b/tests/unit/test_anthropic_beta_gated_fields.py new file mode 100644 index 00000000000..a2b384b4ac4 --- /dev/null +++ b/tests/unit/test_anthropic_beta_gated_fields.py @@ -0,0 +1,301 @@ +import logging +from typing import Final + +import pytest + +import litellm +from litellm.anthropic_beta_gated_fields import ( + ANTHROPIC_INLINE_TOOLS_BETA_HEADER, + ANTHROPIC_MID_CONVERSATION_SYSTEM_CLEAR_AT_BETA_HEADER, + ANTHROPIC_THINKING_BINDING_CONTROLS_BETA_HEADER, + BETA_GATED_FIELDS, + active_beta_values, + log_beta_gated_strips, + strip_beta_gated_native_request, + strip_beta_gated_request_fields, + strip_beta_gated_system_messages, + strip_beta_gated_thinking, +) +from litellm.types.llms.anthropic import ( + ANTHROPIC_MID_CONVERSATION_OUTPUT_CONFIG_BETA_HEADER, + ANTHROPIC_MID_CONVERSATION_TOOL_CHANGES_BETA_HEADER, + ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER, +) + +EVERY_BETA: Final = frozenset(BETA_GATED_FIELDS) +BOUND_THINKING: Final = { + "type": "adaptive", + "block_binding": {"prefix_mismatch_behavior": "error"}, + "display": "updates", +} + + +def _system_turn() -> dict[str, object]: + return { + "role": "system", + "clear_at": "end_turn", + "output_config": {"effort": "low"}, + "content": [ + {"type": "text", "text": "stay terse"}, + {"type": "tool_addition", "tools": [{"name": "lookup"}]}, + {"type": "tool_removal", "tool_names": ["lookup"]}, + ], + } + + +def test_thinking_with_every_beta_active_is_returned_as_is() -> None: + result: Final = strip_beta_gated_thinking(BOUND_THINKING, EVERY_BETA, drop_params=True) + assert result.thinking is BOUND_THINKING + assert result.stripped == () + + +def test_thinking_loses_only_the_field_whose_beta_is_absent() -> None: + without_binding: Final = strip_beta_gated_thinking( + BOUND_THINKING, EVERY_BETA - {ANTHROPIC_THINKING_BINDING_CONTROLS_BETA_HEADER}, drop_params=True + ) + assert without_binding.thinking == {"type": "adaptive", "display": "updates"} + assert without_binding.stripped == ("thinking.block_binding",) + + without_display: Final = strip_beta_gated_thinking( + BOUND_THINKING, EVERY_BETA - {ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER}, drop_params=True + ) + assert without_display.thinking == {"type": "adaptive", "block_binding": {"prefix_mismatch_behavior": "error"}} + assert without_display.stripped == ("thinking.display=updates",) + assert BOUND_THINKING == { + "type": "adaptive", + "block_binding": {"prefix_mismatch_behavior": "error"}, + "display": "updates", + }, "copy-on-write: the caller's thinking dict must survive for the fallback leg" + + +def test_thinking_display_other_than_updates_is_not_gated() -> None: + thinking: Final = {"type": "adaptive", "display": "summarized"} + result: Final = strip_beta_gated_thinking(thinking, frozenset(), drop_params=True) + assert result.thinking is thinking + + +@pytest.mark.parametrize("thinking", [None, "enabled", 12, ["adaptive"]]) +def test_non_mapping_thinking_passes_through(thinking: object) -> None: + result: Final = strip_beta_gated_thinking(thinking, frozenset(), drop_params=True) + assert result.thinking is thinking + assert result.stripped == () + + +def test_system_message_keeps_every_gated_field_when_the_betas_are_active() -> None: + messages: Final = [{"role": "user", "content": "hi"}, _system_turn()] + result: Final = strip_beta_gated_system_messages(messages, EVERY_BETA, drop_params=True) + assert result.messages is messages + assert result.stripped == () + + +def test_system_message_loses_exactly_the_fields_whose_betas_are_absent() -> None: + original: Final = _system_turn() + messages: Final = [{"role": "user", "content": "hi"}, original] + result: Final = strip_beta_gated_system_messages( + messages, + EVERY_BETA + - { + ANTHROPIC_MID_CONVERSATION_SYSTEM_CLEAR_AT_BETA_HEADER, + ANTHROPIC_MID_CONVERSATION_TOOL_CHANGES_BETA_HEADER, + ANTHROPIC_INLINE_TOOLS_BETA_HEADER, + }, + drop_params=True, + ) + assert result.messages == [ + {"role": "user", "content": "hi"}, + { + "role": "system", + "output_config": {"effort": "low"}, + "content": [{"type": "text", "text": "stay terse"}], + }, + ] + assert result.stripped == ( + "messages[].system.clear_at", + "messages[].system.content[].tool_addition|tool_removal", + ) + assert original == _system_turn(), "copy-on-write: the caller's system message must survive" + + +def test_per_turn_control_beta_also_unlocks_system_output_config() -> None: + messages: Final = [{"role": "system", "output_config": {"effort": "low"}, "content": "terse"}] + assert ( + strip_beta_gated_system_messages( + messages, frozenset({"per-turn-control-2026-07-01"}), drop_params=True + ).messages + is messages + ) + assert ( + strip_beta_gated_system_messages( + messages, frozenset({ANTHROPIC_MID_CONVERSATION_OUTPUT_CONFIG_BETA_HEADER}), drop_params=True + ).messages + is messages + ) + assert strip_beta_gated_system_messages(messages, frozenset(), drop_params=True).messages == [ + {"role": "system", "content": "terse"} + ] + + +def test_inline_tools_beta_also_unlocks_system_tool_changes() -> None: + messages: Final = [ + { + "role": "system", + "content": [{"type": "tool_removal", "tool_names": ["lookup"]}, {"type": "text", "text": "x"}], + } + ] + assert ( + strip_beta_gated_system_messages( + messages, frozenset({ANTHROPIC_INLINE_TOOLS_BETA_HEADER}), drop_params=True + ).messages + is messages + ) + assert ( + strip_beta_gated_system_messages( + messages, frozenset({ANTHROPIC_MID_CONVERSATION_TOOL_CHANGES_BETA_HEADER}), drop_params=True + ).messages + is messages + ) + assert strip_beta_gated_system_messages(messages, frozenset(), drop_params=True).messages == [ + {"role": "system", "content": [{"type": "text", "text": "x"}]} + ] + + +def test_system_block_with_unhashable_type_is_kept_for_the_provider_to_reject() -> None: + messages: Final = [{"role": "system", "content": [{"type": [], "text": "x"}]}] + result: Final = strip_beta_gated_system_messages(messages, frozenset(), drop_params=True) + assert result.messages is messages + assert result.stripped == () + + +def test_system_message_emptied_by_the_strip_is_dropped() -> None: + messages: Final = [ + {"role": "user", "content": "hi"}, + {"role": "system", "content": [{"type": "tool_removal", "tool_names": ["lookup"]}]}, + {"role": "assistant", "content": "ok"}, + ] + result: Final = strip_beta_gated_system_messages(messages, frozenset(), drop_params=True) + assert result.messages == [{"role": "user", "content": "hi"}, {"role": "assistant", "content": "ok"}] + assert result.stripped == ( + "messages[].system.content[].tool_addition|tool_removal", + "messages[].system (emptied by the strip)", + ) + + +def test_emptied_trailing_system_turn_after_an_assistant_turn_is_sent_as_is() -> None: + messages: Final = [ + {"role": "user", "content": "hi"}, + {"role": "assistant", "content": "ok"}, + {"role": "system", "content": [{"type": "tool_removal", "tool_names": ["lookup"]}]}, + ] + result: Final = strip_beta_gated_system_messages(messages, frozenset(), drop_params=True) + assert result.messages is messages + assert result.stripped == () + + +def test_emptied_system_turn_is_dropped_when_a_user_turn_still_ends_the_conversation() -> None: + messages: Final = [ + {"role": "user", "content": "hi"}, + {"role": "assistant", "content": "ok"}, + {"role": "system", "content": [{"type": "tool_removal", "tool_names": ["lookup"]}]}, + {"role": "user", "content": "go on"}, + ] + result: Final = strip_beta_gated_system_messages(messages, frozenset(), drop_params=True) + assert [message["role"] for message in result.messages] == ["user", "assistant", "user"] + + +def test_system_turn_emptied_by_the_strip_stays_as_a_directive_when_its_output_config_survives() -> None: + directive: Final = { + "role": "system", + "output_config": {"effort": "low"}, + "content": [{"type": "tool_removal", "tool_names": ["lookup"]}], + } + result: Final = strip_beta_gated_system_messages( + [directive], frozenset({ANTHROPIC_MID_CONVERSATION_OUTPUT_CONFIG_BETA_HEADER}), drop_params=True + ) + assert result.messages == [{"role": "system", "output_config": {"effort": "low"}, "content": []}] + assert result.stripped == ("messages[].system.content[].tool_addition|tool_removal",) + + +def test_system_turn_emptied_by_the_strip_is_dropped_when_its_output_config_is_gated_too() -> None: + directive: Final = { + "role": "system", + "output_config": {"effort": "low"}, + "content": [{"type": "tool_removal", "tool_names": ["lookup"]}], + } + result: Final = strip_beta_gated_system_messages([directive], frozenset(), drop_params=True) + assert result.messages == [] + assert result.stripped[-1] == "messages[].system (emptied by the strip)" + + +def test_request_strip_replaces_only_thinking_and_messages_and_keeps_the_dict_otherwise() -> None: + request: Final = { + "model": "claude", + "thinking": dict(BOUND_THINKING), + "messages": [_system_turn()], + "max_tokens": 5, + } + result: Final = strip_beta_gated_request_fields(request, frozenset(), drop_params=True) + assert result.request == { + "model": "claude", + "thinking": {"type": "adaptive"}, + "messages": [{"role": "system", "content": [{"type": "text", "text": "stay terse"}]}], + "max_tokens": 5, + } + assert result.request is not request + assert request["thinking"] == BOUND_THINKING + assert request["messages"] == [_system_turn()] + + +def test_request_strip_is_identity_when_nothing_is_gated() -> None: + request: Final = { + "thinking": {"type": "enabled", "budget_tokens": 1024}, + "messages": [{"role": "user", "content": "hi"}], + } + result: Final = strip_beta_gated_request_fields(request, frozenset(), drop_params=True) + assert result.request is request + assert result.stripped == () + + +@pytest.mark.parametrize("drop_params", [None, False, "false"]) +def test_without_drop_params_every_gated_field_is_sent_as_is( + drop_params: object, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + messages: Final = [{"role": "user", "content": "hi"}, _system_turn()] + request: Final = {"thinking": BOUND_THINKING, "messages": messages} + + assert strip_beta_gated_thinking(BOUND_THINKING, frozenset(), drop_params=drop_params).thinking is BOUND_THINKING + assert strip_beta_gated_system_messages(messages, frozenset(), drop_params=drop_params).messages is messages + assert strip_beta_gated_request_fields(request, frozenset(), drop_params=drop_params).request is request + native: Final = strip_beta_gated_native_request({}, {"thinking": BOUND_THINKING}, messages, drop_params=drop_params) + assert (native.thinking, list(native.messages), native.thinking_stripped, native.messages_stripped) == ( + BOUND_THINKING, + messages, + False, + False, + ) + + +def test_global_drop_params_strips_without_a_per_call_flag(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(litellm, "drop_params", True) + assert strip_beta_gated_thinking(BOUND_THINKING, frozenset(), drop_params=None).thinking == {"type": "adaptive"} + + +def test_active_beta_values_reads_the_header_case_insensitively_and_merges_body_betas() -> None: + assert active_beta_values({"Anthropic-Beta": "a-2026-01-01, b-2026-01-01 ,"}, ["c-2026-01-01", 7]) == frozenset( + {"a-2026-01-01", "b-2026-01-01", "c-2026-01-01"} + ) + assert active_beta_values({}) == frozenset() + + +def test_every_strip_is_logged_once_at_warning_with_the_fields_model_and_provider( + caplog: pytest.LogCaptureFixture, +) -> None: + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + log_beta_gated_strips(("thinking.block_binding", "thinking.block_binding"), model="claude", provider="azure_ai") + log_beta_gated_strips((), model="claude", provider="azure_ai") + warnings: Final = [record for record in caplog.records if record.levelno == logging.WARNING] + assert len(warnings) == 1 + assert ( + "Dropped thinking.block_binding (unlocked by anthropic-beta thinking-binding-controls-2026-08-01) for " + "model=claude provider=azure_ai" in warnings[0].getMessage() + ) diff --git a/tests/unit/test_lint_workflow_diff_gates.py b/tests/unit/test_lint_workflow_diff_gates.py index 7b1d7802d53..5d3c0d80aa6 100644 --- a/tests/unit/test_lint_workflow_diff_gates.py +++ b/tests/unit/test_lint_workflow_diff_gates.py @@ -1,5 +1,6 @@ from __future__ import annotations +import fnmatch import re import shlex import subprocess @@ -7,6 +8,7 @@ from pathlib import Path from typing import Final import pytest +import yaml WORKFLOW: Final = Path(__file__).resolve().parents[2] / ".github" / "workflows" / "test-linting.yml" DIFF_GATE: Final = re.compile(r'git diff --name-only --diff-filter=\w+ "\$GATE_BASE_SHA" HEAD -- (.+?) \|') @@ -100,3 +102,101 @@ def test_e2e_harness_gate_fires_on_any_e2e_or_harness_file_its_dependency_manife (trigger, "tests/e2e/ui/spec.ts", "tests/e2e/ui/pkg/page.py", "elsewhere/pyproject.toml", "tests/e2e_other/module.py"), ) assert selected == frozenset({trigger}) + + +UI_API_TYPES_DETECT: Final[str] = next( + step["run"] + for step in yaml.safe_load(WORKFLOW.read_text())["jobs"]["ui-api-types"]["steps"] + if step.get("id") == "changes" +) +PRE_COMMIT_LINT: Final = WORKFLOW.parents[2] / "scripts" / "pre_commit_lint.sh" +TERRAFORM_PROVIDER_WORKFLOW: Final = WORKFLOW.with_name("test-terraform-provider.yml") + + +def _ui_api_types_relevant(tmp_path: Path, changed: str) -> bool: + _git(tmp_path, "init", "-q", "-b", "main") + _git(tmp_path, "config", "user.email", "t@t") + _git(tmp_path, "config", "user.name", "t") + _git(tmp_path, "commit", "-q", "--allow-empty", "-m", "base") + _git(tmp_path, "switch", "-q", "-c", "pr") + target: Final = tmp_path / changed + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text("x = 1\n") + _git(tmp_path, "add", "-A") + _git(tmp_path, "commit", "-qm", "change") + _git(tmp_path, "switch", "-q", "main") + _git(tmp_path, "commit", "-q", "--allow-empty", "-m", "main moved") + _git(tmp_path, "merge", "-q", "--no-ff", "--no-edit", "pr") + output: Final = tmp_path / "github_output" + subprocess.run( + ["bash", "-c", UI_API_TYPES_DETECT], + cwd=tmp_path, + check=True, + capture_output=True, + env={"PATH": "/usr/bin:/bin", "GITHUB_OUTPUT": str(output)}, + ) + return output.read_text().strip() == "relevant=true" + + +@pytest.mark.parametrize( + "changed", + ( + "litellm/proxy/proxy_server.py", + "litellm/types/router.py", + "litellm/tracing/generated/models.py", + "litellm/router_strategy/complexity_router/config.py", + "litellm/models/credentials.py", + "enterprise/litellm_enterprise/types/proxy/audit_logging_endpoints.py", + "pyproject.toml", + "uv.lock", + "ui/litellm-dashboard/scripts/gen-api-types.mjs", + "ui/litellm-dashboard/package-lock.json", + "ui/litellm-dashboard/src/lib/http/schema.d.ts", + ), +) +def test_ui_api_types_check_runs_on_a_pull_request_touching_any_spec_input(tmp_path: Path, changed: str) -> None: + assert _ui_api_types_relevant(tmp_path, changed) + + +@pytest.mark.parametrize( + "changed", + ( + "litellm-rust/crates/traces/src/lib.rs", + "litellm-proxy-extras/litellm_proxy_extras/utils.py", + "tests/unit/test_router.py", + "ui/litellm-dashboard/src/app/page.tsx", + "docs/my-website/docs/proxy/config.md", + "litellm-dashboard/pyproject.toml", + ), +) +def test_ui_api_types_check_skips_a_pull_request_that_cannot_change_the_spec(tmp_path: Path, changed: str) -> None: + assert not _ui_api_types_relevant(tmp_path, changed) + + +def _first_single_quoted_capture(pattern: str, text: str) -> str: + match: Final = re.search(pattern, text, re.MULTILINE) + assert match is not None, pattern + return match.group(1) + + +def _tracked_paths_matching(extended_regex: str) -> frozenset[str]: + tracked: Final = _git(WORKFLOW.parents[2], "ls-files") + selected: Final = subprocess.run(["grep", "-E", extended_regex], input=tracked, capture_output=True, text=True) + return frozenset(selected.stdout.splitlines()) + + +def test_make_check_runs_the_api_type_sync_on_the_same_tracked_files_as_ci() -> None: + ci_trigger: Final = _first_single_quoted_capture(r"grep -Eq '([^']+)'", UI_API_TYPES_DETECT) + make_check_trigger: Final = _first_single_quoted_capture(r"^spec_pattern='([^']+)'$", PRE_COMMIT_LINT.read_text()) + ci_selected: Final = _tracked_paths_matching(ci_trigger) + assert "pyproject.toml" in ci_selected + assert _tracked_paths_matching(make_check_trigger) == ci_selected - {".github/workflows/test-linting.yml"} + + +def test_a_pull_request_changing_any_route_file_runs_the_terraform_provider_endpoint_audit() -> None: + triggers: Final = yaml.safe_load(TERRAFORM_PROVIDER_WORKFLOW.read_text())[True]["pull_request"]["paths"] + route_files: Final = _git( + WORKFLOW.parents[2], "grep", "-l", "-E", r"APIRouter\(", "--", "*.py", ":(exclude)tests/**" + ).splitlines() + assert route_files + assert [name for name in route_files if not any(fnmatch.fnmatch(name, pattern) for pattern in triggers)] == [] diff --git a/tests/unit/test_pre_commit_lint.py b/tests/unit/test_pre_commit_lint.py index 7c9b98c1c94..ef84ab9fbc8 100644 --- a/tests/unit/test_pre_commit_lint.py +++ b/tests/unit/test_pre_commit_lint.py @@ -427,6 +427,34 @@ def test_run_ends_with_a_summary_of_ran_and_skipped_blocks(tmp_path: Path) -> No assert "check: FAIL" not in proc.stdout +@pytest.mark.parametrize( + "spec_input", + ( + "litellm/tracing/generated/models.py", + "litellm/models/credentials.py", + "enterprise/litellm_enterprise/types/proxy/audit_logging_endpoints.py", + "pyproject.toml", + "uv.lock", + ), +) +def test_a_staged_spec_input_outside_proxy_and_types_runs_the_api_type_sync(tmp_path: Path, spec_input: str) -> None: + repo, bin_dir = _sandbox(tmp_path) + _commit_all(repo, "base") + _stage_file(repo, spec_input, "x = 1\n") + proc = _run(repo, bin_dir, {}) + assert proc.returncode == 0, proc.stdout + proc.stderr + assert "ran: dashboard API-type sync (npm run gen:api)" in proc.stdout + + +def test_a_staged_rust_crate_change_skips_the_api_type_sync(tmp_path: Path) -> None: + repo, bin_dir = _sandbox(tmp_path) + _commit_all(repo, "base") + _stage_file(repo, "litellm-rust/crates/traces/src/lib.rs", "fn main() {}\n") + proc = _run(repo, bin_dir, {}) + assert proc.returncode == 0, proc.stdout + proc.stderr + assert "skipped: dashboard API-type sync (npm run gen:api)" in proc.stdout + + def test_staged_files_matching_no_check_print_an_explicit_noop_note_and_nonempty_log(tmp_path: Path) -> None: repo, bin_dir = _sandbox(tmp_path) _commit_all(repo, "base") diff --git a/tests/unit/test_router_silent_experiment.py b/tests/unit/test_router_silent_experiment.py index fd648ceb5d2..b696a61aa3c 100644 --- a/tests/unit/test_router_silent_experiment.py +++ b/tests/unit/test_router_silent_experiment.py @@ -1,9 +1,11 @@ import asyncio +import datetime import json +import threading import time -from collections.abc import Callable, Mapping +from collections.abc import Callable, Generator, Mapping from types import SimpleNamespace -from typing import Final +from typing import Final, Literal from unittest.mock import AsyncMock, MagicMock, patch import httpx @@ -22,9 +24,48 @@ class _RecordingLogger(CustomLogger): def __init__(self) -> None: super().__init__() self.success_kwargs: list[dict[str, object]] = [] + self.failure_kwargs: list[dict[str, object]] = [] + self._logged: Final = threading.Condition() - async def async_log_success_event(self, kwargs, response_obj, start_time, end_time): - self.success_kwargs.append(kwargs) + def _record(self, outcome: Literal["success", "failure"], kwargs: dict[str, object]) -> None: + with self._logged: + match outcome: + case "success": + self.success_kwargs.append(kwargs) + case "failure": + self.failure_kwargs.append(kwargs) + self._logged.notify_all() + + async def async_log_success_event( + self, + kwargs: dict[str, object], + response_obj: object, + start_time: datetime.datetime, + end_time: datetime.datetime, + ) -> None: + self._record("success", kwargs) + + async def async_log_failure_event( + self, + kwargs: dict[str, object], + response_obj: object, + start_time: datetime.datetime, + end_time: datetime.datetime, + ) -> None: + self._record("failure", kwargs) + + def wait_until(self, logged_enough: Callable[[], bool], timeout: float = 30.0) -> None: + with self._logged: + self._logged.wait_for(logged_enough, timeout=timeout) + + def judge_flags(self) -> dict[str, object]: + return { + call["litellm_params"]["metadata"]["judge_of"]: call["standard_logging_object"]["metadata"].get( + "is_silent_experiment" + ) + for call in self.success_kwargs + if "judge_of" in call["litellm_params"]["metadata"] + } def shadow_successes(self) -> list[dict[str, object]]: return [ @@ -34,6 +75,17 @@ class _RecordingLogger(CustomLogger): ] +class _SyncAndAsyncRecordingLogger(_RecordingLogger): + def log_success_event( + self, + kwargs: dict[str, object], + response_obj: object, + start_time: datetime.datetime, + end_time: datetime.datetime, + ) -> None: + self._record("success", kwargs) + + async def _settle_shared_logging_worker() -> None: try: await GLOBAL_LOGGING_WORKER.flush() @@ -41,15 +93,46 @@ async def _settle_shared_logging_worker() -> None: await GLOBAL_LOGGING_WORKER.stop() -@pytest.fixture -def recording_logger(): +class _JudgingLogger(_RecordingLogger): + async def _judge(self, kwargs: Mapping[str, object]) -> None: + metadata: Final = kwargs["litellm_params"]["metadata"] + if "judge_of" in metadata: + return + await litellm.acompletion( + model="openai/gpt-5.4-nano", + api_key="fake-key", + messages=[{"role": "user", "content": "score it"}], + mock_response="ok", + metadata={"judge_of": metadata["model_group"]}, + ) + + async def async_log_success_event( + self, + kwargs: dict[str, object], + response_obj: object, + start_time: datetime.datetime, + end_time: datetime.datetime, + ) -> None: + await super().async_log_success_event(kwargs, response_obj, start_time, end_time) + await self._judge(kwargs) + + async def async_log_failure_event( + self, + kwargs: dict[str, object], + response_obj: object, + start_time: datetime.datetime, + end_time: datetime.datetime, + ) -> None: + await self._judge(kwargs) + + +def _installed(logger: _RecordingLogger) -> Generator[_RecordingLogger]: settle_loop: Final = asyncio.new_event_loop() try: settle_loop.run_until_complete(_settle_shared_logging_worker()) finally: settle_loop.close() original_callbacks: Final = litellm.callbacks - logger: Final = _RecordingLogger() litellm.callbacks = [logger] try: yield logger @@ -57,22 +140,44 @@ def recording_logger(): litellm.callbacks = original_callbacks -async def _wait_for_shadow_successes(logger: _RecordingLogger, expected: int, timeout: float = 5.0) -> None: - deadline: Final = time.monotonic() + timeout - while len(logger.shadow_successes()) < expected and time.monotonic() < deadline: - await asyncio.sleep(0.05) +@pytest.fixture +def recording_logger(): + yield from _installed(_RecordingLogger()) -async def _wait_for_all_successes(logger: _RecordingLogger, expected: int, timeout: float = 5.0) -> None: - deadline: Final = time.monotonic() + timeout - while len(logger.success_kwargs) < expected and time.monotonic() < deadline: - await asyncio.sleep(0.05) +@pytest.fixture +def sync_and_async_recording_logger(): + yield from _installed(_SyncAndAsyncRecordingLogger()) -def _wait_for_shadow_successes_sync(logger: _RecordingLogger, expected: int, timeout: float = 5.0) -> None: - deadline: Final = time.monotonic() + timeout - while len(logger.shadow_successes()) < expected and time.monotonic() < deadline: - time.sleep(0.05) +@pytest.fixture +def judging_logger(): + yield from _installed(_JudgingLogger()) + + +async def _wait_for_shadow_successes(logger: _RecordingLogger, expected: int, timeout: float = 30.0) -> None: + await asyncio.to_thread(logger.wait_until, lambda: len(logger.shadow_successes()) >= expected, timeout) + + +async def _wait_for_all_successes(logger: _RecordingLogger, expected: int) -> None: + await asyncio.to_thread(logger.wait_until, lambda: len(logger.success_kwargs) >= expected) + + +async def _wait_for_judges(logger: _RecordingLogger, judged: set[str]) -> None: + await asyncio.to_thread(logger.wait_until, lambda: judged <= logger.judge_flags().keys()) + + +def _silent_flag_by_group(logger: _RecordingLogger) -> dict[str, object]: + return { + call["standard_logging_object"]["model_group"]: call["standard_logging_object"]["metadata"].get( + "is_silent_experiment" + ) + for call in logger.success_kwargs + } + + +def _wait_for_shadow_successes_sync(logger: _RecordingLogger, expected: int) -> None: + logger.wait_until(lambda: len(logger.shadow_successes()) >= expected) def _streaming_model_list(silent_model: object) -> list[dict[str, object]]: @@ -263,6 +368,197 @@ async def test_multiple_shadow_targets_fan_out_async(recording_logger): assert primary_successes[0]["litellm_params"]["metadata"]["model_group"] == "primary-model" +@pytest.mark.asyncio +async def test_standard_logging_payload_marks_only_the_mirrored_calls_as_silent_experiments( + recording_logger: _RecordingLogger, +) -> None: + router: Final = Router(model_list=_streaming_model_list(["shadow-a", "shadow-b"])) + response: Final = await router.acompletion( + model="primary-model", + messages=[{"role": "user", "content": "hi"}], + mock_response="pong", + ) + assert response.choices[0].message.content == "pong" + await _wait_for_all_successes(recording_logger, expected=3) + + payloads: Final = [call["standard_logging_object"] for call in recording_logger.success_kwargs] + assert sorted(payload["model_group"] for payload in payloads) == ["primary-model", "shadow-a", "shadow-b"] + metadata_by_group: Final = {payload["model_group"]: payload["metadata"] for payload in payloads} + assert metadata_by_group["shadow-a"]["is_silent_experiment"] is True + assert metadata_by_group["shadow-b"]["is_silent_experiment"] is True + assert "is_silent_experiment" not in metadata_by_group["primary-model"] + + +def test_standard_logging_payload_marks_only_the_mirrored_calls_on_the_sync_path( + sync_and_async_recording_logger: _SyncAndAsyncRecordingLogger, +) -> None: + router: Final = Router(model_list=_streaming_model_list(["shadow-a", "shadow-b"])) + response: Final = router.completion( + model="primary-model", + messages=[{"role": "user", "content": "hi"}], + mock_response="pong", + ) + assert response.choices[0].message.content == "pong" + sync_and_async_recording_logger.wait_until( + lambda: ( + _silent_flag_by_group(sync_and_async_recording_logger).keys() >= {"primary-model", "shadow-a", "shadow-b"} + ) + ) + + assert _silent_flag_by_group(sync_and_async_recording_logger) == { + "primary-model": None, + "shadow-a": True, + "shadow-b": True, + } + + +@pytest.mark.asyncio +async def test_a_failed_mirrored_call_reaches_failure_callbacks_marked_as_mirrored( + recording_logger: _RecordingLogger, +) -> None: + router: Final = Router( + model_list=[ + { + "model_name": "primary-model", + "litellm_params": { + "model": "openai/gpt-5.4-mini", + "api_key": "fake-key", + "silent_model": "shadow-a", + "mock_response": "pong", + }, + }, + { + "model_name": "shadow-a", + "litellm_params": { + "model": "openai/gpt-5.4-nano", + "api_key": "fake-key", + "mock_response": "litellm.InternalServerError", + }, + }, + ] + ) + await router.acompletion(model="primary-model", messages=[{"role": "user", "content": "hi"}]) + await asyncio.to_thread(recording_logger.wait_until, lambda: len(recording_logger.failure_kwargs) >= 1) + + assert { + ( + call["standard_logging_object"]["model_group"], + call["standard_logging_object"]["metadata"].get("is_silent_experiment"), + ) + for call in recording_logger.failure_kwargs + } == {("shadow-a", True)} + + +_SPEND_ROW_TIME: Final = datetime.datetime(2026, 1, 1, tzinfo=datetime.timezone.utc) + + +def _spend_row_is_silent_experiment(call: Mapping[str, object]) -> object: + from litellm.proxy.spend_tracking.spend_tracking_utils import get_logging_payload + + row: Final = get_logging_payload( + kwargs=dict(call), + response_obj=litellm.ModelResponse(id="chatcmpl-test", choices=[]), + start_time=_SPEND_ROW_TIME, + end_time=_SPEND_ROW_TIME, + ) + return json.loads(row["metadata"]).get("is_silent_experiment") + + +def _through_the_spend_collector(call: Mapping[str, object]) -> Mapping[str, object]: + from litellm.proxy.spend_tracking.spend_event import ( + SpendEvent, + build_spend_event, + decode_spend_event, + spend_event_callback_args, + ) + + line: Final = build_spend_event( + call, + litellm.ModelResponse(id="chatcmpl-test", choices=[]), + _SPEND_ROW_TIME, + _SPEND_ROW_TIME, + store_bodies=False, + ) + assert isinstance(line, bytes) + event: Final = decode_spend_event(line) + assert isinstance(event, SpendEvent) + return spend_event_callback_args(event).kwargs + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "spend_kwargs", + [pytest.param(lambda call: call, id="in_process"), pytest.param(_through_the_spend_collector, id="collector")], +) +async def test_spend_rows_mark_only_the_mirrored_calls_when_the_caller_sends_litellm_metadata( + recording_logger: _RecordingLogger, + spend_kwargs: Callable[[Mapping[str, object]], Mapping[str, object]], +) -> None: + router: Final = Router(model_list=_streaming_model_list(["shadow-a", "shadow-b"])) + await router.acompletion( + model="primary-model", + messages=[{"role": "user", "content": "hi"}], + mock_response="pong", + litellm_metadata={"user_api_key": "test-key"}, + ) + await _wait_for_all_successes(recording_logger, expected=3) + + silent_by_group: Final = { + call["standard_logging_object"]["model_group"]: _spend_row_is_silent_experiment(spend_kwargs(call)) + for call in recording_logger.success_kwargs + } + assert silent_by_group == {"primary-model": None, "shadow-a": True, "shadow-b": True} + + +@pytest.mark.asyncio +async def test_router_counts_a_mirrored_call_once_against_its_target_rate_limits( + recording_logger: _RecordingLogger, +) -> None: + router: Final = Router( + model_list=[ + { + "model_name": "primary-model", + "litellm_params": {"model": "openai/gpt-5.4-mini", "api_key": "fake-key", "silent_model": "shadow-b"}, + }, + { + "model_name": "shadow-b", + "litellm_params": {"model": "openai/gpt-5.4-nano", "api_key": "fake-key", "rpm": 100, "tpm": 100000}, + }, + ] + ) + await router.acompletion(model="primary-model", messages=[{"role": "user", "content": "hi"}], mock_response="pong") + await _wait_for_shadow_successes(recording_logger, expected=1) + await GLOBAL_LOGGING_WORKER.flush() + + shadow_tokens: Final = recording_logger.shadow_successes()[0]["standard_logging_object"]["total_tokens"] + assert await router.get_model_group_usage("shadow-b") == (shadow_tokens, 1) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("bucket", "planted"), + [ + ("metadata", {"is_silent_experiment": True}), + ("litellm_metadata", {"is_silent_experiment": True}), + ], +) +async def test_a_caller_sent_silent_experiment_flag_does_not_mark_a_real_call_as_mirrored( + recording_logger: _RecordingLogger, bucket: str, planted: dict[str, object] +) -> None: + router: Final = Router(model_list=_streaming_model_list(["shadow-a", "shadow-b"])) + await router.acompletion( + model="shadow-b", + messages=[{"role": "user", "content": "hi"}], + mock_response="pong", + **{bucket: planted}, + ) + await _wait_for_all_successes(recording_logger, expected=1) + + call: Final = recording_logger.success_kwargs[0] + assert "is_silent_experiment" not in call["standard_logging_object"]["metadata"] + assert _spend_row_is_silent_experiment(call) is None + + def test_multiple_shadow_targets_fan_out_sync(recording_logger): router = Router(model_list=_streaming_model_list(["shadow-a", "shadow-b"])) response = router.completion( @@ -674,6 +970,8 @@ async def test_aembedding_with_silent_model_sends_provider_body_without_it( request_body: Final = json.loads(route.calls.last.request.read()) assert request_body == {"model": "embed-model", "input": ["black dresses"], "input_type": "query"} assert response.data[0]["embedding"] == [0.1, 0.2] + + @pytest.mark.parametrize("run_silent_experiment", SILENT_EXPERIMENT_RUNNERS) def test_silent_experiment_does_not_launch_from_a_shadow_request(run_silent_experiment): router = Router(model_list=_streaming_model_list(["shadow-a"])) @@ -686,3 +984,63 @@ def test_silent_experiment_does_not_launch_from_a_shadow_request(run_silent_expe ) acompletion.assert_not_awaited() + + +def _judged_router(shadow_outcome: str) -> Router: + return Router( + model_list=[ + { + "model_name": "primary-model", + "litellm_params": { + "model": "openai/gpt-5.4-mini", + "api_key": "fake-key", + "silent_model": "shadow-a", + "mock_response": "pong", + }, + }, + { + "model_name": "shadow-a", + "litellm_params": { + "model": "openai/gpt-5.4-nano", + "api_key": "fake-key", + "mock_response": shadow_outcome, + }, + }, + ] + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("shadow_outcome", ["pong", "litellm.InternalServerError"]) +async def test_calls_a_callback_starts_while_logging_a_mirrored_call_are_not_marked_as_mirrored( + judging_logger: _JudgingLogger, shadow_outcome: str +) -> None: + router: Final = _judged_router(shadow_outcome) + await router.acompletion(model="primary-model", messages=[{"role": "user", "content": "hi"}]) + await _wait_for_judges(judging_logger, {"primary-model", "shadow-a"}) + + assert judging_logger.judge_flags() == {"primary-model": None, "shadow-a": None} + + +def _start_judge_call( + kwargs: Mapping[str, object], completion_response: object, start_time: object, end_time: object +) -> None: + litellm.completion( + model="openai/gpt-5.4-nano", + api_key="fake-key", + messages=[{"role": "user", "content": "score it"}], + mock_response="ok", + metadata={"judge_of": kwargs["litellm_params"]["metadata"]["model_group"]}, + ) + + +@pytest.mark.asyncio +async def test_calls_a_failure_function_callback_starts_for_a_mirrored_call_are_not_marked_as_mirrored( + sync_and_async_recording_logger: _SyncAndAsyncRecordingLogger, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "failure_callback", [_start_judge_call]) + router: Final = _judged_router("litellm.InternalServerError") + await router.acompletion(model="primary-model", messages=[{"role": "user", "content": "hi"}]) + await _wait_for_judges(sync_and_async_recording_logger, {"shadow-a"}) + + assert sync_and_async_recording_logger.judge_flags() == {"shadow-a": None} diff --git a/ui/litellm-dashboard/AGENTS.md b/ui/litellm-dashboard/AGENTS.md index 3453a144e89..274dcff1f48 100644 --- a/ui/litellm-dashboard/AGENTS.md +++ b/ui/litellm-dashboard/AGENTS.md @@ -4,7 +4,7 @@ Never put LiteLLM tokens or API keys in `localStorage`. `localStorage` survives When you fix lint violations that are grandfathered in `eslint-suppressions.json`, run `eslint . --prune-suppressions` and commit the updated baseline so the gate ratchets down instead of leaving a stale suppression -`src/lib/http/schema.d.ts` is generated from the proxy's OpenAPI spec. Never hand-edit it. After changing a backend route or response model that the dashboard consumes, run `npm run gen:api` and commit the result (CI `lint / ui-api-types` enforces this) +`src/lib/http/schema.d.ts` is generated from the proxy's OpenAPI spec. Never hand-edit it. After changing any backend route or request or response model, or bumping a dependency, run `npm run gen:api` and commit the result, since the spec includes every route, not only the ones the dashboard calls (CI `lint / ui-api-types` enforces this) Tests come in three tiers, named by the standard definitions. `Foo.test.tsx` is a unit test: one module, collaborators replaced by doubles, no multi-component tree, and it should run in milliseconds. `Foo.integration.test.tsx` renders a real component tree with real children and only stubs the network boundary. It costs seconds per case, so it earns its place by proving wiring that a unit test cannot reach. Browser-level tests live in `tests/e2e/ui/` as Playwright specs against a live proxy