From 1a2b443a32b15c81ecc54abd0a4329fab117f0e1 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Thu, 29 Oct 2026 15:27:26 -0700 Subject: [PATCH 01/17] ci: run the extensions integration group with 48 workers (#49134) --- .github/workflows/test-integration.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-integration.yml b/.github/workflows/test-integration.yml index 0c5b70a67fe..73159bd64d2 100644 --- a/.github/workflows/test-integration.yml +++ b/.github/workflows/test-integration.yml @@ -51,7 +51,7 @@ jobs: suite: ${{ fromJSON((inputs.suite || 'all') == 'all' && '["providers","extensions","accounting","management","management-replica","database","database-replica","mcp","security","cost","sdk","browser"]' || format('["{0}"]', inputs.suite)) }} env: UV_PYTHON: "3.12" - INTEGRATION_WORKERS: ${{ inputs.workers || fromJSON('{"providers":"32","extensions":"32","accounting":"16","management":"16","management-replica":"16","cost":"16","mcp":"8","security":"8","database":"4","database-replica":"4"}')[matrix.suite] || '' }} + INTEGRATION_WORKERS: ${{ inputs.workers || fromJSON('{"providers":"32","extensions":"48","accounting":"16","management":"16","management-replica":"16","cost":"16","mcp":"8","security":"8","database":"4","database-replica":"4"}')[matrix.suite] || '' }} services: postgres: image: postgres:16@sha256:e17e86066e5ef83e0952a9347f5c792b7ece00972e2aa787a6986f471b3dd3d5 From d8f162ea2267cd6b0dee764f838af850bea23535 Mon Sep 17 00:00:00 2001 From: Joshua Hoblitt <948901+jhoblitt@users.noreply.github.com> Date: Thu, 29 Oct 2026 15:32:37 -0700 Subject: [PATCH 02/17] fix(guardrails): allow MCP call modes on the generic guardrail API (#48060) GenericGuardrailAPI.get_supported_event_hooks only listed pre_call, post_call and during_call, so event hook validation rejected the MCP modes and the guardrail was skipped at proxy startup instead of being attached to MCP tool calls. Add pre_mcp_call, during_mcp_call and post_mcp_call. during_mcp_call runs alongside the tool call, so it can fail the call but cannot rewrite its arguments. --- .../generic_guardrail_api.py | 3 + .../test_generic_guardrail_api.py | 71 +++++++++++++++++++ 2 files changed, 74 insertions(+) diff --git a/litellm/proxy/guardrails/guardrail_hooks/generic_guardrail_api/generic_guardrail_api.py b/litellm/proxy/guardrails/guardrail_hooks/generic_guardrail_api/generic_guardrail_api.py index ecd23d86540..7e052796545 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/generic_guardrail_api/generic_guardrail_api.py +++ b/litellm/proxy/guardrails/guardrail_hooks/generic_guardrail_api/generic_guardrail_api.py @@ -664,4 +664,7 @@ class GenericGuardrailAPI(CustomGuardrail): GuardrailEventHooks.pre_call, GuardrailEventHooks.post_call, GuardrailEventHooks.during_call, + GuardrailEventHooks.pre_mcp_call, + GuardrailEventHooks.during_mcp_call, + GuardrailEventHooks.post_mcp_call, ] diff --git a/tests/unit/proxy/guardrails/guardrail_hooks/test_generic_guardrail_api.py b/tests/unit/proxy/guardrails/guardrail_hooks/test_generic_guardrail_api.py index 8308a7ba386..b5f794cb136 100644 --- a/tests/unit/proxy/guardrails/guardrail_hooks/test_generic_guardrail_api.py +++ b/tests/unit/proxy/guardrails/guardrail_hooks/test_generic_guardrail_api.py @@ -16,12 +16,16 @@ from unittest.mock import AsyncMock, MagicMock, patch import httpx import pytest +from mcp.types import CallToolResult, TextContent import litellm from litellm import ModelResponse from litellm._version import version as litellm_version from litellm.exceptions import GuardrailRaisedException, Timeout from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, MaskedHTTPStatusError +from litellm.proxy._experimental.mcp_server.guardrail_translation.handler import ( + MCPGuardrailTranslationHandler, +) from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.guardrails.guardrail_hooks.generic_guardrail_api import ( GenericGuardrailAPI, @@ -29,6 +33,8 @@ from litellm.proxy.guardrails.guardrail_hooks.generic_guardrail_api import ( from litellm.proxy.guardrails.guardrail_hooks.generic_guardrail_api.generic_guardrail_api import ( _HEADER_PRESENT_PLACEHOLDER, ) +from litellm.proxy.guardrails.guardrail_hooks.unified_guardrail.unified_guardrail import UnifiedLLMGuardrails +from litellm.types.guardrails import GuardrailEventHooks from litellm.types.utils import Choices, GenericGuardrailAPIInputs, Message @@ -2621,3 +2627,68 @@ class TestNumRetries: assert sent.qsize() == num_retries + 1 assert result == {"texts": ["hi"]} + + +def _mcp_guardrail(mode: str) -> GenericGuardrailAPI: + return GenericGuardrailAPI(api_base="http://x", guardrail_name="g", event_hook=mode, default_on=True) + + +def _mock_api_response(payload: dict[str, object]) -> MagicMock: + response: Final = MagicMock() + response.json.return_value = payload + response.raise_for_status = MagicMock() + return response + + +@pytest.mark.parametrize("mode", ["pre_mcp_call", "during_mcp_call", "post_mcp_call"]) +def test_mcp_call_modes_pass_event_hook_validation(mode: str) -> None: + guardrail: Final = _mcp_guardrail(mode) + + assert guardrail.should_run_guardrail(data={}, event_type=GuardrailEventHooks(mode)) is True + + +@pytest.mark.asyncio +async def test_pre_mcp_call_blocked_raises() -> None: + guardrail: Final = _mcp_guardrail("pre_mcp_call") + response: Final = _mock_api_response({"action": "BLOCKED", "blocked_reason": "credential detected in tool input"}) + + with patch.object(guardrail.async_handler, "post", return_value=response): + with pytest.raises(GuardrailRaisedException) as exc_info: + await MCPGuardrailTranslationHandler().process_input_messages( + data={"mcp_tool_name": "t", "mcp_arguments": {"a": "secret"}}, + guardrail_to_apply=guardrail, + ) + + assert exc_info.value.message == "credential detected in tool input" + + +@pytest.mark.asyncio +async def test_post_mcp_call_redacts_result() -> None: + guardrail: Final = _mcp_guardrail("post_mcp_call") + response: Final = _mock_api_response({"action": "GUARDRAIL_INTERVENED", "texts": ["x [R] y"]}) + result: Final = CallToolResult(content=[TextContent(type="text", text="x secret y")], isError=False) + + with patch.object(guardrail.async_handler, "post", return_value=response): + returned: Final = await MCPGuardrailTranslationHandler().process_output_response( + response=result, + guardrail_to_apply=guardrail, + request_data={"mcp_tool_name": "t"}, + ) + + assert returned.content[0].text == "x [R] y" + + +@pytest.mark.asyncio +async def test_during_mcp_call_blocked_raises() -> None: + guardrail: Final = _mcp_guardrail("during_mcp_call") + response: Final = _mock_api_response({"action": "BLOCKED", "blocked_reason": "credential detected in tool input"}) + + with patch.object(guardrail.async_handler, "post", return_value=response): + with pytest.raises(GuardrailRaisedException) as exc_info: + await UnifiedLLMGuardrails().async_moderation_hook( + data={"mcp_tool_name": "t", "mcp_arguments": {"a": "secret"}, "guardrail_to_apply": guardrail}, + user_api_key_dict=UserAPIKeyAuth(), + call_type="call_mcp_tool", + ) + + assert exc_info.value.message == "credential detected in tool input" From fd8b89fbecc5b073de9652be1bbcd7a8fd3d4b7f Mon Sep 17 00:00:00 2001 From: shrey Date: Thu, 29 Oct 2026 15:56:29 -0700 Subject: [PATCH 03/17] fix(mcp)!: restore upstream-token admission for legacy delegate servers (#48939) --- litellm/proxy/auth/oauth2_check.py | 9 +- litellm/proxy/auth/user_api_key_auth.py | 42 +- litellm/proxy/mcp_server/AGENTS.md | 7 +- .../mcp_server/auth/user_api_key_auth_mcp.py | 214 +++++- litellm/proxy/mcp_server/bridge_token_flow.py | 51 +- .../proxy/mcp_server/mcp_server_manager.py | 44 +- litellm/proxy/mcp_server/server.py | 24 +- .../types/mcp_server/mcp_server_manager.py | 8 +- litellm/types/proxy/auth/user_api_key_auth.py | 5 + .../auth/test_user_api_key_auth_mcp.py | 622 +++++++++++++++--- .../test_mcp_server_tool_calls_and_headers.py | 423 +++++++++--- 11 files changed, 1224 insertions(+), 225 deletions(-) diff --git a/litellm/proxy/auth/oauth2_check.py b/litellm/proxy/auth/oauth2_check.py index 3633f7b6127..970f046c3d6 100644 --- a/litellm/proxy/auth/oauth2_check.py +++ b/litellm/proxy/auth/oauth2_check.py @@ -12,6 +12,11 @@ from litellm.llms.custom_httpx.http_handler import ( from litellm.proxy._types import CommonProxyErrors, LitellmUserRoles, UserAPIKeyAuth +class OAuth2TokenValidationError(ValueError): + """The token-info or introspection endpoint did not vouch for the token: it rejected it, reported it + inactive, or could not be reached.""" + + class Oauth2Handler: """ Handles OAuth2 token validation. @@ -207,7 +212,7 @@ class Oauth2Handler: ) except httpx.HTTPStatusError as e: # This will catch any 4xx or 5xx errors - raise ValueError(f"Oauth 2.0 Token validation failed: {e}") + raise OAuth2TokenValidationError(f"Oauth 2.0 Token validation failed: {e}") except Exception as e: # This will catch any other errors (like network issues) - raise ValueError(f"An error occurred during token validation: {e}") + raise OAuth2TokenValidationError(f"An error occurred during token validation: {e}") diff --git a/litellm/proxy/auth/user_api_key_auth.py b/litellm/proxy/auth/user_api_key_auth.py index e5b0beadebf..2b6471952fd 100644 --- a/litellm/proxy/auth/user_api_key_auth.py +++ b/litellm/proxy/auth/user_api_key_auth.py @@ -1160,6 +1160,24 @@ def _matches_routing_override(token_claims: dict, override: "JWTRoutingOverride" ) +def oauth2_introspection_applies( + api_key: str, route: str, *, enable_oauth2_auth: bool, enable_jwt_auth: bool, jwt_handler: JWTHandler +) -> bool: + """Whether LiteLLM validates ``api_key`` on ``route`` by OAuth2 token introspection, on LLM API, info and + lens routes only: a non-JWT token when OAuth2 auth is on globally, or a JWT matched by a routing override.""" + is_jwt: Final = enable_jwt_auth and jwt_handler.is_jwt(token=api_key) + # Routing uses unverified JWT claims only to choose auth path. + # Final authentication is enforced by the selected validator. + route_jwt_to_oauth2: Final = is_jwt and _should_route_jwt_to_oauth2_override(token=api_key, jwt_handler=jwt_handler) + covered_route: Final = ( + RouteChecks.is_llm_api_route(route=route) + or RouteChecks.is_info_route(route=route) + or route == "/lens" + or route.startswith("/lens/") + ) + return covered_route and ((enable_oauth2_auth and not is_jwt) or route_jwt_to_oauth2) + + def _should_route_jwt_to_oauth2_override(token: str, jwt_handler: JWTHandler) -> bool: routing_overrides: Final = jwt_handler.litellm_jwtauth.routing_overrides if not routing_overrides: @@ -2278,23 +2296,13 @@ async def user_api_key_auth_builder( enable_oauth2_auth: Final = general_settings.get("enable_oauth2_auth", False) is True enable_jwt_auth: Final = general_settings.get("enable_jwt_auth", False) is True - is_jwt = jwt_handler.is_jwt(token=api_key) if enable_jwt_auth else False - - # Routing uses unverified JWT claims only to choose auth path. - # Final authentication is enforced by the selected validator. - route_jwt_to_oauth2 = is_jwt and _should_route_jwt_to_oauth2_override(token=api_key, jwt_handler=jwt_handler) - - # OAuth2 applies for: - # 1) when global OAuth2 auth is enabled on LLM + info routes - # 2) JWT tokens that explicitly match routing_overrides on LLM + info routes - route_is_lens: Final = route == "/lens" or route.startswith("/lens/") - should_apply_override_oauth2: Final = route_jwt_to_oauth2 and ( - RouteChecks.is_llm_api_route(route=route) or RouteChecks.is_info_route(route=route) or route_is_lens - ) - should_apply_global_oauth2: Final = enable_oauth2_auth and ( - RouteChecks.is_llm_api_route(route=route) or RouteChecks.is_info_route(route=route) or route_is_lens - ) - if (should_apply_global_oauth2 and not is_jwt) or should_apply_override_oauth2: + if oauth2_introspection_applies( + api_key, + route, + enable_oauth2_auth=enable_oauth2_auth, + enable_jwt_auth=enable_jwt_auth, + jwt_handler=jwt_handler, + ): from litellm.proxy.proxy_server import premium_user if premium_user is not True: diff --git a/litellm/proxy/mcp_server/AGENTS.md b/litellm/proxy/mcp_server/AGENTS.md index 25150c11f1d..b66128454b8 100644 --- a/litellm/proxy/mcp_server/AGENTS.md +++ b/litellm/proxy/mcp_server/AGENTS.md @@ -68,8 +68,11 @@ module materially harder to understand. behind a single generic branch unless tests prove every mode still behaves correctly. - Be especially careful with legacy `delegate_auth_to_upstream: true`. `auth_type: oauth2` - with `delegate_auth_to_upstream: true` is deprecated: LiteLLM admission is required - for matching MCP routes. Use `auth_type: oauth_delegate` for client-forwarded OAuth. + with `delegate_auth_to_upstream: true` is deprecated but still supported. A bare + `Authorization` that is a LiteLLM credential is validated and never forwarded. Any other + bearer is the upstream token: it is forwarded, and the caller is admitted as the JWT or + custom-auth user it also proves when that user is granted the server, else anonymously. + New servers should use `auth_type: true_passthrough` or `auth_type: oauth_delegate`. OAuth discovery endpoints stay public so clients can start the RFC 9728 flow. - Keep database-backed fields in sync across migrations, typed models under `litellm/types/mcp.py` or `litellm/types/mcp_server/`, config loading, this diff --git a/litellm/proxy/mcp_server/auth/user_api_key_auth_mcp.py b/litellm/proxy/mcp_server/auth/user_api_key_auth_mcp.py index cc7080738d0..db640b6d689 100644 --- a/litellm/proxy/mcp_server/auth/user_api_key_auth_mcp.py +++ b/litellm/proxy/mcp_server/auth/user_api_key_auth_mcp.py @@ -2,14 +2,15 @@ import asyncio import functools import re import secrets -from collections.abc import Awaitable, Callable, Mapping, Sequence +from collections.abc import Awaitable, Callable, Iterator, Mapping, Sequence from collections.abc import Set as AbstractSet from dataclasses import dataclass from datetime import datetime, timezone from itertools import chain from types import MappingProxyType -from typing import TYPE_CHECKING, Final, Literal +from typing import TYPE_CHECKING, Final, Literal, TypeAlias +import jwt from fastapi import HTTPException from pydantic import SecretStr, TypeAdapter from starlette.datastructures import Headers @@ -45,9 +46,11 @@ from litellm.proxy.agent_endpoints.auth.agent_access_groups import ( from litellm.proxy.agent_endpoints.auth.agent_caller import agent_caller_auth from litellm.proxy.agent_endpoints.auth.managed_authorization import managed_agent_policy from litellm.proxy.auth.ip_address_utils import IPAddressUtils +from litellm.proxy.auth.oauth2_check import OAuth2TokenValidationError from litellm.proxy.auth.user_api_key_auth import ( # noqa: F401 # legacy module exports _get_bearer_token_or_received_api_key, # pyright: ignore[reportPrivateUsage] # shared x-litellm-api-key parser lives with user_api_key_auth _run_centralized_common_checks, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export + oauth2_introspection_applies, run_centralized_common_checks, user_api_key_auth, ) @@ -100,6 +103,9 @@ if TYPE_CHECKING: _EMPTY_TOOLSET_GRANTS: Final[Mapping[str, Sequence[str]]] = MappingProxyType({}) OPTIONAL_STRING_ADAPTER: Final[TypeAdapter[str | None]] = TypeAdapter(str | None) +_PROXY_SETTINGS_ADAPTER: Final = TypeAdapter(Mapping[str, object]) +_CAUSE_CHAIN_MAX_DEPTH: Final = 16 +_IdentityCheck: TypeAlias = Literal["oauth2", "oauth2_proxy", "jwt"] _CACHED_VALUE: Final = TypeAdapter(object) @@ -233,6 +239,136 @@ def _is_legacy_delegate_cold_start(mcp_servers: list[str] | None, client_ip: str return True +def _cause_chain(exc: BaseException) -> Iterator[BaseException]: + link: BaseException | None = exc # rebind-ok: advances one link of the cause chain per iteration + for _ in range(_CAUSE_CHAIN_MAX_DEPTH): + if link is None: + return + yield link + link = link.__cause__ or link.__context__ + + +def _caused_by(exc: BaseException, error_type: type[BaseException]) -> bool: + return any(isinstance(link, error_type) for link in _cause_chain(exc)) + + +def _signed_with(token: str, secret: str) -> bool: + """True when ``token`` is a JWT whose HS256 signature verifies with ``secret``, whatever its claims say.""" + try: + jwt.decode( + token, + secret, + algorithms=["HS256"], + options={"verify_exp": False, "verify_nbf": False, "verify_iat": False, "verify_aud": False}, + ) + except jwt.PyJWTError: + return False + return True + + +async def _bearer_is_litellm_credential(token: str) -> bool: + """True when ``token`` is, or may be, a credential LiteLLM issued: the master key or a JWT it signed + (dashboard sessions, BYOK sessions), a sealed gateway bearer, or a virtual key the key table holds (a + lookup fault counts as held). Only ``sk-`` tokens are looked up, because no other token can be a + virtual key.""" + from litellm.proxy.mcp_server.bridge_token_flow import ( # noqa: PLC0415 # bridge flow imports this module + is_sealed_gateway_bearer, + virtual_key_may_exist, + ) + from litellm.proxy.mcp_server.mcp_server_manager import ( # noqa: PLC0415 # manager imports this module + LITELLM_VIRTUAL_KEY_PREFIX, + ) + from litellm.proxy.proxy_server import master_key # noqa: PLC0415 # startup owns the master key + + if master_key and (secrets.compare_digest(token.encode(), master_key.encode()) or _signed_with(token, master_key)): + return True + if is_sealed_gateway_bearer(token): + return True + return token.startswith(LITELLM_VIRTUAL_KEY_PREFIX) and await virtual_key_may_exist(token) + + +def _identity_check(token: str, route: str, settings: Mapping[str, object]) -> _IdentityCheck | None: + """The identity check LiteLLM's own auth would run on ``token`` for ``route``, in its order: OAuth2 + introspection, then trusted reverse-proxy identity headers, then a JWT whose issuer may be LiteLLM's.""" + from litellm.proxy.auth.handle_jwt import JWTHandler # noqa: PLC0415 # proxy import cycle + from litellm.proxy.mcp_server.bridge_token_flow import ( # noqa: PLC0415 # bridge flow imports this module + jwt_issuer_may_be_litellm, + ) + from litellm.proxy.proxy_server import ( # noqa: PLC0415 # startup owns the active auth configuration + jwt_handler, + premium_user, + ) + + enable_jwt_auth: Final = settings.get("enable_jwt_auth") is True + if oauth2_introspection_applies( + token, + route, + enable_oauth2_auth=settings.get("enable_oauth2_auth") is True, + enable_jwt_auth=enable_jwt_auth, + jwt_handler=jwt_handler, + ): + return "oauth2" if premium_user is True else None + if settings.get("enable_oauth2_proxy_auth") is True: + return "oauth2_proxy" + if enable_jwt_auth and premium_user is True and JWTHandler.is_jwt(token=token) and jwt_issuer_may_be_litellm(token): + return "jwt" + return None + + +async def _verify_identity(check: Literal["oauth2_proxy", "jwt"], token: str, request: Request) -> None: + from litellm.proxy.auth.oauth2_proxy_hook import ( # noqa: PLC0415 # proxy import cycle + handle_oauth2_proxy_request, + ) + from litellm.proxy.proxy_server import jwt_handler # noqa: PLC0415 # startup owns the active auth configuration + + match check: + case "oauth2_proxy": + await handle_oauth2_proxy_request(request=request) + case "jwt": + await jwt_handler.auth_jwt(token=token) # pyright: ignore[reportUnknownMemberType] # returns a bare dict; only whether it raises matters + case _: # pragma: no cover - exhaustiveness guard, unreachable while the Literal holds + assert_never(check) # pragma: no cover + + +async def _upstream_bearer_identity(token: str, litellm_api_key: str, request: Request) -> UserAPIKeyAuth | None: + """The LiteLLM user an upstream bearer also proves, or ``None`` when it proves none. + + Custom auth decides alone, so any rejection is ``None``. Otherwise the identity check LiteLLM would run + must vouch for the caller. Reverse-proxy headers and a JWT are verified first, and OAuth2 introspection + runs once inside the full check, where a provider that does not vouch is ``None``. Past that, only a + user LiteLLM does not know is ``None``; every other refusal (budget, blocked team, disallowed route) + propagates.""" + from litellm.proxy.auth.auth_utils import get_request_route # noqa: PLC0415 # proxy import cycle + from litellm.proxy.mcp_server.bridge_token_flow import ( # noqa: PLC0415 # bridge flow imports this module + custom_auth_configured, + ) + from litellm.proxy.proxy_server import ( # noqa: PLC0415 # startup owns the active auth configuration + general_settings, # pyright: ignore[reportUnknownVariableType] # legacy global is declared as a bare dict + ) + + if custom_auth_configured(): + try: + return await user_api_key_auth(api_key=litellm_api_key, request=request) + except (HTTPException, ProxyException): + return None + check: Final = _identity_check( + token, get_request_route(request), _PROXY_SETTINGS_ADAPTER.validate_python(general_settings) + ) + if check is None: + return None + if check != "oauth2": + try: + await _verify_identity(check, token, request) + except Exception: # noqa: BLE001 # any verification failure means LiteLLM does not vouch for this bearer + return None + try: + return await user_api_key_auth(api_key=litellm_api_key, request=request) + except (HTTPException, ProxyException, OAuth2TokenValidationError) as exc: + if _caused_by(exc, UserNotFoundError) or _caused_by(exc, OAuth2TokenValidationError): + return None + raise + + def _is_litellm_auth_admission_error(exc: Exception) -> bool: if isinstance(exc, HTTPException): return exc.status_code == 401 @@ -704,6 +840,19 @@ class MCPRequestHandler: route=request_route, mcp_servers=mcp_servers, ) + elif oauth2_headers and ( + legacy_delegate_targets := MCPRequestHandler._legacy_delegate_targets( + path=request_route, + mcp_servers=mcp_servers, + client_ip=IPAddressUtils.get_mcp_client_ip(request), + ) + ): + validated_user_api_key_auth = await MCPRequestHandler._admit_legacy_delegate_bearer( # rebind-ok: one assignment per admission arm, like its siblings + authorization_value=oauth2_headers["Authorization"], + litellm_api_key=litellm_api_key, + targets=legacy_delegate_targets, + request=request, + ) elif oauth2_headers: # Authorization on a non-delegated server: the bearer must be a real # LiteLLM credential, so a failed validation is a genuine 401/403 and @@ -741,10 +890,14 @@ class MCPRequestHandler: custom_key_header_name: Final = OPTIONAL_STRING_ADAPTER.validate_python( general_settings.get("litellm_key_header_name") ) - admitted_credential: Final = MCPRequestHandler.caller_admission_credential( - headers, - validated_user_api_key_auth, - custom_key_header_name=custom_key_header_name, + admitted_credential: Final = ( + None + if validated_user_api_key_auth.mcp_forwards_caller_authorization + else MCPRequestHandler.caller_admission_credential( + headers, + validated_user_api_key_auth, + custom_key_header_name=custom_key_header_name, + ) ) ( oauth2_headers, @@ -934,6 +1087,53 @@ class MCPRequestHandler: return False return True + @staticmethod + def _legacy_delegate_targets( + path: str, mcp_servers: Sequence[str] | None, client_ip: str | None + ) -> tuple[MCPServer, ...]: + """Every server the request targets when ALL of them are legacy ``delegate_auth_to_upstream`` + servers, else ``()``. Names resolve the way downstream routing resolves them, so a permissive + ``x-mcp-servers`` header cannot open the gate for a stricter path target.""" + from litellm.proxy.mcp_server.mcp_server_manager import ( # noqa: PLC0415 # manager imports this module + global_mcp_server_manager, + is_legacy_upstream_delegate, + ) + + servers: Final = tuple( + global_mcp_server_manager.get_mcp_server_by_name(name, client_ip=client_ip) + for name in MCPRequestHandler._resolve_target_server_names(path=path, mcp_servers_header=mcp_servers) + ) + delegates: Final = tuple( + server for server in servers if server is not None and is_legacy_upstream_delegate(server) + ) + return delegates if servers and len(delegates) == len(servers) else () + + @staticmethod + async def _admit_legacy_delegate_bearer( + authorization_value: str, + litellm_api_key: str, + targets: tuple[MCPServer, ...], + request: Request, + ) -> UserAPIKeyAuth: + """Admit a bare ``Authorization`` bearer sent to legacy ``delegate_auth_to_upstream`` servers. + + A LiteLLM credential runs normal admission, and its errors propagate. Any other bearer is the + upstream token and is marked for forwarding: a user it also proves who is granted every target is + admitted as that user, and every other caller is admitted anonymously.""" + from litellm.proxy.mcp_server.mcp_server_manager import ( # noqa: PLC0415 # manager imports this module + global_mcp_server_manager, + ) + + token: Final = _normalize_caller_admission_credential(authorization_value) + if await _bearer_is_litellm_credential(token): + return await user_api_key_auth(api_key=litellm_api_key, request=request) + identity: Final = await _upstream_bearer_identity(token, litellm_api_key, request) + if identity is not None: + allowed: Final = frozenset(await global_mcp_server_manager.get_allowed_mcp_servers(identity)) + if all(server.server_id in allowed for server in targets): + return identity.model_copy(update={"mcp_forwards_caller_authorization": True}) + return UserAPIKeyAuth().model_copy(update={"mcp_forwards_caller_authorization": True}) + @staticmethod def _single_dcr_bridge_delegate_target( path: str, mcp_servers: list[str] | None, client_ip: str | None @@ -1553,7 +1753,7 @@ class MCPRequestHandler: return expiry >= datetime.now(timezone.utc) @staticmethod - def _resolve_target_server_names(path: str, mcp_servers_header: list[str] | None) -> list[str]: + def _resolve_target_server_names(path: str, mcp_servers_header: Sequence[str] | None) -> Sequence[str]: """ Resolve the target MCP server names exactly as downstream routing does (``server.py::extract_mcp_auth_context``). diff --git a/litellm/proxy/mcp_server/bridge_token_flow.py b/litellm/proxy/mcp_server/bridge_token_flow.py index 63428dfb28a..dd9be4cb3e1 100644 --- a/litellm/proxy/mcp_server/bridge_token_flow.py +++ b/litellm/proxy/mcp_server/bridge_token_flow.py @@ -61,7 +61,6 @@ async def oauth_authorization_uses_gateway_credential(request: Request) -> bool: from litellm.proxy.proxy_server import ( # noqa: PLC0415 # startup owns the active auth configuration jwt_handler, master_key, - user_custom_auth, ) if "x-litellm-api-key" in request.headers: @@ -71,10 +70,26 @@ async def oauth_authorization_uses_gateway_credential(request: Request) -> bool: return "authorization" in request.headers if token.startswith("sk-") or (master_key and secrets.compare_digest(token.encode(), master_key.encode())): return True - if user_custom_auth is not None or jwt_handler.litellm_jwtauth.oidc_userinfo_enabled: + if custom_auth_configured() or jwt_handler.litellm_jwtauth.oidc_userinfo_enabled: return True if not JWTHandler.is_jwt(token): - return await _opaque_bearer_is_gateway_credential(token) + return is_sealed_gateway_bearer(token) or await virtual_key_may_exist(token) + return jwt_issuer_may_be_litellm(token) + + +def custom_auth_configured() -> bool: + from litellm.proxy.proxy_server import ( # noqa: PLC0415 # startup owns the active auth configuration + user_custom_auth, + ) + + return user_custom_auth is not None + + +def jwt_issuer_may_be_litellm(token: str) -> bool: + """False only when the JWT names an issuer LiteLLM's JWT auth is not configured to accept.""" + from litellm.proxy.auth.handle_jwt import JWTHandler # noqa: PLC0415 # proxy import cycle + from litellm.proxy.proxy_server import jwt_handler # noqa: PLC0415 # startup owns the active auth configuration + claims: Final = JWTHandler.get_unverified_claims(token) issuer: Final = claims.get("iss") if claims is not None else None global_issuer: Final = os.getenv("JWT_ISSUER") @@ -86,25 +101,35 @@ async def oauth_authorization_uses_gateway_credential(request: Request) -> bool: ) -async def _opaque_bearer_is_gateway_credential(token: str) -> bool: - from litellm.proxy._types import hash_token # noqa: PLC0415 # proxy import cycle +def is_sealed_gateway_bearer(token: str) -> bool: + """True for a bearer only the gateway issues: a bridge or refresh envelope, a v2 GCM blob, or a + dashboard UI token. A UI token that cannot be read still counts, so a lookup fault never reads as foreign.""" from litellm.proxy.auth.auth_checks import ExperimentalUIJWTToken # noqa: PLC0415 # proxy import cycle - from litellm.proxy.auth.resolvers.exceptions import KeyNotFoundError # noqa: PLC0415 # proxy import cycle - from litellm.proxy.auth.resolvers.store import IdentityStore # noqa: PLC0415 # proxy import cycle from litellm.proxy.mcp_server.outbound_credentials.envelope import ( # noqa: PLC0415 # envelope imports bridge types is_envelope, is_refresh_envelope, ) - from litellm.proxy.proxy_server import ( # noqa: PLC0415 # startup owns the identity store dependencies - prisma_client, - user_api_key_cache, - ) if is_envelope(token) or is_refresh_envelope(token) or token.startswith(V2_GCM_PREFIX): return True try: - if ExperimentalUIJWTToken.get_key_object_from_ui_hash_key(token) is not None: - return True + return ExperimentalUIJWTToken.get_key_object_from_ui_hash_key(token) is not None + except Exception as exc: # noqa: BLE001 # an identity lookup fault must not permit cookie fallback + verbose_logger.debug("OAuth bearer ownership could not be checked (%s)", type(exc).__name__) + return True + + +async def virtual_key_may_exist(token: str) -> bool: + """False only when the key table answers that ``token`` is not a virtual key; a lookup fault counts as held.""" + from litellm.proxy._types import hash_token # noqa: PLC0415 # proxy import cycle + from litellm.proxy.auth.resolvers.exceptions import KeyNotFoundError # noqa: PLC0415 # proxy import cycle + from litellm.proxy.auth.resolvers.store import IdentityStore # noqa: PLC0415 # proxy import cycle + from litellm.proxy.proxy_server import ( # noqa: PLC0415 # startup owns the identity store dependencies + prisma_client, + user_api_key_cache, + ) + + try: await IdentityStore(prisma_client, user_api_key_cache).resolve(hashed_token=hash_token(token)) except KeyNotFoundError: return False diff --git a/litellm/proxy/mcp_server/mcp_server_manager.py b/litellm/proxy/mcp_server/mcp_server_manager.py index ceac2b2a8e1..c772e2e9d09 100644 --- a/litellm/proxy/mcp_server/mcp_server_manager.py +++ b/litellm/proxy/mcp_server/mcp_server_manager.py @@ -1196,8 +1196,11 @@ def should_strip_caller_authorization( pass-through cold-start case (RFC 9728) the bearer in ``Authorization`` is the upstream OAuth token and must be forwarded, so we keep it. - - **Delegated OAuth servers**: admission always runs and there is no - anonymous path, so the caller's separate ``Authorization`` is + - **Legacy delegate servers**: forward when MCP admission marked the + caller's ``Authorization`` as the upstream token + (``mcp_forwards_caller_authorization``); otherwise follow the + delegated OAuth rule below. + - **Delegated OAuth servers**: the caller's separate ``Authorization`` is forwarded only when a distinct ``x-litellm-api-key`` carried admission. Without that header the ``Authorization`` *was* the admission credential — a virtual key, an IdP JWT, or an SSO / OIDC / @@ -1206,6 +1209,12 @@ def should_strip_caller_authorization( - **Every other auth-enabled mode**: strip when ``Authorization`` is the credential admission validated. """ + if ( + user_api_key_auth is not None + and user_api_key_auth.mcp_forwards_caller_authorization + and is_legacy_upstream_delegate(mcp_server) + ): + return False if _authorization_is_admission_jwt(raw_headers, user_api_key_auth): return True if mcp_server.auth_type == MCPAuth.oauth2_token_exchange: @@ -1557,6 +1566,18 @@ async def _materialize_auth_headers(auth: httpx2.Auth | None) -> dict[str, str] return {header_name: header_value} if header_value else None +def is_legacy_upstream_delegate(server: MCPServer) -> bool: + """True for an interactive ``auth_type=oauth2`` server with ``delegate_auth_to_upstream=true``, whose + caller ``Authorization`` is the upstream token. The flow is resolved, not read from the column, so an + unstamped M2M-shaped row stays excluded: its stored client credentials drive egress, and admitting it + anonymously would let any caller act as LiteLLM's service account.""" + return ( + server.auth_type == MCPAuth.oauth2 + and server.delegate_auth_to_upstream is True + and MCPServerManager.effective_oauth2_flow(server) != "client_credentials" + ) + + def _consumes_caller_authorization(server: MCPServer) -> bool: """True when this server's egress forwards the caller's request-wide ``Authorization`` upstream: the client-forwarded token modes, legacy OAuth pass-through, and legacy upstream-delegated @@ -1726,18 +1747,15 @@ def _warn_overlapping_tool_policies(server: MCPServer) -> None: def _warn_legacy_delegate_auth_if_applicable(server: MCPServer, *, source: str) -> None: - """Direct legacy delegated OAuth configurations to the admitted replacement.""" - if server.auth_type != MCPAuth.oauth2: - return - if getattr(server, "delegate_auth_to_upstream", False) is not True: - return - if server.has_client_credentials: + """Point legacy delegated OAuth configurations at the client-forwarded replacements.""" + if not is_legacy_upstream_delegate(server): return label: Final = get_server_prefix(server) verbose_logger.warning( "MCP server %r (id=%s, source=%s) uses deprecated auth_type=oauth2 with " - "delegate_auth_to_upstream=true. LiteLLM admission is now required; migrate to " - "auth_type=oauth_delegate for client-forwarded OAuth.", + "delegate_auth_to_upstream=true. It still works: a caller without a LiteLLM credential is " + "admitted anonymously and its upstream token is forwarded. Migrate to auth_type=true_passthrough " + "or auth_type=oauth_delegate.", label, server.server_id, source, @@ -3766,12 +3784,12 @@ class MCPServerManager: ) ) if is_anonymous: - passthrough_server_ids: Final = [ + anonymous_server_ids: Final = [ server.server_id for server in self.get_registry().values() - if server.auth_type == MCPAuth.true_passthrough + if server.auth_type == MCPAuth.true_passthrough or is_legacy_upstream_delegate(server) ] - combined_servers.update(passthrough_server_ids) + combined_servers.update(anonymous_server_ids) restrict_allow_all: Final = ( resolved_general_settings.get("mcp_allow_all_keys_respects_mcp_scope", False) diff --git a/litellm/proxy/mcp_server/server.py b/litellm/proxy/mcp_server/server.py index 32f790a8e81..2105d875b58 100644 --- a/litellm/proxy/mcp_server/server.py +++ b/litellm/proxy/mcp_server/server.py @@ -588,6 +588,7 @@ if MCP_AVAILABLE: from litellm.proxy.mcp_server.mcp_server_manager import ( MCPServerManager, global_mcp_server_manager, + is_legacy_upstream_delegate, ) ###################################################### @@ -2162,8 +2163,9 @@ if MCP_AVAILABLE: oauth2_headers: Mapping[str, str] | None, mcp_server_auth_headers: dict[str, dict[str, str]] | None, raw_headers: Mapping[str, str] | None, + requested_servers: Sequence[str] | None = None, ) -> None: - """Probe pass-through upstream servers in parallel before the MCP session starts. + """Probe pass-through and legacy delegate upstream servers in parallel before the MCP session starts. Only servers the caller's key is already authorized to reach are probed — the list is derived from _get_allowed_mcp_servers so that a user cannot @@ -2186,6 +2188,18 @@ if MCP_AVAILABLE: mcp_servers=mcp_servers, client_ip=client_ip, ) + # A legacy delegate is probed only when the single requested name resolves to it the way admission + # resolves it, so neither a multi-server connect nor an access group can be failed by one rejected + # token; its challenge echoes the requested name, matching the tokenless challenge. + requested_scope: Final = requested_servers if requested_servers is not None else mcp_servers + single_requested_name: Final = ( + requested_scope[0] if requested_scope is not None and len(requested_scope) == 1 else None + ) + requested_server: Final = ( + operations.global_mcp_server_manager.get_mcp_server_by_name(single_requested_name, client_ip=client_ip) + if single_requested_name is not None + else None + ) prepared_headers: Final = tuple( ( srv, @@ -2201,6 +2215,11 @@ if MCP_AVAILABLE: ) for srv in allowed_servers if srv.is_oauth_passthrough + or ( + requested_server is not None + and srv.server_id == requested_server.server_id + and is_legacy_upstream_delegate(srv) + ) ) probe_headers: Final = tuple( ( @@ -2213,7 +2232,7 @@ if MCP_AVAILABLE: for srv, (auth, extra) in prepared_headers ) probe_targets: Final[tuple[tuple[MCPServer, str, str], ...]] = tuple( - (srv, auth_header, srv.name) + (srv, auth_header, srv.name if srv.is_oauth_passthrough else single_requested_name or srv.name) for srv, headers in probe_headers if ( auth_header := next( @@ -2311,6 +2330,7 @@ if MCP_AVAILABLE: if context.mcp_server_auth_headers is not None else None, raw_headers=context.raw_headers, + requested_servers=context.mcp_servers, ) return None diff --git a/litellm/types/mcp_server/mcp_server_manager.py b/litellm/types/mcp_server/mcp_server_manager.py index d29e4db55a4..0d39b0a0616 100644 --- a/litellm/types/mcp_server/mcp_server_manager.py +++ b/litellm/types/mcp_server/mcp_server_manager.py @@ -204,9 +204,11 @@ class MCPServer(LiteLLMBaseModel): access_groups: list[str] | None = None allow_all_keys: bool = False available_on_public_internet: bool = True - # Legacy opt-in to upstream-delegated authentication for ``oauth2`` - # servers. LiteLLM admission still applies; use ``oauth_delegate`` for the - # supported client-forwarded OAuth flow. + # Deprecated opt-in to upstream-delegated authentication for ``oauth2`` + # servers. A bare ``Authorization`` that is not a LiteLLM credential is the + # upstream token and is forwarded; see + # ``MCPRequestHandler._admit_legacy_delegate_bearer``. New servers should use + # ``true_passthrough`` or ``oauth_delegate``. # # Honored only for ``auth_type == oauth2``; ignored for any other # ``auth_type``. OAuth pass-through for non-oauth2 servers diff --git a/litellm/types/proxy/auth/user_api_key_auth.py b/litellm/types/proxy/auth/user_api_key_auth.py index 2c1c75fbe04..1183c36eda7 100644 --- a/litellm/types/proxy/auth/user_api_key_auth.py +++ b/litellm/types/proxy/auth/user_api_key_auth.py @@ -169,6 +169,10 @@ class UserAPIKeyAuth(LiteLLM_VerificationTokenView): # the expected response ob # above; a forged value could at most narrow, but the stripping keeps the field's provenance # single-owner so its meaning stays trustworthy. mcp_session_resource_server_ids: tuple[str, ...] | None = Field(default=None, exclude=True) + # Set only by MCP admission when a legacy delegate_auth_to_upstream caller's Authorization is the + # upstream token rather than a LiteLLM credential, so egress forwards it. Stripped from validated + # input like the markers above: a forged value would forward a LiteLLM credential upstream. + mcp_forwards_caller_authorization: bool = Field(default=False, exclude=True) mcp_toolset_id: str | None = Field(default=None, exclude=True) authenticated_by_custom_auth: bool = Field(default=False, exclude=True) via_virtual_key: bool = Field( @@ -230,6 +234,7 @@ class UserAPIKeyAuth(LiteLLM_VerificationTokenView): # the expected response ob values.pop("mcp_explicit_grants_only", None) values.pop("mcp_source_team_rpm_limits", None) values.pop("mcp_session_resource_server_ids", None) + values.pop("mcp_forwards_caller_authorization", None) values.pop("mcp_toolset_id", None) values.pop("via_virtual_key", None) values.pop("authenticated_by_custom_auth", None) diff --git a/tests/unit/proxy/mcp_server/auth/test_user_api_key_auth_mcp.py b/tests/unit/proxy/mcp_server/auth/test_user_api_key_auth_mcp.py index b7795b4d2f0..ee3ef3f5323 100644 --- a/tests/unit/proxy/mcp_server/auth/test_user_api_key_auth_mcp.py +++ b/tests/unit/proxy/mcp_server/auth/test_user_api_key_auth_mcp.py @@ -1,7 +1,7 @@ import contextlib import json import os -from collections.abc import Sequence +from collections.abc import Callable, Iterator, Sequence from collections.abc import Set as AbstractSet from datetime import datetime, timedelta, timezone from types import SimpleNamespace @@ -1599,9 +1599,7 @@ class TestMCPRequestHandler: team_id="test-team-id", ) - with patch( - "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth" - ) as mock_user_api_key_auth: + with patch("litellm.proxy.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth") as mock_user_api_key_auth: mock_user_api_key_auth.return_value = mock_auth_result # Call the method @@ -1663,10 +1661,10 @@ class TestMCPOAuth2AuthFlow: as LiteLLM API keys, causing auth failures and empty tool listings. """ - async def test_oauth2_token_in_authorization_header_requires_litellm_admission(self): + async def test_oauth2_upstream_token_on_legacy_delegate_is_admitted_anonymously(self) -> None: """ - A bare Authorization token on the legacy delegated mode must establish - a LiteLLM principal rather than entering anonymously. + A bare upstream OAuth token on the legacy delegated mode is admitted + anonymously without LiteLLM key validation and kept for egress. """ from litellm.types.mcp import MCPAuth @@ -1702,10 +1700,11 @@ class TestMCPOAuth2AuthFlow: raw_headers, ) = await MCPRequestHandler.process_mcp_request(scope) - assert isinstance(auth_result, UserAPIKeyAuth) - assert auth_result.user_id == "admitted-user" - mock_auth.assert_awaited_once() - assert oauth2_headers.get("Authorization") == "Bearer atlassian-oauth2-access-token-xyz" + assert auth_result.user_id is None + assert auth_result.api_key is None + assert auth_result.mcp_forwards_caller_authorization is True + mock_auth.assert_not_awaited() + assert oauth2_headers == {"Authorization": "Bearer atlassian-oauth2-access-token-xyz"} async def test_explicit_litellm_key_with_oauth2_authorization(self): """ @@ -2545,9 +2544,10 @@ class TestMCPDelegateAuthToUpstream: """ Tests for the ``delegate_auth_to_upstream`` per-server flag. - The legacy flag no longer bypasses LiteLLM admission. OAuth discovery may - still use the anonymous cold-start challenge, but a presented bearer must - authenticate to LiteLLM unless a separate admission credential is supplied. + A bare bearer that is a LiteLLM credential takes normal admission and is + never forwarded. Any other bearer is the upstream token: it is forwarded, + and the caller is admitted as the JWT or custom-auth user it also proves + when that user is granted the server, else anonymously. """ @staticmethod @@ -2652,37 +2652,497 @@ class TestMCPDelegateAuthToUpstream: assert isinstance(auth_result, UserAPIKeyAuth) mock_auth.assert_awaited_once() - async def test_delegate_with_only_upstream_token_requires_litellm_auth(self): - """ - An upstream token cannot establish a LiteLLM principal and must not - reopen anonymous admission. - """ - from litellm.types.mcp import MCPAuth - - scope = { + @staticmethod + def _bearer_scope(token: str) -> dict[str, object]: + return { "type": "http", "method": "POST", "path": "/mcp/delegated_oauth_server", - "headers": [(b"authorization", b"Bearer upstream-pkce-token")], + "headers": [(b"authorization", f"Bearer {token}".encode())], } + @staticmethod + def _jwt(issuer: str = "https://idp.example") -> str: + import jwt + + return jwt.encode({"iss": issuer, "sub": "jwt-user"}, "test-signing-secret", algorithm="HS256") + + @staticmethod + @contextlib.contextmanager + def _delegate_admission( + admission: AsyncMock, allowed_server_ids: Sequence[str] = ("test-server-id",) + ) -> Iterator[None]: + from litellm.types.mcp import MCPAuth + with ( - patch( - "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", - new_callable=AsyncMock, - side_effect=HTTPException(status_code=401, detail="Invalid API key"), - ) as mock_auth, + patch("litellm.proxy.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", admission), patch("litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, ): mock_mgr.get_mcp_server_by_name.return_value = TestMCPDelegateAuthToUpstream._make_server( auth_type=MCPAuth.oauth2, delegate_auth_to_upstream=True, ) - with pytest.raises(HTTPException) as exc_info: - await MCPRequestHandler.process_mcp_request(scope) + mock_mgr.get_allowed_mcp_servers = AsyncMock(return_value=list(allowed_server_ids)) + yield - assert exc_info.value.status_code == 401 - mock_auth.assert_awaited_once() + @staticmethod + @contextlib.contextmanager + def _jwt_auth_enabled(auth_jwt: AsyncMock) -> Iterator[None]: + from litellm.proxy import proxy_server + from litellm.proxy._types import LiteLLM_JWTAuth + + with ( + patch.dict(proxy_server.general_settings, {"enable_jwt_auth": True}), + patch.object(proxy_server, "premium_user", True), + patch.object(proxy_server, "user_custom_auth", None), + patch.object(proxy_server.jwt_handler, "auth_jwt", auth_jwt), + patch.object(proxy_server.jwt_handler, "litellm_jwtauth", LiteLLM_JWTAuth(), create=True), + patch.dict(os.environ), + ): + os.environ.pop("JWT_ISSUER", None) + yield + + async def test_delegate_forwards_unknown_sk_token_after_key_table_miss(self) -> None: + """An sk- token the key table does not hold, such as a provider API key, belongs to the upstream: + it is forwarded anonymously rather than rejected for its prefix.""" + from litellm.proxy.auth.resolvers.exceptions import KeyNotFoundError + + admission: Final = AsyncMock() + with ( + TestMCPDelegateAuthToUpstream._delegate_admission(admission), + patch( + "litellm.proxy.auth.resolvers.store.IdentityStore.resolve", + new_callable=AsyncMock, + side_effect=KeyNotFoundError("hashed-token"), + ), + ): + auth_result, _, _, _, oauth2_headers, _ = await MCPRequestHandler.process_mcp_request( + TestMCPDelegateAuthToUpstream._bearer_scope("sk-proj-upstream-provider-key") + ) + + assert auth_result.api_key is None + assert auth_result.mcp_forwards_caller_authorization is True + assert oauth2_headers == {"Authorization": "Bearer sk-proj-upstream-provider-key"} + admission.assert_not_awaited() + + @pytest.mark.parametrize( + "lookup_effect", + [None, RuntimeError("key table unreachable")], + ids=["key-held", "lookup-fault"], + ) + async def test_delegate_litellm_key_refusal_propagates(self, lookup_effect: Exception | None) -> None: + """A key the table holds, or one the lookup could not check, takes normal admission, so its + refusal reaches the caller instead of becoming an anonymous request that forwards the key.""" + refusal: Final = ProxyException( + message="Budget has been exceeded", type="budget_exceeded", param=None, code=400 + ) + with ( + TestMCPDelegateAuthToUpstream._delegate_admission(AsyncMock(side_effect=refusal)), + patch( + "litellm.proxy.auth.resolvers.store.IdentityStore.resolve", + new_callable=AsyncMock, + side_effect=lookup_effect, + ), + ): + with pytest.raises(ProxyException) as exc_info: + await MCPRequestHandler.process_mcp_request(TestMCPDelegateAuthToUpstream._bearer_scope("sk-held-key")) + + assert exc_info.value is refusal + + async def test_delegate_master_key_is_admitted_and_never_forwarded(self) -> None: + """The master key is not in the key table, so only recognizing it directly stops it from being + treated as the upstream's token and sent to the upstream server.""" + from litellm.proxy import proxy_server + from litellm.proxy.auth.resolvers.exceptions import KeyNotFoundError + + admission: Final = AsyncMock(return_value=UserAPIKeyAuth(api_key="sk-master-secret", user_role="proxy_admin")) + with ( + patch.object(proxy_server, "master_key", "sk-master-secret"), + TestMCPDelegateAuthToUpstream._delegate_admission(admission), + patch( + "litellm.proxy.auth.resolvers.store.IdentityStore.resolve", + new_callable=AsyncMock, + side_effect=KeyNotFoundError("hashed-master-key"), + ), + ): + auth_result, _, _, _, oauth2_headers, _ = await MCPRequestHandler.process_mcp_request( + TestMCPDelegateAuthToUpstream._bearer_scope("sk-master-secret") + ) + + admission.assert_awaited_once() + assert auth_result.mcp_forwards_caller_authorization is False + assert oauth2_headers is None + + @pytest.mark.parametrize( + "signing_secret, expires_in, forwarded", + [ + ("sk-master-secret", timedelta(hours=1), False), + ("sk-master-secret", timedelta(hours=-1), False), + ("upstream-idp-signing-secret", timedelta(hours=1), True), + ], + ids=["master-key-signed", "master-key-signed-expired", "signed-elsewhere"], + ) + async def test_delegate_master_key_signed_jwt_is_never_forwarded( + self, signing_secret: str, expires_in: timedelta, forwarded: bool + ) -> None: + """LiteLLM signs dashboard and OAuth tokens with the master key, so such a JWT is LiteLLM's own even + once expired: it takes normal admission and never reaches the upstream. A JWT signed by anyone + else is the upstream's token.""" + import jwt + + from litellm.proxy import proxy_server + + token: Final = jwt.encode( + {"sub": "dashboard-user", "exp": datetime.now(timezone.utc) + expires_in}, signing_secret, algorithm="HS256" + ) + admission: Final = AsyncMock(return_value=UserAPIKeyAuth(user_id="dashboard-user")) + with ( + patch.object(proxy_server, "master_key", "sk-master-secret"), + TestMCPDelegateAuthToUpstream._delegate_admission(admission), + ): + auth_result, *_ = await MCPRequestHandler.process_mcp_request( + TestMCPDelegateAuthToUpstream._bearer_scope(token) + ) + + assert auth_result.mcp_forwards_caller_authorization is forwarded + assert admission.await_count == (0 if forwarded else 1) + + @pytest.mark.parametrize( + "token, ui_token_lookup", + [ + ("llm_env_sealed-bridge-envelope", None), + ("llm_refresh_sealed-refresh-envelope", None), + ("v2:gcm:sealed-credential-blob", None), + ("dashboard-login-token", lambda: MagicMock(return_value=UserAPIKeyAuth(user_id="dashboard-user"))), + ("unreadable-dashboard-token", lambda: MagicMock(side_effect=ValueError("Invalid hash key"))), + ], + ids=["bridge-envelope", "refresh-envelope", "gcm-blob", "dashboard-token", "unreadable-dashboard-token"], + ) + async def test_delegate_sealed_litellm_token_takes_normal_admission( + self, token: str, ui_token_lookup: Callable[[], MagicMock] | None + ) -> None: + """A token only LiteLLM issues is LiteLLM's own, even one it fails to read, so it goes through + normal admission and is never marked for forwarding upstream.""" + from litellm.proxy.auth.auth_checks import ExperimentalUIJWTToken + + admission: Final = AsyncMock(return_value=UserAPIKeyAuth(user_id="litellm-user")) + ui_lookup: Final = ( + patch.object(ExperimentalUIJWTToken, "get_key_object_from_ui_hash_key", ui_token_lookup()) + if ui_token_lookup is not None + else contextlib.nullcontext() + ) + with TestMCPDelegateAuthToUpstream._delegate_admission(admission), ui_lookup: + auth_result, *_ = await MCPRequestHandler.process_mcp_request( + TestMCPDelegateAuthToUpstream._bearer_scope(token) + ) + + admission.assert_awaited_once() + assert auth_result.mcp_forwards_caller_authorization is False + + @staticmethod + def _unknown_user_refusal() -> ProxyException: + from litellm.types.proxy.auth.auth_checks import UserNotFoundError + + refusal: Final = ProxyException(message="User doesn't exist in db", type="auth_error", param=None, code=401) + refusal.__cause__ = UserNotFoundError(user_id="jwt-user") + return refusal + + @pytest.mark.parametrize( + "auth_jwt_effect, admission, allowed_server_ids, expected_user_id", + [ + ( + None, + lambda: AsyncMock(return_value=UserAPIKeyAuth(user_id="jwt-user", jwt_claims={"sub": "jwt-user"})), + ["test-server-id"], + "jwt-user", + ), + ( + None, + lambda: AsyncMock(return_value=UserAPIKeyAuth(user_id="jwt-user", jwt_claims={"sub": "jwt-user"})), + [], + None, + ), + ( + None, + lambda: AsyncMock(side_effect=TestMCPDelegateAuthToUpstream._unknown_user_refusal()), + ["test-server-id"], + None, + ), + ( + ValueError("Signature verification failed"), + lambda: AsyncMock(return_value=UserAPIKeyAuth(user_id="jwt-user")), + ["test-server-id"], + None, + ), + ], + ids=["granted-user", "user-not-granted", "user-unknown-to-litellm", "signature-rejected"], + ) + async def test_delegate_jwt_is_forwarded_as_granted_user_or_anonymously( + self, + auth_jwt_effect: Exception | None, + admission: Callable[[], AsyncMock], + allowed_server_ids: Sequence[str], + expected_user_id: str | None, + ) -> None: + """A JWT LiteLLM also accepts admits that user when granted the server; a user who is not + granted, unknown to LiteLLM, or whose signature fails is admitted anonymously. The JWT is always + forwarded, because on this server it is the upstream's token.""" + token: Final = TestMCPDelegateAuthToUpstream._jwt() + with ( + TestMCPDelegateAuthToUpstream._jwt_auth_enabled(AsyncMock(side_effect=auth_jwt_effect)), + TestMCPDelegateAuthToUpstream._delegate_admission(admission(), allowed_server_ids), + ): + auth_result, _, _, _, oauth2_headers, _ = await MCPRequestHandler.process_mcp_request( + TestMCPDelegateAuthToUpstream._bearer_scope(token) + ) + + assert auth_result.user_id == expected_user_id + assert auth_result.mcp_forwards_caller_authorization is True + assert oauth2_headers == {"Authorization": f"Bearer {token}"} + + @pytest.mark.parametrize( + "refusal", + [ + ProxyException(message="Budget has been exceeded", type="budget_exceeded", param=None, code=400), + ProxyException(message="Team=t1 is blocked", type="auth_error", param=None, code=401), + ], + ids=["over-budget", "blocked-team"], + ) + async def test_delegate_jwt_refusal_after_valid_signature_propagates(self, refusal: ProxyException) -> None: + """Once the signature verifies, LiteLLM knows who the caller is, so a budget or an explicit + disallowment is enforced and the request fails instead of continuing anonymously.""" + with ( + TestMCPDelegateAuthToUpstream._jwt_auth_enabled(AsyncMock(return_value={"sub": "jwt-user"})), + TestMCPDelegateAuthToUpstream._delegate_admission(AsyncMock(side_effect=refusal)), + ): + with pytest.raises(ProxyException) as exc_info: + await MCPRequestHandler.process_mcp_request( + TestMCPDelegateAuthToUpstream._bearer_scope(TestMCPDelegateAuthToUpstream._jwt()) + ) + + assert exc_info.value is refusal + + @staticmethod + @contextlib.contextmanager + def _idp_auth_enabled( + mechanism: Literal["oauth2", "oauth2_proxy"], verify: AsyncMock, premium: bool = True + ) -> Iterator[None]: + from litellm.proxy import proxy_server + from litellm.proxy.auth.oauth2_check import Oauth2Handler + + settings: Final = {"enable_oauth2_auth": True} if mechanism == "oauth2" else {"enable_oauth2_proxy_auth": True} + verifier: Final = ( + patch.object(Oauth2Handler, "check_oauth2_token", verify) + if mechanism == "oauth2" + else patch("litellm.proxy.auth.oauth2_proxy_hook.handle_oauth2_proxy_request", verify) + ) + with ( + patch.dict(proxy_server.general_settings, settings), + patch.object(proxy_server, "premium_user", premium), + patch.object(proxy_server, "user_custom_auth", None), + verifier, + ): + yield + + @pytest.mark.parametrize( + "verify_effect, allowed_server_ids, expected_user_id", + [ + (None, ["test-server-id"], "idp-user"), + (None, [], None), + (ValueError("Missing trusted proxy header"), ["test-server-id"], None), + ], + ids=["granted-user", "user-not-granted", "identity-rejected"], + ) + async def test_delegate_proxy_header_identity_follows_the_jwt_rules( + self, + verify_effect: Exception | None, + allowed_server_ids: Sequence[str], + expected_user_id: str | None, + ) -> None: + """Trusted reverse-proxy headers admit a granted user by name, and a user who is not granted or not + vouched for anonymously. The bearer is forwarded either way.""" + admission: Final = AsyncMock(return_value=UserAPIKeyAuth(user_id="idp-user")) + with ( + TestMCPDelegateAuthToUpstream._idp_auth_enabled("oauth2_proxy", AsyncMock(side_effect=verify_effect)), + TestMCPDelegateAuthToUpstream._delegate_admission(admission, allowed_server_ids), + ): + auth_result, _, _, _, oauth2_headers, _ = await MCPRequestHandler.process_mcp_request( + TestMCPDelegateAuthToUpstream._bearer_scope("opaque-idp-token") + ) + + assert auth_result.user_id == expected_user_id + assert auth_result.mcp_forwards_caller_authorization is True + assert oauth2_headers == {"Authorization": "Bearer opaque-idp-token"} + + @pytest.mark.parametrize( + "provider_status, active, allowed_server_ids, expected_user_id", + [ + (200, True, ["test-server-id"], "idp-user"), + (200, True, [], None), + (200, False, ["test-server-id"], None), + (401, True, ["test-server-id"], None), + ], + ids=["granted-user", "user-not-granted", "token-inactive", "provider-rejects-token"], + ) + async def test_delegate_oauth2_introspects_once_and_follows_the_jwt_rules( + self, provider_status: int, active: bool, allowed_server_ids: Sequence[str], expected_user_id: str | None + ) -> None: + """The provider is asked about the token once per request. A token it vouches for admits a granted + user by name, and one it reports inactive or rejects outright is forwarded anonymously instead of + failing the request.""" + import httpx + + from litellm.proxy import proxy_server + from litellm.types.mcp import MCPAuth + + async def introspect(outbound: httpx.Request, **kwargs: object) -> httpx.Response: + return httpx.Response(provider_status, request=outbound, json={"active": active, "sub": "idp-user"}) + + with ( + patch.dict(proxy_server.general_settings, {"enable_oauth2_auth": True}), + patch.object(proxy_server, "premium_user", True), + patch.object(proxy_server, "user_custom_auth", None), + patch.object(proxy_server, "master_key", "sk-fixture-master"), + patch.object(proxy_server, "prisma_client", None), + patch.object(proxy_server, "user_api_key_cache", DualCache()), + patch.dict( + os.environ, + { + "OAUTH_TOKEN_INFO_ENDPOINT": "https://idp.fixture.test/introspect", + "OAUTH_CLIENT_ID": "fixture-client", + "OAUTH_CLIENT_SECRET": "fixture-client-secret", + }, + ), + patch("httpx.AsyncClient.send", side_effect=introspect) as network, + patch("litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, + ): + mock_mgr.get_mcp_server_by_name.return_value = TestMCPDelegateAuthToUpstream._make_server( + auth_type=MCPAuth.oauth2, delegate_auth_to_upstream=True + ) + mock_mgr.get_allowed_mcp_servers = AsyncMock(return_value=list(allowed_server_ids)) + auth_result, _, _, _, oauth2_headers, _ = await MCPRequestHandler.process_mcp_request( + TestMCPDelegateAuthToUpstream._bearer_scope("opaque-idp-token") + ) + + assert network.call_count == 1 + assert auth_result.user_id == expected_user_id + assert auth_result.mcp_forwards_caller_authorization is True + assert oauth2_headers == {"Authorization": "Bearer opaque-idp-token"} + + async def test_delegate_oauth2_rejection_admits_anonymously_whether_or_not_auth_wraps_it(self) -> None: + """A provider rejection is the provider declining to vouch, so the bearer is forwarded anonymously + even when it reaches admission as the bare validation error rather than a wrapped auth error.""" + from litellm.proxy.auth.oauth2_check import OAuth2TokenValidationError + + with ( + TestMCPDelegateAuthToUpstream._idp_auth_enabled("oauth2", AsyncMock()), + TestMCPDelegateAuthToUpstream._delegate_admission( + AsyncMock(side_effect=OAuth2TokenValidationError("Token is not active")) + ), + ): + auth_result, _, _, _, oauth2_headers, _ = await MCPRequestHandler.process_mcp_request( + TestMCPDelegateAuthToUpstream._bearer_scope("opaque-idp-token") + ) + + assert auth_result.user_id is None + assert auth_result.mcp_forwards_caller_authorization is True + assert oauth2_headers == {"Authorization": "Bearer opaque-idp-token"} + + @pytest.mark.parametrize("mechanism", ["oauth2", "oauth2_proxy"]) + async def test_delegate_idp_user_refusal_propagates(self, mechanism: Literal["oauth2", "oauth2_proxy"]) -> None: + """Once the provider vouches for the caller, LiteLLM's budget and permission refusals apply.""" + refusal: Final = ProxyException( + message="Budget has been exceeded", type="budget_exceeded", param=None, code=400 + ) + with ( + TestMCPDelegateAuthToUpstream._idp_auth_enabled(mechanism, AsyncMock()), + TestMCPDelegateAuthToUpstream._delegate_admission(AsyncMock(side_effect=refusal)), + ): + with pytest.raises(ProxyException) as exc_info: + await MCPRequestHandler.process_mcp_request( + TestMCPDelegateAuthToUpstream._bearer_scope("opaque-idp-token") + ) + + assert exc_info.value is refusal + + async def test_delegate_oauth2_without_premium_never_calls_the_provider(self) -> None: + """OAuth2 introspection is enterprise-only, so without a license the bearer is the upstream's alone.""" + introspect: Final = AsyncMock() + with ( + TestMCPDelegateAuthToUpstream._idp_auth_enabled("oauth2", introspect, premium=False), + TestMCPDelegateAuthToUpstream._delegate_admission( + AsyncMock(return_value=UserAPIKeyAuth(user_id="idp-user")) + ), + ): + auth_result, *_ = await MCPRequestHandler.process_mcp_request( + TestMCPDelegateAuthToUpstream._bearer_scope("opaque-idp-token") + ) + + assert auth_result.user_id is None + assert auth_result.mcp_forwards_caller_authorization is True + introspect.assert_not_awaited() + + async def test_delegate_jwt_from_unconfigured_issuer_is_forwarded_without_jwt_auth(self) -> None: + """A JWT from an issuer LiteLLM's JWT auth is not configured for never reaches JWT auth.""" + auth_jwt: Final = AsyncMock(return_value={"sub": "jwt-user"}) + admission: Final = AsyncMock(return_value=UserAPIKeyAuth(user_id="jwt-user")) + with ( + TestMCPDelegateAuthToUpstream._jwt_auth_enabled(auth_jwt), + patch.dict(os.environ, {"JWT_ISSUER": "https://litellm-idp.example"}), + TestMCPDelegateAuthToUpstream._delegate_admission(admission), + ): + auth_result, *_ = await MCPRequestHandler.process_mcp_request( + TestMCPDelegateAuthToUpstream._bearer_scope( + TestMCPDelegateAuthToUpstream._jwt(issuer="https://upstream-idp.example") + ) + ) + + assert auth_result.user_id is None + assert auth_result.mcp_forwards_caller_authorization is True + auth_jwt.assert_not_awaited() + + @pytest.mark.parametrize( + "admission, expected_user_id", + [ + ( + lambda: AsyncMock(return_value=UserAPIKeyAuth(user_id="custom-user", api_key="upstream-custom-token")), + "custom-user", + ), + ( + lambda: AsyncMock( + side_effect=ProxyException(message="custom auth rejected", type="auth_error", param=None, code=401) + ), + None, + ), + ], + ids=["accepted", "rejected"], + ) + async def test_delegate_custom_auth_decides_alone( + self, admission: Callable[[], AsyncMock], expected_user_id: str | None + ) -> None: + """With custom auth configured, its answer alone decides: accepted admits that user, any failure + admits anonymously. The bearer is forwarded either way.""" + from litellm.proxy import proxy_server + + with ( + patch.object(proxy_server, "user_custom_auth", AsyncMock()), + TestMCPDelegateAuthToUpstream._delegate_admission(admission()), + ): + auth_result, _, _, _, oauth2_headers, _ = await MCPRequestHandler.process_mcp_request( + TestMCPDelegateAuthToUpstream._bearer_scope("upstream-custom-token") + ) + + assert auth_result.user_id == expected_user_id + assert auth_result.mcp_forwards_caller_authorization is True + assert oauth2_headers == {"Authorization": "Bearer upstream-custom-token"} + + def test_forwarding_marker_cannot_be_forged_from_input(self) -> None: + """Only admission may mark a caller's Authorization for forwarding; caller-shaped input never can.""" + assert UserAPIKeyAuth(mcp_forwards_caller_authorization=True).mcp_forwards_caller_authorization is False + assert ( + UserAPIKeyAuth.model_validate({"mcp_forwards_caller_authorization": True}).mcp_forwards_caller_authorization + is False + ) async def test_delegate_off_still_requires_litellm_auth(self): """ @@ -2753,7 +3213,12 @@ class TestMCPDelegateAuthToUpstream: await MCPRequestHandler.process_mcp_request(scope) assert exc_info.value.status_code == 401 - async def test_delegate_mixed_targets_fail_closed(self): + @pytest.mark.parametrize( + "bearer_headers", + [[], [(b"authorization", b"Bearer upstream-pkce-token")]], + ids=["no-bearer", "upstream-bearer"], + ) + async def test_delegate_mixed_targets_fail_closed(self, bearer_headers: list[tuple[bytes, bytes]]) -> None: """ x-mcp-servers can list multiple targets. If ANY of them does not opt in to delegate_auth_to_upstream, the bypass must NOT fire — otherwise an @@ -2769,6 +3234,7 @@ class TestMCPDelegateAuthToUpstream: "path": "/mcp", "headers": [ (b"x-mcp-servers", b"delegated_oauth,plain_oauth"), + *bearer_headers, ], } @@ -2863,9 +3329,8 @@ class TestMCPDelegateAuthToUpstream: async def test_authorization_bearer_on_delegate_server_establishes_litellm_principal(self): """ - A bare Authorization bearer now follows normal LiteLLM admission. A - separate x-litellm-api-key is required when Authorization is intended - for the upstream server. + A bare sk- bearer the key table may hold takes normal LiteLLM admission + and is never marked for forwarding upstream. """ from litellm.types.mcp import MCPAuth @@ -2898,6 +3363,7 @@ class TestMCPDelegateAuthToUpstream: ) = await MCPRequestHandler.process_mcp_request(scope) assert isinstance(auth_result, UserAPIKeyAuth) assert auth_result.user_id == "real-user" + assert auth_result.mcp_forwards_caller_authorization is False assert oauth2_headers.get("Authorization") == "Bearer sk-9876" mock_auth.assert_awaited_once() @@ -3079,8 +3545,9 @@ class TestMCPDelegateAuthToUpstream: mock_auth.assert_awaited_once() assert auth.api_key is None - async def test_get_allowed_servers_excludes_legacy_delegates(self): - """Legacy delegated servers are never added to anonymous access.""" + async def test_get_allowed_servers_includes_interactive_legacy_delegates_for_anonymous(self) -> None: + """Anonymous callers reach interactive legacy delegates, never an M2M server whose stored + client credentials would let them act as LiteLLM's service account.""" from litellm.proxy.mcp_server.mcp_server_manager import ( MCPServerManager, ) @@ -3118,7 +3585,7 @@ class TestMCPDelegateAuthToUpstream: ): result = await manager.get_allowed_mcp_servers(None) - assert result == [] + assert result == ["pkce-server"] async def test_true_passthrough_skips_litellm_auth_anonymously(self): """auth_type=true_passthrough performs no admission auth: the caller's Authorization is an @@ -7644,9 +8111,7 @@ class TestMCPDcrBridgeDualCredential: new_callable=AsyncMock, return_value=UserAPIKeyAuth(user_id=presented_user, api_key="any-hash"), ), - patch( - "litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager" - ) as mock_mgr, + patch("litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, patch("litellm.proxy.proxy_server.master_key", self._MASTER_KEY), ): mock_mgr.get_mcp_server_by_name.return_value = self._DELEGATE._bridge_delegate_server() @@ -7697,7 +8162,9 @@ class TestMCPDcrBridgeDualCredential: @pytest.mark.parametrize("root_path", ["", "/litellm"]) @pytest.mark.parametrize("invalid_token", [False, True]) def test_gateway_challenge_metadata_contains_the_public_resource( - public_path: str, root_path: str, invalid_token: bool, + public_path: str, + root_path: str, + invalid_token: bool, ) -> None: from mcp.shared.auth_utils import check_resource_allowed from starlette.requests import Request @@ -7710,8 +8177,12 @@ def test_gateway_challenge_metadata_contains_the_public_resource( ) scope: Final[Scope] = { - "type": "http", "method": "POST", "scheme": "https", "path": "/mcp", - "root_path": root_path, "_original_path": f"{root_path}{public_path}", + "type": "http", + "method": "POST", + "scheme": "https", + "path": "/mcp", + "root_path": root_path, + "_original_path": f"{root_path}{public_path}", "headers": [(b"host", b"gateway.example.com")], } request: Final = Request(scope) @@ -7720,8 +8191,7 @@ def test_gateway_challenge_metadata_contains_the_public_resource( metadata_url: Final = header.split('resource_metadata="', 1)[1].split('"', 1)[0] toolset: Final = public_path in ("/toolset/analysis/mcp", "/analysis/mcp") assert metadata_url == ( - f"https://gateway.example.com{root_path}/.well-known/oauth-protected-resource" - + ("" if toolset else "/mcp") + f"https://gateway.example.com{root_path}/.well-known/oauth-protected-resource" + ("" if toolset else "/mcp") ) metadata: Final = ( oauth_protected_resource_root(request) if toolset else _build_aggregate_protected_resource_response(request) @@ -7924,9 +8394,7 @@ class TestAggregateGatewayDcrChallenge: ): with ( patch(self._AUTH_PATCH_TARGET, side_effect=self._auth_401()), - patch( - "litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager" - ) as mock_mgr, + patch("litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, ): mock_mgr.get_mcp_server_by_name.return_value = server with pytest.raises(HTTPException) as exc_info: @@ -7972,9 +8440,7 @@ class TestAggregateGatewayDcrChallenge: with ( patch.dict(os.environ, {"SERVER_ROOT_PATH": "/litellm"}), patch(self._AUTH_PATCH_TARGET, side_effect=self._auth_401()), - patch( - "litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager" - ) as mock_mgr, + patch("litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, ): mock_mgr.get_mcp_server_by_name.return_value = server with pytest.raises(HTTPException) as exc_info: @@ -8005,9 +8471,7 @@ class TestAggregateGatewayDcrChallenge: ): with ( patch(self._AUTH_PATCH_TARGET, side_effect=self._auth_401()), - patch( - "litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager" - ) as mock_mgr, + patch("litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, ): mock_mgr.get_mcp_server_by_name.return_value = resolved mock_mgr.get_mcp_server_by_identifier.return_value = resolved @@ -8052,9 +8516,7 @@ class TestAggregateGatewayDcrChallenge: (None, None), ] for resolved, expected in cases: - with patch( - "litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager" - ) as mock_mgr: + with patch("litellm.proxy.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr: mock_mgr.get_mcp_server_by_identifier.return_value = resolved assert _gateway_dcr_challenge_target("/mcp/srv", None, None) == expected, resolved assert _gateway_dcr_challenge_target("/mcp/a,b", None, None) is None @@ -10950,14 +11412,15 @@ async def test_catalog_refresh_uses_current_virtual_key_policy_and_keeps_session from litellm.proxy import proxy_server from litellm.types.mcp_server.mcp_toolset import MCPToolset - permission = LiteLLM_ObjectPermissionTable( - object_permission_id="current-policy", mcp_toolsets=["session-toolset"] - ) + permission = LiteLLM_ObjectPermissionTable(object_permission_id="current-policy", mcp_toolsets=["session-toolset"]) toolset: Final = MCPToolset( - toolset_id="session-toolset", toolset_name="session-toolset", + toolset_id="session-toolset", + toolset_name="session-toolset", tools=[{"server_id": "current-server", "tool_name": "read"}], ) - tables: Final = SimpleNamespace(litellm_mcptoolsettable=SimpleNamespace(find_many=AsyncMock(return_value=[toolset]))) + tables: Final = SimpleNamespace( + litellm_mcptoolsettable=SimpleNamespace(find_many=AsyncMock(return_value=[toolset])) + ) monkeypatch.setattr(proxy_server, "prisma_client", SimpleNamespace(db=tables, writer_db=tables)) monkeypatch.setattr(proxy_server, "user_api_key_cache", DualCache()) current = UserAPIKeyAuth( @@ -10984,10 +11447,13 @@ async def test_catalog_refresh_uses_current_virtual_key_policy_and_keeps_session caller.mcp_toolset_id = "session-toolset" refreshed = await MCPRequestHandler.refresh_catalog_authority(caller) reload_key.assert_awaited_once_with("owned-key-hash", check_db_only=True) - assert refreshed.object_permission == permission.model_copy(update={ - "mcp_servers": ["current-server"], "mcp_toolsets": [], - "mcp_tool_permissions": {"current-server": ["read"]}, - }) + assert refreshed.object_permission == permission.model_copy( + update={ + "mcp_servers": ["current-server"], + "mcp_toolsets": [], + "mcp_tool_permissions": {"current-server": ["read"]}, + } + ) assert refreshed.object_permission_id == "current-policy" assert (refreshed.team_id, refreshed.org_id, refreshed.project_id, refreshed.user_id) == ( "new-team", @@ -12091,9 +12557,7 @@ async def test_personal_key_guardrail_auth_preserves_native_tool_name_matching_s from litellm.proxy.mcp_server.mcp_server_manager import global_mcp_server_manager async def _get_team_object(team_id: str, **_: object) -> LiteLLM_TeamTable: - return _personal_key_team_granting_tools( - team_id, ("math-multiply",) if team_id == "team-b" else ("other",) - ) + return _personal_key_team_granting_tools(team_id, ("math-multiply",) if team_id == "team-b" else ("other",)) monkeypatch.setattr(auth_checks, "get_team_object", _get_team_object) monkeypatch.setattr(auth_checks, "get_org_object", AsyncMock(return_value=None)) @@ -12132,9 +12596,7 @@ async def test_personal_key_mcp_billing_inherits_adopted_organization_limits( ) async def _get_team_object(team_id: str, **_: object) -> LiteLLM_TeamTable: - return _personal_key_team_granting_tools(team_id, ("divide",)).model_copy( - update={"organization_id": "org-1"} - ) + return _personal_key_team_granting_tools(team_id, ("divide",)).model_copy(update={"organization_id": "org-1"}) org_loader: Final = AsyncMock(return_value=organization) monkeypatch.setattr(auth_checks, "get_team_object", _get_team_object) @@ -12180,16 +12642,12 @@ async def test_personal_key_mcp_billing_keeps_key_organization_limits( ) async def _get_team_object(team_id: str, **_: object) -> LiteLLM_TeamTable: - return _personal_key_team_granting_tools(team_id, ("divide",)).model_copy( - update={"organization_id": "org-1"} - ) + return _personal_key_team_granting_tools(team_id, ("divide",)).model_copy(update={"organization_id": "org-1"}) org_loader: Final = AsyncMock(return_value=organization) monkeypatch.setattr(auth_checks, "get_team_object", _get_team_object) monkeypatch.setattr(auth_checks, "get_org_object_for_request", org_loader) - auth: Final = personal_key_owner_teams.model_copy( - update={"org_id": "org-k", "organization_rpm_limit": 9} - ) + auth: Final = personal_key_owner_teams.model_copy(update={"org_id": "org-k", "organization_rpm_limit": 9}) server: Final = global_mcp_server_manager.registry["srv-b"] with patch.object(global_mcp_server_manager, "get_mcp_server_from_tool_name", return_value=server): @@ -12321,9 +12779,7 @@ async def test_personal_key_mcp_billing_uses_fresh_team_and_membership_for_stric } ) - async def _get_team_membership( - user_id: str, team_id: str, **kwargs: object - ) -> LiteLLM_TeamMembership: + async def _get_team_membership(user_id: str, team_id: str, **kwargs: object) -> LiteLLM_TeamMembership: database_value: Final = bool(kwargs.get("check_db_only")) return LiteLLM_TeamMembership( user_id=user_id, diff --git a/tests/unit/proxy/mcp_server/test_mcp_server_tool_calls_and_headers.py b/tests/unit/proxy/mcp_server/test_mcp_server_tool_calls_and_headers.py index 78c37184d6c..621a7b65005 100644 --- a/tests/unit/proxy/mcp_server/test_mcp_server_tool_calls_and_headers.py +++ b/tests/unit/proxy/mcp_server/test_mcp_server_tool_calls_and_headers.py @@ -3,6 +3,7 @@ import contextlib import contextvars import json import os +from collections.abc import Coroutine, Iterator, Sequence from datetime import datetime, timedelta from types import SimpleNamespace from typing import Callable, Final, Literal @@ -669,6 +670,75 @@ def test_prepare_mcp_server_headers_legacy_delegate_strips_repeated_admission_ke assert extra_headers is None +@pytest.mark.parametrize( + "caller_fields", + [{}, {"user_id": "jwt-user", "api_key": "upstream-jwt", "jwt_claims": {"sub": "jwt-user"}}], + ids=["anonymous", "admitted-jwt-user"], +) +def test_prepare_mcp_server_headers_legacy_delegate_forwards_marked_upstream_authorization( + caller_fields: dict[str, object], +) -> None: + """When admission marked the caller's Authorization as the upstream token, a legacy delegate forwards + it, including a JWT that also admitted the caller and would otherwise be stripped as an admission JWT.""" + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.mcp_server.server import ( + _prepare_mcp_server_headers, + ) + + server = MCPServer( + server_id="legacy-delegate-marked", + name="legacy-delegate", + transport=MCPTransport.http, + auth_type=MCPAuth.oauth2, + delegate_auth_to_upstream=True, + ) + caller: Final = UserAPIKeyAuth.model_validate(caller_fields).model_copy( + update={"mcp_forwards_caller_authorization": True} + ) + + server_auth_header, extra_headers = _prepare_mcp_server_headers( + server=server, + mcp_server_auth_headers=None, + mcp_auth_header=None, + oauth2_headers={"Authorization": "Bearer upstream-jwt"}, + raw_headers={"authorization": "Bearer upstream-jwt"}, + user_api_key_auth=caller, + ) + + assert server_auth_header is None + assert extra_headers == {"Authorization": "Bearer upstream-jwt"} + + +def test_prepare_mcp_server_headers_forwarding_marker_is_ignored_off_legacy_delegates() -> None: + """The forwarding marker only applies to legacy delegates, so it can never carry a LiteLLM key to an + oauth_delegate upstream.""" + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.mcp_server.server import ( + _prepare_mcp_server_headers, + ) + + server = MCPServer( + server_id="oauth-delegate-marked", + name="oauth-delegate", + transport=MCPTransport.http, + auth_type=MCPAuth.oauth_delegate, + ) + caller: Final = UserAPIKeyAuth(api_key="sk-litellm-key").model_copy( + update={"mcp_forwards_caller_authorization": True} + ) + + _, extra_headers = _prepare_mcp_server_headers( + server=server, + mcp_server_auth_headers=None, + mcp_auth_header=None, + oauth2_headers={"Authorization": "Bearer sk-litellm-key"}, + raw_headers={"authorization": "Bearer sk-litellm-key"}, + user_api_key_auth=caller, + ) + + assert extra_headers is None + + def test_prepare_mcp_server_headers_m2m_skips_authorization_from_raw_extra_headers(): """M2M must not merge caller Authorization from raw_headers when extra_headers lists it.""" try: @@ -7265,15 +7335,19 @@ async def test_legacy_delegate_bare_token_is_not_probed_upstream(): # test-qual @pytest.mark.asyncio -async def test_legacy_delegate_dual_credentials_are_not_probed_upstream(): # test-quality-ok: this removed security-sensitive egress has no return value; non-invocation is the contract - """A separate upstream bearer never triggers the removed legacy probe.""" +@pytest.mark.usefixtures("httpx_transport") +async def test_legacy_delegate_dual_credentials_probe_only_the_upstream_token() -> None: + """With a separate x-litellm-api-key, the preflight probes the upstream with its own token and never + with the LiteLLM key.""" + import respx + from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.mcp_server.server import ( _check_passthrough_upstream_auth, ) - server = _delegate_auth_mcp_server() - scope = _delegate_scope( + server: Final = _delegate_auth_mcp_server() + scope: Final = _delegate_scope( [ (b"x-litellm-api-key", b"sk-litellm-proxy-key"), (b"authorization", b"Bearer upstream-token"), @@ -7281,15 +7355,14 @@ async def test_legacy_delegate_dual_credentials_are_not_probed_upstream(): # te ) with ( + _patch_delegate_resolver(server, "delegate_test"), patch( # test-quality-ok: isolate authorized-server resolution so this test targets the preflight boundary "litellm.proxy.mcp_server.operations._get_allowed_mcp_servers", new=AsyncMock(return_value=[server]), ), - patch( # test-quality-ok: the removed probe call is the security regression under test - "litellm.proxy.mcp_server.server._probe_upstream_auth", - new=AsyncMock(), - ) as probe, + respx.mock as upstream, ): + probe: Final = upstream.post(server.url).respond(200) await _check_passthrough_upstream_auth( scope=scope, user_api_key_auth=UserAPIKeyAuth(user_id="admitted-user"), @@ -7300,7 +7373,160 @@ async def test_legacy_delegate_dual_credentials_are_not_probed_upstream(): # te raw_headers={"x-litellm-api-key": "sk-litellm-proxy-key", "Authorization": "Bearer upstream-token"}, ) - probe.assert_not_awaited() + assert probe.call_count == 1 + sent_headers: Final = probe.calls[0].request.headers + assert sent_headers["Authorization"] == "Bearer upstream-token" + assert all("sk-litellm-proxy-key" not in value for value in sent_headers.values()) + + +def _expired_delegate_token_preflight(mcp_servers: Sequence[str]) -> Coroutine[object, object, None]: + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.mcp_server.server import ( + _check_passthrough_upstream_auth, + ) + + return _check_passthrough_upstream_auth( + scope=_delegate_scope([(b"authorization", b"Bearer expired-upstream-token")]), + user_api_key_auth=UserAPIKeyAuth().model_copy(update={"mcp_forwards_caller_authorization": True}), + mcp_servers=list(mcp_servers), + client_ip=None, + oauth2_headers={"Authorization": "Bearer expired-upstream-token"}, + mcp_server_auth_headers=None, + raw_headers={"authorization": "Bearer expired-upstream-token"}, + ) + + +@contextlib.contextmanager +def _rejecting_delegate_upstream() -> Iterator[None]: + server: Final = _delegate_auth_mcp_server() + with ( + _patch_delegate_resolver(server, "delegate_test"), + patch( # test-quality-ok: isolate authorized-server resolution so this test targets the preflight boundary + "litellm.proxy.mcp_server.operations._get_allowed_mcp_servers", + new=AsyncMock(return_value=[server]), + ), + patch( # test-quality-ok: stands in for the upstream rejecting an expired token + "litellm.proxy.mcp_server.server._probe_upstream_auth", + new=AsyncMock(return_value=(401, 'Bearer error="invalid_token"')), + ), + ): + yield + + +@pytest.mark.asyncio +async def test_legacy_delegate_expired_upstream_token_gets_reauthorization_challenge() -> None: + """An expired upstream token on a single-server route gets the RFC 9728 re-authorization challenge + for the requested name, so the client signs in again instead of seeing an empty tool list.""" + with _rejecting_delegate_upstream(), pytest.raises(HTTPException) as exc_info: + await _expired_delegate_token_preflight(["delegate_test"]) + + assert exc_info.value.status_code == 401 + challenge: Final = exc_info.value.headers["www-authenticate"] + assert 'error="invalid_token"' in challenge + assert ( + 'resource_metadata="http://localhost:4000/.well-known/oauth-protected-resource/mcp/delegate_test"' in challenge + ) + + +@pytest.mark.asyncio +async def test_legacy_delegate_multi_server_scope_is_not_failed_by_one_upstream_token() -> None: + """A multi-server scope is not probed, so one rejected upstream token cannot fail the whole connect.""" + with _rejecting_delegate_upstream(): + assert await _expired_delegate_token_preflight(["delegate_test", "other_server"]) is None + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("httpx_transport") +async def test_legacy_delegate_probe_checks_the_callers_token_not_a_static_authorization() -> None: + """The probe is what checks a delegate caller's own token, so a static Authorization the server sends on + tool calls cannot stand in for it: a junk token still gets the re-authorization challenge.""" + import respx + + server: Final = _delegate_auth_mcp_server().model_copy( + update={"static_headers": {"Authorization": "Bearer operator-token"}} + ) + with ( + _patch_delegate_resolver(server, "delegate_test"), + patch( # test-quality-ok: isolate authorized-server resolution so this test targets the preflight boundary + "litellm.proxy.mcp_server.operations._get_allowed_mcp_servers", + new=AsyncMock(return_value=[server]), + ), + respx.mock as upstream, + ): + upstream.post(server.url).mock( + side_effect=lambda request: httpx.Response( + 200 if request.headers["Authorization"] == "Bearer operator-token" else 401 + ) + ) + outcome: Final = await asyncio.gather( + _expired_delegate_token_preflight(["delegate_test"]), return_exceptions=True + ) + + assert getattr(outcome[0], "status_code", None) == 401 + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("httpx_transport") +@pytest.mark.parametrize( + "requested, expected_status", + [(("delegate_test",), 401), (("delegate_test", "other_server"), None)], + ids=["single-server", "two-requested-one-reachable"], +) +async def test_modern_interaction_probes_legacy_delegate_only_for_a_single_requested_server( + requested: tuple[str, ...], expected_status: int | None +) -> None: + """The single-server rule follows what the caller asked for, not what it can reach, so a request + naming two servers is not failed by one expired delegate token.""" + import respx + + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.mcp_server import server + from litellm.proxy.mcp_server.contracts import OperationContext + + target: Final = _delegate_auth_mcp_server() + scope: Final = _delegate_scope( + [ + (b"authorization", b"Bearer expired-upstream-token"), + (b"mcp-protocol-version", b"2026-07-28"), + (b"mcp-method", b"tools/call"), + (b"mcp-name", b"delegate-add"), + ] + ) + context: Final = OperationContext( + _caller=UserAPIKeyAuth().model_copy(update={"mcp_forwards_caller_authorization": True}), + mcp_servers=requested, + oauth2_headers={"Authorization": "Bearer expired-upstream-token"}, + raw_headers={"authorization": "Bearer expired-upstream-token"}, + ) + body: Final = json.dumps( + { + "jsonrpc": "2.0", + "id": 1, + "method": "tools/call", + "params": { + "name": "delegate-add", + "arguments": {"a": 2, "b": 3}, + "_meta": { + "io.modelcontextprotocol/protocolVersion": "2026-07-28", + "io.modelcontextprotocol/clientCapabilities": {}, + }, + }, + } + ).encode() + with ( + _patch_delegate_resolver(target, "delegate_test"), + patch( # test-quality-ok: isolate authorized-server resolution so this test targets the preflight boundary + "litellm.proxy.mcp_server.operations._get_allowed_mcp_servers", + new=AsyncMock(return_value=[target]), + ), + respx.mock as upstream, + ): + upstream.post(target.url).respond(401, headers={"www-authenticate": 'Bearer error="invalid_token"'}) + outcome: Final = await asyncio.gather( + server._preflight_modern_interaction(scope, body, context), return_exceptions=True + ) + + assert getattr(outcome[0], "status_code", None) == expected_status @pytest.mark.parametrize( @@ -7537,39 +7763,47 @@ async def test_delegate_not_probed_when_named_only_via_server_id(): @pytest.mark.asyncio -async def test_delegate_probe_not_fanned_out_to_access_group_members(): - """A single access-group name passes the one-target route gate but must not fan - the delegate probe out to group-expanded member servers; the group name resolves - to no server under admission's resolver, so no probe fires.""" +@pytest.mark.usefixtures("httpx_transport") +async def test_delegate_probe_not_fanned_out_to_access_group_members() -> None: + """A single access-group name resolves to no server under admission's resolver, so the delegate probe is + not fanned out to the group's members and one member rejecting the token cannot fail the whole group.""" + import respx + from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.mcp_server.server import ( _check_passthrough_upstream_auth, ) - group_member = _delegate_auth_mcp_server() - + rejecting: Final = _delegate_auth_mcp_server() + accepting: Final = _delegate_auth_mcp_server("delegate-2").model_copy( + update={"name": "delegate_two", "url": "http://upstream-two:9402/mcp"} + ) with ( - _patch_delegate_resolver(group_member, "delegate_test"), - patch( + _patch_delegate_resolver(rejecting, "delegate_test"), + patch( # test-quality-ok: isolate authorized-server resolution so this test targets the preflight boundary "litellm.proxy.mcp_server.operations._get_allowed_mcp_servers", - new=AsyncMock(return_value=[group_member]), + new=AsyncMock(return_value=[rejecting, accepting]), ), - patch( - "litellm.proxy.mcp_server.server._probe_upstream_auth", - new=AsyncMock(return_value=(401, None)), - ) as probe, + respx.mock as upstream, ): - await _check_passthrough_upstream_auth( - scope=_delegate_scope([(b"authorization", b"Bearer bogus-token")]), - user_api_key_auth=UserAPIKeyAuth(), - mcp_servers=["prod_tools_group"], - client_ip=None, - oauth2_headers=None, - mcp_server_auth_headers=None, - raw_headers=None, + upstream.post(rejecting.url).respond(401, headers={"www-authenticate": 'Bearer error="invalid_token"'}) + upstream.post(accepting.url).respond(200) + assert ( + await _check_passthrough_upstream_auth( + scope=_delegate_scope( + [(b"x-litellm-api-key", b"sk-litellm-proxy-key"), (b"authorization", b"Bearer upstream-token")] + ), + user_api_key_auth=UserAPIKeyAuth(user_id="admitted-user"), + mcp_servers=["prod_tools_group"], + client_ip=None, + oauth2_headers={"Authorization": "Bearer upstream-token"}, + mcp_server_auth_headers=None, + raw_headers={"x-litellm-api-key": "sk-litellm-proxy-key", "Authorization": "Bearer upstream-token"}, + ) + is None ) - probe.assert_not_awaited() + assert upstream.calls.call_count == 0 @pytest.mark.asyncio @@ -8846,8 +9080,7 @@ async def test_get_allowed_mcp_servers_from_mcp_server_names_unknown_name_fails_ ] with patch( - "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp." - "MCPRequestHandler.get_mcp_servers_from_access_groups", + "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp.MCPRequestHandler.get_mcp_servers_from_access_groups", new_callable=AsyncMock, return_value=[], ): @@ -8905,8 +9138,7 @@ async def test_get_allowed_mcp_servers_from_mcp_server_names_known_alias_returns ] with patch( - "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp." - "MCPRequestHandler.get_mcp_servers_from_access_groups", + "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp.MCPRequestHandler.get_mcp_servers_from_access_groups", new_callable=AsyncMock, return_value=[], ): @@ -8938,8 +9170,7 @@ async def test_get_allowed_mcp_servers_from_mcp_server_names_mixed_known_and_unk ] with patch( - "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp." - "MCPRequestHandler.get_mcp_servers_from_access_groups", + "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp.MCPRequestHandler.get_mcp_servers_from_access_groups", new_callable=AsyncMock, return_value=[], ): @@ -8971,8 +9202,7 @@ async def test_get_allowed_mcp_servers_from_mcp_server_names_access_group_resolv ] with patch( - "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp." - "MCPRequestHandler.get_mcp_servers_from_access_groups", + "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp.MCPRequestHandler.get_mcp_servers_from_access_groups", new_callable=AsyncMock, return_value=["id-b"], ): @@ -9240,15 +9470,11 @@ async def test_stateful_mcp_tracing_follows_each_message_exclusion( is_otel_v2_enabled.cache_clear() notified: Final = asyncio.Event() - async def call_tool( - ctx: ServerRequestContext[object, object], params: CallToolRequestParams - ) -> CallToolResult: + async def call_tool(ctx: ServerRequestContext[object, object], params: CallToolRequestParams) -> CallToolResult: with tracer.start_as_current_span("tool-handler"): return CallToolResult(content=[TextContent(type="text", text="done")]) - async def notification( - ctx: ServerRequestContext[object, object], params: NotificationParams - ) -> HandlerResult: + async def notification(ctx: ServerRequestContext[object, object], params: NotificationParams) -> HandlerResult: with tracer.start_as_current_span("notification-handler"): notified.set() return None @@ -9266,51 +9492,85 @@ async def test_stateful_mcp_tracing_follows_each_message_exclusion( try: async with manager.run(): async with httpx2.AsyncClient(transport=httpx2.ASGITransport(app=app)) as client: - initialized: Final = await client.post(initial_url, headers=headers, json={ - "jsonrpc": "2.0", "id": 0, "method": "initialize", "params": { - "protocolVersion": "2025-11-25", "capabilities": {}, - "clientInfo": {"name": "trace-exclusion", "version": "1"}, - }, - }) - assert initialized.status_code == 200, initialized.text - assert any( - span.name == "initialize" for span in exporter.get_finished_spans() - ) is not initialized_excluded - exporter.clear() - session_headers: Final = {**headers, "MCP-Session-Id": initialized.headers["mcp-session-id"]} - ready: Final = await client.post(initial_url, headers=session_headers, - json={"jsonrpc": "2.0", "method": "notifications/initialized"}) - assert ready.status_code == 202, ready.text - response: Final = await client.post(current_url, headers=session_headers, json={ - "jsonrpc": "2.0", "id": 1, "method": "tools/call", "params": { - "name": "probe", "arguments": {}, "_meta": { - "traceparent": "00-11111111111111111111111111111111-2222222222222222-01", + initialized: Final = await client.post( + initial_url, + headers=headers, + json={ + "jsonrpc": "2.0", + "id": 0, + "method": "initialize", + "params": { + "protocolVersion": "2025-11-25", + "capabilities": {}, + "clientInfo": {"name": "trace-exclusion", "version": "1"}, }, }, - }) + ) + assert initialized.status_code == 200, initialized.text + assert ( + any(span.name == "initialize" for span in exporter.get_finished_spans()) is not initialized_excluded + ) + exporter.clear() + session_headers: Final = {**headers, "MCP-Session-Id": initialized.headers["mcp-session-id"]} + ready: Final = await client.post( + initial_url, headers=session_headers, json={"jsonrpc": "2.0", "method": "notifications/initialized"} + ) + assert ready.status_code == 202, ready.text + response: Final = await client.post( + current_url, + headers=session_headers, + json={ + "jsonrpc": "2.0", + "id": 1, + "method": "tools/call", + "params": { + "name": "probe", + "arguments": {}, + "_meta": { + "traceparent": "00-11111111111111111111111111111111-2222222222222222-01", + }, + }, + }, + ) assert response.status_code == 200 and '"text":"done"' in response.text, response.text names: Final = {span.name for span in exporter.get_finished_spans()} assert ("tools/call probe" in names) is initialized_excluded assert ("tool-handler" in names) is initialized_excluded exporter.clear() - notification_response: Final = await client.post(current_url, headers=session_headers, - json={"jsonrpc": "2.0", "method": "notifications/exclusion-probe"}) + notification_response: Final = await client.post( + current_url, + headers=session_headers, + json={"jsonrpc": "2.0", "method": "notifications/exclusion-probe"}, + ) assert notification_response.status_code == 202, notification_response.text await notified.wait() - assert any( - span.name == "notification-handler" for span in exporter.get_finished_spans() - ) is initialized_excluded + assert ( + any(span.name == "notification-handler" for span in exporter.get_finished_spans()) + is initialized_excluded + ) exporter.clear() - modern_response: Final = await client.post(current_url, headers={ - "Accept": "application/json, text/event-stream", "MCP-Protocol-Version": "2026-07-28", - "MCP-Method": "tools/call", "MCP-Name": "probe", - }, json={ - "jsonrpc": "2.0", "id": 2, "method": "tools/call", "params": { - "name": "probe", "arguments": {}, "_meta": { - PROTOCOL_VERSION_META_KEY: "2026-07-28", CLIENT_CAPABILITIES_META_KEY: {}, + modern_response: Final = await client.post( + current_url, + headers={ + "Accept": "application/json, text/event-stream", + "MCP-Protocol-Version": "2026-07-28", + "MCP-Method": "tools/call", + "MCP-Name": "probe", + }, + json={ + "jsonrpc": "2.0", + "id": 2, + "method": "tools/call", + "params": { + "name": "probe", + "arguments": {}, + "_meta": { + PROTOCOL_VERSION_META_KEY: "2026-07-28", + CLIENT_CAPABILITIES_META_KEY: {}, + }, }, }, - }) + ) assert modern_response.status_code == 200, modern_response.text assert '"text":"done"' in modern_response.text, modern_response.text modern_names: Final = {span.name for span in exporter.get_finished_spans()} @@ -9413,8 +9673,7 @@ async def test_get_allowed_mcp_servers_includes_active_servers_submitted_by_user return_value={"submitted-1": submitted_server}, ), patch( - "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp." - "MCPRequestHandler.get_allowed_mcp_servers", + "litellm.proxy.mcp_server.auth.user_api_key_auth_mcp.MCPRequestHandler.get_allowed_mcp_servers", AsyncMock(return_value=[]), ), patch("litellm.proxy.proxy_server.prisma_client", MagicMock()), @@ -10374,9 +10633,7 @@ async def test_handle_list_tools_all_faults_on_continuation_stays_degraded( outcomes={"a": ServerListFault(tag="upstream_error", status_code=429)}, ) - result: Final = await _list_tools_with_listing( - _mcp_request_ctx, listing, PaginatedRequestParams(cursor="cursor-1") - ) + result: Final = await _list_tools_with_listing(_mcp_request_ctx, listing, PaginatedRequestParams(cursor="cursor-1")) assert isinstance(result, ListToolsResult) assert result.tools == [] From e93e01ca4b66b8a675da0af25193b50e6bb65a91 Mon Sep 17 00:00:00 2001 From: yujonglee Date: Thu, 29 Oct 2026 16:56:44 -0700 Subject: [PATCH 04/17] fix(ci): build base native extension for OpenAPI lint (#49136) * fix(openapi): build native extension for base counts * test(openapi): exercise native import in base worktree --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- scripts/openapi_docs_gate.py | 18 +++++++++++++ .../configuration/test_openapi_docs_gate.py | 9 +++++++ tests/unit/test_openapi_docs_gate.py | 25 +++++++++++++++++++ 3 files changed, 52 insertions(+) create mode 100644 tests/integration/configuration/test_openapi_docs_gate.py diff --git a/scripts/openapi_docs_gate.py b/scripts/openapi_docs_gate.py index 5f5d1021c22..de340bf84e7 100755 --- a/scripts/openapi_docs_gate.py +++ b/scripts/openapi_docs_gate.py @@ -101,6 +101,23 @@ def _install_termination_handlers() -> None: signal.signal(termination, _exit_on_termination) +def _install_base_native(worktree: Path, repo_root: Path) -> None: + env: Final = { + **os.environ, + "CARGO_TARGET_DIR": str( + Path(os.environ.get("CARGO_TARGET_DIR", repo_root / "litellm-rust" / "target")).resolve() + ), + } + _run( + [ + "uv", "pip", "install", "--python", sys.executable, "--no-deps", + "--target", str(worktree.parent / "native-install"), "--editable", str(worktree), + ], + cwd=worktree, + env=env, + ) + + def base_counts(ref: str, repo_root: Path = REPO_ROOT, checker: Path = CHECKER) -> Mapping[str, int]: """Rule counts at `ref`, measured with the *current* checker over the base tree.""" _install_termination_handlers() @@ -111,6 +128,7 @@ def base_counts(ref: str, repo_root: Path = REPO_ROOT, checker: Path = CHECKER) (worktree / "scripts").mkdir(parents=True, exist_ok=True) base_checker: Final = worktree / "scripts" / "check_openapi_docs.py" shutil.copy(checker, base_checker) + _install_base_native(worktree, repo_root) return count_by_rule(_check(worktree, base_checker)) finally: # Teardown must never raise, or it masks the real error when the body failed. diff --git a/tests/integration/configuration/test_openapi_docs_gate.py b/tests/integration/configuration/test_openapi_docs_gate.py new file mode 100644 index 00000000000..eefa663cb10 --- /dev/null +++ b/tests/integration/configuration/test_openapi_docs_gate.py @@ -0,0 +1,9 @@ +from typing import Final + +import openapi_docs_gate as gate + + +def test_base_counts_imports_native_extension_from_detached_worktree() -> None: + expected_counts: Final = dict(gate.count_by_rule(gate.head_violations())) + assert expected_counts + assert dict(gate.base_counts("HEAD")) == expected_counts diff --git a/tests/unit/test_openapi_docs_gate.py b/tests/unit/test_openapi_docs_gate.py index fa8a5b1c7e3..b376178c8a9 100644 --- a/tests/unit/test_openapi_docs_gate.py +++ b/tests/unit/test_openapi_docs_gate.py @@ -5,10 +5,35 @@ here is the checker output parse and `introduced`, the diff scan that names the endpoint behind a breach. """ +import subprocess from pathlib import Path from typing import Final +from unittest.mock import patch import openapi_docs_gate as gate +import pytest + + +def test_base_counts_stops_when_the_native_build_fails(tmp_path: Path) -> None: + parent: Final = tmp_path / "oa_base" + parent.mkdir() + checker: Final = tmp_path / "check_openapi_docs.py" + checker.write_text("print('POST /v1/test OA001')\n") + with ( + patch("openapi_docs_gate.tempfile.mkdtemp", return_value=str(parent)), + patch( + "subprocess.run", + side_effect=( + subprocess.CompletedProcess([], 0, stdout="", stderr=""), + subprocess.CompletedProcess([], 1, stdout="", stderr="build failed"), + subprocess.CompletedProcess([], 0, stdout="", stderr=""), + ), + ) as run, + patch("signal.signal"), + ): + with pytest.raises(SystemExit, match="uv exited 1"): + gate.base_counts("HEAD", repo_root=tmp_path, checker=checker) + assert run.call_count == 3 def test_editing_the_checker_rekeys_the_base_counts(tmp_path: Path) -> None: From 08b539e6a8f2ddbf72c0cf7e060943b28a8355f5 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 29 Oct 2026 17:18:51 -0700 Subject: [PATCH 05/17] fix(proxy)!: bill streamed /v1/completions without stream_options like chat (#45366) * fix(proxy): bill streamed /v1/completions without stream_options like chat * test(proxy): mark the text completion stream usage test variable Final * fix(completions): drop Azure's empty prompt-filter frame when the proxy asked for stream usage * test(spend): drop the module docstring from the text completion stream usage test * fix(proxy): keep the internal stream usage strip flag out of the logged request body --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- .../litellm_core_utils/stream_usage_chunks.py | 27 +++ litellm/main.py | 2 + litellm/proxy/common_request_processing.py | 3 +- litellm/proxy/litellm_pre_call_utils.py | 1 + litellm/proxy/proxy_server.py | 27 +-- .../text_completions/streaming_iterator.py | 28 ++- .../test_text_completion_stream_usage_cost.py | 185 ++++++++++++++++++ .../proxy/test_common_request_processing.py | 8 + .../unit/proxy/test_litellm_pre_call_utils.py | 17 +- ...test_proxy_server_endpoints_and_startup.py | 18 +- tests/unit/test_main.py | 99 +++++++++- .../test_streaming_iterator.py | 87 ++++++++ 12 files changed, 454 insertions(+), 48 deletions(-) create mode 100644 litellm/litellm_core_utils/stream_usage_chunks.py create mode 100644 tests/integration/spend/test_text_completion_stream_usage_cost.py diff --git a/litellm/litellm_core_utils/stream_usage_chunks.py b/litellm/litellm_core_utils/stream_usage_chunks.py new file mode 100644 index 00000000000..eda57c7ea12 --- /dev/null +++ b/litellm/litellm_core_utils/stream_usage_chunks.py @@ -0,0 +1,27 @@ +from collections.abc import Iterable +from typing import Final + +from litellm.types.utils import ModelResponseStream, StreamingChoices + + +def is_empty_streaming_choice(choice: StreamingChoices) -> bool: + if choice.finish_reason is not None: + return False + if getattr(choice, "logprobs", None) is not None: + return False + delta: Final = choice.delta if hasattr(choice, "delta") else None + if delta is None: + return True + return _all_values_none(delta.model_dump().values()) + + +def _all_values_none(values: Iterable[object]) -> bool: + return all(value is None for value in values) + + +def is_injected_stream_usage_artifact(chunk: object) -> bool: + if not isinstance(chunk, ModelResponseStream): + return False + if chunk.provider_specific_fields is not None: + return False + return all(is_empty_streaming_choice(choice) for choice in chunk.choices or []) diff --git a/litellm/main.py b/litellm/main.py index 50ab274a060..0162742de18 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -7975,6 +7975,8 @@ async def atext_completion(*args, **kwargs) -> TextCompletionResponse | TextComp model=model, custom_llm_provider=custom_llm_provider, stream_options=kwargs.get("stream_options"), + usage_injected_by_proxy="_litellm_strip_stream_usage" in kwargs + and kwargs["_litellm_strip_stream_usage"] is True, ) else: ## OpenAI / Azure Text Completion Returns here diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index e810a5ab329..666269be2eb 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -588,6 +588,7 @@ def model_deployments_support_stream_options( _STREAM_USAGE_FLAGS: Final = ("_litellm_strip_stream_usage", STREAM_USAGE_INJECTED_FIELD) +_STREAM_USAGE_TRACKED_ROUTES: Final = frozenset({"acompletion", "atext_completion"}) def stream_usage_tracking_updates( @@ -613,7 +614,7 @@ def stream_usage_tracking_updates( return scrub if always_include is False and route_type == "agenerate_content_stream" and "stream_options" not in data: return {**scrub, "stream_options": {"include_usage": False}} - if always_include is False or route_type != "acompletion": + if always_include is False or route_type not in _STREAM_USAGE_TRACKED_ROUTES: return scrub if isinstance(stream_options, dict) and stream_options.get("include_usage") is True: return scrub diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index 8f60311e491..a87fc815402 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -2505,6 +2505,7 @@ def refresh_proxy_server_request_body_snapshot( "litellm_logging_obj", VECTOR_STORE_CALLER_METADATA_KWARG, STREAM_USAGE_INJECTED_FIELD, + "_litellm_strip_stream_usage", } ) | _TRANSPORT_ONLY_CREDENTIAL_KEYS diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index d4ec26614c0..e6d157a7f81 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -201,7 +201,6 @@ from litellm.types.utils import ( CountContextManagement, ModelResponse, ModelResponseStream, - StreamingChoices, TextCompletionResponse, TokenCountMessagesFormat, TokenCountResponse, @@ -384,6 +383,7 @@ from litellm.litellm_core_utils.sensitive_data_masker import ( SensitiveDataMasker, mask_sensitive_keys, ) +from litellm.litellm_core_utils.stream_usage_chunks import is_injected_stream_usage_artifact from litellm.llms.anthropic.messages.convertible_blocks import ( deployment_drops_params, drop_non_string_tool_result_texts, @@ -10214,29 +10214,6 @@ def _serialize_streaming_chunk(chunk: BaseModel) -> str | bytes: return chunk.model_dump_json(exclude_none=True, exclude_unset=True) -def _is_injected_stream_usage_artifact(chunk: object) -> bool: - if not isinstance(chunk, ModelResponseStream): - return False - if chunk.provider_specific_fields is not None: - return False - return all(_is_empty_streaming_choice(choice) for choice in chunk.choices or []) - - -def _is_empty_streaming_choice(choice: StreamingChoices) -> bool: - if choice.finish_reason is not None: - return False - if getattr(choice, "logprobs", None) is not None: - return False - delta: Final = choice.delta if hasattr(choice, "delta") else None - if delta is None: - return True - return _all_values_none(delta.model_dump().values()) - - -def _all_values_none(values: Iterable[object]) -> bool: - return all(value is None for value in values) - - async def _apply_streaming_chunk_hooks( *, chunk: Any, @@ -10616,7 +10593,7 @@ async def async_data_generator( fallback_model_from_metadata=fallback_model_from_metadata, ) - if strip_stream_usage and _is_injected_stream_usage_artifact(chunk): + if strip_stream_usage and is_injected_stream_usage_artifact(chunk): if pending_fallback_event: yield _format_fallback_metadata_sse_event( fallback_model=fallback_model_from_metadata, diff --git a/litellm/text_completions/streaming_iterator.py b/litellm/text_completions/streaming_iterator.py index ec1b96fd432..61e47e9d7af 100644 --- a/litellm/text_completions/streaming_iterator.py +++ b/litellm/text_completions/streaming_iterator.py @@ -1,6 +1,7 @@ from typing import Final -from litellm.types.utils import Choices, ModelResponse, TextChoices, TextCompletionResponse +from litellm.litellm_core_utils.stream_usage_chunks import is_injected_stream_usage_artifact +from litellm.types.utils import Choices, ModelResponse, ModelResponseStream, TextChoices, TextCompletionResponse class TextCompletionStreamWrapper: @@ -10,11 +11,18 @@ class TextCompletionStreamWrapper: model, stream_options: dict | None = None, custom_llm_provider: str | None = None, + usage_injected_by_proxy: bool = False, ): self.completion_stream = completion_stream self.model = model self.stream_options = stream_options self.custom_llm_provider = custom_llm_provider + self.usage_injected_by_proxy: Final = usage_injected_by_proxy + self.include_usage: Final = ( + not usage_injected_by_proxy + and stream_options is not None + and stream_options.get("include_usage", False) is True + ) def __iter__(self): return self @@ -22,7 +30,7 @@ class TextCompletionStreamWrapper: def __aiter__(self): return self - def convert_to_text_completion_object(self, chunk: ModelResponse): + def convert_to_text_completion_object(self, chunk: ModelResponse | ModelResponseStream): try: response: Final = TextCompletionResponse() response["id"] = chunk.get("id", None) @@ -43,8 +51,7 @@ class TextCompletionStreamWrapper: else: response["choices"] = [] - # only pass usage when stream_options["include_usage"] is True - if self.stream_options and self.stream_options.get("include_usage", False) is True: + if self.include_usage: response.usage = chunk.get("usage", None) else: response.usage = None @@ -53,6 +60,9 @@ class TextCompletionStreamWrapper: except Exception as e: raise Exception(f"Error occurred converting to text completion object - chunk: {chunk}; Error: {e}") + def _is_injected_usage_artifact(self, chunk: ModelResponseStream) -> bool: + return self.usage_injected_by_proxy and is_injected_stream_usage_artifact(chunk) + def __next__(self): # model_response = ModelResponse(stream=True, model=self.model) TextCompletionResponse() @@ -60,8 +70,9 @@ class TextCompletionStreamWrapper: for chunk in self.completion_stream: if chunk == "None" or chunk is None: raise Exception - processed_chunk = self.convert_to_text_completion_object(chunk=chunk) - return processed_chunk + if isinstance(chunk, ModelResponseStream) and self._is_injected_usage_artifact(chunk): + continue + return self.convert_to_text_completion_object(chunk=chunk) raise StopIteration except StopIteration: raise StopIteration @@ -81,8 +92,9 @@ class TextCompletionStreamWrapper: async for chunk in self.completion_stream: if chunk == "None" or chunk is None: raise Exception - processed_chunk = self.convert_to_text_completion_object(chunk=chunk) - return processed_chunk + if isinstance(chunk, ModelResponseStream) and self._is_injected_usage_artifact(chunk): + continue + return self.convert_to_text_completion_object(chunk=chunk) raise StopIteration except StopIteration: raise StopAsyncIteration diff --git a/tests/integration/spend/test_text_completion_stream_usage_cost.py b/tests/integration/spend/test_text_completion_stream_usage_cost.py new file mode 100644 index 00000000000..4d66e60f3e7 --- /dev/null +++ b/tests/integration/spend/test_text_completion_stream_usage_cost.py @@ -0,0 +1,185 @@ +import json +from pathlib import Path +from typing import Final +from uuid import uuid4 + +import pytest +from integration._support.client import Gateway, eventually +from integration._support.database import read_rows +from integration._support.process import owned_proxy +from integration._support.wire import Reply, Request, wire_server +from pydantic import JsonValue + +PROMPT_TOKENS: Final = 19 +COMPLETION_TOKENS: Final = 5 +INPUT_RATE: Final = 0.001 +OUTPUT_RATE: Final = 0.002 +EXPECTED_SPEND: Final = PROMPT_TOKENS * INPUT_RATE + COMPLETION_TOKENS * OUTPUT_RATE +USAGE: Final[dict[str, JsonValue]] = { + "prompt_tokens": PROMPT_TOKENS, + "completion_tokens": COMPLETION_TOKENS, + "total_tokens": PROMPT_TOKENS + COMPLETION_TOKENS, +} +PROMPT: Final = "Say hi" + + +def _sse_frame(payload: dict[str, JsonValue]) -> bytes: + return f"data: {json.dumps(payload, separators=(',', ':'))}\n\n".encode() + + +def _chunk(request_id: str, model: str, delta: dict[str, JsonValue], finish_reason: str | None) -> dict[str, JsonValue]: + return { + "id": request_id, + "object": "chat.completion.chunk", + "created": 1, + "model": model, + "choices": [{"index": 0, "delta": delta, "finish_reason": finish_reason}], + } + + +def _respond(request: Request) -> Reply: + if request.method == "GET" and request.target == "/v1/models": + return Reply(body=json.dumps({"object": "list", "data": [{"id": "gpt-5.4-mini", "object": "model"}]}).encode()) + assert request.method == "POST", request.method + assert request.target == "/v1/chat/completions", request.target + body: Final = json.loads(request.body) + assert body["messages"] == [{"role": "user", "content": PROMPT}], body + request_id: Final = f"chatcmpl-{uuid4().hex}" + model: Final = str(body["model"]) + if body.get("stream") is not True: + return Reply( + body=json.dumps( + { + "id": request_id, + "object": "chat.completion", + "created": 1, + "model": model, + "choices": [ + {"index": 0, "message": {"role": "assistant", "content": "Hi there"}, "finish_reason": "length"} + ], + "usage": USAGE, + } + ).encode() + ) + stream_options: Final = body.get("stream_options") or {} + usage_frames: Final = ( + ( + _sse_frame( + { + "id": request_id, + "object": "chat.completion.chunk", + "created": 1, + "model": model, + "choices": [], + "usage": USAGE, + } + ), + ) + if stream_options.get("include_usage") is True + else () + ) + return Reply( + content_type="text/event-stream", + chunks=( + _sse_frame(_chunk(request_id, model, {"role": "assistant", "content": "Hi"}, None)), + _sse_frame(_chunk(request_id, model, {"content": " there"}, None)), + _sse_frame(_chunk(request_id, model, {}, "length")), + *usage_frames, + b"data: [DONE]\n\n", + ), + ) + + +def _proxy_config(directory: Path, model: str, upstream_url: str) -> Path: + config: Final = directory / "text_completion_stream_usage_cost_config.yaml" + config.write_text( + json.dumps( + { + "model_list": [ + { + "model_name": model, + "litellm_params": { + "model": "openai/gpt-5.4-mini", + "api_base": f"{upstream_url}/v1", + "api_key": "scripted-upstream", + "input_cost_per_token": INPUT_RATE, + "output_cost_per_token": OUTPUT_RATE, + }, + } + ], + "general_settings": { + "master_key": "os.environ/LITELLM_MASTER_KEY", + "database_url": "os.environ/DATABASE_URL", + "store_model_in_db": True, + "disable_spend_logs": False, + "proxy_batch_write_at": 1, + "proxy_batch_polling_interval": 1, + }, + "router_settings": {"disable_cooldowns": True}, + } + ) + ) + return config + + +def _data_events(body: str) -> tuple[dict[str, JsonValue], ...]: + return tuple( + json.loads(line.removeprefix("data:")) + for line in body.splitlines() + if line.startswith("data:") and not line.endswith("[DONE]") + ) + + +def _carries_text_or_finish(frame: dict[str, JsonValue]) -> bool: + choices: Final = frame["choices"] + assert isinstance(choices, list) and len(choices) == 1, frame + choice: Final = choices[0] + assert isinstance(choice, dict), frame + return choice.get("text") is not None or choice.get("finish_reason") is not None + + +def _billed(row: dict[str, object]) -> tuple[object, object, float]: + return (row["prompt_tokens"], row["completion_tokens"], float(str(row["spend"]))) + + +def test_text_completion_stream_without_stream_options_bills_like_its_twins(gateway: Gateway, tmp_path: Path) -> None: + model: Final = f"gpt-5.4-mini-{uuid4().hex}" + body: Final[dict[str, JsonValue]] = {"model": model, "prompt": PROMPT, "max_tokens": COMPLETION_TOKENS} + + with wire_server(_respond) as wire: + config: Final = _proxy_config(tmp_path, model, wire.url) + with owned_proxy(gateway, tmp_path, {}, config=config) as candidate: + silent: Final = candidate.request("POST", "/v1/completions", {**body, "stream": True}) + asked: Final = candidate.request( + "POST", "/v1/completions", {**body, "stream": True, "stream_options": {"include_usage": True}} + ) + whole: Final = candidate.request("POST", "/v1/completions", body) + assert (silent.status_code, asked.status_code, whole.status_code) == (200, 200, 200), ( + silent.text, + asked.text, + whole.text, + ) + silent_frames: Final = _data_events(silent.text) + asked_frames: Final = _data_events(asked.text) + request_ids: Final = (silent_frames[0]["id"], asked_frames[0]["id"], whole.json()["id"]) + rows: Final = eventually( + lambda: read_rows( + 'SELECT request_id, prompt_tokens, completion_tokens, spend FROM "LiteLLM_SpendLogs" ' + "WHERE request_id = ANY(%s)", + (list(request_ids),), + database_url=candidate.database_url, + ), + lambda values: len(values) == 3, + seconds=70, + ) + upstream_stream_options: Final = [ + json.loads(sent.body).get("stream_options") for sent in wire.drain() if sent.method == "POST" + ] + + assert upstream_stream_options == [{"include_usage": True}, {"include_usage": True}, None] + assert all(_carries_text_or_finish(frame) for frame in silent_frames), silent.text + assert all("usage" not in frame for frame in silent_frames), silent.text + assert asked_frames[-1]["usage"]["total_tokens"] == PROMPT_TOKENS + COMPLETION_TOKENS, asked.text + billed: Final = {str(row["request_id"]): _billed(row) for row in rows} + assert billed[str(request_ids[0])] == billed[str(request_ids[1])] == billed[str(request_ids[2])], billed + assert billed[str(request_ids[0])] == (PROMPT_TOKENS, COMPLETION_TOKENS, pytest.approx(EXPECTED_SPEND)), billed diff --git a/tests/unit/proxy/test_common_request_processing.py b/tests/unit/proxy/test_common_request_processing.py index 55b913aa7e4..cc1fc5cd6c3 100644 --- a/tests/unit/proxy/test_common_request_processing.py +++ b/tests/unit/proxy/test_common_request_processing.py @@ -9400,6 +9400,14 @@ class TestApplyStreamUsageTracking: assert data["_litellm_strip_stream_usage"] is True assert data["_litellm_stream_usage_injected"] is True + def test_default_injects_usage_and_marks_strip_for_text_completions(self): + data: Final = {"stream": True, "model": "gpt-5.4-nano"} + + _apply_stream_usage_tracking(data=data, general_settings={}, route_type="atext_completion") + + assert data["stream_options"] == {"include_usage": True} + assert data["_litellm_strip_stream_usage"] is True + def test_default_preserves_other_client_stream_options_keys(self): data = {"stream": True, "stream_options": {"include_obfuscation": True}} diff --git a/tests/unit/proxy/test_litellm_pre_call_utils.py b/tests/unit/proxy/test_litellm_pre_call_utils.py index 00051462b3c..1ce61a8e9b3 100644 --- a/tests/unit/proxy/test_litellm_pre_call_utils.py +++ b/tests/unit/proxy/test_litellm_pre_call_utils.py @@ -1203,24 +1203,35 @@ def test_body_snapshot_excludes_team_callback_credentials() -> None: }, proxy_request -def test_body_snapshot_keeps_the_stream_usage_injection_marker_out_of_the_stored_request() -> None: +@pytest.mark.parametrize( + ("route_type", "request_input"), + [ + ("acompletion", {"messages": [{"role": "user", "content": "hi"}]}), + ("atext_completion", {"prompt": "hi"}), + ], +) +def test_body_snapshot_keeps_the_stream_usage_injection_marker_out_of_the_stored_request( + route_type: str, request_input: dict[str, object] +) -> None: from litellm.proxy.common_request_processing import stream_usage_tracking_updates from litellm.proxy.litellm_pre_call_utils import refresh_proxy_server_request_body_snapshot from litellm.types.litellm_params import STREAM_USAGE_INJECTED_FIELD - request: Final = {"model": "gpt-5-mini", "messages": [{"role": "user", "content": "hi"}], "stream": True} + request: Final = {"model": "gpt-5-mini", **request_input, "stream": True} proxy_request: Final = {"body": dict(request)} data: Final = { **request, - **stream_usage_tracking_updates(request, {}, "acompletion", lambda: True), + **stream_usage_tracking_updates(request, {}, route_type, lambda: True), "proxy_server_request": proxy_request, } refresh_proxy_server_request_body_snapshot(data) assert data[STREAM_USAGE_INJECTED_FIELD] is True + assert data["_litellm_strip_stream_usage"] is True assert proxy_request["body"]["stream_options"] == {"include_usage": True} assert STREAM_USAGE_INJECTED_FIELD not in proxy_request["body"] + assert "_litellm_strip_stream_usage" not in proxy_request["body"] @pytest.mark.asyncio diff --git a/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py b/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py index 02f88d444e4..023ef0b4606 100644 --- a/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py +++ b/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py @@ -13583,29 +13583,29 @@ def _stream_usage_generator_chunks(): def test_is_injected_stream_usage_artifact(): - from litellm.proxy.proxy_server import _is_injected_stream_usage_artifact + from litellm.litellm_core_utils.stream_usage_chunks import is_injected_stream_usage_artifact from litellm.types.utils import ModelResponseStream, Usage content_chunk, finish_chunk, empty_choices_usage_chunk = _stream_usage_test_chunks() - assert _is_injected_stream_usage_artifact(empty_choices_usage_chunk) is True + assert is_injected_stream_usage_artifact(empty_choices_usage_chunk) is True synthetic_final_chunk = ModelResponseStream(model="gpt-5.4-nano") synthetic_final_chunk.usage = Usage(prompt_tokens=50, completion_tokens=188, total_tokens=238) - assert _is_injected_stream_usage_artifact(synthetic_final_chunk) is True + assert is_injected_stream_usage_artifact(synthetic_final_chunk) is True azure_prompt_filter_chunk = ModelResponseStream(model="gpt-5.4-nano", choices=[]) - assert _is_injected_stream_usage_artifact(azure_prompt_filter_chunk) is True + assert is_injected_stream_usage_artifact(azure_prompt_filter_chunk) is True - assert _is_injected_stream_usage_artifact(content_chunk) is False - assert _is_injected_stream_usage_artifact(finish_chunk) is False + assert is_injected_stream_usage_artifact(content_chunk) is False + assert is_injected_stream_usage_artifact(finish_chunk) is False content_chunk_with_usage, finish_chunk_with_usage, _ = _stream_usage_test_chunks() content_chunk_with_usage.usage = Usage(prompt_tokens=50, completion_tokens=188, total_tokens=238) finish_chunk_with_usage.usage = Usage(prompt_tokens=50, completion_tokens=188, total_tokens=238) - assert _is_injected_stream_usage_artifact(content_chunk_with_usage) is False - assert _is_injected_stream_usage_artifact(finish_chunk_with_usage) is False + assert is_injected_stream_usage_artifact(content_chunk_with_usage) is False + assert is_injected_stream_usage_artifact(finish_chunk_with_usage) is False - assert _is_injected_stream_usage_artifact({"usage": {"prompt_tokens": 1}}) is False + assert is_injected_stream_usage_artifact({"usage": {"prompt_tokens": 1}}) is False async def _collect_async_data_generator_frames(request_data: dict) -> list: diff --git a/tests/unit/test_main.py b/tests/unit/test_main.py index e10352e0b3d..11104cf7bca 100644 --- a/tests/unit/test_main.py +++ b/tests/unit/test_main.py @@ -48,7 +48,14 @@ from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler from litellm.types.litellm_params import ControlOptions from litellm.types.llms.openai import AllMessageValues, HttpxBinaryResponseContent from litellm.types.prompts.init_prompts import PromptSpec -from litellm.types.utils import Delta, ModelResponseStream, StandardCallbackDynamicParams, StreamingChoices, Usage +from litellm.types.utils import ( + Delta, + ModelResponseStream, + StandardCallbackDynamicParams, + StreamingChoices, + TextCompletionResponse, + Usage, +) from tests.unit.llms.bedrock.slow_upstream import ( CONVERSE_ANSWER, STREAM_TIMEOUT_SECONDS, @@ -4136,6 +4143,95 @@ def test_mock_text_completion_stream_and_non_stream_report_the_same_zero_admissi assert non_stream.usage.prompt_tokens == 0 +def _carries_text_or_finish(frame: TextCompletionResponse) -> bool: + return frame.choices[0].text is not None or frame.choices[0].finish_reason is not None + + +@pytest.mark.asyncio +async def test_mock_atext_completion_stream_hides_the_usage_chunk_the_proxy_asked_for_on_the_clients_behalf(): + response: Final = await litellm.atext_completion( + model="openai/gpt-5.4-mini", + prompt="hi", + mock_response="ok then", + api_key="mock", + stream=True, + stream_options={"include_usage": True}, + _litellm_strip_stream_usage=True, + ) + + frames: Final[list[TextCompletionResponse]] = [frame async for frame in response] + + assert len(frames) == 4 + assert all(_carries_text_or_finish(frame) for frame in frames) + assert all(frame.usage is None for frame in frames) + + +def _native_completions_reply(outbound: list[dict[str, object]]) -> Callable[[httpx.Request], httpx.Response]: + def respond(request: httpx.Request) -> httpx.Response: + outbound.append(json.loads(request.content)) + base: Final = {"id": "cmpl-native", "object": "text_completion", "created": 1, "model": "code-model"} + frames: Final = ( + {**base, "choices": [{"index": 0, "text": "def ", "finish_reason": None, "logprobs": None}]}, + {**base, "choices": [{"index": 0, "text": "main", "finish_reason": "length", "logprobs": None}]}, + {**base, "choices": [], "usage": {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15}}, + ) + content: Final = "".join(f"data: {json.dumps(frame)}\n\n" for frame in frames) + "data: [DONE]\n\n" + return httpx.Response(200, headers={"content-type": "text/event-stream"}, content=content.encode()) + + return respond + + +def _native_completions_upstream(outbound: list[dict[str, object]]) -> OpenAI: + transport: Final = httpx.MockTransport(_native_completions_reply(outbound)) + return OpenAI(api_key="transport-only", http_client=httpx.Client(transport=transport)) + + +def _async_native_completions_upstream(outbound: list[dict[str, object]]) -> AsyncOpenAI: + transport: Final = httpx.MockTransport(_native_completions_reply(outbound)) + return AsyncOpenAI(api_key="transport-only", http_client=httpx.AsyncClient(transport=transport)) + + +@pytest.mark.asyncio +async def test_native_atext_completion_stream_hides_the_usage_chunk_the_proxy_asked_for_on_the_clients_behalf(): + outbound: Final[list[dict[str, object]]] = [] + response: Final = await litellm.atext_completion( + model="text-completion-openai/code-model", + prompt="hi", + client=_async_native_completions_upstream(outbound), + api_key="transport-only", + stream=True, + stream_options={"include_usage": True}, + _litellm_strip_stream_usage=True, + ) + + frames: Final[list[TextCompletionResponse]] = [frame async for frame in response] + + assert [sent.get("stream_options") for sent in outbound] == [{"include_usage": True}] + assert [(frame.choices[0].text, frame.choices[0].finish_reason) for frame in frames] == [ + ("def ", None), + ("main", None), + (None, "length"), + ] + assert all(frame.usage is None for frame in frames) + + +def test_native_text_completion_stream_keeps_the_usage_chunk_the_client_asked_for(): + frames: Final = list( + litellm.text_completion( + model="text-completion-openai/code-model", + prompt="hi", + client=_native_completions_upstream([]), + api_key="transport-only", + stream=True, + stream_options={"include_usage": True}, + ) + ) + + assert len(frames) == 4 + assert all(_carries_text_or_finish(frame) for frame in frames[:3]) + assert (frames[-1].usage.prompt_tokens, frames[-1].usage.completion_tokens) == (10, 5) + + def test_mock_completion_stream_with_model_response(): """Test that mock_completion correctly handles stream=True with a ModelResponse as mock_response.""" from litellm import completion @@ -4293,7 +4389,6 @@ class TestCallTypesOCR: def test_stream_chunk_builder_text_completion_combines_text_and_usage(): from litellm.main import stream_chunk_builder_text_completion - from litellm.types.utils import TextCompletionResponse chunks = [ TextCompletionResponse( diff --git a/tests/unit/text_completions/test_streaming_iterator.py b/tests/unit/text_completions/test_streaming_iterator.py index b7edffc0753..e5459a5a0cd 100644 --- a/tests/unit/text_completions/test_streaming_iterator.py +++ b/tests/unit/text_completions/test_streaming_iterator.py @@ -142,3 +142,90 @@ def test_sync_text_completion_stream_maps_upstream_failure_after_content() -> No next(wrapper) assert (error.value.model, error.value.llm_provider) == ("test-model", "openai") + + +def _stream_with_trailing_usage_chunk() -> tuple[ModelResponseStream, ...]: + return ( + ModelResponseStream(choices=[StreamingChoices(index=0, delta=Delta(content="Hi"))]), + ModelResponseStream(choices=[StreamingChoices(index=0, delta=Delta(), finish_reason="length")]), + ModelResponseStream( + choices=[StreamingChoices(index=0, delta=Delta())], + usage=Usage(prompt_tokens=19, completion_tokens=5, total_tokens=24), + ), + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("async_stream", (False, True)) +@pytest.mark.parametrize( + ("usage_injected_by_proxy", "expected_frames", "expected_usage"), + ( + (True, [("Hi", None), (None, "length")], [None, None]), + (False, [("Hi", None), (None, "length"), (None, None)], [None, None, (19, 5)]), + ), +) +async def test_text_completion_stream_hides_only_the_usage_chunk_the_proxy_asked_for( + async_stream: bool, + usage_injected_by_proxy: bool, + expected_frames: list[tuple[str | None, str | None]], + expected_usage: list[tuple[int, int] | None], +) -> None: + async def chunks() -> AsyncIterator[ModelResponseStream]: + for chunk in _stream_with_trailing_usage_chunk(): + yield chunk + + wrapper: Final = TextCompletionStreamWrapper( + completion_stream=chunks() if async_stream else iter(_stream_with_trailing_usage_chunk()), + model="gpt-5.4-mini", + stream_options={"include_usage": True}, + usage_injected_by_proxy=usage_injected_by_proxy, + ) + frames: Final = [frame async for frame in wrapper] if async_stream else list(wrapper) + + assert [(frame.choices[0].text, frame.choices[0].finish_reason) for frame in frames] == expected_frames + assert [ + None if frame.usage is None else (frame.usage.prompt_tokens, frame.usage.completion_tokens) for frame in frames + ] == expected_usage + + +def test_text_completion_stream_keeps_a_finish_frame_but_hides_the_usage_the_proxy_asked_for() -> None: + finish_with_usage: Final = ModelResponseStream( + choices=[StreamingChoices(index=0, delta=Delta(), finish_reason="stop")], + usage=Usage(prompt_tokens=27, completion_tokens=5, total_tokens=32), + ) + wrapper: Final = TextCompletionStreamWrapper( + completion_stream=iter((finish_with_usage,)), + model="claude-haiku-4-5", + stream_options={"include_usage": True}, + usage_injected_by_proxy=True, + ) + + frames: Final = list(wrapper) + + assert [(frame.choices[0].finish_reason, frame.usage) for frame in frames] == [("stop", None)] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("async_stream", (False, True)) +async def test_text_completion_stream_hides_the_empty_prompt_filter_frame_when_the_proxy_asked_for_usage( + async_stream: bool, +) -> None: + prompt_filter_frame: Final = ModelResponseStream(id="", choices=[]) + upstream: Final = (prompt_filter_frame, *_stream_with_trailing_usage_chunk()) + + async def chunks() -> AsyncIterator[ModelResponseStream]: + for chunk in upstream: + yield chunk + + wrapper: Final = TextCompletionStreamWrapper( + completion_stream=chunks() if async_stream else iter(upstream), + model="gpt-5.4-mini", + stream_options={"include_usage": True}, + usage_injected_by_proxy=True, + ) + frames: Final = [frame async for frame in wrapper] if async_stream else list(wrapper) + + assert [(frame.choices[0].text, frame.choices[0].finish_reason) for frame in frames] == [ + ("Hi", None), + (None, "length"), + ] From 68b5a737a17a59f1f3f3b2682b3c3f0d474ea2f7 Mon Sep 17 00:00:00 2001 From: zoveki <349028329+zoveki@users.noreply.github.com> Date: Thu, 29 Oct 2026 17:18:59 -0700 Subject: [PATCH 06/17] fix(anthropic): retry /v1/messages without strict when Claude rejects strict tools under drop_params (#49102) * fix(anthropic): retry /v1/messages without strict when Claude rejects strict tools under drop_params The native /v1/messages path sent tools with `strict` straight through, so when Claude's strict-tool limits rejected them with a 400 the caller got that 400 even with drop_params set. Under drop_params, a 400 that matches Claude's strict-limit phrases on a request with a strict tool is now sent once more with `strict` removed from every tool. Without drop_params nothing changes. The Claude strict-limit matcher moves from bedrock/common_utils.py to anthropic/common_utils.py so Bedrock converse and the base messages config share it, and the Rust bridge routes strict tools under drop_params to Python. * fix(anthropic): retry without strict only when a tool sends strict true A tool with only strict false no longer triggers a resend of an identical body, the warning no longer blames tool schemas when the rejected schema may be the output format, and requests with only additional_drop_params stay on the Rust messages route since the Python retry ignores them * fix(anthropic): keep sync messages calls with strict tools on Rust Python has no sync /v1/messages handler, so routing a sync call with a strict tool to Python raised ValueError where Rust served it before. The strict-tools detector now applies to async calls only. The retry warning also names the strict tool count without a plural. --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- litellm/llms/anthropic/common_utils.py | 43 +++++ .../anthropic_messages/transformation.py | 24 +++ .../bedrock/chat/native_output_fallback.py | 4 +- litellm/llms/bedrock/common_utils.py | 11 -- litellm/llms/custom_httpx/llm_http_handler.py | 3 +- litellm/messages/dispatch.py | 1 + litellm/rust_bridge/catalog.py | 3 + litellm/rust_bridge/messages/detectors.py | 8 + .../messages/test_dispatch.py | 46 +++++ .../native/test_anthropic_messages_handler.py | 61 ++++++- .../anthropic/test_anthropic_common_utils.py | 26 ++- .../test_anthropic_claude_transformation.py | 160 +++++++++++++++++- tests/unit/rust_bridge/test_detectors.py | 16 ++ 13 files changed, 387 insertions(+), 19 deletions(-) diff --git a/litellm/llms/anthropic/common_utils.py b/litellm/llms/anthropic/common_utils.py index 3b7fcdea0e4..0bc519187f4 100644 --- a/litellm/llms/anthropic/common_utils.py +++ b/litellm/llms/anthropic/common_utils.py @@ -6,6 +6,7 @@ import copy import json import re from collections.abc import Container, Mapping, MutableMapping, Sequence +from dataclasses import dataclass from datetime import datetime, timezone from types import MappingProxyType from typing import Annotated, Any, ClassVar, Final, Literal, TypeVar @@ -1774,6 +1775,48 @@ def anthropic_request_errors_on_thinking_binding_mismatch(request_body: object) return binding is not None and binding.get("prefix_mismatch_behavior") == "error" +_CLAUDE_STRICT_TOOLS_REJECTION: Final = re.compile( + r"Too many strict tools \(\d+\)" + r"|Schemas contains too many (?:optional parameters|parameters with union types)" + r"|[Cc]ompiled grammar (?:is too large|size)" + r"|tools\.\d+\.custom(?:\.strict)?: " + r"|Grammar compilation timed out" +) +_CLAUDE_TOOLS_ADAPTER: Final = TypeAdapter(tuple[Mapping[str, object], ...]) + + +def is_claude_strict_tools_rejection(status_code: int, error_text: str) -> bool: + return status_code == 400 and _CLAUDE_STRICT_TOOLS_REJECTION.search(error_text) is not None + + +@dataclass(frozen=True, slots=True) +class ToolsWithoutStrict: + tools: tuple[Mapping[str, object], ...] + dropped_count: int + + +def tools_without_strict(tools: object) -> ToolsWithoutStrict | None: + try: + validated: Final = _CLAUDE_TOOLS_ADAPTER.validate_python(tools) + except ValidationError: + return None + dropped_count: Final = sum(tool.get("strict") is True for tool in validated) + if dropped_count == 0: + return None + return ToolsWithoutStrict( + tools=tuple({key: value for key, value in tool.items() if key != "strict"} for tool in validated), + dropped_count=dropped_count, + ) + + +def warn_claude_strict_tools_dropped(dropped_count: int) -> None: + litellm.verbose_logger.warning( + "Claude rejected the request's strict schemas, so litellm removed `strict` from every tool (%d had " + "strict true) and sent the request again because drop_params is set.", + dropped_count, + ) + + def strip_thinking_blocks_from_anthropic_messages(messages: Sequence[object]) -> list[object]: """ Return a new message list with thinking / redacted_thinking content blocks removed diff --git a/litellm/llms/base_llm/anthropic_messages/transformation.py b/litellm/llms/base_llm/anthropic_messages/transformation.py index bda7015ab2b..d234ac40c3d 100644 --- a/litellm/llms/base_llm/anthropic_messages/transformation.py +++ b/litellm/llms/base_llm/anthropic_messages/transformation.py @@ -4,6 +4,7 @@ from typing import TYPE_CHECKING, Any, Final import httpx +from litellm.litellm_core_utils.core_helpers import drops_unsupported_params from litellm.types.llms.anthropic_messages.anthropic_response import ( AnthropicMessagesResponse, ) @@ -201,11 +202,23 @@ class BaseAnthropicMessagesConfig(ABC): anthropic_request_errors_on_thinking_binding_mismatch, is_anthropic_invalid_thinking_block_error, is_anthropic_thinking_binding_mismatch_error, + is_claude_strict_tools_rejection, + tools_without_strict, + validated_object_mapping, ) if e.response.status_code != 400: return False error_text: Final[str] = e.response.text + body: Final = validated_object_mapping(request_body) + drops_rejected_strict_tools: Final = ( + drops_unsupported_params(litellm_params.get("drop_params")) + and is_claude_strict_tools_rejection(e.response.status_code, error_text) + and body is not None + and tools_without_strict(body.get("tools")) is not None + ) + if drops_rejected_strict_tools: + return True caller_asked_for_this_error: Final = anthropic_request_errors_on_thinking_binding_mismatch( request_body ) and is_anthropic_thinking_binding_mismatch_error(error_text) @@ -219,11 +232,22 @@ class BaseAnthropicMessagesConfig(ABC): """ from litellm.llms.anthropic.common_utils import ( is_anthropic_invalid_thinking_block_error, + is_claude_strict_tools_rejection, strip_thinking_blocks_from_anthropic_messages_request_dict, + tools_without_strict, + warn_claude_strict_tools_dropped, ) if e.response.status_code == 400 and is_anthropic_invalid_thinking_block_error(e.response.text): strip_thinking_blocks_from_anthropic_messages_request_dict(request_data) + without_strict: Final = ( + tools_without_strict(request_data.get("tools")) + if is_claude_strict_tools_rejection(e.response.status_code, e.response.text) + else None + ) + if without_strict is not None: + warn_claude_strict_tools_dropped(without_strict.dropped_count) + request_data["tools"] = list(without_strict.tools) # rebind-ok: the retry loop re-signs this same dict return request_data def native_output_fallback_litellm_params( diff --git a/litellm/llms/bedrock/chat/native_output_fallback.py b/litellm/llms/bedrock/chat/native_output_fallback.py index 6892cd37beb..8069c3cf7e7 100644 --- a/litellm/llms/bedrock/chat/native_output_fallback.py +++ b/litellm/llms/bedrock/chat/native_output_fallback.py @@ -2,11 +2,11 @@ from collections.abc import Callable, Coroutine from functools import partial from typing import Final, TypeAlias +from litellm.llms.anthropic.common_utils import is_claude_strict_tools_rejection from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.bedrock.common_utils import ( NativeOutputFallback, is_bedrock_native_output_rejection, - is_bedrock_strict_tools_rejection, warn_bedrock_native_output_rejected, warn_bedrock_strict_tools_dropped, ) @@ -34,7 +34,7 @@ def _native_output_retry( def _strict_tools_retry( error: BaseLLMException, fallback: NativeOutputFallbackFactory | None, model: str ) -> NativeOutputFallback | None: - if fallback is None or not is_bedrock_strict_tools_rejection(error.status_code, error.message): + if fallback is None or not is_claude_strict_tools_rejection(error.status_code, error.message): return None retry: Final = fallback() if retry is not None: diff --git a/litellm/llms/bedrock/common_utils.py b/litellm/llms/bedrock/common_utils.py index 4175479f068..d88d7e6280c 100644 --- a/litellm/llms/bedrock/common_utils.py +++ b/litellm/llms/bedrock/common_utils.py @@ -366,21 +366,10 @@ class NativeOutputFallback: litellm_params: Mapping[str, object] -_BEDROCK_STRICT_TOOLS_REJECTION: Final = re.compile( - r"Too many strict tools \(\d+\)" - r"|Schemas contains too many (?:optional parameters|parameters with union types)" - r"|[Cc]ompiled grammar (?:is too large|size)" - r"|tools\.\d+\.custom(?:\.strict)?: " - r"|Grammar compilation timed out" -) _TOOLS_ADAPTER: Final = TypeAdapter(tuple[Mapping[str, object], ...]) _TOOL_SPEC_ADAPTER: Final = TypeAdapter(Mapping[str, object]) -def is_bedrock_strict_tools_rejection(status_code: int, error_text: str) -> bool: - return status_code == 400 and _BEDROCK_STRICT_TOOLS_REJECTION.search(error_text) is not None - - def warn_bedrock_strict_tools_dropped(model: str) -> None: verbose_logger.warning( "Bedrock rejected the strict tool schemas for model=%s, so litellm sent the request again without `strict` " diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index 4178095c9ef..1f25235a651 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -2467,8 +2467,7 @@ class BaseLLMHTTPHandler: ) else: verbose_logger.debug( - "Anthropic /v1/messages: invalid thinking signature; " - "stripping thinking blocks and retrying (attempt %s/%s).", + "Anthropic /v1/messages: retrying after a recoverable HTTP error (attempt %s/%s).", len(recovered_errors) + 2, max_attempts, ) diff --git a/litellm/messages/dispatch.py b/litellm/messages/dispatch.py index 8e6becc60b4..35136c41dc2 100644 --- a/litellm/messages/dispatch.py +++ b/litellm/messages/dispatch.py @@ -121,6 +121,7 @@ _ASYNC_FEATURES: Final[detectors.Detectors] = MappingProxyType( Feature.PRE_REQUEST_HOOK: route_detectors.pre_request_hook, Feature.WEBSEARCH_INTERCEPTION: route_detectors.websearch_interception, Feature.MESSAGES_INTERCEPTOR: route_detectors.intercepted, + Feature.STRICT_TOOLS_RETRY: route_detectors.strict_tools_retry, } ) diff --git a/litellm/rust_bridge/catalog.py b/litellm/rust_bridge/catalog.py index fe52f8416d0..629d9098970 100644 --- a/litellm/rust_bridge/catalog.py +++ b/litellm/rust_bridge/catalog.py @@ -98,6 +98,8 @@ class Feature(str, Enum): """Replayed thinking blocks, which Python strips and retries once if Anthropic rejects their signature.""" TOOL_CUSTOM_FIELD = "tool_custom_field" """Tools carrying a non-spec custom object, which Python drops or hoists for the provider.""" + STRICT_TOOLS_RETRY = "strict_tools_retry" + """Strict tools under drop_params, which Python sends again without strict if Claude rejects their limits.""" CLIENT_HEADERS = "client_headers" """Proxy-forwarded headers that Python merges into extra_headers.""" DEPLOYMENT_HOOK = "deployment_hook" @@ -179,6 +181,7 @@ MESSAGES_PYTHON_FEATURES: Final = COMMON_PYTHON_FEATURES | { Feature.STREAM_COST_INJECTION, Feature.THINKING_SIGNATURE_RETRY, Feature.TOOL_CUSTOM_FIELD, + Feature.STRICT_TOOLS_RETRY, } RESPONSES_PYTHON_FEATURES: Final = COMMON_PYTHON_FEATURES | { diff --git a/litellm/rust_bridge/messages/detectors.py b/litellm/rust_bridge/messages/detectors.py index 52917126b3b..8edd2707405 100644 --- a/litellm/rust_bridge/messages/detectors.py +++ b/litellm/rust_bridge/messages/detectors.py @@ -5,6 +5,7 @@ from __future__ import annotations from typing import Final import litellm +from litellm.litellm_core_utils.core_helpers import drops_unsupported_params from litellm.llms.opencode.common_utils import is_opencode_api_base from litellm.rust_bridge.detectors import api_base, overridden, present, streaming from litellm.rust_bridge.public_call import NativeCall, optional_mapping, optional_sequence, optional_str @@ -48,6 +49,13 @@ def tool_custom_field(request: NativeCall) -> bool: return any("custom" in (optional_mapping(tool) or {}) for tool in tools) +def strict_tools_retry(request: NativeCall) -> bool: + tools: Final = optional_sequence(request.resolved.get("tools")) or () + return drops_unsupported_params(request.resolved.get("drop_params")) and any( + (optional_mapping(tool) or {}).get("strict") is True for tool in tools + ) + + def stream_cost_injection(request: NativeCall) -> bool: """Python adds response cost to streaming usage when the global setting is truthy.""" return streaming(request) and bool(litellm.include_cost_in_streaming_usage) diff --git a/tests/test_litellm_rust/messages/test_dispatch.py b/tests/test_litellm_rust/messages/test_dispatch.py index 15f58afb4e5..5b11e628b8d 100644 --- a/tests/test_litellm_rust/messages/test_dispatch.py +++ b/tests/test_litellm_rust/messages/test_dispatch.py @@ -442,6 +442,7 @@ class _Agentic(CustomLogger): ADVISOR_TOOL: Final = {"type": ANTHROPIC_ADVISOR_TOOL_TYPE, "name": "advisor", "model": "claude-opus-5-5"} +STRICT_TOOL: Final = {"name": "a", "input_schema": {"type": "object"}, "strict": True} @pytest.mark.parametrize( @@ -527,6 +528,45 @@ def test_each_detected_feature_reaches_its_rule( assert served(kwargs) is python_result +@pytest.mark.asyncio +@pytest.mark.parametrize( + "kwargs", + ( + pytest.param({"tools": [STRICT_TOOL]}, id="drop-params-unset"), + pytest.param({"drop_params": False, "tools": [STRICT_TOOL]}, id="drop-params-off"), + pytest.param( + {"drop_params": True, "tools": [{"name": "a", "input_schema": {"type": "object"}}]}, id="no-strict" + ), + ), +) +async def test_strict_tools_stay_native_unless_python_could_drop_strict( + monkeypatch: pytest.MonkeyPatch, kwargs: Mapping[str, object] +) -> None: + rules: Final[Rules] = ( + RouteRule(Route.MESSAGES, Rollout.PYTHON_ONLY, features=frozenset({Feature.STRICT_TOOLS_RETRY})), + RouteRule(Route.MESSAGES, Rollout.RUST_REQUIRED), + ) + native_result: Final = response() + monkeypatch.setattr(litellm, "drop_params", False) + + async def python(*call_args: object, **python_kwargs: object) -> AnthropicMessagesResponse: # kwargs-ok: stub + pytest.fail("Python cannot drop strict here") + + async def native(request: NativeCall) -> AnthropicMessagesResponse: + return native_result + + served: Final = await _ADISPATCH.arun( + (16, MESSAGES, "anthropic/claude-sonnet-4-5"), + kwargs, + python=python, + binding=amessages_binding(native), + native=lambda hook, request, call_args, native_kwargs: hook(request), + rules=rules, + ) + + assert served is native_result + + ASYNC_ONLY_FEATURES: Final = ( pytest.param(Feature.PRE_REQUEST_HOOK, {}, {"callbacks": [_Rewrite()]}, id="pre-request-hook"), pytest.param( @@ -538,6 +578,12 @@ ASYNC_ONLY_FEATURES: Final = ( {}, id="messages-interceptor", ), + pytest.param( + Feature.STRICT_TOOLS_RETRY, {"drop_params": True, "tools": [STRICT_TOOL]}, {}, id="strict-tools-retry" + ), + pytest.param( + Feature.STRICT_TOOLS_RETRY, {"tools": [STRICT_TOOL]}, {"drop_params": True}, id="strict-tools-retry-global" + ), ) diff --git a/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_handler.py b/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_handler.py index c11f58faec4..5ce6d03f477 100644 --- a/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_handler.py +++ b/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_handler.py @@ -3,6 +3,7 @@ import copy import json import os import uuid +from dataclasses import dataclass from typing import Any, Dict, Final, List import httpx @@ -10,7 +11,7 @@ import responses import respx import pytest from fastapi.testclient import TestClient -from pydantic import ValidationError +from pydantic import TypeAdapter, ValidationError from unittest.mock import AsyncMock, MagicMock, patch @@ -2396,3 +2397,61 @@ def test_native_messages_tool_config_point_marks_last_tool_and_drops_the_key(mon assert "cache_control" not in captured["params"]["tools"][0] assert captured["messages"][0]["content"][-1]["cache_control"] == {"type": "ephemeral"} assert tools == tools_before + + +# Anthropic's 400 for a request past Claude's strict tool limits, seen live on claude-sonnet-4-6 on 2026-10-29 +_TOO_MANY_STRICT_TOOLS: Final = { + "type": "error", + "error": { + "type": "invalid_request_error", + "message": "Too many strict tools (21). The maximum number of strict tools supported is 20. Try reducing the " + "number of tools marked as strict.", + }, + "request_id": "req_011CgWXzv8cmAr9vFs786WV4", +} +_JSON_BODY: Final = TypeAdapter(dict[str, object]) + + +@dataclass(slots=True) +class _RejectsStrictToolsOnce: + sent: tuple[dict[str, object], ...] = () + + def __call__(self, request: httpx.Request) -> httpx.Response: + self.sent = (*self.sent, _JSON_BODY.validate_json(request.content)) + if len(self.sent) == 1: + return httpx.Response(400, json=_TOO_MANY_STRICT_TOOLS, request=request) + return httpx.Response( + 200, + json={ + "id": "msg_strict_retry", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-4-6", + "content": [{"type": "text", "text": "done"}], + "stop_reason": "end_turn", + "stop_sequence": None, + "usage": {"input_tokens": 1, "output_tokens": 1}, + }, + request=request, + ) + + +@pytest.mark.asyncio +async def test_anthropic_messages_strict_tools_rejection_is_sent_again_without_strict_under_drop_params() -> None: + anthropic: Final = _RejectsStrictToolsOnce() + tool: Final = {"name": "lookup_record", "input_schema": {"type": "object"}, "strict": True} + + response: Final = await litellm.anthropic_messages( + max_tokens=16, + messages=[{"role": "user", "content": "Use lookup_record to find invoice 42"}], + model="anthropic/claude-sonnet-4-6", + api_key="sk-test", + tools=[tool], + drop_params=True, + client=AsyncHTTPHandler(transport=httpx.MockTransport(anthropic)), + ) + + first, second = anthropic.sent + assert first["tools"] == [tool] + assert second == {**first, "tools": [{"name": "lookup_record", "input_schema": {"type": "object"}}]} + assert _JSON_BODY.validate_python(response)["id"] == "msg_strict_retry" diff --git a/tests/unit/llms/anthropic/test_anthropic_common_utils.py b/tests/unit/llms/anthropic/test_anthropic_common_utils.py index b37b2534b73..3ec72f29d34 100644 --- a/tests/unit/llms/anthropic/test_anthropic_common_utils.py +++ b/tests/unit/llms/anthropic/test_anthropic_common_utils.py @@ -25,7 +25,7 @@ import pytest import respx import litellm -from litellm.llms.anthropic.common_utils import is_claude_tool_target +from litellm.llms.anthropic.common_utils import ToolsWithoutStrict, is_claude_tool_target, tools_without_strict from litellm.llms.custom_httpx.http_handler import HTTPHandler sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "../../../../.."))) @@ -4559,3 +4559,27 @@ def test_shared_legacy_thinking_translation_drops_a_non_numeric_budget_under_dro ) def test_is_claude_tool_target(custom_llm_provider: str, model: str, expected: bool): assert is_claude_tool_target(custom_llm_provider=custom_llm_provider, model=model) is expected + + +@pytest.mark.parametrize( + "tools", + [ + None, + "lookup_record", + [], + [{"name": "a"}], + [{"name": "a", "strict": False}], + [{"name": "a", "strict": True}, "b"], + ], + ids=["absent", "not_a_list", "empty", "no_strict_key", "strict_false_only", "malformed_tool"], +) +def test_tools_without_strict_is_none_unless_every_tool_is_an_object_and_one_is_strict(tools: object) -> None: + assert tools_without_strict(tools) is None + + +def test_tools_without_strict_drops_only_the_strict_key_and_counts_the_strict_tools() -> None: + tools: Final = [{"name": "a", "strict": True}, {"name": "b"}, {"name": "c", "strict": False}] + + assert tools_without_strict(tools) == ToolsWithoutStrict( + tools=({"name": "a"}, {"name": "b"}, {"name": "c"}), dropped_count=1 + ) diff --git a/tests/unit/llms/bedrock/messages/invoke_transformations/test_anthropic_claude_transformation.py b/tests/unit/llms/bedrock/messages/invoke_transformations/test_anthropic_claude_transformation.py index 74f6f3030cb..0bd480dcdbc 100644 --- a/tests/unit/llms/bedrock/messages/invoke_transformations/test_anthropic_claude_transformation.py +++ b/tests/unit/llms/bedrock/messages/invoke_transformations/test_anthropic_claude_transformation.py @@ -8,7 +8,7 @@ import zlib from collections.abc import AsyncIterator, Mapping, Sequence from dataclasses import dataclass from datetime import datetime -from types import SimpleNamespace +from types import MappingProxyType, SimpleNamespace from typing import Final from unittest.mock import Mock @@ -4051,12 +4051,14 @@ _THINKING_HISTORY: Final = ( @dataclass(slots=True) class ScriptedBedrockInvoke: - replies: tuple[tuple[int, Mapping[str, object]], ...] + replies: tuple[tuple[int, Mapping[str, object] | bytes], ...] bodies: tuple[dict[str, object], ...] = () def __call__(self, request: httpx.Request) -> httpx.Response: self.bodies = (*self.bodies, REASONING_BODY_ADAPTER.validate_json(request.content)) status_code, body = self.replies[len(self.bodies) - 1] + if isinstance(body, bytes): + return httpx.Response(status_code, content=body) return httpx.Response(status_code, json=body) @@ -4065,6 +4067,7 @@ async def _send_bedrock_messages( model_info: Mapping[str, object] | None, replies: Sequence[tuple[int, Mapping[str, object]]], messages: Sequence[Mapping[str, object]] = ({"role": "user", "content": "Summarise the weather as JSON."},), + extra: Mapping[str, object] = MappingProxyType({}), ) -> tuple[ScriptedBedrockInvoke, Exception | None]: bedrock: Final = ScriptedBedrockInvoke(replies=tuple(replies)) try: @@ -4080,6 +4083,7 @@ async def _send_bedrock_messages( aws_region_name="us-east-1", client=AsyncHTTPHandler(transport=httpx.MockTransport(bedrock)), **({} if model_info is None else {"model_info": model_info}), + **extra, ) except Exception as error: return bedrock, error @@ -4171,6 +4175,158 @@ async def test_bedrock_messages_400_is_not_retried_unless_it_rejects_opt_in_nati assert len(bedrock.bodies) == 1 +# Bedrock's 400 bodies for requests past Claude's strict tool limits, seen live on us.anthropic.claude-sonnet-4-6 on 2026-10-29 +_STRICT_TOOLS_REJECTIONS: Final = ( + { + "message": "Too many strict tools (21). The maximum number of strict tools supported is 20. Try reducing the " + "number of tools marked as strict." + }, + { + "message": "Schemas contains too many optional parameters (96), which would make grammar compilation " + "inefficient. Reduce the number of optional parameters in your tool schemas (limit: 24)." + }, + { + "message": "Schemas contains too many parameters with union types (24 parameters with type arrays or anyOf). " + "This causes exponential compilation cost. Reduce the number of nullable or union-typed parameters (limit: 16 " + "parameters with unions)." + }, + { + "message": "The compiled grammar is too large, which would cause performance issues. Simplify your tool schemas " + "or reduce the number of strict tools." + }, +) +_LOOKUP_TOOL: Final = { + "name": "lookup_record", + "description": "Look up records", + "input_schema": { + "type": "object", + "additionalProperties": False, + "required": ["query"], + "properties": {"query": {"type": "string"}}, + }, +} +_STRICT_TOOLS: Final = ({**_LOOKUP_TOOL, "strict": True}, {**_LOOKUP_TOOL, "name": "lookup_invoice", "strict": True}) +_STREAMED_MESSAGE: Final = _bedrock_event_frame( + {"type": "message_start", "message": {**_ANTHROPIC_MESSAGE, "content": [], "stop_reason": None}} +) + _bedrock_event_frame({"type": "message_stop"}) + + +async def _send_strict_tools_to_bedrock( + replies: Sequence[tuple[int, Mapping[str, object] | bytes]], + drop_params: bool | None, + tools: Sequence[Mapping[str, object]] = _STRICT_TOOLS, + stream: bool = False, +) -> tuple[ScriptedBedrockInvoke, object]: + bedrock: Final = ScriptedBedrockInvoke(replies=tuple(replies)) + try: + response: Final = await litellm.anthropic_messages( + model="bedrock/invoke/us.anthropic.claude-sonnet-4-6", + messages=[{"role": "user", "content": "Use lookup_record to find invoice 42"}], + max_tokens=200, + tools=[dict(tool) for tool in tools], + stream=stream, + drop_params=drop_params, + aws_access_key_id="test-access-key", + aws_secret_access_key="test-secret-key", + aws_region_name="us-east-1", + client=AsyncHTTPHandler(transport=httpx.MockTransport(bedrock)), + ) + except litellm.BadRequestError as error: + return bedrock, error + if isinstance(response, AsyncIterator): + return bedrock, b"".join([chunk async for chunk in response]) + return bedrock, response + + +def _assert_resent_without_strict(bedrock: ScriptedBedrockInvoke) -> None: + first, second = bedrock.bodies + first_tools: Final = TypeAdapter(list[dict[str, object]]).validate_python(first["tools"]) + assert [tool["strict"] for tool in first_tools] == [True, True] + assert second == { + **first, + "tools": [{key: value for key, value in tool.items() if key != "strict"} for tool in first_tools], + } + + +@pytest.mark.usefixtures("local_model_cost_map") +@pytest.mark.parametrize("rejection", _STRICT_TOOLS_REJECTIONS, ids=["count", "optional", "union", "grammar"]) +async def test_bedrock_messages_strict_tools_rejection_is_sent_again_without_strict_under_drop_params( + rejection: Mapping[str, object], +) -> None: + bedrock, response = await _send_strict_tools_to_bedrock([(400, rejection), (200, _ANTHROPIC_MESSAGE)], True) + + _assert_resent_without_strict(bedrock) + assert REASONING_BODY_ADAPTER.validate_python(response)["id"] == _ANTHROPIC_MESSAGE["id"] + + +@pytest.mark.usefixtures("local_model_cost_map") +async def test_bedrock_messages_strict_tools_rejection_is_sent_again_under_global_drop_params( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(litellm, "drop_params", True) + + bedrock, response = await _send_strict_tools_to_bedrock( + [(400, _STRICT_TOOLS_REJECTIONS[0]), (200, _ANTHROPIC_MESSAGE)], None + ) + + _assert_resent_without_strict(bedrock) + assert REASONING_BODY_ADAPTER.validate_python(response)["id"] == _ANTHROPIC_MESSAGE["id"] + + +@pytest.mark.usefixtures("local_model_cost_map") +async def test_bedrock_messages_streamed_strict_tools_rejection_is_sent_again_without_strict_under_drop_params() -> ( + None +): + bedrock, streamed = await _send_strict_tools_to_bedrock( + [(400, _STRICT_TOOLS_REJECTIONS[0]), (200, _STREAMED_MESSAGE)], True, stream=True + ) + + _assert_resent_without_strict(bedrock) + assert isinstance(streamed, bytes) and b"event: message_stop" in streamed + + +@pytest.mark.usefixtures("local_model_cost_map") +async def test_bedrock_messages_body_rebuilt_without_native_output_still_drops_the_rejected_strict() -> None: + bedrock, error = await _send_bedrock_messages( + "us.anthropic.claude-opus-4-8", + None, + [(400, _STRICT_TOOLS_REJECTIONS[0]), (400, _NATIVE_OUTPUT_REJECTION), (200, _ANTHROPIC_MESSAGE)], + extra={"tools": [dict(tool) for tool in _STRICT_TOOLS], "drop_params": True}, + ) + + assert error is None + first, _, rebuilt = bedrock.bodies + assert "format" in REASONING_BODY_ADAPTER.validate_python(first["output_config"]) + assert "format" not in REASONING_BODY_ADAPTER.validate_python(rebuilt.get("output_config", {})) + assert rebuilt["tools"] == [ + {key: value for key, value in tool.items() if key != "strict"} for tool in _STRICT_TOOLS + ] + + +@pytest.mark.usefixtures("local_model_cost_map") +@pytest.mark.parametrize( + ("drop_params", "tools", "rejection"), + [ + (None, _STRICT_TOOLS, _STRICT_TOOLS_REJECTIONS[0]), + (False, _STRICT_TOOLS, _STRICT_TOOLS_REJECTIONS[0]), + (True, _STRICT_TOOLS, {"message": "messages.0.content: Field required"}), + (True, (_LOOKUP_TOOL,), _STRICT_TOOLS_REJECTIONS[0]), + (True, ({**_LOOKUP_TOOL, "strict": False},), _STRICT_TOOLS_REJECTIONS[1]), + ], + ids=["drop_params_unset", "drop_params_off", "other_400", "no_strict_tool", "strict_false_only"], +) +async def test_bedrock_messages_400_reaches_the_caller_unless_drop_params_can_drop_the_rejected_strict( + drop_params: bool | None, tools: Sequence[Mapping[str, object]], rejection: Mapping[str, object] +) -> None: + bedrock, raised = await _send_strict_tools_to_bedrock( + [(400, rejection), (200, _ANTHROPIC_MESSAGE)], drop_params, tools + ) + + assert isinstance(raised, litellm.BadRequestError) + assert str(rejection["message"]) in str(raised) + assert len(bedrock.bodies) == 1 + + @pytest.mark.parametrize("trace", ["enabled", "disabled"]) def test_bedrock_messages_applies_the_deployment_guardrail_config_as_invoke_headers(trace: str) -> None: from litellm.llms.bedrock.messages.invoke_transformations.anthropic_claude_transformation import ( diff --git a/tests/unit/rust_bridge/test_detectors.py b/tests/unit/rust_bridge/test_detectors.py index c02d2c12dbf..634e69910b9 100644 --- a/tests/unit/rust_bridge/test_detectors.py +++ b/tests/unit/rust_bridge/test_detectors.py @@ -33,6 +33,7 @@ MAPPED_FILE_INPUT: Final = [{"role": "user", "content": [{"type": "input_file", CACHED_TEXT: Final = {"type": "input_text", "text": "hi", "cache_control": {"type": "ephemeral"}} THINKING_TURN: Final = {"role": "assistant", "content": [{"type": "thinking", "thinking": "x", "signature": "s"}]} REDACTED: Final = {"type": "redacted_thinking", "data": "x"} +STRICT_TOOL: Final = {"name": "b", "input_schema": {"type": "object"}, "strict": True} TOOL_SEARCH_TYPE: Final = sorted(ANTHROPIC_TOOL_SEARCH_TOOL_TYPES)[0] @@ -192,6 +193,21 @@ def _wrap_nested_level(value: object, depth: int) -> object: (messages_detectors.tool_custom_field, {"tools": [{"name": "a"}, {"name": "b", "custom": {}}]}, True), (messages_detectors.tool_custom_field, {"tools": [{"name": "a", "defer_loading": True}]}, False), (messages_detectors.tool_custom_field, {}, False), + (messages_detectors.strict_tools_retry, {"drop_params": True, "tools": [{"name": "a"}, STRICT_TOOL]}, True), + ( + messages_detectors.strict_tools_retry, + {"drop_params": True, "tools": [{**STRICT_TOOL, "strict": False}]}, + False, + ), + ( + messages_detectors.strict_tools_retry, + {"additional_drop_params": ["metadata.user_id"], "tools": [STRICT_TOOL]}, + False, + ), + (messages_detectors.strict_tools_retry, {"tools": [STRICT_TOOL]}, False), + (messages_detectors.strict_tools_retry, {"drop_params": False, "tools": [STRICT_TOOL]}, False), + (messages_detectors.strict_tools_retry, {"drop_params": True, "tools": [{"name": "a"}]}, False), + (messages_detectors.strict_tools_retry, {"drop_params": True}, False), (detectors.drop_params, {"drop_params": True}, True), (detectors.drop_params, {"drop_params": False}, False), (detectors.drop_params, {"drop_params": " True "}, True), From 9252709cab33d7848b2a4d0761a9431b1ea7baa4 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Thu, 29 Oct 2026 17:25:48 -0700 Subject: [PATCH 07/17] feat(ui): add i18next plumbing and move the Workflows page onto it (#49144) * test(ui): pin the Workflows page copy with characterization tests Locks the header, table, filters drawer, detail drawer, relative time and duration text on the Workflows page so the upcoming move onto i18next can be checked against it without editing any test * feat(ui): add i18next plumbing and move the Workflows page copy onto it Adds an English-only i18next instance with typed keys from src/i18n/locales/en.json, wraps the root layout in I18nProvider, and loads the instance in the test setup. Every string the Workflows page authors now renders through t(), while backend values such as run status, roles, event types and metadata keys stay raw. The rendered English is unchanged --- ui/litellm-dashboard/package-lock.json | 74 ++++- ui/litellm-dashboard/package.json | 2 + .../workflows/WorkflowRuns.test.tsx | 252 ++++++++++++++++-- .../(dashboard)/workflows/WorkflowRuns.tsx | 167 ++++++------ ui/litellm-dashboard/src/app/layout.tsx | 27 +- .../src/i18n/I18nProvider.tsx | 9 + ui/litellm-dashboard/src/i18n/i18n.test-d.ts | 10 + ui/litellm-dashboard/src/i18n/i18n.ts | 23 ++ ui/litellm-dashboard/src/i18n/locales/en.json | 76 ++++++ ui/litellm-dashboard/tests/setupTests.ts | 2 + 10 files changed, 522 insertions(+), 120 deletions(-) create mode 100644 ui/litellm-dashboard/src/i18n/I18nProvider.tsx create mode 100644 ui/litellm-dashboard/src/i18n/i18n.test-d.ts create mode 100644 ui/litellm-dashboard/src/i18n/i18n.ts create mode 100644 ui/litellm-dashboard/src/i18n/locales/en.json diff --git a/ui/litellm-dashboard/package-lock.json b/ui/litellm-dashboard/package-lock.json index 8704dc55c38..48def46e3fe 100644 --- a/ui/litellm-dashboard/package-lock.json +++ b/ui/litellm-dashboard/package-lock.json @@ -25,6 +25,7 @@ "clsx": "^2.1.1", "date-fns": "^4.4.0", "dayjs": "1.11.19", + "i18next": "26.5.11", "jwt-decode": "4.0.0", "lucide-react": "0.513.0", "moment": "2.31.0", @@ -43,6 +44,7 @@ "react-dom": "19.2.8", "react-hook-form": "7.82.0", "react-hotkeys-hook": "5.3.3", + "react-i18next": "17.1.9", "react-json-view-lite": "2.5.0", "react-markdown": "9.1.0", "react-reconciler": "0.33.0", @@ -412,9 +414,9 @@ } }, "node_modules/@babel/runtime": { - "version": "7.29.2", - "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.29.2.tgz", - "integrity": "sha512-JiDShH45zKHWyGe4ZNVRrCjBz8Nh9TMmZG1kh4QTK8hCBTWBi8Da+i7s1fJw7/lYpM4ccepSNfqzZ/QvABBi5g==", + "version": "7.29.10", + "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.29.10.tgz", + "integrity": "sha512-NpugTzbKgGrNe45Ui32cN8vCFFMxVtP/9rtfwv0nP5FRDAjvYBxJcLVpYA9mr1jxP6cnIT7x9LUzuDTbWYYNRA==", "license": "MIT", "engines": { "node": ">=6.9.0" @@ -7319,6 +7321,15 @@ "dev": true, "license": "MIT" }, + "node_modules/html-parse-stringify": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/html-parse-stringify/-/html-parse-stringify-4.0.1.tgz", + "integrity": "sha512-0zHsZJrK7S3K2aucXWL6ycoYJ/iNtIcFHC/nYQgFklPtrv5LpJctIiSCroWZWeuoXvuyFdzp6KzjJQ+OT5MfFw==", + "license": "MIT", + "funding": { + "url": "https://locize.com" + } + }, "node_modules/html-url-attributes": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/html-url-attributes/-/html-url-attributes-3.0.1.tgz", @@ -7357,6 +7368,34 @@ "node": ">= 14" } }, + "node_modules/i18next": { + "version": "26.5.11", + "resolved": "https://registry.npmjs.org/i18next/-/i18next-26.5.11.tgz", + "integrity": "sha512-Wc0klBPKxqZ2tL2qdAnF8xvvnWgr1IMGSY2ikTiAvaFVbz7xiFARtkoU4RidBDw1SyziGROp0qxu6qnJ7l2Wug==", + "funding": [ + { + "type": "individual", + "url": "https://www.locize.com/i18next" + }, + { + "type": "individual", + "url": "https://www.i18next.com/how-to/faq#i18next-is-awesome.-how-can-i-support-the-project" + }, + { + "type": "individual", + "url": "https://www.locize.com" + } + ], + "license": "MIT", + "peerDependencies": { + "typescript": "^5 || ^6 || ^7" + }, + "peerDependenciesMeta": { + "typescript": { + "optional": true + } + } + }, "node_modules/ignore": { "version": "5.3.2", "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.3.2.tgz", @@ -10597,6 +10636,33 @@ "react-dom": ">=16.8.0" } }, + "node_modules/react-i18next": { + "version": "17.1.9", + "resolved": "https://registry.npmjs.org/react-i18next/-/react-i18next-17.1.9.tgz", + "integrity": "sha512-OgYqIH13XHPlWTaaI795Dz+k2AdKnXcOxNQVnf750T8Ij0TYWp1nYvXRpoJBulPqlyk2OAbn1PM4tFKiIj/byA==", + "license": "MIT", + "dependencies": { + "@babel/runtime": "^7.29.7", + "html-parse-stringify": "^4.0.1", + "use-sync-external-store": "^1.6.0" + }, + "peerDependencies": { + "i18next": ">= 26.2.0", + "react": ">= 16.8.0", + "typescript": "^5 || ^6 || ^7" + }, + "peerDependenciesMeta": { + "react-dom": { + "optional": true + }, + "react-native": { + "optional": true + }, + "typescript": { + "optional": true + } + } + }, "node_modules/react-intersection-observer": { "version": "11.0.1", "resolved": "https://registry.npmjs.org/react-intersection-observer/-/react-intersection-observer-11.0.1.tgz", @@ -12105,7 +12171,7 @@ "version": "5.9.3", "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", - "dev": true, + "devOptional": true, "license": "Apache-2.0", "bin": { "tsc": "bin/tsc", diff --git a/ui/litellm-dashboard/package.json b/ui/litellm-dashboard/package.json index b1b8fde44ee..f65316439bf 100644 --- a/ui/litellm-dashboard/package.json +++ b/ui/litellm-dashboard/package.json @@ -42,6 +42,7 @@ "clsx": "^2.1.1", "date-fns": "^4.4.0", "dayjs": "1.11.19", + "i18next": "26.5.11", "jwt-decode": "4.0.0", "lucide-react": "0.513.0", "moment": "2.31.0", @@ -60,6 +61,7 @@ "react-dom": "19.2.8", "react-hook-form": "7.82.0", "react-hotkeys-hook": "5.3.3", + "react-i18next": "17.1.9", "react-json-view-lite": "2.5.0", "react-markdown": "9.1.0", "react-reconciler": "0.33.0", diff --git a/ui/litellm-dashboard/src/app/(dashboard)/workflows/WorkflowRuns.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/workflows/WorkflowRuns.test.tsx index 6707330a54c..f4a299a5dcc 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/workflows/WorkflowRuns.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/workflows/WorkflowRuns.test.tsx @@ -14,7 +14,7 @@ interface FakeRun { status: string; workflow_type: string; created_at: string; - metadata: { title?: string; state?: string } | null; + metadata: Record | null; } const RUNS: FakeRun[] = [ @@ -34,16 +34,16 @@ const RUNS: FakeRun[] = [ }, ]; -function mockFetch(runs: FakeRun[]) { +function mockFetch(runs: FakeRun[], events: FakeEvent[] = [], messages: FakeMessage[] = []) { return vi.fn((url: string) => { if (url.includes("/runs?limit")) { return Promise.resolve({ ok: true, json: () => Promise.resolve({ runs }) }); } if (url.includes("/events")) { - return Promise.resolve({ ok: true, json: () => Promise.resolve({ events: [] }) }); + return Promise.resolve({ ok: true, json: () => Promise.resolve({ events }) }); } if (url.includes("/messages")) { - return Promise.resolve({ ok: true, json: () => Promise.resolve({ messages: [] }) }); + return Promise.resolve({ ok: true, json: () => Promise.resolve({ messages }) }); } return Promise.resolve({ ok: false, status: 404, json: () => Promise.resolve({}) }); }); @@ -114,7 +114,7 @@ interface FakeMessage { created_at: string; } -const DETAIL_RUN = { +const DETAIL_RUN: FakeRun = { run_id: "run-aaaaaaaa-1111", status: "completed", workflow_type: "grill", @@ -151,33 +151,21 @@ const DETAIL_MESSAGES: FakeMessage[] = [ }, ]; -function mockDetailFetch(events: FakeEvent[], messages: FakeMessage[]) { - return vi.fn((url: string) => { - if (url.includes("/runs?limit")) { - return Promise.resolve({ ok: true, json: () => Promise.resolve({ runs: [DETAIL_RUN] }) }); - } - if (url.includes("/events")) { - return Promise.resolve({ ok: true, json: () => Promise.resolve({ events }) }); - } - if (url.includes("/messages")) { - return Promise.resolve({ ok: true, json: () => Promise.resolve({ messages }) }); - } - return Promise.resolve({ ok: false, status: 404, json: () => Promise.resolve({}) }); - }); -} - -async function openDetailDrawer(events = DETAIL_EVENTS, messages = DETAIL_MESSAGES) { +async function openRunDrawer(run: FakeRun, events: FakeEvent[], messages: FakeMessage[]) { const user = userEvent.setup(); - const fetchSpy = mockDetailFetch(events, messages); + const fetchSpy = mockFetch([run], events, messages); vi.stubGlobal("fetch", fetchSpy); render(); - await user.click(await screen.findByText("First run")); + await user.click(await screen.findByText(run.metadata?.title ?? run.workflow_type)); const drawer = await screen.findByRole("dialog"); - await waitFor(() => expect(within(drawer).getByText("Timeline")).toBeInTheDocument()); + expect(await within(drawer).findByText("Timeline")).toBeInTheDocument(); return { user, fetchSpy, drawer }; } +const openDetailDrawer = (events = DETAIL_EVENTS, messages = DETAIL_MESSAGES) => + openRunDrawer(DETAIL_RUN, events, messages); + describe("WorkflowRuns detail drawer", () => { it("shows the run's identity and metadata fields", async () => { const { drawer } = await openDetailDrawer(); @@ -244,3 +232,219 @@ describe("WorkflowRuns detail drawer", () => { await waitFor(() => expect(screen.queryAllByRole("dialog")).toHaveLength(0)); }); }); + +const FULL_RUN: FakeRun = { + run_id: "run-cccccccc-3333", + status: "running", + workflow_type: "autoship", + created_at: "2026-01-01T00:00:00.000Z", + metadata: { + title: "Full run", + state: "reviewing", + pr_url: "https://example.com/pr/2", + worktree_path: "/tmp/wt/full", + grill_session_id: "grill-1", + session_id: "sess-1", + plan_text: "step ".repeat(26), + }, +}; + +const TIMED_EVENTS: FakeEvent[] = [ + { + event_id: "ev-a", + event_type: "step.started", + step_name: "plan", + sequence_number: 1, + created_at: "2026-01-01T00:00:00.100Z", + data: null, + }, + { + event_id: "ev-b", + event_type: "step.started", + step_name: "implement", + sequence_number: 2, + created_at: "2026-01-01T00:00:00.165Z", + data: null, + }, + { + event_id: "ev-c", + event_type: "hook.waiting", + step_name: "review", + sequence_number: 3, + created_at: "2026-01-01T00:00:01.365Z", + data: null, + }, +]; + +describe("WorkflowRuns page copy", () => { + it("announces the loading state while runs are being fetched", async () => { + vi.stubGlobal( + "fetch", + vi.fn(() => new Promise(() => {})), + ); + render(); + + expect(await screen.findByText("Loading workflow runs…")).toBeInTheDocument(); + }); + + it("labels the page header, the table columns and the search box", async () => { + vi.stubGlobal("fetch", mockFetch(RUNS)); + render(); + await screen.findByText("First run"); + + expect(screen.getByText("Workflow Runs")).toBeInTheDocument(); + expect(screen.getByText("Durable state tracking for agents and automated workflows")).toBeInTheDocument(); + expect(screen.getAllByRole("columnheader").map((header) => header.textContent)).toEqual([ + "Run", + "Type", + "Status", + "Created", + ]); + expect(screen.getByPlaceholderText("Search runs…")).toBeInTheDocument(); + }); + + it("names every column in the Columns menu", async () => { + const user = userEvent.setup(); + vi.stubGlobal("fetch", mockFetch(RUNS)); + render(); + await screen.findByText("First run"); + + await user.click(screen.getByRole("button", { name: "Columns" })); + + expect((await screen.findAllByRole("menuitemcheckbox")).map((item) => item.textContent)).toEqual([ + "Run", + "Type", + "Status", + "Created", + ]); + }); + + it("labels the filters drawer and offers every run status as a filter", async () => { + const user = userEvent.setup(); + vi.stubGlobal("fetch", mockFetch(RUNS)); + render(); + await screen.findByText("First run"); + + await user.click(screen.getByRole("button", { name: "Filters" })); + const filters = await screen.findByRole("dialog", { name: "Filters" }); + expect(filters).toHaveAccessibleDescription("Narrow down workflow runs"); + expect(within(filters).getByText("Status")).toBeInTheDocument(); + expect(within(filters).getByText("Type")).toBeInTheDocument(); + expect(within(filters).getByPlaceholderText("Filter by type…")).toBeInTheDocument(); + + const statusSelect = within(filters).getByRole("combobox"); + expect(statusSelect).toHaveTextContent("All statuses"); + await user.click(statusSelect); + expect((await screen.findAllByRole("option")).map((option) => option.textContent)).toEqual([ + "All statuses", + "Pending", + "Running", + "Paused", + "Completed", + "Failed", + ]); + + await user.click(screen.getByRole("option", { name: "Running" })); + expect(statusSelect).toHaveTextContent("Running"); + }); + + it("labels the detail drawer, its controls and every primary metadata field", async () => { + const { user, drawer } = await openRunDrawer(FULL_RUN, [TIMED_EVENTS[0]], []); + + expect(drawer).toHaveAccessibleName("Workflow run details"); + expect(drawer).toHaveAccessibleDescription("Metadata, timeline and messages for the selected workflow run"); + for (const label of ["status", "created", "pr", "state", "worktree", "grill session", "session"]) { + expect(within(drawer).getByText(label)).toBeInTheDocument(); + } + expect(within(drawer).getByRole("button", { name: "close" })).toBeInTheDocument(); + expect(within(drawer).getByRole("button", { name: "Refresh" })).toBeInTheDocument(); + expect(within(drawer).getByText("1 event")).toBeInTheDocument(); + + await user.click(within(drawer).getByRole("button", { name: /Messages/ })); + expect(await within(drawer).findByText("No messages")).toBeInTheDocument(); + }); + + it("counts zero events in the timeline header", async () => { + const { drawer } = await openRunDrawer(FULL_RUN, [], []); + + expect(within(drawer).getByText("0 events")).toBeInTheDocument(); + }); + + it("collapses a long metadata value behind a more toggle that flips to less", async () => { + const { user, drawer } = await openRunDrawer(FULL_RUN, [], []); + + await user.click(within(drawer).getByRole("button", { name: "more" })); + + expect(within(drawer).getByRole("button", { name: "less" })).toBeInTheDocument(); + }); + + it("formats timeline durations in milliseconds under a second and in seconds above it", async () => { + const { drawer } = await openRunDrawer(FULL_RUN, TIMED_EVENTS, []); + + expect(within(drawer).getByText("65ms")).toBeInTheDocument(); + expect(within(drawer).getByText("1.2s")).toBeInTheDocument(); + expect(within(drawer).getAllByText("1.4s")).toHaveLength(2); + }); +}); + +describe("WorkflowRuns relative times", () => { + afterEach(() => { + vi.useRealTimers(); + }); + + it("shows created times in the largest whole unit and passes an unparseable date through", async () => { + const now = Date.parse("2026-03-10T12:00:00Z"); + vi.useFakeTimers({ toFake: ["Date"] }); + vi.setSystemTime(now); + const runCreated = (id: string, createdAt: string): FakeRun => ({ + run_id: `run-${id}`, + status: "completed", + workflow_type: `type-${id}`, + created_at: createdAt, + metadata: null, + }); + const secondsAgo = (seconds: number) => new Date(now - seconds * 1000).toISOString(); + vi.stubGlobal( + "fetch", + mockFetch([ + runCreated("a", secondsAgo(59)), + runCreated("b", secondsAgo(60)), + runCreated("c", secondsAgo(3600)), + runCreated("d", secondsAgo(86400)), + runCreated("e", "not-a-date"), + ]), + ); + render(); + + expect(await screen.findByText("59s ago")).toBeInTheDocument(); + expect(screen.getByText("1m ago")).toBeInTheDocument(); + expect(screen.getByText("1h ago")).toBeInTheDocument(); + expect(screen.getByText("1d ago")).toBeInTheDocument(); + expect(screen.getByText("not-a-date")).toBeInTheDocument(); + }); +}); + +describe("WorkflowRuns detail drawer relative times", () => { + afterEach(() => { + vi.useRealTimers(); + }); + + it("labels the run's age, each message's age and every field of a hovered timeline event", async () => { + vi.useFakeTimers({ toFake: ["Date"] }); + vi.setSystemTime(Date.parse("2026-01-01T00:00:10Z")); + const { user, drawer } = await openDetailDrawer(); + + expect(within(drawer).getByText("10s ago")).toBeInTheDocument(); + + await user.hover(within(drawer).getByText("step.started")); + const tooltip = await screen.findByText( + (_, element) => element?.getAttribute("data-slot") === "tooltip-content", + {}, + { timeout: 3000 }, + ); + expect(tooltip).toHaveTextContent('type: step.startedstep: planseq: 1time: 9s agodata: {"attempt":1}'); + + await user.click(within(drawer).getByRole("button", { name: /Messages/ })); + expect(await within(drawer).findByText("8s ago")).toBeInTheDocument(); + }); +}); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/workflows/WorkflowRuns.tsx b/ui/litellm-dashboard/src/app/(dashboard)/workflows/WorkflowRuns.tsx index ba4c23fcc9c..f02bbd0688c 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/workflows/WorkflowRuns.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/workflows/WorkflowRuns.tsx @@ -1,6 +1,8 @@ import React, { useState, useEffect, useCallback, useMemo } from "react"; import { ArrowLeft, ChevronDown, RefreshCw } from "lucide-react"; import type { ColumnDef, ColumnFiltersState } from "@tanstack/react-table"; +import type { ParseKeys, TFunction } from "i18next"; +import { useTranslation } from "react-i18next"; import { getGlobalLitellmHeaderName, proxyBaseUrl } from "@/components/networking"; import { DataTable, @@ -70,12 +72,12 @@ const STATUS_DOT: Record = { }; const RUN_STATUS_OPTIONS: RunStatus[] = ["pending", "running", "paused", "completed", "failed"]; -const STATUS_LABELS: Record = { - pending: "Pending", - running: "Running", - paused: "Paused", - completed: "Completed", - failed: "Failed", +const STATUS_LABEL_KEYS: Record = { + pending: "workflows.status.pending", + running: "workflows.status.running", + paused: "workflows.status.paused", + completed: "workflows.status.completed", + failed: "workflows.status.failed", }; const EVENT_COLOR: Record = { @@ -91,22 +93,22 @@ function eventStyle(type: string) { // ── helpers ─────────────────────────────────────────────────────────────────── -function timeAgo(iso: string): string { +function timeAgo(iso: string, t: TFunction): string { const diff = Date.now() - new Date(iso).getTime(); if (isNaN(diff)) return iso; const s = Math.floor(diff / 1000); - if (s < 60) return `${s}s ago`; + if (s < 60) return t("workflows.timeAgo.seconds", { count: s }); const m = Math.floor(s / 60); - if (m < 60) return `${m}m ago`; + if (m < 60) return t("workflows.timeAgo.minutes", { count: m }); const h = Math.floor(m / 60); - if (h < 24) return `${h}h ago`; - return `${Math.floor(h / 24)}d ago`; + if (h < 24) return t("workflows.timeAgo.hours", { count: h }); + return t("workflows.timeAgo.days", { count: Math.floor(h / 24) }); } -function fmtDuration(ms: number): string { +function fmtDuration(ms: number, t: TFunction): string { if (ms < 0) return ""; - if (ms < 1000) return `${ms}ms`; - return `${(ms / 1000).toFixed(1)}s`; + if (ms < 1000) return t("workflows.duration.ms", { count: ms }); + return t("workflows.duration.seconds", { value: (ms / 1000).toFixed(1) }); } function runTitle(run: WorkflowRun): string { @@ -130,6 +132,7 @@ const StatusDot: React.FC<{ status: RunStatus; className?: string }> = ({ status const TRUNCATE_AT = 120; const TruncatedValue: React.FC<{ value: string }> = ({ value }) => { + const { t } = useTranslation(); const [expanded, setExpanded] = useState(false); if (value.length <= TRUNCATE_AT) { return {value}; @@ -138,7 +141,7 @@ const TruncatedValue: React.FC<{ value: string }> = ({ value }) => { {expanded ? value : value.slice(0, TRUNCATE_AT) + "…"} ); @@ -147,13 +150,14 @@ const TruncatedValue: React.FC<{ value: string }> = ({ value }) => { // ── metadata card ───────────────────────────────────────────────────────────── const MetadataCard: React.FC<{ run: WorkflowRun }> = ({ run }) => { + const { t } = useTranslation(); const meta = run.metadata ?? {}; const primaryFields: { key: string; label: string }[] = [ - { key: "state", label: "state" }, - { key: "worktree_path", label: "worktree" }, - { key: "grill_session_id", label: "grill session" }, - { key: "session_id", label: "session" }, + { key: "state", label: t("workflows.metadata.state") }, + { key: "worktree_path", label: t("workflows.metadata.worktree") }, + { key: "grill_session_id", label: t("workflows.metadata.grillSession") }, + { key: "session_id", label: t("workflows.metadata.session") }, ]; const primaryKeys = new Set(["title", ...primaryFields.map((f) => f.key)]); @@ -175,15 +179,15 @@ const MetadataCard: React.FC<{ run: WorkflowRun }> = ({ run }) => { {/* key fields grid */}
- + {run.status} - - {timeAgo(run.created_at)} + + {timeAgo(run.created_at, t)} {meta.pr_url && ( - + = ({ run, events }) => { + const { t } = useTranslation(); if (events.length === 0) { - return
No events recorded
; + return
{t("workflows.drawer.noEvents")}
; } const runStart = new Date(run.created_at).getTime(); const eventTimes = events.map((e) => new Date(e.created_at).getTime()); const lastTime = Math.max(...eventTimes); const totalSpan = Math.max(lastTime - runStart, 1); - const totalDur = fmtDuration(lastTime - runStart); + const totalDur = fmtDuration(lastTime - runStart, t); return ( @@ -275,7 +280,7 @@ const GanttTimeline: React.FC<{ : lastTime + Math.max(totalSpan * 0.12, 500); const widthPct = Math.max(8, ((nextTime - evTime) / totalSpan) * 100); const style = eventStyle(ev.event_type); - const dur = fmtDuration(nextTime - evTime); + const dur = fmtDuration(nextTime - evTime, t); return ( @@ -303,24 +308,24 @@ const GanttTimeline: React.FC<{
- type: + {t("workflows.tooltip.type")} {ev.event_type}
- step: + {t("workflows.tooltip.step")} {ev.step_name}
- seq: + {t("workflows.tooltip.seq")} {ev.sequence_number}
- time: - {timeAgo(ev.created_at)} + {t("workflows.tooltip.time")} + {timeAgo(ev.created_at, t)}
{ev.data && Object.keys(ev.data).length > 0 && (
- data: + {t("workflows.tooltip.data")} {JSON.stringify(ev.data)}
)} @@ -346,15 +351,18 @@ const ROLE_COLOR: Record = { tool_result: "text-warning", }; -const MessageRow: React.FC<{ msg: WorkflowRunMessage }> = ({ msg }) => ( -
- [{msg.role}] -
- {msg.content} - {timeAgo(msg.created_at)} +const MessageRow: React.FC<{ msg: WorkflowRunMessage }> = ({ msg }) => { + const { t } = useTranslation(); + return ( +
+ [{msg.role}] +
+ {msg.content} + {timeAgo(msg.created_at, t)} +
-
-); + ); +}; // ── collapsible section ─────────────────────────────────────────────────────── @@ -379,6 +387,7 @@ const DetailSection: React.FC<{ // ── main component ──────────────────────────────────────────────────────────── const WorkflowRuns: React.FC = ({ accessToken }) => { + const { t } = useTranslation(); const [runs, setRuns] = useState([]); const [loadingRuns, setLoadingRuns] = useState(false); const [selectedRun, setSelectedRun] = useState(null); @@ -455,8 +464,8 @@ const WorkflowRuns: React.FC = ({ accessToken }) => { { id: "run", accessorFn: (row) => `${runTitle(row)} ${row.run_id}`, - header: "Run", - meta: { title: "Run", skeleton: "twoLine" }, + header: t("workflows.table.columns.run"), + meta: { title: t("workflows.table.columns.run"), skeleton: "twoLine" }, cell: ({ row }) => { const run = row.original; return ( @@ -472,8 +481,8 @@ const WorkflowRuns: React.FC = ({ accessToken }) => { }, { accessorKey: "workflow_type", - header: "Type", - meta: { title: "Type" }, + header: t("workflows.table.columns.type"), + meta: { title: t("workflows.table.columns.type") }, filterFn: "includesString", cell: ({ row }) => ( {row.original.workflow_type} @@ -482,8 +491,8 @@ const WorkflowRuns: React.FC = ({ accessToken }) => { { id: "status", accessorKey: "status", - header: "Status", - meta: { title: "Status" }, + header: t("workflows.table.columns.status"), + meta: { title: t("workflows.table.columns.status") }, filterFn: "equalsString", cell: ({ row }) => { const run = row.original; @@ -497,22 +506,22 @@ const WorkflowRuns: React.FC = ({ accessToken }) => { }, { accessorKey: "created_at", - header: "Created", - meta: { title: "Created" }, - cell: ({ row }) => {timeAgo(row.original.created_at)}, + header: t("workflows.table.columns.created"), + meta: { title: t("workflows.table.columns.created") }, + cell: ({ row }) => {timeAgo(row.original.created_at, t)}, }, ], - [], + [t], ); + const statusLabels = Object.fromEntries(RUN_STATUS_OPTIONS.map((status) => [status, t(STATUS_LABEL_KEYS[status])])); + return (
{/* page header */}
-
Workflow Runs
-
- Durable state tracking for agents and automated workflows -
+
{t("workflows.header.title")}
+
{t("workflows.header.subtitle")}
= ({ accessToken }) => { columns={columns} getRowId={(run) => run.run_id} isLoading={loadingRuns} - loadingMessage="Loading workflow runs…" - noDataMessage={
No workflow runs yet
} + loadingMessage={t("workflows.table.loading")} + noDataMessage={ +
{t("workflows.table.empty")}
+ } paginationMode="client" pageSizeOptions={[50, 100]} filterMode="client" @@ -537,7 +548,7 @@ const WorkflowRuns: React.FC = ({ accessToken }) => { table={table} searchValue={globalFilter} onSearchChange={setGlobalFilter} - searchPlaceholder="Search runs…" + searchPlaceholder={t("workflows.table.searchPlaceholder")} onRefresh={fetchRuns} isRefreshing={loadingRuns} onOpenFilters={() => setFiltersOpen(true)} @@ -546,35 +557,35 @@ const WorkflowRuns: React.FC = ({ accessToken }) => { table={table} open={filtersOpen} onOpenChange={setFiltersOpen} - title="Filters" - description="Narrow down workflow runs" + title={t("workflows.filters.title")} + description={t("workflows.filters.description")} > {({ get, set }) => ( <> - + - + set("workflow_type", event.target.value)} - placeholder="Filter by type…" + placeholder={t("workflows.filters.typePlaceholder")} /> @@ -590,10 +601,8 @@ const WorkflowRuns: React.FC = ({ accessToken }) => { showCloseButton={false} className="overflow-y-auto p-0 data-[side=right]:w-full data-[side=right]:sm:max-w-[680px]" > - Workflow run details - - Metadata, timeline and messages for the selected workflow run - + {t("workflows.drawer.srTitle")} + {t("workflows.drawer.srDescription")} {!selectedRun ? null : loadingDetail ? (
@@ -609,11 +618,11 @@ const WorkflowRuns: React.FC = ({ accessToken }) => { onClick={() => setDrawerOpen(false)} > - close + {t("workflows.drawer.close")}
@@ -623,19 +632,17 @@ const WorkflowRuns: React.FC = ({ accessToken }) => { {/* collapsible sections */}
- {events.length} {events.length === 1 ? "event" : "events"} - - } + title={t("workflows.drawer.timeline")} + meta={t("workflows.drawer.eventCount", { count: events.length })} defaultOpen > - + {messages.length === 0 ? ( -
No messages
+
+ {t("workflows.drawer.noMessages")} +
) : (
{messages.map((msg) => ( diff --git a/ui/litellm-dashboard/src/app/layout.tsx b/ui/litellm-dashboard/src/app/layout.tsx index 3a9d130839c..a8ff93f69f6 100644 --- a/ui/litellm-dashboard/src/app/layout.tsx +++ b/ui/litellm-dashboard/src/app/layout.tsx @@ -10,6 +10,7 @@ import ReactQueryProvider from "@/contexts/ReactQueryProvider"; import HotkeysProvider from "@/contexts/HotkeysProvider"; import { TerminologyProvider } from "@/contexts/TerminologyContext"; import { Toaster } from "@/components/ui/sonner"; +import I18nProvider from "@/i18n/I18nProvider"; const inter = Inter({ subsets: ["latin"] }); @@ -29,18 +30,20 @@ export default function RootLayout({ // cannot predict; suppressHydrationWarning confines that mismatch to this element. - - - - - - {children} - - - - - - + + + + + + + {children} + + + + + + + ); diff --git a/ui/litellm-dashboard/src/i18n/I18nProvider.tsx b/ui/litellm-dashboard/src/i18n/I18nProvider.tsx new file mode 100644 index 00000000000..f0ecdfe2568 --- /dev/null +++ b/ui/litellm-dashboard/src/i18n/I18nProvider.tsx @@ -0,0 +1,9 @@ +"use client"; + +import { I18nextProvider } from "react-i18next"; + +import i18n from "./i18n"; + +export default function I18nProvider({ children }: { children: React.ReactNode }) { + return {children}; +} diff --git a/ui/litellm-dashboard/src/i18n/i18n.test-d.ts b/ui/litellm-dashboard/src/i18n/i18n.test-d.ts new file mode 100644 index 00000000000..baa8c757311 --- /dev/null +++ b/ui/litellm-dashboard/src/i18n/i18n.test-d.ts @@ -0,0 +1,10 @@ +import type { TFunction } from "i18next"; +import { expectTypeOf, test } from "vitest"; + +test("translation keys come from the English message file", () => { + const t = {} as TFunction; + + expectTypeOf(t("workflows.drawer.eventCount", { count: 2 })).toBeString(); + // @ts-expect-error + t("workflows.header.titel"); +}); diff --git a/ui/litellm-dashboard/src/i18n/i18n.ts b/ui/litellm-dashboard/src/i18n/i18n.ts new file mode 100644 index 00000000000..0e9c364a0b3 --- /dev/null +++ b/ui/litellm-dashboard/src/i18n/i18n.ts @@ -0,0 +1,23 @@ +import i18n, { type InitOptions } from "i18next"; +import { initReactI18next } from "react-i18next"; + +import en from "./locales/en.json"; + +declare module "i18next" { + interface CustomTypeOptions { + defaultNS: "translation"; + resources: { translation: typeof en }; + } +} + +const options: InitOptions = { + lng: "en", + fallbackLng: "en", + interpolation: { escapeValue: false }, + resources: { en: { translation: en } }, + initAsync: false, +}; + +void i18n.use(initReactI18next).init(options); + +export default i18n; diff --git a/ui/litellm-dashboard/src/i18n/locales/en.json b/ui/litellm-dashboard/src/i18n/locales/en.json new file mode 100644 index 00000000000..9a65e074b25 --- /dev/null +++ b/ui/litellm-dashboard/src/i18n/locales/en.json @@ -0,0 +1,76 @@ +{ + "workflows": { + "header": { + "title": "Workflow Runs", + "subtitle": "Durable state tracking for agents and automated workflows" + }, + "table": { + "columns": { + "run": "Run", + "type": "Type", + "status": "Status", + "created": "Created" + }, + "loading": "Loading workflow runs…", + "empty": "No workflow runs yet", + "searchPlaceholder": "Search runs…" + }, + "filters": { + "title": "Filters", + "description": "Narrow down workflow runs", + "status": "Status", + "type": "Type", + "allStatuses": "All statuses", + "typePlaceholder": "Filter by type…" + }, + "status": { + "pending": "Pending", + "running": "Running", + "paused": "Paused", + "completed": "Completed", + "failed": "Failed" + }, + "drawer": { + "srTitle": "Workflow run details", + "srDescription": "Metadata, timeline and messages for the selected workflow run", + "close": "close", + "refresh": "Refresh", + "timeline": "Timeline", + "eventCount_one": "{{count}} event", + "eventCount_other": "{{count}} events", + "messages": "Messages", + "noMessages": "No messages", + "noEvents": "No events recorded" + }, + "metadata": { + "status": "status", + "created": "created", + "pr": "pr", + "state": "state", + "worktree": "worktree", + "grillSession": "grill session", + "session": "session" + }, + "truncate": { + "more": "more", + "less": "less" + }, + "tooltip": { + "type": "type: ", + "step": "step: ", + "seq": "seq: ", + "time": "time: ", + "data": "data: " + }, + "timeAgo": { + "seconds": "{{count}}s ago", + "minutes": "{{count}}m ago", + "hours": "{{count}}h ago", + "days": "{{count}}d ago" + }, + "duration": { + "ms": "{{count}}ms", + "seconds": "{{value}}s" + } + } +} diff --git a/ui/litellm-dashboard/tests/setupTests.ts b/ui/litellm-dashboard/tests/setupTests.ts index fe637e0473a..2b6b4c7484c 100644 --- a/ui/litellm-dashboard/tests/setupTests.ts +++ b/ui/litellm-dashboard/tests/setupTests.ts @@ -2,6 +2,8 @@ import "@testing-library/jest-dom"; import { cleanup } from "@testing-library/react"; import { afterEach, vi } from "vitest"; +import "@/i18n/i18n"; + const ensureTestLocalStorage = () => { if (typeof window === "undefined" || typeof window.Storage === "undefined") { return; From ea51e77dc5bd718e16de370ae32badf955d3ce7f Mon Sep 17 00:00:00 2001 From: mika-lt <349045450+mika-lt@users.noreply.github.com> Date: Thu, 29 Oct 2026 17:39:56 -0700 Subject: [PATCH 08/17] fix(responses)!: keep the proxy's error body on litellm.responses errors from litellm_proxy models (#49073) * fix(responses): keep the proxy's error body on litellm.responses errors from litellm_proxy models litellm.responses() with a litellm_proxy/ model raised BadRequestError with body None when the proxy answered a 4xx, while litellm.completion() on the same proxy carries the decoded error object. The litellm_proxy Responses config now decodes the error JSON into the exception body the way the OpenAI SDK does, so body, code, type and the message match chat completions on sync, async and streaming calls. * fix(proxy): relay a downstream proxy's provider_specific_fields in error responses A LiteLLM proxy routing to another LiteLLM proxy now copies the downstream's provider_specific_fields (which guardrail fired) from the exception body into its own error answer, on chat and Responses alike. Adds tests for the two _openai_sdk_error_body branches the existing tests left open. * fix(proxy): redact internal details from relayed provider_specific_fields * fix(proxy): redact relayed provider_specific_fields per key so credential-named values stay hidden * fix(proxy): redact numeric credentials in relayed provider_specific_fields --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- litellm/_logging.py | 28 +++++ .../litellm_proxy/responses/transformation.py | 32 ++++++ litellm/proxy/common_request_processing.py | 4 +- .../common_utils/openai_error_payload.py | 16 +++ .../code_coverage_tests/recursive_detector.py | 1 + .../responses/test_transformation.py | 84 +++++++++++++++ .../proxy/test_common_request_processing.py | 101 ++++++++++++++++++ 7 files changed, 265 insertions(+), 1 deletion(-) diff --git a/litellm/_logging.py b/litellm/_logging.py index 5279cb233b4..6a38eea2b12 100644 --- a/litellm/_logging.py +++ b/litellm/_logging.py @@ -28,9 +28,11 @@ from litellm.litellm_core_utils.env_utils import get_env_int from litellm.litellm_core_utils.safe_json_dumps import UNSERIALIZABLE_OBJECT, safe_dumps, safe_json_structure from litellm.litellm_core_utils.safe_json_loads import safe_json_loads from litellm.litellm_core_utils.secret_redaction import ( + REDACTED, python_redact_string, python_redact_structured_value, redact_internal_details, + redact_structured_value, ) from litellm.litellm_core_utils.secret_redaction import ( redact_string as redact_secret_string, @@ -140,6 +142,32 @@ def redact_internal_details_from_client_message(value: str) -> str: return redact_internal_details(value) +_CLIENT_FIELDS: Final = TypeAdapter(dict[str, object]) +_CLIENT_ITEMS: Final = TypeAdapter(list[object]) + + +def _redacted_client_field(key: str | None, value: object) -> object: + if isinstance(value, str): + return redact_internal_details(redact_structured_value(key, value)) + if isinstance(value, dict): + return { + child_key: _redacted_client_field(child_key, child) + for child_key, child in _CLIENT_FIELDS.validate_python(value).items() + } + if isinstance(value, (list, tuple)): + return [_redacted_client_field(key, child) for child in _CLIENT_ITEMS.validate_python(value)] + if not isinstance(value, (int, float)): + return value + text: Final = str(value) + return value if redact_structured_value(key, text) == text else REDACTED + + +def redact_internal_details_from_client_fields(fields: Mapping[str, object]) -> object: + if not _ENABLE_SECRET_REDACTION: + return fields + return _redacted_client_field(None, dict(fields)) + + def _substituted_color_message(record: logging.LogRecord) -> str | None: """Render a record's ``color_message`` against its args, or None if absent. diff --git a/litellm/llms/litellm_proxy/responses/transformation.py b/litellm/llms/litellm_proxy/responses/transformation.py index 3092b49b743..fc94eb30504 100644 --- a/litellm/llms/litellm_proxy/responses/transformation.py +++ b/litellm/llms/litellm_proxy/responses/transformation.py @@ -5,11 +5,30 @@ LiteLLM Proxy supports the OpenAI Responses API natively when the underlying mod This config enables pass-through behavior to the proxy's /v1/responses endpoint. """ +from typing import Final + +import httpx +from pydantic import TypeAdapter, ValidationError + from litellm.litellm_core_utils.url_utils import join_api_base +from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig from litellm.secret_managers.main import get_secret_str from litellm.types.utils import LlmProviders +_ERROR_BODY: Final = TypeAdapter(dict[str, object]) + + +def _openai_sdk_error_body( + error_message: str, +) -> dict[str, object] | None: # mutable-ok: BaseLLMException.body is a dict + try: + decoded: Final = _ERROR_BODY.validate_json(error_message) + except ValidationError: + return None + nested: Final = decoded.get("error") + return _ERROR_BODY.validate_python(nested) if isinstance(nested, dict) else decoded + class LiteLLMProxyResponsesAPIConfig(OpenAIResponsesAPIConfig): """ @@ -43,6 +62,19 @@ class LiteLLMProxyResponsesAPIConfig(OpenAIResponsesAPIConfig): return join_api_base(api_base, "responses") + def get_error_class( + self, + error_message: str, + status_code: int, + headers: dict[str, object] | httpx.Headers, # mutable-ok: inherited contract + ) -> BaseLLMException: + return BaseLLMException( + status_code=status_code, + message=error_message, + headers=headers, + body=_openai_sdk_error_body(error_message), + ) + def supports_native_websocket(self) -> bool: """LiteLLM Proxy does not support native WebSocket for Responses API""" return False diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index 666269be2eb..7cd752c2297 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -114,6 +114,7 @@ from litellm.proxy.common_utils.openai_error_payload import ( error_status_code, openai_error_param, openai_error_type, + relayed_provider_specific_fields, ) from litellm.proxy.common_utils.sse_keepalive import ( SSE_COMMENT_PING_BYTES, @@ -4462,7 +4463,8 @@ class ProxyBaseLLMRequestProcessing: param=openai_error_param(e), openai_code=getattr(e, "code", None), code=_code, - provider_specific_fields=getattr(e, "provider_specific_fields", None), + provider_specific_fields=getattr(e, "provider_specific_fields", None) + or relayed_provider_specific_fields(e), headers=safe_headers, ) diff --git a/litellm/proxy/common_utils/openai_error_payload.py b/litellm/proxy/common_utils/openai_error_payload.py index 3c91e87ee51..f92fbeef9a1 100644 --- a/litellm/proxy/common_utils/openai_error_payload.py +++ b/litellm/proxy/common_utils/openai_error_payload.py @@ -8,7 +8,9 @@ from types import MappingProxyType from typing import Final from fastapi import HTTPException, status +from pydantic import TypeAdapter +from litellm._logging import redact_internal_details_from_client_fields from litellm.constants import STRINGIFIED_NONE from litellm.proxy._types import ProxyErrorTypes, ProxyException, proxy_exception_status_code from litellm.proxy.db.db_lookup_gate import DBLookupDeadlineExceeded @@ -26,6 +28,8 @@ _OPENAI_ERROR_TYPE_BY_STATUS: Final[Mapping[int, str]] = MappingProxyType( _AUTH_FAILURE_STATUS_CODES: Final = frozenset({status.HTTP_401_UNAUTHORIZED, status.HTTP_403_FORBIDDEN}) +_ERROR_BODY: Final = TypeAdapter(dict[str, object]) + _FAILOVER_SQLSTATES: Final = frozenset({"25006", "57P01", "57P02", "57P03"}) _FAILOVER_REFUSALS: Final = ( @@ -77,6 +81,18 @@ def openai_error_param(exc: object) -> str | None: return carried if isinstance(carried, str) and carried != STRINGIFIED_NONE else None +def relayed_provider_specific_fields( + exc: object, +) -> dict[str, object] | None: # mutable-ok: ProxyException.provider_specific_fields is a dict + body: Final = attribute_of(exc, "body") + if not isinstance(body, dict): + return None + fields: Final = _ERROR_BODY.validate_python(body).get("provider_specific_fields") + if not isinstance(fields, dict): + return None + return _ERROR_BODY.validate_python(redact_internal_details_from_client_fields(_ERROR_BODY.validate_python(fields))) + + def proxy_exception_for(exc: Exception, default_status_code: int) -> ProxyException: match exc: case ProxyException(): diff --git a/tests/code_coverage_tests/recursive_detector.py b/tests/code_coverage_tests/recursive_detector.py index a5bb5e0b340..c943ff2cbd0 100644 --- a/tests/code_coverage_tests/recursive_detector.py +++ b/tests/code_coverage_tests/recursive_detector.py @@ -81,6 +81,7 @@ IGNORE_FUNCTIONS = [ "completion_cost", # max depth 1: recursion only fires for mixed-tier Responses WS logging objects, and each split part carries a single service_tier so _split_responses_ws_logging_object_by_service_tier returns None. "_string_leaves", # bounded by the nesting depth of a safe_json_structure output (a finite JSON tree, no cycles possible). "_replace_string_leaves", # bounded by the nesting depth of a safe_json_structure output (a finite JSON tree, no cycles possible). + "_redacted_client_field", # bounded by the nesting depth of a JSON-decoded error body (a finite JSON tree, no cycles possible). "_sort_processed_sets", # bounded by the nesting depth of the log-record extra it walks (a finite JSON tree, no cycles possible). "scrub_json_strings", # max depth set (MAX_SCRUB_DEPTH); fails closed by returning "[Filtered]" for anything nested past the cap. "_without_requester_ip_address", # walks only the fresh tree _sanitize_request_body_for_spend_logs_payload returns (mapping cycles already cut), so it never goes deeper than that sanitizer did. diff --git a/tests/unit/llms/litellm_proxy/responses/test_transformation.py b/tests/unit/llms/litellm_proxy/responses/test_transformation.py index 3d774b79eed..5537d9d1916 100644 --- a/tests/unit/llms/litellm_proxy/responses/test_transformation.py +++ b/tests/unit/llms/litellm_proxy/responses/test_transformation.py @@ -1,4 +1,5 @@ import json +from collections.abc import Callable, Iterable from typing import Final import httpx @@ -94,3 +95,86 @@ def test_compact_request_forwards_metadata_and_user_to_the_downstream_proxy(drop "user": "end-user-1", "temperature": 0.2, } + + +_GUARDRAIL_BLOCK: Final = { + "error": { + "message": "Content blocked: secret_project_codename pattern detected", + "type": "invalid_request_error", + "param": None, + "code": "400", + "provider_specific_fields": {"guardrail_name": "block-secret-project", "guardrail_mode": "pre_call"}, + } +} + + +def _proxy_rejection(route: str, response: httpx.Response, call: Callable[[], object]) -> litellm.BadRequestError: + with respx.mock(assert_all_called=True) as mock: + _ = mock.post(f"https://my-proxy.example.com/{route}").mock(return_value=response) + with pytest.raises(litellm.BadRequestError) as caught: + _ = call() + return caught.value + + +def _consume(stream: Iterable[object]) -> None: + for _ in stream: + pass + + +_PROXY_PARAMS: Final = {"api_base": "https://my-proxy.example.com/", "api_key": "sk-proxy-key"} + + +@pytest.mark.usefixtures(httpx_transport.__name__) +@pytest.mark.parametrize("stream", [False, True]) +def test_proxy_400_reaches_responses_caller_with_the_same_body_as_chat_completions(stream: bool) -> None: + chat_error: Final = _proxy_rejection( + "chat/completions", + httpx.Response(400, json=_GUARDRAIL_BLOCK), + lambda: litellm.completion( + model="litellm_proxy/gpt-5.5", messages=[{"role": "user", "content": "project bluebird"}], **_PROXY_PARAMS + ), + ) + responses_error: Final = _proxy_rejection( + "responses", + httpx.Response(400, json=_GUARDRAIL_BLOCK), + lambda: ( + _consume(litellm.responses(model="litellm_proxy/gpt-5.5", input="project bluebird", stream=True, **_PROXY_PARAMS)) + if stream + else litellm.responses(model="litellm_proxy/gpt-5.5", input="project bluebird", **_PROXY_PARAMS) + ), + ) + assert responses_error.body == chat_error.body == _GUARDRAIL_BLOCK["error"] + assert (responses_error.code, responses_error.type) == (chat_error.code, chat_error.type) + assert str(responses_error) == str(chat_error) + + +@pytest.mark.usefixtures(httpx_transport.__name__) +def test_proxy_error_without_a_json_object_body_still_raises_with_its_text() -> None: + error: Final = _proxy_rejection( + "responses", + httpx.Response(400, text="upstream proxy rejected the request"), + lambda: litellm.responses(model="litellm_proxy/gpt-5.5", input="hi", **_PROXY_PARAMS), + ) + assert error.body is None + assert "upstream proxy rejected the request" in str(error) + + +@pytest.mark.usefixtures(httpx_transport.__name__) +def test_proxy_error_object_without_an_error_member_is_the_body_as_sent() -> None: + sent: Final = {"message": "upstream proxy rejected the request", "type": "invalid_request_error"} + error: Final = _proxy_rejection( + "responses", + httpx.Response(400, json=sent), + lambda: litellm.responses(model="litellm_proxy/gpt-5.5", input="hi", **_PROXY_PARAMS), + ) + assert error.body == sent + + +@pytest.mark.usefixtures(httpx_transport.__name__) +def test_proxy_error_whose_error_member_is_text_still_raises_with_that_text() -> None: + error: Final = _proxy_rejection( + "responses", + httpx.Response(400, json={"error": "upstream proxy rejected the request"}), + lambda: litellm.responses(model="litellm_proxy/gpt-5.5", input="hi", **_PROXY_PARAMS), + ) + assert "upstream proxy rejected the request" in str(error) diff --git a/tests/unit/proxy/test_common_request_processing.py b/tests/unit/proxy/test_common_request_processing.py index cc1fc5cd6c3..aeab7a3d348 100644 --- a/tests/unit/proxy/test_common_request_processing.py +++ b/tests/unit/proxy/test_common_request_processing.py @@ -14,6 +14,7 @@ from tests.unit.proxy.router_entry_point_stubs import route_entry_points_to_stub import anthropic import httpx import pytest +import respx from fastapi import HTTPException, Request, Response, status from fastapi.responses import JSONResponse, StreamingResponse @@ -12516,3 +12517,103 @@ def test_maybe_get_model_id_prefers_litellm_metadata_over_caller_metadata_withou def test_maybe_get_model_id_returns_none_before_the_router_picked_a_deployment(): assert ProxyBaseLLMRequestProcessing(data={"metadata": {}}).maybe_get_model_id(None) is None + + +_DOWNSTREAM_GUARDRAIL_FIELDS: Final = MappingProxyType( + {"guardrail_name": "block-secret-project", "guardrail_mode": "pre_call", "pattern": "secret_project_codename"} +) + + +def _downstream_proxy_rejection( + route: str, call: Callable[[], object], fields: Mapping[str, object] = _DOWNSTREAM_GUARDRAIL_FIELDS +) -> Exception: + body: Final = { + "error": { + "message": "Content blocked: secret_project_codename pattern detected", + "type": "invalid_request_error", + "param": None, + "code": "400", + "provider_specific_fields": dict(fields), + } + } + with respx.mock(assert_all_called=True) as mock: + _ = mock.post(f"https://downstream-proxy.example.com/{route}").mock(return_value=httpx.Response(400, json=body)) + with pytest.raises(litellm.BadRequestError) as caught: + _ = call() + return caught.value + + +_DOWNSTREAM: Final = MappingProxyType( + {"api_base": "https://downstream-proxy.example.com/", "api_key": "sk-downstream-key"} +) + + +@pytest.mark.usefixtures("httpx_transport") +@pytest.mark.parametrize( + ("route", "call"), + [ + ( + "responses", + lambda: litellm.responses(model="litellm_proxy/gpt-5.5", input="project bluebird", **_DOWNSTREAM), + ), + ( + "chat/completions", + lambda: litellm.completion( + model="litellm_proxy/gpt-5.5", messages=[{"role": "user", "content": "project bluebird"}], **_DOWNSTREAM + ), + ), + ], +) +async def test_a_chained_proxy_relays_which_downstream_guardrail_blocked_the_request( + route: str, call: Callable[[], object] +) -> None: + rejection: Final = _downstream_proxy_rejection(route, call) + with pytest.raises(ProxyException) as relayed: + await ProxyBaseLLMRequestProcessing(data={"model": "chained"})._handle_llm_api_exception( + e=rejection, + user_api_key_dict=ProxyUserAPIKeyAuth(api_key="sk-front"), + proxy_logging_obj=ProxyLogging(user_api_key_cache=DualCache()), + ) + assert relayed.value.to_dict()["provider_specific_fields"] == dict(_DOWNSTREAM_GUARDRAIL_FIELDS) + + +_DOWNSTREAM_LEAKABLE_VALUES: Final = ( + "guard-scan.internal.corp", + "/srv/secret/app/guard.py", + "10.0.4.17", + "abcd1234efgh5678", + "cs-0123456789", + "tok-0123456789abcdef", + "4815162342108", +) + + +@pytest.mark.usefixtures("httpx_transport") +async def test_a_chained_proxy_redacts_secrets_and_internal_details_from_relayed_guardrail_fields() -> None: + fields: Final = { + **_DOWNSTREAM_GUARDRAIL_FIELDS, + "exception": "connect to http://guard-scan.internal.corp:9000/v1/check failed at /srv/secret/app/guard.py", + "api_key": "abcd1234efgh5678", + "access_token": ["tok-0123456789abcdef"], + "scanner": { + "hosts": ["10.0.4.17:8080"], + "client_secret": "cs-0123456789", + "api_secret": 4815162342108, + "attempts": 2, + }, + } + rejection: Final = _downstream_proxy_rejection( + "responses", + lambda: litellm.responses(model="litellm_proxy/gpt-5.5", input="project bluebird", **_DOWNSTREAM), + fields, + ) + with pytest.raises(ProxyException) as relayed: + await ProxyBaseLLMRequestProcessing(data={"model": "chained"})._handle_llm_api_exception( + e=rejection, + user_api_key_dict=ProxyUserAPIKeyAuth(api_key="sk-front"), + proxy_logging_obj=ProxyLogging(user_api_key_cache=DualCache()), + ) + answered: Final = json.loads(json.dumps(relayed.value.to_dict()["provider_specific_fields"])) + assert {key: answered[key] for key in _DOWNSTREAM_GUARDRAIL_FIELDS} == dict(_DOWNSTREAM_GUARDRAIL_FIELDS) + assert answered["scanner"]["attempts"] == 2 + assert [value for value in _DOWNSTREAM_LEAKABLE_VALUES if value in json.dumps(answered)] == [] From 2b4cb92c8b0e470b1fd8a4ae0d91c8193353abde Mon Sep 17 00:00:00 2001 From: Alon Sokolovski Date: Fri, 30 Oct 2026 02:42:11 +0200 Subject: [PATCH 09/17] fix(context_management)!: run clear_tool_uses in the gateway on the Responses API path (#47861) * fix(context_management): run clear_tool_uses in the gateway on the Responses API path /v1/messages to openai/ models goes through the Responses API, which translated only compact_20260112 (to its native compaction) and dropped every other edit without a warning, so clear_tool_uses_20250919 never cleared anything. Edits without a Responses API form now run in the same polyfill as the chat completions path, before the call, and their applied_edits come back on the response and the final message_delta. compact stays native. * fix(context_management): check the sync polyfill result instead of casting it The strict ruff gate (TID251) bans typing.cast. * test(context_management): cover the Responses path's polyfill error handling A polyfill crash is a 500 before the call, a rejected edit under drop_params is skipped, and the sync stream reports its edits. normalize_spec never raises for a dict spec, so its guard is gone. * fix(context_management): count unreadable blocks and check tool_result text before clearing clear_tool_uses now maps blocks the token counter cannot read (server tool, compaction) to JSON text, so an input_tokens trigger no longer 500s on that history. The Responses API path checks malformed tool_result text on the original messages before clearing, as the chat completions path does. Also removes comments that repeated the code and makes the test fixtures immutable. * fix(context_management): count only blocks the model receives and keep list specs unsent clear_tool_uses now leaves blocks token_counter cannot read, like server tool results and compaction, out of its count instead of counting them as JSON text. No adapter sends them to the model, so an input_tokens trigger no longer fires on tokens the model never sees. On the Responses API path, an OpenAI-style list context_management stays unsent, as it was before, instead of reaching OpenAI's native compaction, where a threshold under 1000 is a 400. On the chat completions path, a clear edit listed before compact_20260112 now runs on the turns after the latest compaction block, as Anthropic does, instead of counting and clearing turns the compaction block drops. The crash test that replaced apply_context_management with a mock is removed. * fix(context_management): apply clear edits in OpenAI count_tokens and skip nested off-spec blocks when counting * fix(context_management): run edits on the history drop_params sends, so count_tokens and /v1/messages clear alike * fix(context_management): count clear_at_least freed tokens over sent blocks only * fix(context_management): count tools with JSON Schema shorthands and reject a non-object trigger or keep A tool whose schema has an array without items, a boolean subschema, or a search_result block without a title made the token counter raise, so a clear edit answered 500. A string trigger or keep also answered 500, even under drop_params. Both now count or get the API's 400, and drop_params drops the malformed field. * fix(token_counter): count a malformed tool schema keyword as absent and move shorthand framed-estimate cases * fix(count_tokens): pass the system prompt into clearing so count_tokens matches inference * fix(context_management): keep a malformed system prompt from failing clear_tool_uses on the Responses API A number, an object, or a list holding non-object items as system made the polyfill answer 500. The polyfill now keeps a list's object items and treats any other shape as no system, on both /v1/messages and count_tokens, so the two paths still count the same text --------- Co-authored-by: AlonSoko <25619863+AlonSoko@users.noreply.github.com> Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- litellm/litellm_core_utils/token_counter.py | 25 +- .../anthropic/messages/adapters/handler.py | 3 +- .../messages/context_management/dispatcher.py | 5 +- .../editors/clear_tool_uses.py | 75 +++- .../context_management/editors/compact.py | 8 +- .../messages/responses_adapters/handler.py | 153 ++++++++- .../responses_adapters/streaming_iterator.py | 10 +- .../responses/count_tokens/token_counter.py | 45 ++- .../litellm_core_utils/test_token_counter.py | 51 +++ .../test_clear_tool_uses.py | 130 ++++++- .../context_management/test_compact.py | 6 +- .../context_management/test_dispatcher.py | 55 +++ .../test_responses_adapters_handler.py | 325 +++++++++++++++++- .../count_tokens/test_token_counter.py | 120 ++++++- .../vertex_ai/count_tokens/test_context.py | 2 +- tests/unit/proxy/test_proxy_token_counter.py | 54 ++- 16 files changed, 993 insertions(+), 74 deletions(-) diff --git a/litellm/litellm_core_utils/token_counter.py b/litellm/litellm_core_utils/token_counter.py index 591d77bdb16..5ad4e130e62 100644 --- a/litellm/litellm_core_utils/token_counter.py +++ b/litellm/litellm_core_utils/token_counter.py @@ -1224,10 +1224,6 @@ def _validate_anthropic_content(content: Mapping[str, object]) -> type: if expected_cls is None: raise ValueError(f"Unknown Anthropic content type: '{content_type}'") - missing: Final = [k for k in getattr(expected_cls, "__required_keys__", set()) if k not in content] - if missing: - raise ValueError(f"Missing required fields in {content_type} block: {', '.join(missing)}") - return expected_cls @@ -1552,12 +1548,13 @@ def _format_single_function_definition(function: Mapping[str, object]) -> tuple[ def _format_object_parameters(parameters, indent): properties: Final = parameters.get("properties") - if not properties: + if not isinstance(properties, Mapping) or not properties: return "" - required_params: Final = parameters.get("required", []) + required: Final = parameters.get("required") + required_params: Final = tuple(required) if isinstance(required, list) else () lines: Final = [] for key, props in properties.items(): - description = props.get("description") + description = props.get("description") if isinstance(props, Mapping) else None if description: lines.append(f"// {description}") question = "?" @@ -1568,19 +1565,21 @@ def _format_object_parameters(parameters, indent): def _format_type(props, indent): + if not isinstance(props, Mapping): + return "any" type: Final = props.get("type") + enum: Final = props.get("enum") if type == "string": - if "enum" in props: - return " | ".join([f'"{item}"' for item in props["enum"]]) + if isinstance(enum, list): + return " | ".join([f'"{item}"' for item in enum]) return "string" elif type == "array": - # items is required, OpenAI throws an error if it's missing - return f"{_format_type(props['items'], indent)}[]" + return f"{_format_type(props.get('items'), indent)}[]" elif type == "object": return f"{{\n{_format_object_parameters(props, indent + 2)}\n}}" elif type in ["integer", "number"]: - if "enum" in props: - return " | ".join([f'"{item}"' for item in props["enum"]]) + if isinstance(enum, list): + return " | ".join([f'"{item}"' for item in enum]) return "number" elif type == "boolean": return "boolean" diff --git a/litellm/llms/anthropic/messages/adapters/handler.py b/litellm/llms/anthropic/messages/adapters/handler.py index 71ccfbaf915..704bec9056f 100644 --- a/litellm/llms/anthropic/messages/adapters/handler.py +++ b/litellm/llms/anthropic/messages/adapters/handler.py @@ -160,6 +160,7 @@ async def prepare_context_managed_request( """Apply client compaction history, then optional context_management polyfill.""" from litellm.llms.anthropic.messages.context_management.editors.compact import ( apply_client_compaction_block_history, + slice_around_compaction_block, ) # Skip the client-history pre-processing when a ``compact_20260112`` @@ -175,7 +176,7 @@ async def prepare_context_managed_request( if polyfill_will_run: history_result: PolyfillResult | None = None - working_messages: _AnthropicMessages = messages + working_messages: _AnthropicMessages = list(slice_around_compaction_block(messages)[0]) working_system: _AnthropicSystem = system else: history_result = apply_client_compaction_block_history( diff --git a/litellm/llms/anthropic/messages/context_management/dispatcher.py b/litellm/llms/anthropic/messages/context_management/dispatcher.py index 1557cf206ac..2b713646d50 100644 --- a/litellm/llms/anthropic/messages/context_management/dispatcher.py +++ b/litellm/llms/anthropic/messages/context_management/dispatcher.py @@ -8,6 +8,7 @@ from litellm._logging import verbose_logger from litellm.litellm_core_utils.asyncify import asyncify from litellm.types.llms.anthropic import AppliedEdit +from ..convertible_blocks import drop_non_string_tool_result_texts from .constants import CLEAR_TOOL_USES_EDIT_TYPE, COMPACT_EDIT_TYPE from .editors import ( CheckedEdit, @@ -103,7 +104,7 @@ async def apply_context_management( Every ``clear_tool_uses_20250919`` edit is validated before any editor runs, so a malformed one raises its 400 before a ``compact_20260112`` summary call is paid for, or runs without its malformed fields under - ``drop_invalid_edits``. + ``drop_invalid_edits``, which also drops non-string ``tool_result`` text first, as the provider never receives it. The dispatcher is async so async editors (``compact_20260112``) can ``await`` the configured summarization model. Sync editors run in a @@ -119,7 +120,7 @@ async def apply_context_management( raise first_error well_formed_edits: Final = tuple(check.edit for check in checked) - current_messages = messages + current_messages = list(drop_non_string_tool_result_texts(messages, drop_invalid_edits)) current_system = system aggregated_applied: Final[list[AppliedEdit]] = [] aggregated_compaction_block = None diff --git a/litellm/llms/anthropic/messages/context_management/editors/clear_tool_uses.py b/litellm/llms/anthropic/messages/context_management/editors/clear_tool_uses.py index c4f7966f429..fabc112d443 100644 --- a/litellm/llms/anthropic/messages/context_management/editors/clear_tool_uses.py +++ b/litellm/llms/anthropic/messages/context_management/editors/clear_tool_uses.py @@ -39,6 +39,21 @@ if TYPE_CHECKING: T = TypeVar("T") # api.anthropic.com, measured 2026-10-19: text results of 1 to 46 bytes are never cleared or counted toward keep _TINY_RESULT_BYTES: Final = 46 +_TOKEN_COUNTER_BLOCK_TYPES: Final = frozenset( + { + "text", + "image_url", + "image", + "document", + "file", + "tool_use", + "tool_result", + "thinking", + "redacted_thinking", + "tool_reference", + "search_result", + } +) ToolNames = Literal["all"] | frozenset[str] @@ -52,16 +67,19 @@ class _ClearAtLeast(BaseModel): class _ClearToolUsesFields(BaseModel): model_config = ConfigDict(frozen=True) + trigger: Mapping[str, object] | None = None + keep: Mapping[str, object] | None = None clear_at_least: _ClearAtLeast | None = None exclude_tools: tuple[str, ...] | None = None clear_tool_inputs: bool | tuple[str, ...] | None = None _BLOCKS: Final = TypeAdapter(tuple[Mapping[str, object], ...]) -# api.anthropic.com, measured 2026-10-19: its wording for the two shapes pydantic words differently +# api.anthropic.com, measured 2026-10-19 and 2026-10-28: its wording for the shapes pydantic words differently _API_MESSAGES: Final = MappingProxyType( { "tuple_type": "Input should be a valid array", + "dict_type": "Input should be an object", "model_type": "Input does not match the expected shape.", } ) @@ -178,10 +196,12 @@ def _cleared_result(block: Mapping[str, object]) -> dict[str, object]: def _freed_tokens(model: str, result: Mapping[str, object], use: Mapping[str, object], clears_input: bool) -> int: """Tokens that clearing this use frees: its result, and its input when ``clear_tool_inputs`` covers it.""" - before: Final = [result, use] if clears_input else [result] - after: Final = [_cleared_result(result), {**use, "input": {}}] if clears_input else [_cleared_result(result)] + before: Final[Sequence[Mapping[str, object]]] = (result, use) if clears_input else (result,) + after: Final[Sequence[Mapping[str, object]]] = ( + (_cleared_result(result), {**use, "input": {}}) if clears_input else (_cleared_result(result),) + ) counts: Final = tuple( - litellm.token_counter(model=model, messages=[{"role": "user", "content": blocks}]) for blocks in (before, after) + _count_tokens(model, [{"role": "user", "content": blocks}], None) for blocks in (before, after) ) return max(counts[0] - counts[1], 0) @@ -199,6 +219,35 @@ def _select_in_batches(candidates: Sequence[str], tokens: Mapping[str, int], cle return tuple(candidates[:batch_end]) +def _without_uncountable_blocks(block: Mapping[str, object]) -> Mapping[str, object]: + nested: Final = _parse(_BLOCKS, block.get("content")) + if nested is None: + return block + return {**block, "content": [inner for inner in nested if inner.get("type") in _TOKEN_COUNTER_BLOCK_TYPES]} + + +def _countable_message(message: Mapping[str, object]) -> Mapping[str, object]: + blocks: Final = _parse(_BLOCKS, message.get("content")) + if blocks is None: + return message + return { + **message, + "content": [ + _without_uncountable_blocks(block) for block in blocks if block.get("type") in _TOKEN_COUNTER_BLOCK_TYPES + ], + } + + +def _count_tokens( + model: str, messages: Sequence[Mapping[str, object]], tools: Sequence[Mapping[str, object]] | None +) -> int: + return litellm.token_counter( + model=model, + messages=[_countable_message(message) for message in messages], + tools=cast("list[ChatCompletionToolParam] | None", tools), + ) + + def _trigger_met( trigger: dict[str, object], model: str, @@ -218,11 +267,7 @@ def _trigger_met( if not isinstance(threshold, int): threshold = DEFAULT_INPUT_TOKENS_TRIGGER - message_tokens: Final = litellm.token_counter( - model=model, - messages=messages, - tools=cast("list[ChatCompletionToolParam] | None", tools), - ) + message_tokens: Final = _count_tokens(model, messages, tools) if message_tokens > threshold: return True, message_tokens current_tokens: Final = message_tokens + count_system_tokens(model, system) @@ -375,17 +420,9 @@ def apply_clear_tool_uses_20250919( [_message_without_inputs(message, ids_to_clear, input_tools) for message in edited] if input_tools else edited ) - tokens_before: Final = ( - counted_tokens - if counted_tokens is not None - else litellm.token_counter( - model=model, messages=kept, tools=cast("list[ChatCompletionToolParam] | None", tools) - ) - ) + tokens_before: Final = counted_tokens if counted_tokens is not None else _count_tokens(model, kept, tools) kept_after, _ = history_after_client_compaction(final_messages, system) - tokens_after: Final = litellm.token_counter( - model=model, messages=kept_after, tools=cast("list[ChatCompletionToolParam] | None", tools) - ) + tokens_after: Final = _count_tokens(model, kept_after, tools) cleared_input_tokens: Final = max(tokens_before - tokens_after, 0) return final_messages, { diff --git a/litellm/llms/anthropic/messages/context_management/editors/compact.py b/litellm/llms/anthropic/messages/context_management/editors/compact.py index 0bf8cac7344..d1cfcc2f2bc 100644 --- a/litellm/llms/anthropic/messages/context_management/editors/compact.py +++ b/litellm/llms/anthropic/messages/context_management/editors/compact.py @@ -137,7 +137,7 @@ def _find_latest_compaction_index( return None, None -def _slice_around_compaction_block( +def slice_around_compaction_block( messages: Sequence[_MsgT], ) -> tuple[Sequence[_MsgT | dict[str, object]], dict[str, object] | None]: """Apply Anthropic's "drop everything before the compaction block" rule. @@ -194,7 +194,7 @@ def history_after_client_compaction( system: str | Sequence[Mapping[str, object]] | None, ) -> tuple[list[dict[str, object]], str | Sequence[Mapping[str, object]] | None]: """Return the history the API keeps after the latest compaction block, without it, and the system with its summary.""" - sliced, block = _slice_around_compaction_block(messages) + sliced, block = slice_around_compaction_block(messages) if block is None: return list(messages), system tail: Final = _strip_compaction_blocks(sliced) @@ -615,7 +615,7 @@ def apply_client_compaction_block_history( tail is forwarded unchanged (with compaction blocks stripped) so recent turns the summary does not cover are preserved. """ - effective_messages, prior_compaction_block = _slice_around_compaction_block(messages) + effective_messages, prior_compaction_block = slice_around_compaction_block(messages) if prior_compaction_block is None: return None @@ -686,7 +686,7 @@ async def apply_compact_20260112( # opt-in gate below so that even when summarization is disabled we still # strip Anthropic-only ``compaction`` blocks from messages going to # non-Anthropic backends (which would reject them). - effective_messages, prior_compaction_block = _slice_around_compaction_block(messages) + effective_messages, prior_compaction_block = slice_around_compaction_block(messages) prior_summary_text: Final = prior_compaction_block.get("content") if prior_compaction_block else None augmented_system: str | list[dict[str, object]] | None = system if isinstance(prior_summary_text, str) and prior_summary_text: diff --git a/litellm/llms/anthropic/messages/responses_adapters/handler.py b/litellm/llms/anthropic/messages/responses_adapters/handler.py index 9b22170c9db..0eef3fd87c0 100644 --- a/litellm/llms/anthropic/messages/responses_adapters/handler.py +++ b/litellm/llms/anthropic/messages/responses_adapters/handler.py @@ -10,11 +10,15 @@ from typing import Final, TypeAlias from pydantic import TypeAdapter, ValidationError import litellm +from litellm._logging import verbose_logger from litellm.integrations.anthropic_cache_control_hook import ( AnthropicCacheControlHook, supports_openai_prompt_cache_breakpoint, ) -from litellm.litellm_core_utils.core_helpers import drops_unsupported_params +from litellm.litellm_core_utils.asyncify import ( + run_async_function, # pyright: ignore[reportUnknownVariableType] # untyped helper +) +from litellm.litellm_core_utils.core_helpers import as_str_mapping, drops_unsupported_params from litellm.litellm_core_utils.prompt_templates.common_utils import ( cache_control_as_prompt_cache_breakpoint, prompt_cache_breakpoint_for_wire, @@ -28,14 +32,19 @@ from litellm.types.llms.anthropic import ( AnthropicMessagesRequest, AnthropicOutputConfig, AnthropicOutputSchema, + ContextManagementResponse, ) from litellm.types.llms.anthropic_messages.anthropic_response import ( AnthropicMessagesResponse, ) from litellm.types.llms.openai import ResponsesAPIResponse +from litellm.types.utils import CountContextManagement from litellm.utils import ProviderConfigManager -from ..adapters.handler import request_safeguards_evaluator +from ..adapters.handler import context_management_explicitly_dropped, request_safeguards_evaluator +from ..context_management import AnthropicContextManagementError, PolyfillResult, apply_context_management +from ..context_management.constants import COMPACT_EDIT_TYPE +from ..context_management.dispatcher import normalize_spec from ..convertible_blocks import reject_malformed_tool_result_text from ..safeguards import with_safeguard_results from ..utils import litellm_logging_obj_from_kwargs, local_model_name @@ -45,10 +54,111 @@ from .transformation import LiteLLMAnthropicToResponsesAPIAdapter AnthropicRequestMessages: TypeAlias = list[AllAnthropicMessageValues] | list[dict[str, object]] BlockRewrite: TypeAlias = Callable[[Mapping[str, object]], Mapping[str, object]] _BLOCK_LIST: Final = TypeAdapter(list[dict[str, object]]) +_SYSTEM_ITEMS: Final = TypeAdapter(list[object]) +_SPEC: Final = TypeAdapter(dict[str, object]) +_SplitEdits: TypeAlias = tuple[dict[str, object] | None, Sequence[Mapping[str, object]]] +AnthropicContextManagementSpec: TypeAlias = dict[str, object] | list[dict[str, object]] | None _ADAPTER: Final = LiteLLMAnthropicToResponsesAPIAdapter() +def _split_context_management( + spec: AnthropicContextManagementSpec, + kwargs: Mapping[str, object], +) -> _SplitEdits: + """Return (the edits the Responses API runs natively, the edits the gateway polyfill runs). + + The Responses API compacts natively (``compact_20260112`` becomes its ``compaction``); any other + edit, such as ``clear_tool_uses_20250919``, has no Responses API form and runs in the polyfill, as + it does on the chat completions path. An OpenAI-style list spec is not Anthropic's shape and stays + unsent, as it was before the polyfill ran here. + """ + if not isinstance(spec, dict): + return None, () + if not spec or context_management_explicitly_dropped(kwargs.get("additional_drop_params")): + return spec, () + edits: Final = normalize_spec(spec) or [] + native: Final = [edit for edit in edits if edit.get("type") == COMPACT_EDIT_TYPE] + return ({"edits": native} if native else None), tuple( + edit for edit in edits if edit.get("type") != COMPACT_EDIT_TYPE + ) + + +def _system_blocks(system: object) -> str | Sequence[Mapping[str, object]] | None: + if system is None or isinstance(system, str): + return system + try: + items: Final = _SYSTEM_ITEMS.validate_python(system) + except ValidationError: + return None + return tuple(block for block in map(as_str_mapping, items) if block is not None) + + +async def _polyfilled( + *, + model: str, + messages: AnthropicRequestMessages, + tools: Sequence[AllAnthropicToolsValues | Mapping[str, object]] | None, + system: object, + edits: Sequence[Mapping[str, object]], + kwargs: Mapping[str, object], +) -> PolyfillResult | None: + """Run the edits the Responses API can't run natively; raise like the chat completions path does.""" + if not edits: + return None + reject_malformed_tool_result_text(messages, model, kwargs) + drop_invalid_edits: Final = drops_unsupported_params(kwargs.get("drop_params")) + blocks: Final = _system_blocks(system) + try: + return await apply_context_management( + model=model, + messages=_BLOCK_LIST.validate_python(messages), + tools=_BLOCK_LIST.validate_python(tools) if tools else None, + system=blocks if blocks is None or isinstance(blocks, str) else _BLOCK_LIST.validate_python(blocks), + context_management_spec={"edits": [dict(edit) for edit in edits]}, + drop_invalid_edits=drop_invalid_edits, + ) + except AnthropicContextManagementError: + raise + except Exception as e: + verbose_logger.exception("context_management polyfill failed on the Responses API path: %s", e) + raise AnthropicContextManagementError( + status_code=500, message=f"context_management polyfill failed: {e}" + ) from e + + +async def count_view_messages( + *, + model: str, + messages: Sequence[Mapping[str, object]], + tools: Sequence[Mapping[str, object]] | None, + system: object, + context_management: CountContextManagement | None, +) -> Sequence[Mapping[str, object]]: + """The messages /v1/messages sends the Responses API once the polyfill has run the edits it can't run natively.""" + if context_management is None or not isinstance(context_management.spec, dict): + return messages + _, edits = _split_context_management(_SPEC.validate_python(context_management.spec), {}) + result: Final = await _polyfilled( + model=model, + messages=_BLOCK_LIST.validate_python(messages), + tools=tools, + system=system, + edits=edits, + kwargs={"drop_params": context_management.drop_invalid_edits}, + ) + return messages if result is None else result.messages + + +def _with_applied_edits( + response: AnthropicMessagesResponse, polyfill_result: PolyfillResult | None +) -> AnthropicMessagesResponse: + applied_edits: Final = polyfill_result.applied_edits_for_response() if polyfill_result else None + if not applied_edits: + return response + return {**response, "context_management": ContextManagementResponse(applied_edits=list(applied_edits))} + + def _record_bridged_response(responses_kwargs: Mapping[str, object], result: ResponsesAPIResponse) -> None: logging_obj: Final = litellm_logging_obj_from_kwargs(responses_kwargs) if logging_obj is not None: @@ -281,7 +391,7 @@ class LiteLLMMessagesToResponsesAPIHandler: max_tokens: int, messages: AnthropicRequestMessages, model: str, - context_management: dict[str, object] | None = None, + context_management: AnthropicContextManagementSpec = None, metadata: dict[str, object] | None = None, output_config: AnthropicOutputConfig | None = None, stop_sequences: list[str] | None = None, @@ -296,11 +406,15 @@ class LiteLLMMessagesToResponsesAPIHandler: output_format: AnthropicOutputSchema | None = None, **kwargs: object, ) -> AnthropicMessagesResponse | AsyncIterator[bytes]: + native_context_management, polyfilled_edits = _split_context_management(context_management, kwargs) + polyfill_result: Final = await _polyfilled( + model=model, messages=messages, tools=tools, system=system, edits=polyfilled_edits, kwargs=kwargs + ) responses_kwargs: Final = _build_responses_kwargs( max_tokens=max_tokens, - messages=messages, + messages=polyfill_result.messages if polyfill_result is not None else messages, model=model, - context_management=context_management, + context_management=native_context_management, metadata=metadata, output_config=output_config, stop_sequences=stop_sequences, @@ -325,13 +439,16 @@ class LiteLLMMessagesToResponsesAPIHandler: model=local_model_name(model, kwargs.get("custom_llm_provider")), litellm_logging_obj=litellm_logging_obj_from_kwargs(responses_kwargs), safeguards_evaluator=safeguards_evaluator, + applied_edits=polyfill_result.applied_edits_for_response() if polyfill_result else None, ) return wrapper.async_anthropic_sse_wrapper() if not isinstance(result, ResponsesAPIResponse): raise ValueError(f"Expected ResponsesAPIResponse, got {type(result)}") - served: Final = await with_safeguard_results(_ADAPTER.translate_response(result), safeguards_evaluator) + served: Final = await with_safeguard_results( + _with_applied_edits(_ADAPTER.translate_response(result), polyfill_result), safeguards_evaluator + ) _record_bridged_response(responses_kwargs, result) return served @@ -340,7 +457,7 @@ class LiteLLMMessagesToResponsesAPIHandler: max_tokens: int, messages: AnthropicRequestMessages, model: str, - context_management: dict[str, object] | None = None, + context_management: AnthropicContextManagementSpec = None, metadata: dict[str, object] | None = None, output_config: AnthropicOutputConfig | None = None, stop_sequences: list[str] | None = None, @@ -382,11 +499,26 @@ class LiteLLMMessagesToResponsesAPIHandler: ) # Sync path + native_context_management, polyfilled_edits = _split_context_management(context_management, kwargs) + ran: Final[object] = ( # pyright: ignore[reportUnknownVariableType] # run_async_function is untyped + run_async_function( + _polyfilled, + model=model, + messages=messages, + tools=tools, + system=system, + edits=polyfilled_edits, + kwargs=kwargs, + ) + if polyfilled_edits + else None + ) + polyfill_result: Final = ran if isinstance(ran, PolyfillResult) else None responses_kwargs: Final = _build_responses_kwargs( max_tokens=max_tokens, - messages=messages, + messages=polyfill_result.messages if polyfill_result is not None else messages, model=model, - context_management=context_management, + context_management=native_context_management, metadata=metadata, output_config=output_config, stop_sequences=stop_sequences, @@ -409,10 +541,11 @@ class LiteLLMMessagesToResponsesAPIHandler: responses_stream=result, model=local_model_name(model, kwargs.get("custom_llm_provider")), litellm_logging_obj=litellm_logging_obj_from_kwargs(responses_kwargs), + applied_edits=polyfill_result.applied_edits_for_response() if polyfill_result else None, ) return wrapper.async_anthropic_sse_wrapper() if not isinstance(result, ResponsesAPIResponse): raise ValueError(f"Expected ResponsesAPIResponse, got {type(result)}") - return _ADAPTER.translate_response(result) + return _with_applied_edits(_ADAPTER.translate_response(result), polyfill_result) diff --git a/litellm/llms/anthropic/messages/responses_adapters/streaming_iterator.py b/litellm/llms/anthropic/messages/responses_adapters/streaming_iterator.py index 44f233f2254..dae96ba4ce1 100644 --- a/litellm/llms/anthropic/messages/responses_adapters/streaming_iterator.py +++ b/litellm/llms/anthropic/messages/responses_adapters/streaming_iterator.py @@ -2,7 +2,7 @@ ## Translates OpenAI call to Anthropic `/v1/messages` format import asyncio from collections import deque -from collections.abc import AsyncGenerator, Awaitable, Callable, Iterator, Mapping +from collections.abc import AsyncGenerator, Awaitable, Callable, Iterator, Mapping, Sequence from typing import TYPE_CHECKING, Any, Final import anyio @@ -27,6 +27,7 @@ from litellm.responses.streaming_iterator import ( partial_responses_incomplete_event_on_disconnect, stream_error_status_and_message, ) +from litellm.types.llms.anthropic import AppliedEdit, ContextManagementResponse from litellm.types.llms.anthropic_messages.anthropic_response import AnthropicUsage from litellm.types.llms.base import LiteLLMBaseModel @@ -129,8 +130,10 @@ class AnthropicResponsesStreamWrapper: model: str, litellm_logging_obj: "LiteLLMLoggingObject | None" = None, safeguards_evaluator: "SafeguardsEvaluator | None" = None, + applied_edits: Sequence[AppliedEdit] | None = None, ) -> None: self.responses_stream = responses_stream + self._applied_edits: Final = tuple(applied_edits or ()) self.model = model self.litellm_logging_obj = litellm_logging_obj self.safeguards_evaluator: Final = safeguards_evaluator @@ -445,6 +448,11 @@ class AnthropicResponsesStreamWrapper: "type": "message_delta", "delta": message_delta_payload, "usage": dict(anthropic_usage), + **( + {"context_management": ContextManagementResponse(applied_edits=list(self._applied_edits))} + if self._applied_edits + else {} + ), } ) self._chunk_queue.append({"type": "message_stop"}) diff --git a/litellm/llms/openai/responses/count_tokens/token_counter.py b/litellm/llms/openai/responses/count_tokens/token_counter.py index 6ae61e15953..87a75dae95e 100644 --- a/litellm/llms/openai/responses/count_tokens/token_counter.py +++ b/litellm/llms/openai/responses/count_tokens/token_counter.py @@ -15,7 +15,9 @@ from litellm._logging import verbose_logger from litellm.litellm_core_utils.asyncify import asyncify from litellm.litellm_core_utils.core_helpers import as_str_mapping from litellm.litellm_core_utils.tool_fields import DropPolicy +from litellm.llms.anthropic.messages.context_management import AnthropicContextManagementError from litellm.llms.anthropic.messages.convertible_blocks import deployment_drops_params, unconvertible_blocks_error +from litellm.llms.anthropic.messages.responses_adapters.handler import count_view_messages from litellm.llms.anthropic.messages.responses_adapters.transformation import ( LiteLLMAnthropicToResponsesAPIAdapter, ) @@ -95,6 +97,32 @@ def _count_error(model_to_use: str, request_model: str, message: str, status_cod ) +async def _sent_messages( + model: str, + messages: Sequence[Mapping[str, object]], + tools: Sequence[Mapping[str, object]] | None, + system: object, + messages_format: TokenCountMessagesFormat, + context_management: CountContextManagement | None, +) -> Sequence[Mapping[str, object]] | AnthropicContextManagementError: + match messages_format: + case "openai": + return messages + case "anthropic": + try: + return await count_view_messages( + model=model, + messages=messages, + tools=tools, + system=system, + context_management=context_management, + ) + except AnthropicContextManagementError as error: + return error + case _: + assert_never(messages_format) + + class OpenAITokenCounter(BaseTokenCounter): """Token counter implementation for OpenAI provider using the Responses API.""" @@ -175,13 +203,28 @@ class OpenAITokenCounter(BaseTokenCounter): rejected_before_provider_call=True, ) + sent_messages: Final = await _sent_messages( + model_to_use, messages, tools, system, messages_format, context_management + ) + if isinstance(sent_messages, AnthropicContextManagementError): + return TokenCountResponse( + total_tokens=0, + request_model=request_model, + model_used=model_to_use, + tokenizer_type="openai_api", + error=True, + error_message=sent_messages.message, + status_code=sent_messages.status_code, + rejected_before_provider_call=True, + ) + resolved: Final = _count_credential(deployment, request_model or model_to_use) if isinstance(resolved, OpenAIError): return _count_error(model_to_use, request_model, resolved.message, resolved.status_code) input_items, instructions, counted_tools, text = self._responses_request_parts( model_to_use=model_to_use, - messages=messages, + messages=sent_messages, tools=tools, system=system, messages_format=messages_format, diff --git a/tests/unit/litellm_core_utils/test_token_counter.py b/tests/unit/litellm_core_utils/test_token_counter.py index d97970753f8..f03c7421899 100644 --- a/tests/unit/litellm_core_utils/test_token_counter.py +++ b/tests/unit/litellm_core_utils/test_token_counter.py @@ -523,6 +523,43 @@ def test_token_counter_skips_non_mapping_tools(): assert mixed_tokens == valid_tokens +@pytest.mark.parametrize( + ("shorthand", "explicit"), + [({"type": "array"}, {"type": "array", "items": {}}), (True, {})], + ids=["array_without_items", "boolean_subschema"], +) +def test_token_counter_counts_json_schema_shorthands_like_their_explicit_form(shorthand: object, explicit: object) -> None: + assert _count_with_property_schema(shorthand) == _count_with_property_schema(explicit) + + +@pytest.mark.parametrize( + ("malformed", "without_keyword"), + [ + ( + {"type": "object", "properties": {"id": {"type": "string"}}, "required": True}, + {"type": "object", "properties": {"id": {"type": "string"}}}, + ), + ({"type": "object", "properties": [{"id": {"type": "string"}}]}, {"type": "object"}), + ({"type": "string", "enum": None}, {"type": "string"}), + ({"type": "integer", "enum": 3}, {"type": "integer"}), + ], + ids=["required_true", "properties_list", "string_enum_null", "integer_enum_number"], +) +def test_token_counter_counts_a_malformed_schema_keyword_as_absent(malformed: object, without_keyword: object) -> None: + assert _count_with_property_schema(malformed) == _count_with_property_schema(without_keyword) + + +def _count_with_property_schema(property_schema: object) -> int: + tool: Final = { + "type": "function", + "function": { + "name": "read_files", + "parameters": {"type": "object", "properties": {"paths": property_schema}}, + }, + } + return token_counter_new(model="gpt-4o", messages=[{"role": "user", "content": "Read the config files"}], tools=[tool]) + + _WEATHER_FUNCTION: Final = { "name": "get_weather", "description": "Get the current weather for a city", @@ -2132,6 +2169,20 @@ def test_token_counter_counts_anthropic_search_result_text() -> None: ) +def test_token_counter_counts_a_search_result_without_a_title() -> None: + from litellm.litellm_core_utils.token_counter import _count_content_list + + search_result: Final = { + "type": "search_result", + "source": "https://example.invalid/notes", + "content": [{"type": "text", "text": "The code word is 7f3a."}], + } + + assert _count_content_list(len, [search_result], False, None) == len( + "https://example.invalid/notesThe code word is 7f3a." + ) + + @pytest.mark.parametrize("nested", (False, True)) def test_unknown_content_type_remains_a_value_error_with_an_optional_fallback(nested: bool) -> None: from litellm.litellm_core_utils.token_counter import _count_content_list diff --git a/tests/unit/llms/anthropic/messages/context_management/test_clear_tool_uses.py b/tests/unit/llms/anthropic/messages/context_management/test_clear_tool_uses.py index f03ffb25b2b..4cbf4cfa5f6 100644 --- a/tests/unit/llms/anthropic/messages/context_management/test_clear_tool_uses.py +++ b/tests/unit/llms/anthropic/messages/context_management/test_clear_tool_uses.py @@ -217,6 +217,106 @@ def test_cleared_input_tokens_is_nonnegative(): assert applied["cleared_input_tokens"] >= 0 +SERVER_TOOL_TURN: Final = { + "role": "assistant", + "content": [ + {"type": "server_tool_use", "id": "srvtoolu_01", "name": "web_search", "input": {"query": "Mumbai rain"}}, + { + "type": "web_search_tool_result", + "tool_use_id": "srvtoolu_01", + "content": [{"type": "web_search_result", "url": "https://example.com", "title": "rain " * 300}], + }, + ], +} + + +def _clear_above( + messages: list[dict[str, object]], threshold: int +) -> tuple[list[dict[str, object]], AppliedEdit | None]: + return apply_clear_tool_uses_20250919( + model=MODEL, + messages=messages, + tools=None, + system=None, + edit_spec={ + "type": "clear_tool_uses_20250919", + "trigger": {"type": "input_tokens", "value": threshold}, + "keep": {"type": "tool_uses", "value": 1}, + }, + ) + + +def test_an_input_tokens_trigger_skips_blocks_no_adapter_sends() -> None: + history: Final = _make_history(n_pairs=3) + sent_tokens: Final = litellm.token_counter(model=MODEL, messages=history) + turn: Final = SERVER_TOOL_TURN + messages: Final = [history[0], turn, *history[1:]] + + untouched, not_applied = _clear_above(messages, sent_tokens + 100) + cleared, applied = _clear_above(messages, sent_tokens - 100) + + assert not_applied is None + assert untouched == messages + assert applied is not None + assert applied["cleared_tool_uses"] == 2 + assert _cleared_ids(cleared) == {"toolu_00", "toolu_01"} + assert cleared[1] == turn + + +def _first_result_with(blocks: list[dict[str, object]]) -> dict[str, object]: + return {"role": "user", "content": [{"type": "tool_result", "tool_use_id": "toolu_00", "content": blocks}]} + + +def test_an_input_tokens_trigger_skips_blocks_no_adapter_sends_inside_a_tool_result() -> None: + history: Final = _make_history(n_pairs=3) + result_text: Final = {"type": "text", "text": "Result 0: " + "x" * 200} + search_result: Final = {"type": "web_search_result", "url": "https://example.com", "title": "rain " * 300} + sent_tokens: Final = litellm.token_counter( + model=MODEL, messages=[*history[:2], _first_result_with([result_text]), *history[3:]] + ) + messages: Final = [*history[:2], _first_result_with([result_text, search_result]), *history[3:]] + + untouched, not_applied = _clear_above(messages, sent_tokens + 100) + cleared, applied = _clear_above(messages, sent_tokens - 100) + + assert not_applied is None + assert untouched == messages + assert applied is not None + assert _cleared_ids(cleared) == {"toolu_00", "toolu_01"} + + +def _clear_at_least_one_token( + messages: list[dict[str, object]], +) -> tuple[list[dict[str, object]], AppliedEdit | None]: + return apply_clear_tool_uses_20250919( + model=MODEL, + messages=messages, + tools=None, + system=None, + edit_spec={ + "type": "clear_tool_uses_20250919", + "trigger": {"type": "tool_uses", "value": 0}, + "keep": {"type": "tool_uses", "value": 1}, + "clear_at_least": {"type": "input_tokens", "value": 1}, + }, + ) + + +def test_clear_at_least_frees_only_blocks_an_adapter_sends_inside_a_tool_result() -> None: + history: Final = _make_history(n_pairs=3) + result_text: Final = {"type": "text", "text": "Result 0: " + "x" * 200} + search_result: Final = {"type": "web_search_result", "url": "https://example.com", "title": "rain " * 300} + + _, sent_only = _clear_at_least_one_token([*history[:2], _first_result_with([result_text]), *history[3:]]) + cleared, applied = _clear_at_least_one_token( + [*history[:2], _first_result_with([result_text, search_result]), *history[3:]] + ) + + assert applied is not None + assert applied == sent_only + assert _cleared_ids(cleared) == {"toolu_00", "toolu_01"} + + def _blocks_of(messages: list[dict[str, object]], block_type: str) -> tuple[dict[str, object], ...]: contents: Final = (message["content"] for message in messages) blocks: Final = chain.from_iterable(content for content in contents if isinstance(content, list)) @@ -559,6 +659,9 @@ async def _malformed_spec_error(edits: list[dict[str, object]]) -> AnthropicCont ("exclude_tools", "get_weather"), ("exclude_tools", [1]), ("clear_tool_inputs", "get_weather"), + ("trigger", "soon"), + ("trigger", [1]), + ("keep", 5), ], ) async def test_malformed_fields_are_a_400_even_below_the_trigger(field: str, value: object) -> None: @@ -576,12 +679,15 @@ async def test_malformed_fields_are_a_400_even_below_the_trigger(field: str, val ("clear_at_least", 1200, "clear_at_least: Input does not match the expected shape."), ("clear_at_least", {"type": "input_tokens", "value": "1200"}, "value: Input should be a valid integer"), ("clear_tool_inputs", 5, "clear_tool_inputs.bool: Input should be a valid boolean, unable to interpret input"), + ("trigger", "soon", "trigger: Input should be an object"), + ("keep", 5, "keep: Input should be an object"), ], ) async def test_malformed_field_400_names_the_edit_and_field_as_the_api_does( field: str, value: object, message: str ) -> None: - # api.anthropic.com, measured 2026-10-19, answers these four with these messages for the edit at index 1. + # api.anthropic.com answers these with these messages for the edit at index 1 (trigger and keep measured + # 2026-10-28, the rest 2026-10-19). error = await _malformed_spec_error( [{"type": "clear_thinking_20251015"}, {"type": "clear_tool_uses_20250919", field: value}] ) @@ -625,6 +731,28 @@ async def test_drop_params_drops_only_the_malformed_field(field: str, value: obj assert _tool_inputs(result.messages) == {"toolu_00": {}, "toolu_01": {}, "toolu_02": {"location": "City2"}} +@pytest.mark.parametrize(("field", "value"), [("trigger", "soon"), ("keep", 5)]) +async def test_drop_params_treats_a_malformed_trigger_or_keep_as_never_sent(field: str, value: object) -> None: + async def run(edit: dict[str, object], drop_invalid_edits: bool) -> PolyfillResult | None: + return await _run_polyfill_if_enabled( + model=MODEL, + messages=_make_history(n_pairs=5), + tools=None, + system=None, + context_management_spec={"edits": [edit]}, + litellm_metadata={}, + additional_drop_params=None, + llm_router=None, + drop_invalid_edits=drop_invalid_edits, + ) + + malformed: Final = {**_VALID_EDIT, field: value} + with pytest.raises(AnthropicContextManagementError, match=rf"\.{field}: Input should be an object$"): + await run(malformed, drop_invalid_edits=False) + never_sent: Final = {key: spec for key, spec in _VALID_EDIT.items() if key != field} + assert await run(malformed, drop_invalid_edits=True) == await run(never_sent, drop_invalid_edits=True) + + async def test_malformed_edit_is_rejected_before_a_compact_summary_is_paid_for( monkeypatch: pytest.MonkeyPatch, ) -> None: diff --git a/tests/unit/llms/anthropic/messages/context_management/test_compact.py b/tests/unit/llms/anthropic/messages/context_management/test_compact.py index dff931a6074..32899a8e3c8 100644 --- a/tests/unit/llms/anthropic/messages/context_management/test_compact.py +++ b/tests/unit/llms/anthropic/messages/context_management/test_compact.py @@ -29,7 +29,7 @@ from litellm.llms.anthropic.messages.context_management.editors.compact import ( _augment_system_with_summary, _extract_summary_text, _select_last_user_question, - _slice_around_compaction_block, + slice_around_compaction_block, _strip_compaction_blocks, apply_client_compaction_block_history, apply_compact_20260112, @@ -153,7 +153,7 @@ def test_applied_edits_for_response_includes_compact_when_block_present(): def test_slice_around_compaction_block_found(): messages = _messages_with_compaction("my summary") - sliced, block = _slice_around_compaction_block(messages) + sliced, block = slice_around_compaction_block(messages) assert block is not None assert block["type"] == "compaction" assert block["content"] == "my summary" @@ -164,7 +164,7 @@ def test_slice_around_compaction_block_found(): def test_slice_around_compaction_block_not_found(): messages = _simple_messages() - sliced, block = _slice_around_compaction_block(messages) + sliced, block = slice_around_compaction_block(messages) assert block is None assert sliced is messages # same object, no copy diff --git a/tests/unit/llms/anthropic/messages/context_management/test_dispatcher.py b/tests/unit/llms/anthropic/messages/context_management/test_dispatcher.py index fabb05c2e61..e55af602013 100644 --- a/tests/unit/llms/anthropic/messages/context_management/test_dispatcher.py +++ b/tests/unit/llms/anthropic/messages/context_management/test_dispatcher.py @@ -5,6 +5,7 @@ Unit tests for the context_management polyfill dispatcher. from litellm.llms.anthropic.messages.context_management import ( apply_context_management, ) +from litellm.llms.anthropic.messages.convertible_blocks import drop_non_string_tool_result_texts MODEL = "xai/grok-4" @@ -160,3 +161,57 @@ async def test_sync_editor_counts_tokens_off_the_event_loop(): assert result.messages == messages assert_loop_stayed_free(took, lags) + + +def _history_with_null_text_beside_a_short_first_result(): + return [ + {"role": "user", "content": "Hi"}, + { + "role": "assistant", + "content": [{"type": "tool_use", "id": "t0", "name": "f", "input": {}}], + }, + { + "role": "user", + "content": [ + { + "type": "tool_result", + "tool_use_id": "t0", + "content": [{"type": "text", "text": "ok"}, {"type": "text", "text": None}], + } + ], + }, + *_history_with_two_tool_pairs()[1:], + ] + + +async def test_drop_invalid_edits_clears_as_if_malformed_tool_result_text_was_never_sent(): + spec = { + "edits": [ + { + "type": "clear_tool_uses_20250919", + "trigger": {"type": "tool_uses", "value": 2}, + "keep": {"type": "tool_uses", "value": 1}, + } + ] + } + history = _history_with_null_text_beside_a_short_first_result() + + from_raw = await apply_context_management( + model=MODEL, + messages=history, + tools=None, + system=None, + context_management_spec=spec, + drop_invalid_edits=True, + ) + from_sent = await apply_context_management( + model=MODEL, + messages=list(drop_non_string_tool_result_texts(history, True)), + tools=None, + system=None, + context_management_spec=spec, + drop_invalid_edits=True, + ) + + assert from_raw.applied_edits == from_sent.applied_edits + assert from_raw.messages == from_sent.messages diff --git a/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_handler.py b/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_handler.py index bba5413fbdf..382917fc756 100644 --- a/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_handler.py +++ b/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_handler.py @@ -2,7 +2,8 @@ import datetime import json import os import sys -from collections.abc import Mapping +from collections.abc import AsyncIterator, Mapping +from itertools import chain from typing import Final, cast from unittest.mock import AsyncMock, patch @@ -761,3 +762,325 @@ async def test_recomputed_cost_of_a_served_bridge_response_prices_the_reasoning_ assert logging_obj.cost_breakdown is not None assert logging_obj.cost_breakdown["output_cost"] == pytest.approx(expected_output) assert logging_obj.cost_breakdown["reasoning_cost"] == pytest.approx(expected_reasoning) + + +TEXT_RESPONSE: Final = { + "id": "resp_cleared", + "object": "response", + "created_at": 1, + "status": "completed", + "model": "gpt-5.6-luna", + "output": [ + { + "type": "message", + "id": "msg_1", + "role": "assistant", + "status": "completed", + "content": [{"type": "output_text", "text": "done", "annotations": []}], + } + ], + "parallel_tool_calls": True, + "tool_choice": "auto", + "tools": [], + "usage": {"input_tokens": 3, "output_tokens": 4, "total_tokens": 7}, +} +CLEAR_EDIT: Final = { + "type": "clear_tool_uses_20250919", + "trigger": {"type": "tool_uses", "value": 1}, + "keep": {"type": "tool_uses", "value": 1}, +} +COMPACT_EDIT: Final = {"type": "compact_20260112", "trigger": {"type": "input_tokens", "value": 150000}} + + +def _read(i: int) -> tuple[dict[str, object], dict[str, object]]: + return ( + { + "role": "assistant", + "content": [{"type": "tool_use", "id": f"toolu_{i}", "name": "Bash", "input": {"command": f"cat {i}"}}], + }, + { + "role": "user", + "content": [{"type": "tool_result", "tool_use_id": f"toolu_{i}", "content": f"log {i} " * 40}], + }, + ) + + +def _three_reads() -> list[dict[str, object]]: + return [{"role": "user", "content": "Read the logs."}, *chain.from_iterable(_read(i) for i in range(3))] + + +async def _cleared_bridge( + respx_mock: respx.MockRouter, + monkeypatch: pytest.MonkeyPatch, + *, + stream: bool, + edits: list[dict[str, object]], + drop_params: bool = False, + system: object = None, +) -> tuple[object, Mapping[str, object]]: + monkeypatch.setenv("OPENAI_API_KEY", "sk-clear-test") + monkeypatch.setenv("DISABLE_AIOHTTP_TRANSPORT", "True") + litellm.in_memory_llm_clients_cache.flush_cache() + upstream: Final = respx_mock.post("https://api.openai.com/v1/responses") + if stream: + upstream.respond(status_code=200, headers={"Content-Type": "text/event-stream"}, content=RESPONSES_SSE_BODY) + else: + upstream.respond(status_code=200, json=TEXT_RESPONSE) + response: Final = await LiteLLMMessagesToResponsesAPIHandler.async_anthropic_messages_handler( + max_tokens=1024, + messages=_three_reads(), + model="openai/gpt-5.6-luna", + tools=[dict(BASH_TOOL)], + stream=stream, + custom_llm_provider="openai", + context_management={"edits": edits}, + drop_params=drop_params, + system=system, + ) + body: Final = json.loads(upstream.calls.last.request.content) + if not stream: + return response, body + return [json.loads(chunk.decode().split("data: ", 1)[1]) async for chunk in response], body + + +@pytest.mark.asyncio +async def test_clear_tool_uses_clears_old_results_before_the_responses_api_call( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + from litellm.llms.anthropic.messages.context_management.constants import CLEARED_TOOL_RESULT_PLACEHOLDER + + served, body = await _cleared_bridge(respx_mock, monkeypatch, stream=False, edits=[dict(CLEAR_EDIT)]) + + outputs: Final = { + item["call_id"]: item["output"] for item in body["input"] if item.get("type") == "function_call_output" + } + assert [CLEARED_TOOL_RESULT_PLACEHOLDER in json.dumps(outputs[f"toolu_{i}"]) for i in range(3)] == [ + True, + True, + False, + ] + assert "context_management" not in body + assert isinstance(served, dict) + assert served["context_management"]["applied_edits"][0]["cleared_tool_uses"] == 2 + + +@pytest.mark.asyncio +async def test_streamed_clear_tool_uses_reports_its_edits_on_the_final_message_delta( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + events, _ = await _cleared_bridge(respx_mock, monkeypatch, stream=True, edits=[dict(CLEAR_EDIT)]) + + assert isinstance(events, list) + message_delta: Final = next(event for event in events if event["type"] == "message_delta") + assert message_delta["context_management"]["applied_edits"][0]["cleared_tool_uses"] == 2 + + +@pytest.mark.asyncio +async def test_compact_stays_native_while_clear_tool_uses_runs_in_the_gateway( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + served, body = await _cleared_bridge( + respx_mock, monkeypatch, stream=False, edits=[dict(CLEAR_EDIT), dict(COMPACT_EDIT)] + ) + + assert body["context_management"] == [{"type": "compaction", "compact_threshold": 150000}] + assert isinstance(served, dict) + assert [edit["type"] for edit in served["context_management"]["applied_edits"]] == ["clear_tool_uses_20250919"] + + +@pytest.mark.asyncio +async def test_a_malformed_clear_tool_uses_edit_is_a_400_before_the_responses_api_call( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + from litellm.llms.anthropic.messages.context_management import AnthropicContextManagementError + + upstream: Final = respx_mock.post("https://api.openai.com/v1/responses").respond( + status_code=200, json=TEXT_RESPONSE + ) + monkeypatch.setenv("OPENAI_API_KEY", "sk-clear-test") + with pytest.raises(AnthropicContextManagementError) as raised: + await LiteLLMMessagesToResponsesAPIHandler.async_anthropic_messages_handler( + max_tokens=1024, + messages=_three_reads(), + model="openai/gpt-5.6-luna", + custom_llm_provider="openai", + context_management={"edits": [{**CLEAR_EDIT, "exclude_tools": "Bash"}]}, + ) + assert raised.value.status_code == 400 + assert not upstream.called + + +@pytest.mark.asyncio +async def test_a_malformed_text_in_a_result_clearing_would_drop_is_still_a_400( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + upstream: Final = respx_mock.post("https://api.openai.com/v1/responses").respond( + status_code=200, json=TEXT_RESPONSE + ) + monkeypatch.setenv("OPENAI_API_KEY", "sk-clear-test") + reads: Final = _three_reads() + malformed_oldest_result: Final = { + "role": "user", + "content": [{"type": "tool_result", "tool_use_id": "toolu_0", "content": [{"type": "text", "text": None}]}], + } + with pytest.raises(litellm.BadRequestError) as rejected: + await LiteLLMMessagesToResponsesAPIHandler.async_anthropic_messages_handler( + max_tokens=1024, + messages=[*reads[:2], malformed_oldest_result, *reads[3:]], + model="openai/gpt-5.6-luna", + tools=[dict(BASH_TOOL)], + custom_llm_provider="openai", + context_management={"edits": [dict(CLEAR_EDIT)]}, + ) + assert rejected.value.status_code == 400 + assert "tool_result toolu_0 content.0.text: Input should be a valid string" in rejected.value.message + assert not upstream.called + + +def test_the_sync_path_clears_old_results_too(respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch) -> None: + from litellm.llms.anthropic.messages.context_management.constants import CLEARED_TOOL_RESULT_PLACEHOLDER + + monkeypatch.setenv("OPENAI_API_KEY", "sk-clear-test") + monkeypatch.setenv("DISABLE_AIOHTTP_TRANSPORT", "True") + litellm.in_memory_llm_clients_cache.flush_cache() + upstream: Final = respx_mock.post("https://api.openai.com/v1/responses").respond( + status_code=200, json=TEXT_RESPONSE + ) + + served: Final = LiteLLMMessagesToResponsesAPIHandler.anthropic_messages_handler( + max_tokens=1024, + messages=_three_reads(), + model="openai/gpt-5.6-luna", + tools=[dict(BASH_TOOL)], + custom_llm_provider="openai", + context_management={"edits": [dict(CLEAR_EDIT)]}, + ) + + body: Final = json.loads(upstream.calls.last.request.content) + outputs: Final = { + item["call_id"]: item["output"] for item in body["input"] if item.get("type") == "function_call_output" + } + assert [CLEARED_TOOL_RESULT_PLACEHOLDER in json.dumps(outputs[f"toolu_{i}"]) for i in range(3)] == [ + True, + True, + False, + ] + assert isinstance(served, dict) + assert served["context_management"]["applied_edits"][0]["cleared_tool_uses"] == 2 + + +@pytest.mark.asyncio +async def test_an_openai_style_context_management_list_is_still_not_sent_to_the_responses_api( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + upstream: Final = respx_mock.post("https://api.openai.com/v1/responses").respond( + status_code=200, json=TEXT_RESPONSE + ) + monkeypatch.setenv("OPENAI_API_KEY", "sk-clear-test") + monkeypatch.setenv("DISABLE_AIOHTTP_TRANSPORT", "True") + litellm.in_memory_llm_clients_cache.flush_cache() + + served: Final = await LiteLLMMessagesToResponsesAPIHandler.async_anthropic_messages_handler( + max_tokens=1024, + messages=_three_reads(), + model="openai/gpt-5.6-luna", + custom_llm_provider="openai", + context_management=[{"type": "compaction", "compact_threshold": 500}], + ) + + assert "context_management" not in json.loads(upstream.calls.last.request.content) + assert isinstance(served, dict) + assert "context_management" not in served + + +@pytest.mark.asyncio +async def test_under_drop_params_a_malformed_edit_still_clears_without_its_bad_field( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + from litellm.llms.anthropic.messages.context_management.constants import CLEARED_TOOL_RESULT_PLACEHOLDER + + served, body = await _cleared_bridge( + respx_mock, monkeypatch, stream=False, edits=[{**CLEAR_EDIT, "exclude_tools": "Bash"}], drop_params=True + ) + + outputs: Final = { + item["call_id"]: item["output"] for item in body["input"] if item.get("type") == "function_call_output" + } + assert [CLEARED_TOOL_RESULT_PLACEHOLDER in json.dumps(outputs[f"toolu_{i}"]) for i in range(3)] == [ + True, + True, + False, + ] + assert isinstance(served, dict) + assert served["context_management"]["applied_edits"][0]["cleared_tool_uses"] == 2 + + +def _cleared_outputs(body: Mapping[str, object]) -> list[bool]: + from litellm.llms.anthropic.messages.context_management.constants import CLEARED_TOOL_RESULT_PLACEHOLDER + + outputs: Final = { + item["call_id"]: item["output"] for item in body["input"] if item.get("type") == "function_call_output" + } + return [CLEARED_TOOL_RESULT_PLACEHOLDER in json.dumps(outputs[f"toolu_{i}"]) for i in range(3)] + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "system", + [7, {"type": "text", "text": "Be brief."}, ["Be brief."], ["Be brief.", {"type": "text", "text": "Be kind."}]], + ids=["number", "object", "strings", "mixed_list"], +) +@pytest.mark.parametrize("stream", [False, True], ids=["non_stream", "stream"]) +async def test_clear_tool_uses_still_clears_when_the_system_prompt_is_not_text_or_a_block_list( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, system: object, stream: bool +) -> None: + _, body = await _cleared_bridge(respx_mock, monkeypatch, stream=stream, edits=[dict(CLEAR_EDIT)], system=system) + + assert _cleared_outputs(body) == [True, True, False] + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("system", "expect_cleared"), + [ + (["Stray.", {"type": "text", "text": "Follow the policy. " * 1500}], [True, True, False]), + (["Follow the policy. " * 1500], [False, False, False]), + ], + ids=["text_block_counts", "stray_string_does_not"], +) +async def test_input_tokens_trigger_counts_the_text_blocks_of_a_system_list_with_stray_items( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, system: object, expect_cleared: list[bool] +) -> None: + trigger_between_messages_and_system: Final = litellm.token_counter(model="gpt-4o", messages=_three_reads()) + 1000 + edit: Final = {**CLEAR_EDIT, "trigger": {"type": "input_tokens", "value": trigger_between_messages_and_system}} + + _, body = await _cleared_bridge(respx_mock, monkeypatch, stream=False, edits=[edit], system=system) + + assert _cleared_outputs(body) == expect_cleared + + +@pytest.mark.asyncio +async def test_the_sync_path_reports_its_edits_on_the_streamed_message_delta( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv("OPENAI_API_KEY", "sk-clear-test") + monkeypatch.setenv("DISABLE_AIOHTTP_TRANSPORT", "True") + litellm.in_memory_llm_clients_cache.flush_cache() + respx_mock.post("https://api.openai.com/v1/responses").respond( + status_code=200, headers={"Content-Type": "text/event-stream"}, content=RESPONSES_SSE_BODY + ) + + sse: Final = LiteLLMMessagesToResponsesAPIHandler.anthropic_messages_handler( + max_tokens=1024, + messages=_three_reads(), + model="openai/gpt-5.6-luna", + tools=[dict(BASH_TOOL)], + stream=True, + custom_llm_provider="openai", + context_management={"edits": [dict(CLEAR_EDIT)]}, + ) + events: Final = [ + json.loads(chunk.decode().split("data: ", 1)[1]) async for chunk in cast(AsyncIterator[bytes], sse) + ] + message_delta: Final = next(event for event in events if event["type"] == "message_delta") + assert message_delta["context_management"]["applied_edits"][0]["cleared_tool_uses"] == 2 diff --git a/tests/unit/llms/openai/responses/count_tokens/test_token_counter.py b/tests/unit/llms/openai/responses/count_tokens/test_token_counter.py index 17c1b35e211..29c25c5fdbf 100644 --- a/tests/unit/llms/openai/responses/count_tokens/test_token_counter.py +++ b/tests/unit/llms/openai/responses/count_tokens/test_token_counter.py @@ -2,6 +2,7 @@ import copy import json import operator from collections.abc import Mapping +from itertools import chain from typing import Final import httpx @@ -11,9 +12,10 @@ from pydantic import JsonValue, ValidationError import litellm from litellm.litellm_core_utils.tool_fields import DropPolicy +from litellm.llms.anthropic.messages.context_management.constants import CLEARED_TOOL_RESULT_PLACEHOLDER from litellm.llms.openai.responses.count_tokens.token_counter import OpenAITokenCounter from litellm.responses.utils import ResponsesAPIRequestUtils -from litellm.types.utils import TokenCountMessagesFormat, TokenCountResponse +from litellm.types.utils import CountContextManagement, TokenCountMessagesFormat, TokenCountResponse COUNT_URL: Final = "https://api.openai.com/v1/responses/input_tokens" @@ -484,6 +486,122 @@ async def test_anthropic_format_tool_result_text_block_without_text_still_counts assert {"type": "function_call_output", "call_id": "toolu_log", "output": "build ok\n\ndone"} in sent["input"] +def _tool_pair(index: int) -> tuple[dict[str, object], dict[str, object]]: + return ( + {"role": "assistant", "content": [{"type": "tool_use", "id": f"toolu_{index}", "name": "log", "input": {}}]}, + { + "role": "user", + "content": [{"type": "tool_result", "tool_use_id": f"toolu_{index}", "content": f"log {index} " * 40}], + }, + ) + + +TOOL_PAIRS: Final[list[dict[str, object]]] = [ + {"role": "user", "content": "Compare the logs."}, + *chain.from_iterable(_tool_pair(index) for index in range(3)), +] + + +async def _count_with_edit( + monkeypatch: pytest.MonkeyPatch, edit: dict[str, JsonValue], drop_invalid_edits: bool, system: object = None +) -> tuple[TokenCountResponse | None, respx.Route]: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + with respx.mock: + route: Final = respx.post(COUNT_URL).mock(return_value=httpx.Response(200, json={"input_tokens": 9})) + result: Final = await OpenAITokenCounter().count_tokens( + model_to_use="gpt-4o", + messages=copy.deepcopy(TOOL_PAIRS), + contents=None, + deployment={"litellm_params": {"api_key": "sk-test"}}, + request_model="openai/gpt-4o", + system=system, + messages_format="anthropic", + context_management=CountContextManagement(spec={"edits": [edit]}, drop_invalid_edits=drop_invalid_edits), + ) + return result, route + + +def _sent_outputs(route: respx.Route) -> dict[str, str]: + sent: Final = json.loads(route.calls.last.request.content) + return {item["call_id"]: item["output"] for item in sent["input"] if item["type"] == "function_call_output"} + + +CLEAR_EDIT: Final[dict[str, JsonValue]] = { + "type": "clear_tool_uses_20250919", + "trigger": {"type": "tool_uses", "value": 1}, + "keep": {"type": "tool_uses", "value": 1}, +} + + +@pytest.mark.asyncio +async def test_anthropic_format_counts_the_history_the_clear_edit_leaves_for_inference( + monkeypatch: pytest.MonkeyPatch, +) -> None: + result, route = await _count_with_edit(monkeypatch, CLEAR_EDIT, drop_invalid_edits=False) + + assert result is not None + assert (result.error, result.total_tokens) == (False, 9) + assert _sent_outputs(route) == { + "toolu_0": CLEARED_TOOL_RESULT_PLACEHOLDER, + "toolu_1": CLEARED_TOOL_RESULT_PLACEHOLDER, + "toolu_2": "log 2 " * 40, + } + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "system", + [ + "Follow the policy. " * 1500, + [{"type": "text", "text": "Follow the policy. " * 1500}], + ["Stray.", {"type": "text", "text": "Follow the policy. " * 1500}], + ], + ids=["string", "blocks", "blocks_beside_a_stray_string"], +) +async def test_anthropic_format_input_tokens_trigger_counts_the_system_prompt_like_inference( + monkeypatch: pytest.MonkeyPatch, system: object +) -> None: + message_tokens: Final = litellm.token_counter(model="gpt-4o", messages=TOOL_PAIRS) + trigger_between_messages_and_system: Final = message_tokens + 1000 + edit: Final[dict[str, JsonValue]] = { + **CLEAR_EDIT, + "trigger": {"type": "input_tokens", "value": trigger_between_messages_and_system}, + } + + _, without_system = await _count_with_edit(monkeypatch, edit, drop_invalid_edits=False) + _, with_system = await _count_with_edit(monkeypatch, edit, drop_invalid_edits=False, system=system) + + assert _sent_outputs(without_system)["toolu_0"] == "log 0 " * 40 + assert _sent_outputs(with_system) == { + "toolu_0": CLEARED_TOOL_RESULT_PLACEHOLDER, + "toolu_1": CLEARED_TOOL_RESULT_PLACEHOLDER, + "toolu_2": "log 2 " * 40, + } + + +@pytest.mark.asyncio +async def test_anthropic_format_malformed_clear_edit_answers_400_without_calling_openai( + monkeypatch: pytest.MonkeyPatch, +) -> None: + result, route = await _count_with_edit(monkeypatch, {**CLEAR_EDIT, "exclude_tools": "log"}, drop_invalid_edits=False) + + assert route.called is False + assert result is not None + assert (result.error, result.status_code) == (True, 400) + assert "exclude_tools" in str(result.error_message) + + +@pytest.mark.asyncio +async def test_anthropic_format_drop_params_counts_the_clear_without_its_malformed_field( + monkeypatch: pytest.MonkeyPatch, +) -> None: + result, route = await _count_with_edit(monkeypatch, {**CLEAR_EDIT, "exclude_tools": "log"}, drop_invalid_edits=True) + + assert result is not None + assert result.error is False + assert _sent_outputs(route)["toolu_0"] == CLEARED_TOOL_RESULT_PLACEHOLDER + + @pytest.mark.asyncio async def test_count_uses_the_global_openai_key_chat_would_use(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) diff --git a/tests/unit/llms/vertex_ai/count_tokens/test_context.py b/tests/unit/llms/vertex_ai/count_tokens/test_context.py index b2a0b735b0f..38eb87debfd 100644 --- a/tests/unit/llms/vertex_ai/count_tokens/test_context.py +++ b/tests/unit/llms/vertex_ai/count_tokens/test_context.py @@ -649,7 +649,7 @@ async def test_anthropic_view_counts_the_unedited_body_without_an_error_log_when assert not any(record.levelno >= logging.ERROR for record in caplog.records) -async def test_anthropic_view_counts_the_turns_after_a_compaction_block_when_an_edit_before_compact_fails() -> None: +async def test_anthropic_view_clears_only_the_turns_after_a_compaction_block_when_clear_runs_before_compact() -> None: clear_before_compact: Final[JsonValue] = { "edits": [ { diff --git a/tests/unit/proxy/test_proxy_token_counter.py b/tests/unit/proxy/test_proxy_token_counter.py index 43c017468ff..c182cd1ec75 100644 --- a/tests/unit/proxy/test_proxy_token_counter.py +++ b/tests/unit/proxy/test_proxy_token_counter.py @@ -2064,22 +2064,6 @@ async def test_framing_keeps_the_400_for_blocks_the_provider_counter_cannot_conv }, id="web_search_blocks", ), - pytest.param( - { - "messages": [{"role": "user", "content": "What is new?"}], - "tools": [{"name": "search", "input_schema": {"type": "object", "properties": {"query": True}}}], - }, - id="boolean_subschema_tool", - ), - pytest.param( - { - "messages": [{"role": "user", "content": "What is new?"}], - "tools": [ - {"name": "search", "input_schema": {"type": "object", "properties": {"tags": {"type": "array"}}}} - ], - }, - id="array_schema_without_items", - ), pytest.param( {"messages": [], "contents": [{"role": "user", "parts": [{"text": "What is new?"}]}]}, id="google_contents", @@ -2138,6 +2122,44 @@ async def test_framed_estimate_that_cannot_read_the_body_answers_the_provider_er assert readable.count_source == "local_estimate" +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("shorthand", "explicit"), + [ + pytest.param({"type": "array"}, {"type": "array", "items": {}}, id="array_schema_without_items"), + pytest.param(True, {}, id="boolean_subschema"), + ], +) +async def test_framed_estimate_counts_a_json_schema_shorthand_like_its_explicit_form( + monkeypatch: pytest.MonkeyPatch, shorthand: object, explicit: dict[str, object] +): + counter: Final = _ScriptedCounter({"us.moonshotai.kimi-k3": _provider_reply("kimi", status_code=400)}) + monkeypatch.setattr( + litellm.proxy.proxy_server, + "llm_router", + Router(model_list=[{"model_name": "kimi", "litellm_params": {"model": "bedrock/us.moonshotai.kimi-k3"}}]), + ) + monkeypatch.setattr( + litellm.proxy.proxy_server, "_get_provider_token_counter", lambda deployment, model: (counter, model, "bedrock") + ) + + async def count(schema: object) -> TokenCountResponse: + return await token_counter( + request=TokenCountRequest( + model="kimi", + messages=[{"role": "user", "content": "What is new?"}], + tools=[{"name": "search", "input_schema": {"type": "object", "properties": {"tags": schema}}}], + ), + call_endpoint=True, + ) + + from_shorthand: Final = await count(shorthand) + from_explicit: Final = await count(explicit) + + assert from_shorthand.count_source == "local_estimate" + assert from_shorthand.total_tokens == from_explicit.total_tokens + + @pytest.mark.asyncio @pytest.mark.parametrize( "contents", From a6fda5a17e1b330c42890ab6fd3cf1c8bacf77c2 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Thu, 29 Oct 2026 17:45:37 -0700 Subject: [PATCH 10/17] bump: litellm-enterprise 0.1.80 -> 0.1.81, litellm-proxy-extras 0.4.111 -> 0.4.112 (#49149) --- enterprise/pyproject.toml | 4 ++-- litellm-proxy-extras/pyproject.toml | 4 ++-- pyproject.toml | 4 ++-- uv.lock | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/enterprise/pyproject.toml b/enterprise/pyproject.toml index 4653f919e17..7db2de1f874 100644 --- a/enterprise/pyproject.toml +++ b/enterprise/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "litellm-enterprise" -version = "0.1.80" +version = "0.1.81" description = "Package for LiteLLM Enterprise features" readme = "README.md" requires-python = ">=3.9" @@ -26,7 +26,7 @@ required-version = ">=0.10.9" module-root = "" [tool.commitizen] -version = "0.1.80" +version = "0.1.81" version_files = [ "pyproject.toml:^version", "../pyproject.toml:litellm-enterprise==", diff --git a/litellm-proxy-extras/pyproject.toml b/litellm-proxy-extras/pyproject.toml index 6de44be0c12..a12e1e27cfc 100644 --- a/litellm-proxy-extras/pyproject.toml +++ b/litellm-proxy-extras/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "litellm-proxy-extras" -version = "0.4.111" +version = "0.4.112" description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package." readme = "README.md" requires-python = ">=3.9" @@ -30,7 +30,7 @@ required-version = ">=0.10.9" module-root = "" [tool.commitizen] -version = "0.4.111" +version = "0.4.112" version_files = [ "pyproject.toml:^version", "../pyproject.toml:litellm-proxy-extras==", diff --git a/pyproject.toml b/pyproject.toml index 44dfa498904..6e4732b4f7a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -84,8 +84,8 @@ proxy = [ "mcp>=2.3.0,<3", "httpx2>=2.10.0,<3", "pydantic>=2.12.0,<3", - "litellm-proxy-extras==0.4.111", - "litellm-enterprise==0.1.80", + "litellm-proxy-extras==0.4.112", + "litellm-enterprise==0.1.81", "RestrictedPython>=8.5,<9.0", "rich>=13.9.4,<14.0", "InquirerPy>=0.3.4,<1.0", diff --git a/uv.lock b/uv.lock index 5be97af9ed2..b3e8ddc492c 100644 --- a/uv.lock +++ b/uv.lock @@ -5478,12 +5478,12 @@ provides-extras = ["test"] [[package]] name = "litellm-enterprise" -version = "0.1.80" +version = "0.1.81" source = { editable = "enterprise" } [[package]] name = "litellm-proxy-extras" -version = "0.4.111" +version = "0.4.112" source = { editable = "litellm-proxy-extras" } dependencies = [ { name = "psycopg" }, From bbc68e21f92f017853859d76fb43ca5e4641a2f2 Mon Sep 17 00:00:00 2001 From: zoveki <349028329+zoveki@users.noreply.github.com> Date: Thu, 29 Oct 2026 17:48:38 -0700 Subject: [PATCH 11/17] feat(dashscope): add image edits and text-to-speech (#48998) * feat(dashscope): add image edits and text-to-speech * fix(dashscope): bill reported speech characters and input images, follow the chat region for edits, keep the call timeout on the audio download * fix(dashscope): bill deployment image prices as main does and route edits to the host speech uses A deployment that sets its own input_cost_per_image, output_cost_per_image or input_cost_per_pixel bills through the default image calculator again, so an override is never read as a per-input-image price and pixel pricing on qwen-image-3.0-pro no longer raises. Edits now resolve the caller's api_base or DASHSCOPE_API_BASE through the same API-root helper speech uses, on any host, with DASHSCOPE_API_BASE_IMAGE still winning. An image response whose usage is null or not an object keeps its images instead of raising. * fix(images): bill a deployment's own prices on image edits through the proxy The proxy builds the logging object before the router picks a deployment, so image_edit never handed the deployment's price fields to its cost calculation and billed the cost-map price instead. Pass the custom pricing params with the pre-call update, as the Responses API does. * fix(images): bill per-pixel image prices at a DashScope W*H size DashScope's mapped size reads 1328*1328, which the cost size reader rejected, so a deployment's input_cost_per_pixel billed every DashScope generation or edit at the 1024x1024 default. * fix(dashscope): bill image prices by their meaning, report the rendered size, prefer the deployment edit base, fetch audio over https * fix(dashscope): read the rendered image size from output_width and output_height * test(images): bill a fal edit at its deployment's own image price * fix(dashscope): keep the qwen-image-3.0 input-image fee out of the shared cost map row Released proxies read the cost map from main and bill input_cost_per_image as the flat price of each DashScope image, so the row's $0.003 input fee billed every qwen-image-3.0 image there at a tenth of its $0.03 price. A deployment can still set input_cost_per_image to bill DashScope's input images --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- litellm/cost_calculator.py | 10 +- litellm/images/main.py | 3 +- .../litellm_core_utils/llm_cost_calc/utils.py | 24 +- .../base_llm/text_to_speech/transformation.py | 12 + litellm/llms/custom_httpx/llm_http_handler.py | 2 +- litellm/llms/dashscope/common_utils.py | 22 +- litellm/llms/dashscope/image_edit/__init__.py | 0 .../dashscope/image_edit/transformation.py | 160 +++++++ .../image_generation/cost_calculator.py | 19 + .../image_generation/transformation.py | 94 ++-- .../llms/dashscope/text_to_speech/__init__.py | 0 .../text_to_speech/transformation.py | 297 +++++++++++++ litellm/main.py | 9 +- ...odel_prices_and_context_window_backup.json | 67 ++- .../provider_endpoints_support_backup.json | 4 +- litellm/utils.py | 57 ++- model_prices_and_context_window.json | 67 ++- provider_endpoints_support.json | 4 +- ...st_dashscope_image_edit_and_speech_wire.py | 259 +++++++++++ tests/unit/images/test_image_edit.py | 119 +++++ ...est_dashscope_image_edit_transformation.py | 358 +++++++++++++++ ...dashscope_text_to_speech_transformation.py | 415 ++++++++++++++++++ .../test_text_to_speech_transformation.py | 38 ++ .../test_get_complete_url_trailing_slash.py | 1 + tests/unit/test_dashscope_image_generation.py | 61 ++- 25 files changed, 2009 insertions(+), 93 deletions(-) create mode 100644 litellm/llms/dashscope/image_edit/__init__.py create mode 100644 litellm/llms/dashscope/image_edit/transformation.py create mode 100644 litellm/llms/dashscope/image_generation/cost_calculator.py create mode 100644 litellm/llms/dashscope/text_to_speech/__init__.py create mode 100644 litellm/llms/dashscope/text_to_speech/transformation.py create mode 100644 tests/integration/providers/test_dashscope_image_edit_and_speech_wire.py create mode 100644 tests/unit/llms/dashscope/test_dashscope_image_edit_transformation.py create mode 100644 tests/unit/llms/dashscope/test_dashscope_text_to_speech_transformation.py diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index a4a9a5bee39..defe1e63b86 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -1887,7 +1887,10 @@ def completion_cost( video_resolution=video_resolution, ) elif call_type in _SPEECH_CALL_TYPES: - prompt_characters = litellm.utils.count_characters(text=prompt) + prompt_characters = _speech_billed_characters( + completion_response if isinstance(completion_response, HttpxBinaryResponseContent) else None, + prompt, + ) elif call_type in _TRANSCRIPTION_CALL_TYPES: # Check _hidden_params first (duration stored there to # avoid polluting the response body), then fall back to @@ -2366,6 +2369,11 @@ def response_cost_calculator( raise e +def _speech_billed_characters(speech_response: HttpxBinaryResponseContent | None, prompt: str) -> int: + reported: Final = speech_response.hidden_params.get("billed_characters") if speech_response is not None else None + return reported if isinstance(reported, int) else litellm.utils.count_characters(text=prompt) + + def _video_input_image_cost( model: str, custom_llm_provider: str | None, diff --git a/litellm/images/main.py b/litellm/images/main.py index 4aad5210028..165543aadba 100644 --- a/litellm/images/main.py +++ b/litellm/images/main.py @@ -53,7 +53,7 @@ from litellm.main import ( from litellm.secret_managers.main import get_secret_str from litellm.types.images.main import ImageEditOptionalRequestParams from litellm.types.llms.openai import ImageGenerationRequestQuality -from litellm.types.router import GenericLiteLLMParams +from litellm.types.router import CustomPricingLiteLLMParams, GenericLiteLLMParams from litellm.types.utils import ( LITELLM_IMAGE_VARIATION_PROVIDERS, LlmProviders, @@ -1010,6 +1010,7 @@ def image_edit( optional_params=dict(image_edit_request_params), litellm_params={ **image_edit_request_params, + **litellm_params.model_dump(include=set(CustomPricingLiteLLMParams.model_fields), exclude_none=True), "litellm_call_id": litellm_call_id, "model_info": model_info, "vertex_location": litellm_params.vertex_location, diff --git a/litellm/litellm_core_utils/llm_cost_calc/utils.py b/litellm/litellm_core_utils/llm_cost_calc/utils.py index 4b0b875af6d..404dfab0a6d 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/utils.py +++ b/litellm/litellm_core_utils/llm_cost_calc/utils.py @@ -75,6 +75,14 @@ _IMAGE_TOKEN_RATE_KEYS: Final[tuple[str, ...]] = ( "output_cost_per_image_token", ) + +def _deployment_prices_images_without_an_output_price(pricing: ModelInfo | None) -> bool: + return pricing is not None and ( + "input_cost_per_pixel" in pricing + or ("input_cost_per_image" in pricing and "output_cost_per_image" not in pricing) + ) + + # Pre-resolved service-tier cost-key suffixes (e.g. "_priority"). Used per # request in the cost-calc path, so the f-strings are built once here instead # of being rebuilt for every model_info key on every call. Longest-first so a @@ -127,7 +135,7 @@ def _get_token_detail_value(details: object, key: str) -> int | None: return value if isinstance(value, int) else None -_IMAGE_SIZE_PATTERN: Final = re.compile(r"\d+(?:x|-x-)\d+") +_IMAGE_SIZE_PATTERN: Final = re.compile(r"\d+(?:x|-x-|\*)\d+") def _requested_image_param(optional_params: Mapping[str, object] | None, key: str) -> str | None: @@ -137,7 +145,7 @@ def _requested_image_param(optional_params: Mapping[str, object] | None, key: st def _requested_image_size(optional_params: Mapping[str, object] | None) -> str | None: value: Final = _requested_image_param(optional_params, "size") - return value if value is not None and _IMAGE_SIZE_PATTERN.fullmatch(value) else None + return value.replace("*", "x") if value is not None and _IMAGE_SIZE_PATTERN.fullmatch(value) else None def get_web_search_requests(server_tool_use: object) -> int | None: @@ -2090,7 +2098,7 @@ class _ModelPricedImageCostCalculator(Protocol): def _model_priced_image_cost_calculator( - custom_llm_provider: str | None, model: str + custom_llm_provider: str | None, model: str, pricing: ModelInfo | None ) -> _ModelPricedImageCostCalculator | None: match custom_llm_provider: case litellm.LlmProviders.RECRAFT.value: @@ -2116,6 +2124,14 @@ def _model_priced_image_cost_calculator( from litellm.llms.tencent.image_generation.transformation import is_tokenhub_image_model return tencent_image_cost_calculator if is_tokenhub_image_model(model) else None + case litellm.LlmProviders.DASHSCOPE.value: + if _deployment_prices_images_without_an_output_price(pricing): + return None + from litellm.llms.dashscope.image_generation.cost_calculator import ( + cost_calculator as dashscope_image_cost_calculator, + ) + + return dashscope_image_cost_calculator case _: return None @@ -2178,7 +2194,7 @@ class CostCalculatorUtils: ) resolved_n: Final = n if n is not None else (len(completion_response.data) if completion_response.data else 0) pricing: Final = deployment_pricing(model_info) - model_priced_calculator: Final = _model_priced_image_cost_calculator(custom_llm_provider, model) + model_priced_calculator: Final = _model_priced_image_cost_calculator(custom_llm_provider, model, pricing) if model_priced_calculator is not None: return model_priced_calculator(model=model, image_response=completion_response, model_info=pricing) diff --git a/litellm/llms/base_llm/text_to_speech/transformation.py b/litellm/llms/base_llm/text_to_speech/transformation.py index be49098b62b..6df83966bf8 100644 --- a/litellm/llms/base_llm/text_to_speech/transformation.py +++ b/litellm/llms/base_llm/text_to_speech/transformation.py @@ -135,6 +135,18 @@ class BaseTextToSpeechConfig(ABC): Transform provider response to standard format """ + async def async_transform_text_to_speech_response( + self, + model: str, + raw_response: httpx.Response, + logging_obj: LiteLLMLoggingObj, + ) -> "HttpxBinaryResponseContent": + return self.transform_text_to_speech_response( + model=model, + raw_response=raw_response, + logging_obj=logging_obj, + ) + def get_error_class(self, error_message: str, status_code: int, headers: dict) -> BaseLLMException: from ..chat.transformation import BaseLLMException diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index 1f25235a651..c92315e53b3 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -12849,7 +12849,7 @@ class BaseLLMHTTPHandler: provider_config=text_to_speech_provider_config, ) - speech_response: Final = text_to_speech_provider_config.transform_text_to_speech_response( + speech_response: Final = await text_to_speech_provider_config.async_transform_text_to_speech_response( model=model, raw_response=response, logging_obj=logging_obj, diff --git a/litellm/llms/dashscope/common_utils.py b/litellm/llms/dashscope/common_utils.py index 5ad2aa17a3a..d3e1fb8eada 100644 --- a/litellm/llms/dashscope/common_utils.py +++ b/litellm/llms/dashscope/common_utils.py @@ -24,7 +24,8 @@ DASHSCOPE_ANTHROPIC_PATH: Final = "/apps/anthropic" DASHSCOPE_NATIVE_V1_PATH: Final = "/api/v1" DASHSCOPE_COMPATIBLE_API_PATH: Final = "/compatible-api/v1" DASHSCOPE_RERANK_PATH: Final = f"{DASHSCOPE_COMPATIBLE_API_PATH}/reranks" -DASHSCOPE_IMAGE_PATH: Final = "/api/v1/services/aigc/multimodal-generation/generation" +DASHSCOPE_MULTIMODAL_GENERATION_PATH: Final = "/api/v1/services/aigc/multimodal-generation/generation" +DASHSCOPE_IMAGE_PATH: Final = DASHSCOPE_MULTIMODAL_GENERATION_PATH DASHSCOPE_DEFAULT_ORIGIN: Final = "https://dashscope.aliyuncs.com" _STANDARD_DASHSCOPE_HOSTS: Final = frozenset( @@ -100,6 +101,21 @@ def is_dashscope_native_endpoint_url(api_base: str) -> bool: return urlparse(api_base).path.rstrip("/").startswith(f"{DASHSCOPE_NATIVE_V1_PATH}/") +def dashscope_api_root(api_base: str) -> str: + endpoints: Final = resolve_dashscope_endpoints(api_base) + if endpoints is not None: + return endpoints.origin + trimmed: Final = api_base.rstrip("/") + if trimmed.endswith(DASHSCOPE_CHAT_COMPATIBLE_PATH): + return trimmed.removesuffix(DASHSCOPE_CHAT_COMPATIBLE_PATH) + root, separator, _ = trimmed.partition(DASHSCOPE_NATIVE_V1_PATH) + return root if separator else trimmed + + +def is_aliyun_host(host: str | None) -> bool: + return host is not None and (host == "aliyuncs.com" or host.endswith(".aliyuncs.com")) + + def resolve_dashscope_openai_api_base( api_base: str | None, env_var: str = "DASHSCOPE_API_BASE", @@ -140,9 +156,7 @@ def _chat_shaped_aliyun_origin(api_base: str | None) -> str | None: if api_base is None: return None parsed: Final = urlparse(api_base) - host: Final = parsed.hostname or "" - on_aliyun_host: Final = host == "aliyuncs.com" or host.endswith(".aliyuncs.com") - if not on_aliyun_host or parsed.path.rstrip("/") != DASHSCOPE_CHAT_COMPATIBLE_PATH: + if not is_aliyun_host(parsed.hostname) or parsed.path.rstrip("/") != DASHSCOPE_CHAT_COMPATIBLE_PATH: return None return f"{parsed.scheme}://{parsed.netloc}" diff --git a/litellm/llms/dashscope/image_edit/__init__.py b/litellm/llms/dashscope/image_edit/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/litellm/llms/dashscope/image_edit/transformation.py b/litellm/llms/dashscope/image_edit/transformation.py new file mode 100644 index 00000000000..0d34017a321 --- /dev/null +++ b/litellm/llms/dashscope/image_edit/transformation.py @@ -0,0 +1,160 @@ +from collections.abc import Mapping +from typing import TYPE_CHECKING, Final + +import httpx +from httpx._types import RequestFiles +from pydantic import TypeAdapter +from typing_extensions import override + +import litellm +from litellm.images.utils import image_to_data_url +from litellm.llms.base_llm.image_edit.transformation import BaseImageEditConfig +from litellm.llms.dashscope.common_utils import ( + DASHSCOPE_CHAT_COMPATIBLE_PATH, + DASHSCOPE_MULTIMODAL_GENERATION_PATH, + dashscope_api_root, + is_dashscope_native_endpoint_url, + resolve_dashscope_endpoints, +) +from litellm.llms.dashscope.image_generation.transformation import ( + DEFAULT_API_BASE, + OPENAI_TO_DASHSCOPE_SIZE, + DashScopeImageGenerationConfig, +) +from litellm.secret_managers.main import get_secret_str +from litellm.types.images.main import ImageEditOptionalRequestParams +from litellm.types.router import GenericLiteLLMParams +from litellm.types.utils import ImageResponse + +if TYPE_CHECKING: + from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + +MAX_INPUT_IMAGES: Final = 3 +DASHSCOPE_IMAGE_EDIT_PARAMS: Final = ("n", "size", "negative_prompt", "prompt_extend", "watermark", "seed") + +_IMAGES: Final = TypeAdapter(tuple[object, ...]) + + +def _dashscope_size(size: object) -> object: + if not isinstance(size, str): + return size + return OPENAI_TO_DASHSCOPE_SIZE.get(size, size.replace("x", "*")) + + +def _deployment_edit_url(api_base: str) -> str: + derives_from_root: Final = not is_dashscope_native_endpoint_url(api_base) and ( + resolve_dashscope_endpoints(api_base) is not None + or api_base.rstrip("/").endswith(DASHSCOPE_CHAT_COMPATIBLE_PATH) + ) + return f"{dashscope_api_root(api_base)}{DASHSCOPE_MULTIMODAL_GENERATION_PATH}" if derives_from_root else api_base + + +class DashScopeImageEditConfig(BaseImageEditConfig): + def __init__(self, generation_config: DashScopeImageGenerationConfig | None = None) -> None: + super().__init__() + self._generation_config: Final = generation_config or DashScopeImageGenerationConfig() + + @override + def get_supported_openai_params(self, model: str) -> list[str]: # mutable-ok: mirrors override contract + return list(DASHSCOPE_IMAGE_EDIT_PARAMS) + + @override + def map_openai_params( + self, + image_edit_optional_params: ImageEditOptionalRequestParams, + model: str, + drop_params: bool, + ) -> dict[str, object]: # mutable-ok: mirrors override contract + return { + key: _dashscope_size(value) if key == "size" else value + for key, value in image_edit_optional_params.items() + if key in DASHSCOPE_IMAGE_EDIT_PARAMS + } + + @override + def validate_environment( + self, + headers: dict[str, str], # mutable-ok: mirrors override contract + model: str, + api_key: str | None = None, + litellm_params: dict[str, object] | None = None, # mutable-ok: mirrors override contract + api_base: str | None = None, + ) -> dict[str, str]: # mutable-ok: mirrors override contract + return self._generation_config.validate_environment( + headers=dict(headers), + model=model, + messages=[], + optional_params={}, + litellm_params=dict(litellm_params or {}), + api_key=api_key, + api_base=api_base, + ) + + @override + def get_complete_url( + self, + model: str, + api_base: str | None, + litellm_params: dict[str, object], # mutable-ok: mirrors override contract + ) -> str: + if api_base is not None: + return _deployment_edit_url(api_base) + env_api_base: Final = get_secret_str("DASHSCOPE_API_BASE") + derived: Final = ( + f"{dashscope_api_root(env_api_base)}{DASHSCOPE_MULTIMODAL_GENERATION_PATH}" if env_api_base else None + ) + return get_secret_str("DASHSCOPE_API_BASE_IMAGE") or derived or DEFAULT_API_BASE + + @override + def use_multipart_form_data(self) -> bool: + return False + + @override + def transform_image_edit_request( + self, + model: str, + prompt: str | None, + image: object, + image_edit_optional_request_params: Mapping[str, object], + litellm_params: GenericLiteLLMParams, + headers: Mapping[str, str], + ) -> tuple[dict[str, object], RequestFiles]: # mutable-ok: mirrors override contract + candidates: Final = _IMAGES.validate_python(image) if isinstance(image, list) else (image,) + images: Final = tuple(candidate for candidate in candidates if candidate is not None) + if len(images) > MAX_INPUT_IMAGES: + raise litellm.BadRequestError( + message=f"DashScope image edit accepts at most {MAX_INPUT_IMAGES} input images, got {len(images)}", + model=model, + llm_provider="dashscope", + ) + prompt_content: Final = ({"text": prompt},) if prompt is not None else () + return { + "model": model, + "input": { + "messages": [ + { + "role": "user", + "content": [*({"image": image_to_data_url(item)} for item in images), *prompt_content], + } + ] + }, + "parameters": dict(image_edit_optional_request_params), + }, () + + @override + def transform_image_edit_response( + self, + model: str, + raw_response: httpx.Response, + logging_obj: "LiteLLMLoggingObj", + ) -> ImageResponse: + return self._generation_config.transform_image_generation_response( + model=model, + raw_response=raw_response, + model_response=ImageResponse(), + logging_obj=logging_obj, + request_data={}, + optional_params={}, + litellm_params={}, + encoding=None, + ) diff --git a/litellm/llms/dashscope/image_generation/cost_calculator.py b/litellm/llms/dashscope/image_generation/cost_calculator.py new file mode 100644 index 00000000000..f699ef7e1c5 --- /dev/null +++ b/litellm/llms/dashscope/image_generation/cost_calculator.py @@ -0,0 +1,19 @@ +from typing import Final + +import litellm +from litellm.litellm_core_utils.llm_cost_calc.utils import resolve_image_model_info +from litellm.types.utils import ImageResponse, ModelInfo + + +def cost_calculator(model: str, image_response: ImageResponse, model_info: ModelInfo | None = None) -> float: + prices: Final = resolve_image_model_info( + model=model, custom_llm_provider=litellm.LlmProviders.DASHSCOPE.value, model_info=model_info + ) + output_cost_per_image: Final = prices.get("output_cost_per_image") + if output_cost_per_image is None: + raise ValueError(f"No output_cost_per_image found for dashscope/{model}") + input_images: Final = image_response.hidden_params.get("billable_input_image_count") + input_image_cost: Final = (prices.get("input_cost_per_image") or 0.0) * ( + input_images if isinstance(input_images, int) else 0 + ) + return output_cost_per_image * len(image_response.data or ()) + input_image_cost diff --git a/litellm/llms/dashscope/image_generation/transformation.py b/litellm/llms/dashscope/image_generation/transformation.py index 2f528b5a2c3..d1b5a7b40e7 100644 --- a/litellm/llms/dashscope/image_generation/transformation.py +++ b/litellm/llms/dashscope/image_generation/transformation.py @@ -23,7 +23,7 @@ Response format: } """ -from collections.abc import Iterable, Mapping +from collections.abc import Iterable, Iterator, Mapping from typing import TYPE_CHECKING, Any, Final import httpx @@ -61,7 +61,7 @@ _DECODED_JSON: Final = TypeAdapter(object) DEFAULT_API_BASE: Final = "https://dashscope-intl.aliyuncs.com/api/v1/services/aigc/multimodal-generation/generation" # Maps OpenAI size strings (WxH) to DashScope size strings (W*H) -OPENAI_TO_DASHSCOPE_SIZE: Final[dict] = { +OPENAI_TO_DASHSCOPE_SIZE: Final[Mapping[str, str]] = { "256x256": "256*256", "512x512": "512*512", "1024x1024": "1024*1024", @@ -71,6 +71,24 @@ OPENAI_TO_DASHSCOPE_SIZE: Final[dict] = { } +def _message_image_urls(message: Mapping[str, object]) -> Iterator[str]: + contents: Final = _JSON_OBJECTS.validate_python(message.get("content", [])) + return (url for url in (item.get("image") for item in contents) if isinstance(url, str) and url) + + +def _image_urls(response_object: Mapping[str, object]) -> Iterator[str]: + output: Final = _JSON_OBJECT.validate_python(response_object.get("output", {})) + choices: Final = _JSON_OBJECTS.validate_python(output.get("choices", [])) + for choice in choices: + yield from _message_image_urls(_JSON_OBJECT.validate_python(choice.get("message", {}))) + + +def _dashscope_size(size: object) -> object: + if not isinstance(size, str): + return size + return OPENAI_TO_DASHSCOPE_SIZE.get(size, size.replace("x", "*")) + + class DashScopeImageGenerationConfig(BaseImageGenerationConfig): """ Configuration for DashScope image generation (qwen-image-2.0, qwen-image-2.0-pro, @@ -82,24 +100,16 @@ class DashScopeImageGenerationConfig(BaseImageGenerationConfig): def map_openai_params( self, - non_default_params: dict, - optional_params: dict, + non_default_params: dict[str, object], + optional_params: dict[str, object], model: str, drop_params: bool, - ) -> dict: + ) -> dict[str, object]: supported_params: Final = self.get_supported_openai_params(model) - mapped: Final[dict] = {} - for k, v in non_default_params.items(): - if k in optional_params: - continue - if k not in supported_params: - continue - if k == "size": - # Convert "WxH" → "W*H" - mapped["size"] = OPENAI_TO_DASHSCOPE_SIZE.get(v, v.replace("x", "*")) - else: - mapped[k] = v - return mapped + requested: Final = { + k: v for k, v in non_default_params.items() if k in supported_params and k not in optional_params + } + return {k: _dashscope_size(v) if k == "size" else v for k, v in requested.items()} def _resolve_api_key(self, api_key: str | None) -> str: resolved_api_key: Final = api_key or get_secret_str("DASHSCOPE_API_KEY") @@ -121,8 +131,8 @@ class DashScopeImageGenerationConfig(BaseImageGenerationConfig): api_base: str | None, api_key: str | None, model: str, - optional_params: dict, - litellm_params: dict, + optional_params: dict[str, object], + litellm_params: dict[str, object], stream: bool | None = None, ) -> str: image_env_api_base: Final = self._image_api_base_env() @@ -140,14 +150,14 @@ class DashScopeImageGenerationConfig(BaseImageGenerationConfig): def validate_environment( self, - headers: dict, + headers: dict[str, str], model: str, messages: list[AllMessageValues], - optional_params: dict, - litellm_params: dict, + optional_params: dict[str, object], + litellm_params: dict[str, object], api_key: str | None = None, api_base: str | None = None, - ) -> dict: + ) -> dict[str, str]: headers["Authorization"] = f"Bearer {self._resolve_api_key(api_key)}" headers["Content-Type"] = "application/json" return headers @@ -186,9 +196,9 @@ class DashScopeImageGenerationConfig(BaseImageGenerationConfig): raw_response: httpx.Response, model_response: ImageResponse, logging_obj: LiteLLMLoggingObj, - request_data: dict, - optional_params: dict, - litellm_params: dict, + request_data: dict[str, object], + optional_params: dict[str, object], + litellm_params: dict[str, object], encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, @@ -225,17 +235,21 @@ class DashScopeImageGenerationConfig(BaseImageGenerationConfig): headers=raw_response.headers, ) - if not model_response.data: - model_response.data = [] - - output: Final = _JSON_OBJECT.validate_python(response_object.get("output", {})) - choices: Final = _JSON_OBJECTS.validate_python(output.get("choices", [])) - for choice in choices: - message = _JSON_OBJECT.validate_python(choice.get("message", {})) - content_list = _JSON_OBJECTS.validate_python(message.get("content", [])) - for content_item in content_list: - image_url = content_item.get("image") - if image_url: - model_response.data.append(ImageObject(url=image_url)) - - return model_response + images: Final = [*(model_response.data or ()), *(ImageObject(url=url) for url in _image_urls(response_object))] + usage: Final = response_object.get("usage") + usage_fields: Final[Mapping[str, object]] = usage if isinstance(usage, Mapping) else {} + width: Final = usage_fields.get("output_width", usage_fields.get("width")) + height: Final = usage_fields.get("output_height", usage_fields.get("height")) + image_response: Final = model_response.model_copy( + update={ + "data": images, + **({"size": f"{width}x{height}"} if isinstance(width, int) and isinstance(height, int) else {}), + } + ) + input_image_count: Final = usage_fields.get("input_image_count") + if isinstance(input_image_count, int) and input_image_count > 0: + image_response.hidden_params = { + **model_response.hidden_params, + "billable_input_image_count": input_image_count, + } + return image_response diff --git a/litellm/llms/dashscope/text_to_speech/__init__.py b/litellm/llms/dashscope/text_to_speech/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/litellm/llms/dashscope/text_to_speech/transformation.py b/litellm/llms/dashscope/text_to_speech/transformation.py new file mode 100644 index 00000000000..053682040bd --- /dev/null +++ b/litellm/llms/dashscope/text_to_speech/transformation.py @@ -0,0 +1,297 @@ +from collections.abc import Callable, Mapping +from dataclasses import dataclass +from itertools import chain +from typing import TYPE_CHECKING, Final +from urllib.parse import urlparse + +import httpx +from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError +from typing_extensions import override + +import litellm +from litellm.exceptions import UnsupportedParamsError +from litellm.litellm_core_utils.core_helpers import drops_unsupported_params +from litellm.litellm_core_utils.url_utils import SSRFError, async_safe_get, safe_get +from litellm.llms.base_llm.chat.transformation import BaseLLMException +from litellm.llms.base_llm.text_to_speech.transformation import ( + BaseTextToSpeechConfig, + TextToSpeechRequestData, +) +from litellm.llms.custom_httpx.http_handler import ( + AsyncHTTPHandler, + HTTPHandler, + get_async_httpx_client, # pyright: ignore[reportUnknownVariableType] # factory stub leaves params untyped + get_httpx_client, # pyright: ignore[reportUnknownVariableType] # factory stub leaves params untyped +) +from litellm.llms.dashscope.common_utils import ( + DASHSCOPE_MULTIMODAL_GENERATION_PATH, + DashScopeError, + dashscope_api_root, + is_aliyun_host, + missing_dashscope_family_key_message, + resolve_dashscope_family_api_key, +) +from litellm.secret_managers.main import get_secret_str +from litellm.types.llms.openai import HttpxBinaryResponseContent +from litellm.types.utils import Usage + +if TYPE_CHECKING: + from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + +DASHSCOPE_TTS_DEFAULT_API_BASE: Final = "https://dashscope-intl.aliyuncs.com" +_SPEECH_SYNTHESIZER_MODEL_PREFIX: Final = "qwen-audio-" +_TIMEOUT_PHASES: Final = TypeAdapter(dict[str, float | None]) + + +@dataclass(frozen=True, slots=True) +class _TextToSpeechFamily: + path: str + response_formats: frozenset[str] + sends_format: bool + takes_instructions: bool + + +MULTIMODAL_GENERATION_FAMILY: Final = _TextToSpeechFamily( + path=DASHSCOPE_MULTIMODAL_GENERATION_PATH, + response_formats=frozenset({"wav"}), + sends_format=False, + takes_instructions=False, +) +SPEECH_SYNTHESIZER_FAMILY: Final = _TextToSpeechFamily( + path="/api/v1/services/audio/tts/SpeechSynthesizer", + response_formats=frozenset({"mp3", "wav", "opus", "pcm"}), + sends_format=True, + takes_instructions=True, +) + + +class _Audio(BaseModel): + model_config = ConfigDict(frozen=True) + url: str + + +class _Output(BaseModel): + model_config = ConfigDict(frozen=True) + audio: _Audio + + +class _Usage(BaseModel): + model_config = ConfigDict(frozen=True) + input_tokens: int | None = None + output_tokens: int | None = None + characters: int | None = None + + +class _SpeechResponse(BaseModel): + model_config = ConfigDict(frozen=True) + output: _Output | None = None + usage: _Usage | None = None + code: str | None = None + message: str | None = None + + +def text_to_speech_family(model: str) -> _TextToSpeechFamily: + if model.startswith(_SPEECH_SYNTHESIZER_MODEL_PREFIX): + return SPEECH_SYNTHESIZER_FAMILY + return MULTIMODAL_GENERATION_FAMILY + + +def _mapped_field(family: _TextToSpeechFamily, key: str, value: object) -> Mapping[str, object] | None: + match key: + case "response_format" if value in family.response_formats: + return {"format": value} if family.sends_format else {} + case "instructions" if family.takes_instructions: + return {"instruction": value} + case _: + return None + + +def _unsupported(model: str, unsupported: tuple[str, ...]) -> UnsupportedParamsError: + return UnsupportedParamsError( + message=f"DashScope {model} does not support {', '.join(unsupported)}, set drop_params=True to ignore it", + model=model, + llm_provider="dashscope", + ) + + +def _speech_response(raw_response: httpx.Response) -> _SpeechResponse: + try: + return _SpeechResponse.model_validate_json(raw_response.content) + except ValidationError as e: + raise DashScopeError( + status_code=502, + message=f"Unexpected DashScope text-to-speech response: {e}", + headers=raw_response.headers, + ) from e + + +def _https_on_aliyun(url: str) -> str: + parsed: Final = urlparse(url) + return ( + parsed._replace(scheme="https").geturl() if parsed.scheme == "http" and is_aliyun_host(parsed.hostname) else url + ) + + +def _audio_url(raw_response: httpx.Response) -> tuple[str, _Usage | None]: + parsed: Final = _speech_response(raw_response) + if parsed.output is None: + raise DashScopeError( + status_code=max(raw_response.status_code, 400), + message=parsed.message or parsed.code or raw_response.text, + headers=raw_response.headers, + ) + return _https_on_aliyun(parsed.output.audio.url), parsed.usage + + +def _request_timeout(raw_response: httpx.Response) -> httpx.Timeout | None: + extension: Final = raw_response.request.extensions.get("timeout") + if extension is None: + return None + phases: Final = _TIMEOUT_PHASES.validate_python(extension) + return httpx.Timeout( + connect=phases.get("connect"), read=phases.get("read"), write=phases.get("write"), pool=phases.get("pool") + ) + + +def _token_usage(usage: _Usage | None) -> Usage | None: + if usage is None or usage.input_tokens is None or usage.output_tokens is None: + return None + return Usage( + prompt_tokens=usage.input_tokens, + completion_tokens=usage.output_tokens, + total_tokens=usage.input_tokens + usage.output_tokens, + ) + + +def _refused_audio_url(error: SSRFError) -> DashScopeError: + return DashScopeError(status_code=502, message=f"DashScope returned an audio URL that was refused: {error}") + + +def _binary_audio(audio_response: httpx.Response, usage: _Usage | None) -> HttpxBinaryResponseContent: + if audio_response.status_code != 200: + raise DashScopeError( + status_code=502, + message=f"DashScope audio download failed with HTTP {audio_response.status_code}", + headers=audio_response.headers, + ) + binary: Final = HttpxBinaryResponseContent(audio_response) + binary.usage = _token_usage(usage) + if usage is not None and usage.characters is not None: + binary.hidden_params = {**binary.hidden_params, "billed_characters": usage.characters} + return binary + + +def _shared_async_client() -> AsyncHTTPHandler: + return get_async_httpx_client(llm_provider=litellm.LlmProviders.DASHSCOPE) + + +class DashScopeTextToSpeechConfig(BaseTextToSpeechConfig): + def __init__( + self, + sync_client: Callable[[], HTTPHandler] = get_httpx_client, + async_client: Callable[[], AsyncHTTPHandler] = _shared_async_client, + ) -> None: + super().__init__() + self._sync_client: Final = sync_client + self._async_client: Final = async_client + + @override + def get_supported_openai_params(self, model: str) -> list[str]: # mutable-ok: mirrors override contract + family: Final = text_to_speech_family(model) + return ["voice", "response_format", *(("instructions",) if family.takes_instructions else ())] + + @override + def map_openai_params( + self, + model: str, + optional_params: dict[str, object], # mutable-ok: mirrors override contract + voice: str | dict[str, object] | None = None, # mutable-ok: mirrors override contract + drop_params: bool = False, + kwargs: Mapping[str, object] | None = None, + ) -> tuple[str | None, dict[str, object]]: # mutable-ok: mirrors override contract + family: Final = text_to_speech_family(model) + fields: Final = tuple((key, value, _mapped_field(family, key, value)) for key, value in optional_params.items()) + unsupported: Final = ( + *(f"{key}={value!r}" for key, value, field in fields if field is None), + *(("voice as an object",) if isinstance(voice, dict) else ()), + ) + if unsupported and not (drop_params or drops_unsupported_params((kwargs or {}).get("drop_params"))): + raise _unsupported(model, unsupported) + supported: Final = tuple(field for _, _, field in fields if field is not None) + return voice if isinstance(voice, str) else None, dict(chain.from_iterable(f.items() for f in supported)) + + @override + def validate_environment( + self, + headers: dict[str, str], # mutable-ok: mirrors override contract + model: str, + api_key: str | None = None, + api_base: str | None = None, + ) -> dict[str, str]: # mutable-ok: mirrors override contract + resolved_api_key: Final = resolve_dashscope_family_api_key("dashscope", api_key) + if resolved_api_key is None: + raise ValueError(missing_dashscope_family_key_message("dashscope")) + return {**headers, "Authorization": f"Bearer {resolved_api_key}", "Content-Type": "application/json"} + + @override + def get_complete_url( + self, + model: str, + api_base: str | None, + litellm_params: dict[str, object], # mutable-ok: mirrors override contract + ) -> str: + configured: Final = api_base or get_secret_str("DASHSCOPE_API_BASE") + root: Final = dashscope_api_root(configured) if configured else DASHSCOPE_TTS_DEFAULT_API_BASE + return f"{root}{text_to_speech_family(model).path}" + + @override + def transform_text_to_speech_request( + self, + model: str, + input: str, + voice: str | None, + optional_params: dict[str, object], # mutable-ok: mirrors override contract + litellm_params: dict[str, object], # mutable-ok: mirrors override contract + headers: dict[str, str], # mutable-ok: mirrors override contract + ) -> TextToSpeechRequestData: + voice_field: Final = {"voice": voice} if voice is not None else {} + return {"dict_body": {"model": model, "input": {"text": input, **voice_field, **optional_params}}} + + @override + def transform_text_to_speech_response( + self, + model: str, + raw_response: httpx.Response, + logging_obj: "LiteLLMLoggingObj", + ) -> HttpxBinaryResponseContent: + audio_url, usage = _audio_url(raw_response) + try: + audio_response: Final = safe_get(self._sync_client(), audio_url, timeout=_request_timeout(raw_response)) + except SSRFError as e: + raise _refused_audio_url(e) from e + return _binary_audio(audio_response, usage) + + @override + async def async_transform_text_to_speech_response( + self, + model: str, + raw_response: httpx.Response, + logging_obj: "LiteLLMLoggingObj", + ) -> HttpxBinaryResponseContent: + audio_url, usage = _audio_url(raw_response) + try: + audio_response: Final = await async_safe_get( + self._async_client(), audio_url, timeout=_request_timeout(raw_response) + ) + except SSRFError as e: + raise _refused_audio_url(e) from e + return _binary_audio(audio_response, usage) + + @override + def get_error_class( + self, + error_message: str, + status_code: int, + headers: dict[str, str] | httpx.Headers, # mutable-ok: mirrors override contract + ) -> BaseLLMException: + return DashScopeError(status_code=status_code, message=error_message, headers=httpx.Headers(headers)) diff --git a/litellm/main.py b/litellm/main.py index 0162742de18..d469aa629fe 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -8826,8 +8826,9 @@ def speech( extra_headers: Final = kwargs.get("extra_headers", None) model_info: Final = kwargs.get("model_info", None) shared_session: Final = kwargs.get("shared_session", None) + caller_api_base: Final = api_base or base_url model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( - model=model, custom_llm_provider=custom_llm_provider, api_base=api_base or base_url + model=model, custom_llm_provider=custom_llm_provider, api_base=caller_api_base ) kwargs.pop("tags", []) @@ -8882,13 +8883,13 @@ def speech( custom_llm_provider=custom_llm_provider, ) response: HttpxBinaryResponseContent | Coroutine[object, object, HttpxBinaryResponseContent] | None = None - if custom_llm_provider == "edenai": - litellm_params_dict["api_base"] = api_base + if custom_llm_provider in ("edenai", "dashscope") and text_to_speech_provider_config is not None: + litellm_params_dict["api_base"] = caller_api_base if custom_llm_provider == "dashscope" else api_base response = base_llm_http_handler.text_to_speech_handler( model=model, input=input, voice=voice if isinstance(voice, str) else None, - text_to_speech_provider_config=text_to_speech_provider_config or litellm.EdenAITextToSpeechConfig(), + text_to_speech_provider_config=text_to_speech_provider_config, text_to_speech_optional_params=optional_params, custom_llm_provider=custom_llm_provider, litellm_params=litellm_params_dict, diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index c6bbb63b6fc..50cfc6007ca 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -18346,25 +18346,31 @@ "dashscope/qwen-image-2.0": { "litellm_provider": "dashscope", "mode": "image_generation", - "source": "https://www.alibabacloud.com/help/en/model-studio/models", + "output_cost_per_image": 0.035, + "source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing", "supported_endpoints": [ - "/v1/images/generations" + "/v1/images/generations", + "/v1/images/edits" ] }, "dashscope/qwen-image-2.0-pro": { "litellm_provider": "dashscope", "mode": "image_generation", - "source": "https://www.alibabacloud.com/help/en/model-studio/models", + "output_cost_per_image": 0.075, + "source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing", "supported_endpoints": [ - "/v1/images/generations" + "/v1/images/generations", + "/v1/images/edits" ] }, "dashscope/qwen-image-3.0": { "litellm_provider": "dashscope", "mode": "image_generation", - "source": "https://www.alibabacloud.com/help/en/model-studio/models", + "output_cost_per_image": 0.03, + "source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing", "supported_endpoints": [ - "/v1/images/generations" + "/v1/images/generations", + "/v1/images/edits" ] }, "dashscope/qwen-image-3.0-pro": { @@ -18372,7 +18378,54 @@ "mode": "image_generation", "source": "https://www.alibabacloud.com/help/en/model-studio/models", "supported_endpoints": [ - "/v1/images/generations" + "/v1/images/generations", + "/v1/images/edits" + ] + }, + "dashscope/qwen3-tts-flash": { + "input_cost_per_character": 1e-05, + "litellm_provider": "dashscope", + "mode": "audio_speech", + "source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing", + "supported_endpoints": [ + "/v1/audio/speech" + ] + }, + "dashscope/qwen-audio-3.1-tts-flash": { + "input_cost_per_token": 2.3e-07, + "output_cost_per_token": 1.87e-06, + "litellm_provider": "dashscope", + "mode": "audio_speech", + "source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing", + "supported_endpoints": [ + "/v1/audio/speech" + ] + }, + "dashscope/qwen-audio-3.1-tts-plus": { + "input_cost_per_character": 3e-05, + "litellm_provider": "dashscope", + "mode": "audio_speech", + "source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing", + "supported_endpoints": [ + "/v1/audio/speech" + ] + }, + "dashscope/qwen-audio-3.0-tts-plus": { + "input_cost_per_character": 2e-05, + "litellm_provider": "dashscope", + "mode": "audio_speech", + "source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing", + "supported_endpoints": [ + "/v1/audio/speech" + ] + }, + "dashscope/qwen-audio-3.0-tts-flash": { + "input_cost_per_character": 1.5e-05, + "litellm_provider": "dashscope", + "mode": "audio_speech", + "source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing", + "supported_endpoints": [ + "/v1/audio/speech" ] }, "dashscope/wan3.0-video": { diff --git a/litellm/provider_endpoints_support_backup.json b/litellm/provider_endpoints_support_backup.json index 1d487f33835..069ecfb82b2 100644 --- a/litellm/provider_endpoints_support_backup.json +++ b/litellm/provider_endpoints_support_backup.json @@ -1148,9 +1148,9 @@ "text_completion": true, "embeddings": false, "image_generations": true, - "image_edits": false, + "image_edits": true, "audio_transcriptions": false, - "audio_speech": false, + "audio_speech": true, "rerank": false, "ocr": false, "search": false, diff --git a/litellm/utils.py b/litellm/utils.py index e46e8ca1d52..904d154f8df 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -10923,19 +10923,8 @@ class ProviderConfigManager: return None @staticmethod - def get_provider_image_edit_config( - model: str, - provider: LlmProviders, - ) -> BaseImageEditConfig | None: - if LlmProviders.OPENAI == provider: - from litellm.llms.openai.image_edit import OpenAIImageEditConfig - - return OpenAIImageEditConfig() - elif LlmProviders.HOSTED_VLLM == provider: - from litellm.llms.hosted_vllm.image_edit import get_hosted_vllm_image_edit_config - - return get_hosted_vllm_image_edit_config(model=model) - elif LlmProviders.AZURE == provider: + def _get_model_independent_image_edit_config(provider: LlmProviders) -> BaseImageEditConfig | None: + if LlmProviders.AZURE == provider: from litellm.llms.azure.image_edit.transformation import ( AzureImageEditConfig, ) @@ -10953,6 +10942,36 @@ class ProviderConfigManager: ) return BlackForestLabsImageEditConfig() + elif LlmProviders.LITELLM_PROXY == provider: + from litellm.llms.litellm_proxy.image_edit.transformation import ( + LiteLLMProxyImageEditConfig, + ) + + return LiteLLMProxyImageEditConfig() + elif LlmProviders.DASHSCOPE == provider: + from litellm.llms.dashscope.image_edit.transformation import ( + DashScopeImageEditConfig, + ) + + return DashScopeImageEditConfig() + return None + + @staticmethod + def get_provider_image_edit_config( + model: str, + provider: LlmProviders, + ) -> BaseImageEditConfig | None: + model_independent_config: Final = ProviderConfigManager._get_model_independent_image_edit_config(provider) + if model_independent_config is not None: + return model_independent_config + if LlmProviders.OPENAI == provider: + from litellm.llms.openai.image_edit import OpenAIImageEditConfig + + return OpenAIImageEditConfig() + elif LlmProviders.HOSTED_VLLM == provider: + from litellm.llms.hosted_vllm.image_edit import get_hosted_vllm_image_edit_config + + return get_hosted_vllm_image_edit_config(model=model) elif LlmProviders.FAL_AI == provider: from litellm.llms.fal_ai.image_edit import get_fal_ai_image_edit_config @@ -10965,12 +10984,6 @@ class ProviderConfigManager: from litellm.llms.gemini.image_edit import get_gemini_image_edit_config return get_gemini_image_edit_config(model) - elif LlmProviders.LITELLM_PROXY == provider: - from litellm.llms.litellm_proxy.image_edit.transformation import ( - LiteLLMProxyImageEditConfig, - ) - - return LiteLLMProxyImageEditConfig() elif LlmProviders.VERTEX_AI == provider: from litellm.llms.vertex_ai.image_edit import ( get_vertex_ai_image_edit_config, @@ -11192,6 +11205,12 @@ class ProviderConfigManager: return AWSPollyTextToSpeechConfig() elif litellm.LlmProviders.EDENAI == provider: return litellm.EdenAITextToSpeechConfig() + elif litellm.LlmProviders.DASHSCOPE == provider: + from litellm.llms.dashscope.text_to_speech.transformation import ( + DashScopeTextToSpeechConfig, + ) + + return DashScopeTextToSpeechConfig() return None @staticmethod diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index c6bbb63b6fc..50cfc6007ca 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -18346,25 +18346,31 @@ "dashscope/qwen-image-2.0": { "litellm_provider": "dashscope", "mode": "image_generation", - "source": "https://www.alibabacloud.com/help/en/model-studio/models", + "output_cost_per_image": 0.035, + "source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing", "supported_endpoints": [ - "/v1/images/generations" + "/v1/images/generations", + "/v1/images/edits" ] }, "dashscope/qwen-image-2.0-pro": { "litellm_provider": "dashscope", "mode": "image_generation", - "source": "https://www.alibabacloud.com/help/en/model-studio/models", + "output_cost_per_image": 0.075, + "source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing", "supported_endpoints": [ - "/v1/images/generations" + "/v1/images/generations", + "/v1/images/edits" ] }, "dashscope/qwen-image-3.0": { "litellm_provider": "dashscope", "mode": "image_generation", - "source": "https://www.alibabacloud.com/help/en/model-studio/models", + "output_cost_per_image": 0.03, + "source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing", "supported_endpoints": [ - "/v1/images/generations" + "/v1/images/generations", + "/v1/images/edits" ] }, "dashscope/qwen-image-3.0-pro": { @@ -18372,7 +18378,54 @@ "mode": "image_generation", "source": "https://www.alibabacloud.com/help/en/model-studio/models", "supported_endpoints": [ - "/v1/images/generations" + "/v1/images/generations", + "/v1/images/edits" + ] + }, + "dashscope/qwen3-tts-flash": { + "input_cost_per_character": 1e-05, + "litellm_provider": "dashscope", + "mode": "audio_speech", + "source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing", + "supported_endpoints": [ + "/v1/audio/speech" + ] + }, + "dashscope/qwen-audio-3.1-tts-flash": { + "input_cost_per_token": 2.3e-07, + "output_cost_per_token": 1.87e-06, + "litellm_provider": "dashscope", + "mode": "audio_speech", + "source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing", + "supported_endpoints": [ + "/v1/audio/speech" + ] + }, + "dashscope/qwen-audio-3.1-tts-plus": { + "input_cost_per_character": 3e-05, + "litellm_provider": "dashscope", + "mode": "audio_speech", + "source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing", + "supported_endpoints": [ + "/v1/audio/speech" + ] + }, + "dashscope/qwen-audio-3.0-tts-plus": { + "input_cost_per_character": 2e-05, + "litellm_provider": "dashscope", + "mode": "audio_speech", + "source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing", + "supported_endpoints": [ + "/v1/audio/speech" + ] + }, + "dashscope/qwen-audio-3.0-tts-flash": { + "input_cost_per_character": 1.5e-05, + "litellm_provider": "dashscope", + "mode": "audio_speech", + "source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing", + "supported_endpoints": [ + "/v1/audio/speech" ] }, "dashscope/wan3.0-video": { diff --git a/provider_endpoints_support.json b/provider_endpoints_support.json index 1d487f33835..069ecfb82b2 100644 --- a/provider_endpoints_support.json +++ b/provider_endpoints_support.json @@ -1148,9 +1148,9 @@ "text_completion": true, "embeddings": false, "image_generations": true, - "image_edits": false, + "image_edits": true, "audio_transcriptions": false, - "audio_speech": false, + "audio_speech": true, "rerank": false, "ocr": false, "search": false, diff --git a/tests/integration/providers/test_dashscope_image_edit_and_speech_wire.py b/tests/integration/providers/test_dashscope_image_edit_and_speech_wire.py new file mode 100644 index 00000000000..ea16633d81d --- /dev/null +++ b/tests/integration/providers/test_dashscope_image_edit_and_speech_wire.py @@ -0,0 +1,259 @@ +import base64 +import json +import os +import uuid +from collections.abc import Iterator +from pathlib import Path +from typing import Final +from urllib.parse import urlsplit, urlunsplit + +import pytest +import yaml +from integration._support.client import JSON_OBJECT, Gateway, gateway_from_environment, list_value, object_value +from integration._support.process import graceful_stop_seconds, owned_proxy +from integration._support.wire import Reply, Request, Wire, wire_server +from integration.providers._dashscope_video_support import deployed, json_reply, rate, settled_spend +from pydantic import JsonValue + +_API_KEY: Final = "synthetic-dashscope-key" +_MULTIMODAL_PATH: Final = "/api/v1/services/aigc/multimodal-generation/generation" +_SYNTHESIZER_PATH: Final = "/api/v1/services/audio/tts/SpeechSynthesizer" +_OSS_PATH: Final = "/oss/" +_PNG: Final = base64.b64decode( + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg==" +) +_PNG_DATA_URI: Final = f"data:image/png;base64,{base64.b64encode(_PNG).decode()}" +_EDITED_URL: Final = "https://dashscope-result.example.com/edited.png" +_PROMPT: Final = "Turn the red circle into a blue square" +_TEXT: Final = "Hello from LiteLLM." +_BILLED_CHARACTERS: Final = len(_TEXT) +_WAV: Final = b"RIFF\x24\x00\x00\x00WAVEfmt " + uuid.uuid4().bytes +_MP3: Final = b"ID3\x04\x00\x00\x00\x00\x00\x00" + uuid.uuid4().bytes +_SIGNED_QUERY: Final = "Expires=1793316671&OSSAccessKeyId=LTAI5t&Signature=ab%2Bc%3D" +_QWEN_TTS: Final = "dashscope-qwen-tts" +_QWEN_AUDIO_TTS: Final = "dashscope-qwen-audio-tts" +_INPUT_TOKENS: Final = 8 +_OUTPUT_TOKENS: Final = 30 +_OWNED_TIMEOUT: Final = int(2 * graceful_stop_seconds() + 120) + + +def _edit_api(request: Request) -> Reply: + assert (request.method, request.target) == ("POST", _MULTIMODAL_PATH), (request.method, request.target) + return json_reply( + { + "output": {"choices": [{"message": {"role": "assistant", "content": [{"image": _EDITED_URL}]}}]}, + "usage": {"width": 1024, "height": 1024, "image_count": 1}, + } + ) + + +def _synthesized(request: Request, file_name: str, usage: dict[str, JsonValue]) -> Reply: + audio_url: Final = f"http://{request.headers['host']}{_OSS_PATH}{file_name}?{_SIGNED_QUERY}" + return json_reply({"output": {"audio": {"url": audio_url, "data": "", "id": "audio-1"}}, "usage": usage}) + + +def _speech_api(request: Request) -> Reply: + if request.method == "GET" and request.target.startswith(f"{_OSS_PATH}speech.wav"): + return Reply(body=_WAV, content_type="audio/wav") + if request.method == "GET" and request.target.startswith(f"{_OSS_PATH}speech.mp3"): + return Reply(body=_MP3, content_type="audio/mpeg") + if (request.method, request.target) == ("POST", _MULTIMODAL_PATH): + return _synthesized(request, "speech.wav", {"characters": _BILLED_CHARACTERS}) + assert (request.method, request.target) == ("POST", _SYNTHESIZER_PATH), (request.method, request.target) + return _synthesized( + request, + "speech.mp3", + {"characters": _BILLED_CHARACTERS, "input_tokens": _INPUT_TOKENS, "output_tokens": _OUTPUT_TOKENS}, + ) + + +def _pooled_database_url(url: str) -> str: + parts: Final = urlsplit(url) + query: Final = "&".join(part for part in (parts.query, "connection_limit=5") if part) + return urlunsplit(parts._replace(query=query)) + + +def _tts_deployment(name: str, model: str, api_base: str) -> dict[str, JsonValue]: + return {"model_name": name, "litellm_params": {"model": model, "api_base": api_base, "api_key": _API_KEY}} + + +def _error_message(body: bytes) -> str: + return str(object_value(JSON_OBJECT.validate_json(body)["error"])["message"]) + + +@pytest.fixture(scope="module") +def rig() -> Iterator[Gateway]: + with gateway_from_environment() as gateway: + yield gateway + + +@pytest.fixture(scope="module") +def speech_wire() -> Iterator[Wire]: + with wire_server(_speech_api) as served: + yield served + + +@pytest.fixture(scope="module") +def speech_proxy(rig: Gateway, speech_wire: Wire, tmp_path_factory: pytest.TempPathFactory) -> Iterator[Gateway]: + directory: Final = tmp_path_factory.mktemp("dashscope-speech") + config: Final = JSON_OBJECT.validate_python(yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text())) + path: Final = directory / "dashscope-speech.yaml" + path.write_text( + yaml.safe_dump( + { + **config, + "model_list": [ + _tts_deployment(_QWEN_TTS, "dashscope/qwen3-tts-flash", speech_wire.url), + _tts_deployment(_QWEN_AUDIO_TTS, "dashscope/qwen-audio-3.1-tts-flash", speech_wire.url), + ], + "general_settings": { + **object_value(config["general_settings"]), + "user_url_allowed_hosts": ["127.0.0.1"], + }, + } + ) + ) + with owned_proxy( + rig, directory, {"DATABASE_URL": _pooled_database_url(os.environ["DATABASE_URL"])}, config=path + ) as owned: + yield owned + + +def test_a_multipart_edit_reaches_dashscope_as_inline_images_and_bills_the_edited_image(gateway: Gateway) -> None: + with wire_server(_edit_api) as wire, gateway.scenario() as scenario: + model: Final = deployed( + gateway, + scenario, + model="dashscope/qwen-image-2.0", + api_base=f"{wire.url}{_MULTIMODAL_PATH}", + api_key=_API_KEY, + ) + key: Final = scenario.key(models=[model]) + response: Final = gateway.client.post( + "/v1/images/edits", + data={"model": model, "prompt": _PROMPT, "size": "1024x1024", "negative_prompt": "blurry"}, + files=[("image[]", ("a.png", _PNG, "image/png")), ("image[]", ("b.png", _PNG, "image/png"))], + headers={"Authorization": f"Bearer {key}"}, + ) + + assert response.status_code == 200, response.text + data: Final = list_value(JSON_OBJECT.validate_json(response.content)["data"]) + assert [object_value(item)["url"] for item in data] == [_EDITED_URL] + (request,) = wire.drain() + assert request.headers["authorization"] == f"Bearer {_API_KEY}" + assert json.loads(request.body) == { + "model": "qwen-image-2.0", + "input": { + "messages": [ + { + "role": "user", + "content": [{"image": _PNG_DATA_URI}, {"image": _PNG_DATA_URI}, {"text": _PROMPT}], + } + ] + }, + "parameters": {"size": "1024*1024", "negative_prompt": "blurry"}, + } + settled_spend(key, rate("dashscope", "output_cost_per_image", model="qwen-image-2.0")) + + +def test_a_fourth_image_is_refused_with_a_400_before_dashscope_is_called(gateway: Gateway) -> None: + with wire_server(_edit_api) as wire, gateway.scenario() as scenario: + model: Final = deployed( + gateway, + scenario, + model="dashscope/qwen-image-2.0", + api_base=f"{wire.url}{_MULTIMODAL_PATH}", + api_key=_API_KEY, + ) + response: Final = gateway.client.post( + "/v1/images/edits", + data={"model": model, "prompt": _PROMPT}, + files=[("image[]", (f"{index}.png", _PNG, "image/png")) for index in range(4)], + headers={"Authorization": f"Bearer {gateway.key}"}, + ) + + assert response.status_code == 400, response.text + assert "at most 3 input images, got 4" in _error_message(response.content) + assert wire.drain() == () + + +@pytest.mark.timeout(_OWNED_TIMEOUT) +def test_qwen3_tts_returns_the_downloaded_wav_and_bills_each_input_character( + speech_proxy: Gateway, speech_wire: Wire +) -> None: + speech_wire.drain() + with speech_proxy.scenario() as scenario: + key: Final = scenario.key(models=[_QWEN_TTS]) + response: Final = speech_proxy.request( + "POST", "/v1/audio/speech", {"model": _QWEN_TTS, "input": _TEXT, "voice": "Cherry"}, key=key + ) + + assert response.status_code == 200, response.text + assert response.content == _WAV + assert response.headers["content-type"] == "audio/wav" + synthesis, download = speech_wire.drain() + assert (synthesis.method, synthesis.target) == ("POST", _MULTIMODAL_PATH) + assert synthesis.headers["authorization"] == f"Bearer {_API_KEY}" + assert json.loads(synthesis.body) == {"model": "qwen3-tts-flash", "input": {"text": _TEXT, "voice": "Cherry"}} + assert (download.method, download.target) == ("GET", f"{_OSS_PATH}speech.wav?{_SIGNED_QUERY}") + assert "authorization" not in download.headers + settled_spend(key, _BILLED_CHARACTERS * rate("dashscope", "input_cost_per_character", "qwen3-tts-flash")) + + +@pytest.mark.timeout(_OWNED_TIMEOUT) +def test_qwen_audio_tts_sends_format_and_instruction_returns_the_mp3_and_bills_reported_tokens( + speech_proxy: Gateway, speech_wire: Wire +) -> None: + speech_wire.drain() + model: Final = "qwen-audio-3.1-tts-flash" + with speech_proxy.scenario() as scenario: + key: Final = scenario.key(models=[_QWEN_AUDIO_TTS]) + response: Final = speech_proxy.request( + "POST", + "/v1/audio/speech", + { + "model": _QWEN_AUDIO_TTS, + "input": _TEXT, + "voice": "longanhuan_v3.1", + "response_format": "mp3", + "instructions": "Speak slowly.", + }, + key=key, + ) + + assert response.status_code == 200, response.text + assert response.content == _MP3 + assert response.headers["content-type"] == "audio/mpeg" + synthesis, download = speech_wire.drain() + assert (synthesis.method, synthesis.target) == ("POST", _SYNTHESIZER_PATH) + assert json.loads(synthesis.body) == { + "model": model, + "input": {"text": _TEXT, "voice": "longanhuan_v3.1", "format": "mp3", "instruction": "Speak slowly."}, + } + assert (download.method, download.target) == ("GET", f"{_OSS_PATH}speech.mp3?{_SIGNED_QUERY}") + settled_spend( + key, + _INPUT_TOKENS * rate("dashscope", "input_cost_per_token", model) + + _OUTPUT_TOKENS * rate("dashscope", "output_cost_per_token", model), + ) + + +@pytest.mark.timeout(_OWNED_TIMEOUT) +def test_a_format_qwen3_tts_cannot_produce_answers_400_without_reaching_dashscope( + speech_proxy: Gateway, speech_wire: Wire +) -> None: + speech_wire.drain() + refused: Final = speech_proxy.request( + "POST", "/v1/audio/speech", {"model": _QWEN_TTS, "input": _TEXT, "voice": "Cherry", "response_format": "mp3"} + ) + served: Final = speech_proxy.request( + "POST", "/v1/audio/speech", {"model": _QWEN_TTS, "input": _TEXT, "voice": "Cherry", "response_format": "wav"} + ) + + assert refused.status_code == 400, refused.text + assert "response_format='mp3'" in _error_message(refused.content) + assert served.status_code == 200, served.text + assert [(request.method, request.target.split("?")[0]) for request in speech_wire.drain()] == [ + ("POST", _MULTIMODAL_PATH), + ("GET", f"{_OSS_PATH}speech.wav"), + ] diff --git a/tests/unit/images/test_image_edit.py b/tests/unit/images/test_image_edit.py index 4f858050e3e..06372a29d92 100644 --- a/tests/unit/images/test_image_edit.py +++ b/tests/unit/images/test_image_edit.py @@ -12,6 +12,7 @@ from typing_extensions import ReadOnly, TypedDict, override import litellm from litellm.integrations.custom_logger import CustomLogger +from litellm.litellm_core_utils.litellm_logging import Logging from litellm.types.utils import FileTypes, ImageResponse _PNG_SIGNATURE: Final = b"\x89PNG\r\n\x1a\n" @@ -34,6 +35,26 @@ _EDIT_RESPONSE: Final = { }, } +_DASHSCOPE_IMAGE_URL: Final = ( + "https://dashscope-intl.aliyuncs.com/api/v1/services/aigc/multimodal-generation/generation" +) +_DASHSCOPE_EDIT_RESPONSE: Final = { + "output": { + "choices": [ + { + "finish_reason": "stop", + "message": {"role": "assistant", "content": [{"image": "https://example.com/edited.png"}]}, + } + ] + }, + "usage": {"image_count": 1}, +} + +_FAL_EDIT_URL: Final = "https://fal.run/fal-ai/flux-2/edit" +_FAL_EDIT_RESPONSE: Final = { + "images": [{"url": "https://example.com/edited.png", "width": 1024, "height": 768, "content_type": "image/png"}] +} + class _LoggedImageEdit(TypedDict): model: ReadOnly[str] @@ -229,3 +250,101 @@ async def test_router_image_edit_returns_the_provider_image( assert isinstance(result, ImageResponse) assert result.data is not None and result.data[0].b64_json == _EDITED_IMAGE_B64 assert route.call_count == 1 + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("deployment_price", "size", "expected_cost"), + ( + ({"input_cost_per_image": 0.0123}, None, 0.0123), + ({"input_cost_per_pixel": 1e-08}, "2048x2048", 2048 * 2048 * 1e-08), + ), +) +async def test_an_edit_bills_the_deployment_price_at_its_size_on_a_logging_object_made_before_routing( + respx_mock: respx.MockRouter, + httpx_transport: None, + monkeypatch: pytest.MonkeyPatch, + deployment_price: dict[str, float], + size: str | None, + expected_cost: float, +) -> None: + monkeypatch.delenv("DASHSCOPE_API_BASE", raising=False) + monkeypatch.delenv("DASHSCOPE_API_BASE_IMAGE", raising=False) + respx_mock.post(_DASHSCOPE_IMAGE_URL).mock(return_value=httpx.Response(200, json=_DASHSCOPE_EDIT_RESPONSE)) + logging_made_before_routing: Final = Logging( + model="qwen-image-edit", + messages=[], + stream=False, + call_type="aimage_edit", + start_time=datetime.now(), + litellm_call_id="edit-call", + function_id="edit-call", + ) + + router: Final = litellm.Router( + model_list=[ + { + "model_name": "qwen-image-edit", + "litellm_params": { + "model": "dashscope/qwen-image-2.0", + "api_key": "fake-key", + **deployment_price, + }, + } + ] + ) + + result: Final = await router.aimage_edit( + model="qwen-image-edit", + image=_FIRST_IMAGE, + prompt="Turn the circle into a square", + size=size, + litellm_logging_obj=logging_made_before_routing, + litellm_call_id="edit-call", + ) + + assert isinstance(result, ImageResponse) + assert result._hidden_params["response_cost"] == pytest.approx(expected_cost) + + +@pytest.mark.asyncio +async def test_a_fal_edit_bills_the_deployment_price_per_image_on_a_logging_object_made_before_routing( + respx_mock: respx.MockRouter, + httpx_transport: None, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.delenv("FAL_AI_API_BASE", raising=False) + respx_mock.post(_FAL_EDIT_URL).mock(return_value=httpx.Response(200, json=_FAL_EDIT_RESPONSE)) + logging_made_before_routing: Final = Logging( + model="flux-edit", + messages=[], + stream=False, + call_type="aimage_edit", + start_time=datetime.now(), + litellm_call_id="fal-edit-call", + function_id="fal-edit-call", + ) + + router: Final = litellm.Router( + model_list=[ + { + "model_name": "flux-edit", + "litellm_params": { + "model": "fal_ai/fal-ai/flux-2/edit", + "api_key": "fake-key", + "output_cost_per_image": 0.0456, + }, + } + ] + ) + + result: Final = await router.aimage_edit( + model="flux-edit", + image=_FIRST_IMAGE, + prompt="Turn the circle into a square", + litellm_logging_obj=logging_made_before_routing, + litellm_call_id="fal-edit-call", + ) + + assert isinstance(result, ImageResponse) + assert result._hidden_params["response_cost"] == pytest.approx(0.0456) diff --git a/tests/unit/llms/dashscope/test_dashscope_image_edit_transformation.py b/tests/unit/llms/dashscope/test_dashscope_image_edit_transformation.py new file mode 100644 index 00000000000..c3d186fdd8a --- /dev/null +++ b/tests/unit/llms/dashscope/test_dashscope_image_edit_transformation.py @@ -0,0 +1,358 @@ +import base64 +import json +from collections.abc import Callable +from typing import Final + +import httpx +import pytest + +import litellm +from litellm.litellm_core_utils.llm_cost_calc.utils import CostCalculatorUtils +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler +from litellm.llms.dashscope.image_edit.transformation import DashScopeImageEditConfig +from litellm.llms.dashscope.text_to_speech.transformation import DashScopeTextToSpeechConfig +from litellm.types.utils import ModelInfo + +PNG: Final = base64.b64decode( + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg==" +) +PNG_DATA_URI: Final = f"data:image/png;base64,{base64.b64encode(PNG).decode()}" +INTL_ENDPOINT: Final = "https://dashscope-intl.aliyuncs.com/api/v1/services/aigc/multimodal-generation/generation" +EDITED_URL: Final = "https://dashscope-result.example.com/edited.png" +EDITED: Final = { + "output": {"choices": [{"message": {"role": "assistant", "content": [{"image": EDITED_URL}]}}]}, + "usage": {"width": 1024, "height": 1024, "image_count": 1}, +} + + +def _recording( + seen: list[httpx.Request], reply: Callable[[httpx.Request], httpx.Response] +) -> httpx.MockTransport: # mutable-ok: test records the requests the transport receives + def respond(request: httpx.Request) -> httpx.Response: + seen.append(request) + return reply(request) + + return httpx.MockTransport(respond) + + +def _edited(_: httpx.Request) -> httpx.Response: + return httpx.Response(200, json=EDITED) + + +def _sync_client( + seen: list[httpx.Request], reply: Callable[[httpx.Request], httpx.Response] = _edited +) -> HTTPHandler: # mutable-ok: shared request log + return HTTPHandler(client=httpx.Client(transport=_recording(seen, reply))) + + +@pytest.fixture(autouse=True) +def _no_ambient_dashscope_base(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("DASHSCOPE_API_BASE_IMAGE", raising=False) + monkeypatch.delenv("DASHSCOPE_API_BASE", raising=False) + + +@pytest.mark.parametrize("asynchronous", (False, True)) +async def test_an_edit_posts_every_image_inline_with_provider_params_at_the_top_of_parameters( + asynchronous: bool, +) -> None: + seen: Final[list[httpx.Request]] = [] # mutable-ok: request log + arguments: Final = { + "model": "dashscope/qwen-image-2.0", + "image": [PNG, PNG], + "prompt": "Turn the circle into a square", + "size": "1024x1024", + "n": 1, + "negative_prompt": "blurry", + "prompt_extend": False, + "watermark": False, + "seed": 42, + "api_key": "sk-dashscope-test", + } + response: Final = ( + await litellm.aimage_edit(**arguments, client=AsyncHTTPHandler(transport=_recording(seen, _edited))) + if asynchronous + else litellm.image_edit(**arguments, client=_sync_client(seen)) + ) + + assert [image.url for image in response.data or []] == [EDITED_URL] + (request,) = seen + assert str(request.url) == INTL_ENDPOINT + assert request.headers["authorization"] == "Bearer sk-dashscope-test" + assert json.loads(request.content) == { + "model": "qwen-image-2.0", + "input": { + "messages": [ + { + "role": "user", + "content": [ + {"image": PNG_DATA_URI}, + {"image": PNG_DATA_URI}, + {"text": "Turn the circle into a square"}, + ], + } + ] + }, + "parameters": { + "n": 1, + "size": "1024*1024", + "negative_prompt": "blurry", + "prompt_extend": False, + "watermark": False, + "seed": 42, + }, + } + + +@pytest.mark.parametrize(("size", "dashscope_size"), (("1792x1024", "1792*1024"), ("1328x1328", "1328*1328"))) +def test_an_openai_size_reaches_dashscope_in_its_star_format(size: str, dashscope_size: str) -> None: + seen: Final[list[httpx.Request]] = [] # mutable-ok: request log + + litellm.image_edit( + model="dashscope/qwen-image-2.0", image=PNG, prompt="p", size=size, api_key="k", client=_sync_client(seen) + ) + + assert json.loads(seen[0].content)["parameters"] == {"size": dashscope_size} + + +CHINA_ENDPOINT: Final = "https://dashscope.aliyuncs.com/api/v1/services/aigc/multimodal-generation/generation" + + +@pytest.mark.parametrize( + ("api_base", "env_base", "image_base", "endpoint"), + ( + ("https://dashscope.aliyuncs.com/compatible-mode/v1", None, None, CHINA_ENDPOINT), + ("https://dashscope-intl.aliyuncs.com/compatible-mode/v1/", None, None, INTL_ENDPOINT), + ("https://dashscope.aliyuncs.com", None, None, CHINA_ENDPOINT), + ("https://dashscope-intl.aliyuncs.com/apps/anthropic", None, None, INTL_ENDPOINT), + (None, "https://dashscope.aliyuncs.com/apps/anthropic", None, CHINA_ENDPOINT), + ( + "https://gw.example.com/compatible-mode/v1", + None, + None, + "https://gw.example.com/api/v1/services/aigc/multimodal-generation/generation", + ), + (None, "https://dashscope.aliyuncs.com/compatible-mode/v1", None, CHINA_ENDPOINT), + (None, "https://dashscope.aliyuncs.com/api/v1", None, CHINA_ENDPOINT), + (None, None, None, INTL_ENDPOINT), + ( + "https://images.example.com/edit", + "https://dashscope.aliyuncs.com/compatible-mode/v1", + None, + "https://images.example.com/edit", + ), + ("https://dashscope.aliyuncs.com/compatible-mode/v1", None, "https://images.example.com/edit", CHINA_ENDPOINT), + ( + None, + "https://dashscope.aliyuncs.com/compatible-mode/v1", + "https://images.example.com/edit", + "https://images.example.com/edit", + ), + ), +) +def test_an_edit_goes_to_the_image_endpoint_on_the_host_its_dashscope_base_names( + monkeypatch: pytest.MonkeyPatch, api_base: str | None, env_base: str | None, image_base: str | None, endpoint: str +) -> None: + if env_base is not None: + monkeypatch.setenv("DASHSCOPE_API_BASE", env_base) + if image_base is not None: + monkeypatch.setenv("DASHSCOPE_API_BASE_IMAGE", image_base) + seen: Final[list[httpx.Request]] = [] # mutable-ok: request log + + litellm.image_edit( + model="dashscope/qwen-image-2.0", + image=PNG, + prompt="p", + api_key="k", + api_base=api_base, + client=_sync_client(seen), + ) + + assert str(seen[0].url) == endpoint + + +@pytest.mark.parametrize("api_base", ("https://dashscope.aliyuncs.com/compatible-mode/v1", None)) +def test_an_edit_and_a_speech_call_with_the_same_dashscope_base_reach_the_same_host( + monkeypatch: pytest.MonkeyPatch, api_base: str | None +) -> None: + monkeypatch.setenv("DASHSCOPE_API_BASE", "https://gw.example.com/compatible-mode/v1") + edit_url: Final = DashScopeImageEditConfig().get_complete_url( + model="qwen-image-2.0", api_base=api_base, litellm_params={} + ) + speech_url: Final = DashScopeTextToSpeechConfig().get_complete_url( + model="qwen3-tts-flash", api_base=api_base, litellm_params={} + ) + + assert httpx.URL(edit_url).host == httpx.URL(speech_url).host + + +def test_a_fourth_image_is_refused_with_a_400_before_any_request_even_under_drop_params() -> None: + seen: Final[list[httpx.Request]] = [] # mutable-ok: request log + + with pytest.raises(litellm.BadRequestError) as refused: + litellm.image_edit( + model="dashscope/qwen-image-2.0", + image=[PNG, PNG, PNG, PNG], + prompt="p", + api_key="k", + drop_params=True, + client=_sync_client(seen), + ) + + assert refused.value.status_code == 400 + assert "at most 3 input images, got 4" in str(refused.value) + assert seen == [] + + +def test_three_images_are_accepted() -> None: + seen: Final[list[httpx.Request]] = [] # mutable-ok: request log + + litellm.image_edit( + model="dashscope/qwen-image-2.0", image=[PNG, PNG, PNG], prompt="p", api_key="k", client=_sync_client(seen) + ) + + content: Final = json.loads(seen[0].content)["input"]["messages"][0]["content"] + assert content == [{"image": PNG_DATA_URI}] * 3 + [{"text": "p"}] + + +def test_an_unsupported_openai_param_answers_400_and_is_dropped_under_drop_params() -> None: + seen: Final[list[httpx.Request]] = [] # mutable-ok: request log + client: Final = _sync_client(seen) + + with pytest.raises(litellm.UnsupportedParamsError) as refused: + litellm.image_edit( + model="dashscope/qwen-image-2.0", image=PNG, prompt="p", quality="high", api_key="k", client=client + ) + litellm.image_edit( + model="dashscope/qwen-image-2.0", + image=PNG, + prompt="p", + quality="high", + api_key="k", + drop_params=True, + client=client, + ) + + assert refused.value.status_code == 400 + assert "quality" in str(refused.value) + (request,) = seen + assert json.loads(request.content)["parameters"] == {} + + +@pytest.mark.parametrize( + ("reply", "error", "message"), + ( + ( + httpx.Response(200, json={"code": "InvalidParameter", "message": "Image is too small"}), + litellm.APIError, + "Image is too small", + ), + ( + httpx.Response(400, json={"code": "InvalidParameter", "message": "Model not exist."}), + litellm.BadRequestError, + "Model not exist.", + ), + ), +) +def test_a_dashscope_rejection_reaches_the_caller_with_its_message( + reply: httpx.Response, error: type[Exception], message: str +) -> None: + with pytest.raises(error, match=message): + litellm.image_edit( + model="dashscope/qwen-image-2.0", + image=PNG, + prompt="p", + api_key="k", + client=_sync_client([], lambda _: reply), + ) + + +@pytest.mark.parametrize("model", ("qwen-image-2.0", "qwen-image-2.0-pro", "qwen-image-3.0")) +@pytest.mark.usefixtures("local_model_cost_map") +def test_each_edited_image_is_billed_at_the_model_output_image_price(model: str) -> None: + response: Final = litellm.ImageResponse(data=[{"url": EDITED_URL}, {"url": EDITED_URL}]) + + cost: Final = litellm.completion_cost( + completion_response=response, + model=f"dashscope/{model}", + call_type="aimage_edit", + custom_llm_provider="dashscope", + ) + + assert cost == pytest.approx(2 * litellm.model_cost[f"dashscope/{model}"]["output_cost_per_image"]) + assert cost > 0 + + +@pytest.mark.usefixtures("local_model_cost_map") +def test_the_input_images_dashscope_reports_are_billed_on_top_of_the_edited_images() -> None: + def reply(_: httpx.Request) -> httpx.Response: + return httpx.Response(200, json={**EDITED, "usage": {"input_image_count": 2, "output_image_count": 1}}) + + response: Final = litellm.image_edit( + model="dashscope/qwen-image-3.0", image=[PNG, PNG], prompt="p", api_key="k", client=_sync_client([], reply) + ) + + assert isinstance(response, litellm.ImageResponse) + assert CostCalculatorUtils.route_image_generation_cost_calculator( + model="qwen-image-3.0", + completion_response=response, + custom_llm_provider="dashscope", + call_type="aimage_edit", + model_info=ModelInfo(input_cost_per_image=0.004, output_cost_per_image=0.04), + ) == pytest.approx(2 * 0.004 + 0.04) + + +def _edited_images(count: int, input_images: int) -> litellm.ImageResponse: + response: Final = litellm.ImageResponse(data=[{"url": EDITED_URL}] * count) + response.hidden_params = {"billable_input_image_count": input_images} + return response + + +@pytest.mark.parametrize( + ("model", "deployment_price", "size", "expected_cost"), + ( + ("qwen-image-2.0", ModelInfo(input_cost_per_image=0.02), None, 2 * 0.02), + ("qwen-image-3.0-pro", ModelInfo(input_cost_per_pixel=1e-08), "1024x1024", 2 * 1024 * 1024 * 1e-08), + ), +) +@pytest.mark.usefixtures("local_model_cost_map") +def test_a_deployment_image_price_bills_the_edit_the_way_it_bills_any_image_call( + model: str, deployment_price: ModelInfo, size: str | None, expected_cost: float +) -> None: + cost: Final = CostCalculatorUtils.route_image_generation_cost_calculator( + model=model, + completion_response=_edited_images(count=2, input_images=3), + custom_llm_provider="dashscope", + size=size, + call_type="aimage_edit", + model_info=deployment_price, + ) + + assert cost == pytest.approx(expected_cost) + + +@pytest.mark.usefixtures("local_model_cost_map") +def test_a_deployment_that_copies_both_image_prices_bills_edited_and_input_images_at_its_own_prices() -> None: + cost: Final = CostCalculatorUtils.route_image_generation_cost_calculator( + model="qwen-image-3.0", + completion_response=_edited_images(count=2, input_images=3), + custom_llm_provider="dashscope", + call_type="aimage_edit", + model_info=ModelInfo(input_cost_per_image=0.004, output_cost_per_image=0.04), + ) + + assert cost == pytest.approx(2 * 0.04 + 3 * 0.004) + + +@pytest.mark.usefixtures("local_model_cost_map") +def test_a_deployment_token_price_keeps_the_shared_per_image_billing() -> None: + prices: Final = litellm.model_cost["dashscope/qwen-image-3.0"] + + cost: Final = CostCalculatorUtils.route_image_generation_cost_calculator( + model="qwen-image-3.0", + completion_response=_edited_images(count=1, input_images=2), + custom_llm_provider="dashscope", + call_type="aimage_edit", + model_info=ModelInfo(input_cost_per_token=1e-06), + ) + + assert cost == pytest.approx(2 * prices.get("input_cost_per_image", 0.0) + prices["output_cost_per_image"]) diff --git a/tests/unit/llms/dashscope/test_dashscope_text_to_speech_transformation.py b/tests/unit/llms/dashscope/test_dashscope_text_to_speech_transformation.py new file mode 100644 index 00000000000..d47c66be3cd --- /dev/null +++ b/tests/unit/llms/dashscope/test_dashscope_text_to_speech_transformation.py @@ -0,0 +1,415 @@ +import datetime +import json +from collections.abc import Callable +from typing import Final + +import httpx +import pytest + +import litellm +from litellm.litellm_core_utils.litellm_logging import Logging +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler +from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler +from litellm.llms.dashscope.common_utils import DashScopeError +from litellm.llms.dashscope.text_to_speech.transformation import DashScopeTextToSpeechConfig +from litellm.types.llms.openai import HttpxBinaryResponseContent +from litellm.types.utils import Usage + +INTL: Final = "https://dashscope-intl.aliyuncs.com" +MULTIMODAL_PATH: Final = "/api/v1/services/aigc/multimodal-generation/generation" +SYNTHESIZER_PATH: Final = "/api/v1/services/audio/tts/SpeechSynthesizer" +AUDIO_URL: Final = ( + "http://93.184.216.34/prod/qwen-tts/a1.mp3?Expires=1793316671&OSSAccessKeyId=LTAI&Signature=ab%2Bc%3D" +) +AUDIO: Final = b"ID3\x04\x00synthetic-mp3-frames" +SYNTHESIZED: Final = { + "output": {"audio": {"url": AUDIO_URL, "data": "", "id": "audio_1"}}, + "usage": {"characters": 6, "input_tokens": 8, "output_tokens": 30, "total_tokens": 38}, +} + +API_REQUEST: Final = httpx.Request("POST", f"{INTL}{MULTIMODAL_PATH}") + +Reply = Callable[[httpx.Request], httpx.Response] + + +def _api(reply: dict[str, object]) -> Reply: # mutable-ok: JSON body fixture + return lambda _: httpx.Response(200, json=reply) + + +def _audio(_: httpx.Request) -> httpx.Response: + return httpx.Response(200, content=AUDIO, headers={"content-type": "audio/mpeg"}) + + +def _recording(seen: list[httpx.Request], reply: Reply) -> httpx.MockTransport: # mutable-ok: request log + def respond(request: httpx.Request) -> httpx.Response: + seen.append(request) + return reply(request) + + return httpx.MockTransport(respond) + + +def _refusing_blocking_download(request: httpx.Request) -> httpx.Response: + raise AssertionError(f"blocking client used for {request.url} on the event loop") + + +def _logging() -> Logging: + return Logging( + model="dashscope-tts", + messages=[], + stream=False, + call_type="aspeech", + start_time=datetime.datetime(2026, 10, 28, tzinfo=datetime.timezone.utc), + litellm_call_id="call-tts", + function_id="fn-tts", + ) + + +@pytest.fixture(autouse=True) +def _no_ambient_dashscope_base(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("DASHSCOPE_API_BASE", raising=False) + + +@pytest.mark.parametrize( + ("model", "optional_params", "path", "provider_input"), + ( + ( + "qwen3-tts-flash", + {"response_format": "wav"}, + MULTIMODAL_PATH, + {"text": "Hello.", "voice": "Cherry"}, + ), + ( + "qwen-audio-3.1-tts-flash", + {"response_format": "mp3", "instructions": "Speak slowly."}, + SYNTHESIZER_PATH, + {"text": "Hello.", "voice": "Cherry", "format": "mp3", "instruction": "Speak slowly."}, + ), + ( + "qwen-audio-3.0-tts-plus", + {}, + SYNTHESIZER_PATH, + {"text": "Hello.", "voice": "Cherry"}, + ), + ), +) +async def test_each_family_posts_to_its_endpoint_and_the_async_handler_returns_the_downloaded_audio( + model: str, + optional_params: dict[str, object], # mutable-ok: parametrized fixture + path: str, + provider_input: dict[str, object], # mutable-ok: parametrized fixture +) -> None: + api_seen: Final[list[httpx.Request]] = [] # mutable-ok: request log + download_seen: Final[list[httpx.Request]] = [] # mutable-ok: request log + config: Final = DashScopeTextToSpeechConfig( + sync_client=lambda: HTTPHandler( + client=httpx.Client(transport=httpx.MockTransport(_refusing_blocking_download)) + ), + async_client=lambda: AsyncHTTPHandler(transport=_recording(download_seen, _audio)), + ) + voice, mapped = config.map_openai_params(model=model, optional_params=optional_params, voice="Cherry") + + speech: Final = await BaseLLMHTTPHandler().async_text_to_speech_handler( + model=model, + input="Hello.", + voice=voice, + text_to_speech_provider_config=config, + text_to_speech_optional_params=mapped, + custom_llm_provider="dashscope", + litellm_params={"api_key": "sk-dashscope-test", "api_base": None}, + logging_obj=_logging(), + timeout=30, + client=AsyncHTTPHandler(transport=_recording(api_seen, _api(SYNTHESIZED))), + ) + + assert speech.content == AUDIO + (api_request,) = api_seen + assert str(api_request.url) == f"{INTL}{path}" + assert api_request.headers["authorization"] == "Bearer sk-dashscope-test" + assert json.loads(api_request.content) == {"model": model, "input": provider_input} + (download,) = download_seen + assert str(download.url) == AUDIO_URL + assert "authorization" not in download.headers + assert download.extensions["timeout"] == api_request.extensions["timeout"] + + +def test_the_sync_handler_downloads_through_the_injected_blocking_client_within_the_call_timeout() -> None: + api_seen: Final[list[httpx.Request]] = [] # mutable-ok: request log + download_seen: Final[list[httpx.Request]] = [] # mutable-ok: request log + config: Final = DashScopeTextToSpeechConfig( + sync_client=lambda: HTTPHandler(client=httpx.Client(transport=_recording(download_seen, _audio))) + ) + + speech: Final = BaseLLMHTTPHandler().text_to_speech_handler( + model="qwen3-tts-flash", + input="Hello.", + voice="Cherry", + text_to_speech_provider_config=config, + text_to_speech_optional_params={}, + custom_llm_provider="dashscope", + litellm_params={"api_key": "k", "api_base": None}, + logging_obj=_logging(), + timeout=30, + client=HTTPHandler(client=httpx.Client(transport=_recording(api_seen, _api(SYNTHESIZED)))), + ) + + assert isinstance(speech, HttpxBinaryResponseContent) + assert speech.content == AUDIO + (api_request,) = api_seen + (download,) = download_seen + assert str(download.url) == AUDIO_URL + assert download.extensions["timeout"] == api_request.extensions["timeout"] + + +@pytest.mark.parametrize( + ("audio_url", "download_url"), + ( + ( + "http://dashscope-result-sh.oss-cn-shanghai.aliyuncs.com/a1.wav?Expires=1&Signature=ab%2Bc%3D", + "https://dashscope-result-sh.oss-cn-shanghai.aliyuncs.com/a1.wav?Expires=1&Signature=ab%2Bc%3D", + ), + ( + "https://dashscope-result-sh.oss-cn-shanghai.aliyuncs.com/a1.wav", + "https://dashscope-result-sh.oss-cn-shanghai.aliyuncs.com/a1.wav", + ), + ("http://audio.notaliyuncs.com/a1.wav", "http://audio.notaliyuncs.com/a1.wav"), + (AUDIO_URL, AUDIO_URL), + ), +) +def test_an_alibaba_cloud_audio_url_is_downloaded_over_https( + monkeypatch: pytest.MonkeyPatch, audio_url: str, download_url: str +) -> None: + monkeypatch.setattr(litellm, "user_url_validation", False) + download_seen: Final[list[httpx.Request]] = [] # mutable-ok: request log + config: Final = DashScopeTextToSpeechConfig( + sync_client=lambda: HTTPHandler(client=httpx.Client(transport=_recording(download_seen, _audio))) + ) + + speech: Final = BaseLLMHTTPHandler().text_to_speech_handler( + model="qwen3-tts-flash", + input="Hello.", + voice="Cherry", + text_to_speech_provider_config=config, + text_to_speech_optional_params={}, + custom_llm_provider="dashscope", + litellm_params={"api_key": "k", "api_base": None}, + logging_obj=_logging(), + timeout=30, + client=HTTPHandler( + client=httpx.Client( + transport=_recording([], _api({**SYNTHESIZED, "output": {"audio": {"url": audio_url}}})) + ) + ), + ) + + assert speech.content == AUDIO + (download,) = download_seen + assert str(download.url) == download_url + + +@pytest.mark.parametrize( + ("api_base", "environment", "model", "url"), + ( + (None, None, "qwen3-tts-flash", f"{INTL}{MULTIMODAL_PATH}"), + (None, None, "qwen-audio-3.1-tts-plus", f"{INTL}{SYNTHESIZER_PATH}"), + ( + "https://dashscope.aliyuncs.com/compatible-mode/v1", + None, + "qwen-audio-3.1-tts-plus", + f"https://dashscope.aliyuncs.com{SYNTHESIZER_PATH}", + ), + (f"{INTL}{SYNTHESIZER_PATH}", None, "qwen3-tts-flash", f"{INTL}{MULTIMODAL_PATH}"), + ( + "https://dashscope.aliyuncs.com/apps/anthropic", + None, + "qwen3-tts-flash", + f"https://dashscope.aliyuncs.com{MULTIMODAL_PATH}", + ), + ( + "https://relay.example.com/dashscope/", + None, + "qwen3-tts-flash", + f"https://relay.example.com/dashscope{MULTIMODAL_PATH}", + ), + ( + None, + "https://dashscope.aliyuncs.com/compatible-mode/v1", + "qwen3-tts-flash", + f"https://dashscope.aliyuncs.com{MULTIMODAL_PATH}", + ), + ), +) +def test_the_endpoint_follows_the_model_family_on_the_configured_host( + monkeypatch: pytest.MonkeyPatch, api_base: str | None, environment: str | None, model: str, url: str +) -> None: + if environment is not None: + monkeypatch.setenv("DASHSCOPE_API_BASE", environment) + + assert DashScopeTextToSpeechConfig().get_complete_url(model=model, api_base=api_base, litellm_params={}) == url + + +@pytest.mark.parametrize( + ("model", "optional_params", "voice", "refused"), + ( + ("qwen3-tts-flash", {"response_format": "mp3"}, "Cherry", "response_format='mp3'"), + ("qwen3-tts-flash", {"instructions": "Whisper."}, "Cherry", "instructions='Whisper.'"), + ("qwen-audio-3.1-tts-flash", {"response_format": "aac"}, "Cherry", "response_format='aac'"), + ("qwen-audio-3.1-tts-flash", {"speed": 1.5}, "Cherry", "speed=1.5"), + ("qwen-audio-3.1-tts-flash", {}, {"voice_id": "Cherry"}, "voice as an object"), + ), +) +def test_an_unsupported_param_answers_400_and_is_dropped_under_drop_params( + model: str, + optional_params: dict[str, object], # mutable-ok: parametrized fixture + voice: str | dict[str, object], # mutable-ok: parametrized fixture + refused: str, +) -> None: + config: Final = DashScopeTextToSpeechConfig() + + with pytest.raises(litellm.UnsupportedParamsError) as rejected: + config.map_openai_params(model=model, optional_params=optional_params, voice=voice) + dropped_voice, dropped = config.map_openai_params( + model=model, optional_params=optional_params, voice=voice, kwargs={"drop_params": True} + ) + + assert rejected.value.status_code == 400 + assert refused in str(rejected.value) + assert dropped == {} + assert dropped_voice == (voice if isinstance(voice, str) else None) + + +async def test_aspeech_uses_the_native_endpoint_on_the_caller_api_base_and_refuses_a_private_audio_url() -> None: + api_seen: Final[list[httpx.Request]] = [] # mutable-ok: request log + private_audio: Final = {"output": {"audio": {"url": "http://169.254.169.254/latest/meta-data"}}} + + with pytest.raises(litellm.BadGatewayError) as refused: + await litellm.aspeech( + model="dashscope/qwen-audio-3.1-tts-flash", + input="Hello.", + voice="longanhuan_v3.1", + response_format="opus", + api_key="sk-dashscope-test", + api_base="https://dashscope.aliyuncs.com/compatible-mode/v1", + client=AsyncHTTPHandler(transport=_recording(api_seen, _api(private_audio))), + ) + + assert refused.value.status_code == 502 + assert "audio URL that was refused" in str(refused.value) + (api_request,) = api_seen + assert str(api_request.url) == f"https://dashscope.aliyuncs.com{SYNTHESIZER_PATH}" + assert json.loads(api_request.content) == { + "model": "qwen-audio-3.1-tts-flash", + "input": {"text": "Hello.", "voice": "longanhuan_v3.1", "format": "opus"}, + } + + +@pytest.mark.parametrize( + ("api_reply", "audio_reply", "status", "message"), + ( + ( + httpx.Response(200, json={"code": "InvalidParameter", "message": "The voice property is required."}), + _audio, + 400, + "The voice property is required.", + ), + ( + httpx.Response(200, content=b"gateway"), + _audio, + 502, + "Unexpected DashScope text-to-speech response", + ), + ( + httpx.Response(200, json=SYNTHESIZED, request=API_REQUEST), + lambda _: httpx.Response(403), + 502, + "audio download failed with HTTP 403", + ), + ), +) +async def test_a_failed_synthesis_or_download_raises_a_dashscope_error( + api_reply: httpx.Response, audio_reply: Reply, status: int, message: str +) -> None: + config: Final = DashScopeTextToSpeechConfig( + async_client=lambda: AsyncHTTPHandler(transport=httpx.MockTransport(audio_reply)) + ) + + with pytest.raises(DashScopeError) as failed: + await config.async_transform_text_to_speech_response( + model="qwen3-tts-flash", raw_response=api_reply, logging_obj=_logging() + ) + + assert failed.value.status_code == status + assert message in failed.value.message + + +async def test_provider_token_usage_rides_on_the_audio_so_token_priced_speech_is_billed() -> None: + config: Final = DashScopeTextToSpeechConfig( + async_client=lambda: AsyncHTTPHandler(transport=httpx.MockTransport(_audio)) + ) + + speech: Final = await config.async_transform_text_to_speech_response( + model="qwen-audio-3.1-tts-flash", + raw_response=httpx.Response(200, json=SYNTHESIZED, request=API_REQUEST), + logging_obj=_logging(), + ) + + assert speech.usage == Usage(prompt_tokens=8, completion_tokens=30, total_tokens=38) + + +@pytest.mark.usefixtures("local_model_cost_map") +async def test_character_priced_speech_bills_the_characters_dashscope_reports() -> None: + text: Final = "你好 世界。" + reported: Final = {**SYNTHESIZED, "usage": {"characters": 10}} + config: Final = DashScopeTextToSpeechConfig( + async_client=lambda: AsyncHTTPHandler(transport=httpx.MockTransport(_audio)) + ) + speech: Final = await config.async_transform_text_to_speech_response( + model="qwen3-tts-flash", + raw_response=httpx.Response(200, json=reported, request=API_REQUEST), + logging_obj=_logging(), + ) + + cost: Final = litellm.completion_cost( + completion_response=speech, + model="dashscope/qwen3-tts-flash", + call_type="aspeech", + prompt=text, + custom_llm_provider="dashscope", + ) + + assert litellm.utils.count_characters(text) != 10 + assert cost == pytest.approx(10 * litellm.model_cost["dashscope/qwen3-tts-flash"]["input_cost_per_character"]) + + +@pytest.mark.parametrize( + "model", ("qwen3-tts-flash", "qwen-audio-3.1-tts-plus", "qwen-audio-3.0-tts-plus", "qwen-audio-3.0-tts-flash") +) +@pytest.mark.usefixtures("local_model_cost_map") +def test_character_priced_speech_bills_each_input_character_at_the_model_rate(model: str) -> None: + cost: Final = litellm.completion_cost( + completion_response=HttpxBinaryResponseContent(httpx.Response(200, content=AUDIO)), + model=f"dashscope/{model}", + call_type="aspeech", + prompt="Hello.", + custom_llm_provider="dashscope", + ) + + assert cost == pytest.approx(6 * litellm.model_cost[f"dashscope/{model}"]["input_cost_per_character"]) + assert cost > 0 + + +@pytest.mark.usefixtures("local_model_cost_map") +def test_token_priced_speech_bills_the_provider_reported_input_and_output_tokens() -> None: + speech: Final = HttpxBinaryResponseContent(httpx.Response(200, content=AUDIO)) + speech.usage = Usage(prompt_tokens=8, completion_tokens=30, total_tokens=38) + row: Final = litellm.model_cost["dashscope/qwen-audio-3.1-tts-flash"] + + cost: Final = litellm.completion_cost( + completion_response=speech, + model="dashscope/qwen-audio-3.1-tts-flash", + call_type="aspeech", + prompt="Hello.", + custom_llm_provider="dashscope", + ) + + assert cost == pytest.approx(8 * row["input_cost_per_token"] + 30 * row["output_cost_per_token"]) + assert row["output_cost_per_token"] > row["input_cost_per_token"] > 0 diff --git a/tests/unit/llms/runwayml/test_text_to_speech_transformation.py b/tests/unit/llms/runwayml/test_text_to_speech_transformation.py index 2e4d68a02da..981ac3e4855 100644 --- a/tests/unit/llms/runwayml/test_text_to_speech_transformation.py +++ b/tests/unit/llms/runwayml/test_text_to_speech_transformation.py @@ -2,13 +2,51 @@ Test RunwayML text-to-speech transformation """ +from collections.abc import Iterator +from typing import Final +import httpx +import pytest +import respx +import litellm from litellm.llms.runwayml.text_to_speech.transformation import ( RunwayMLTextToSpeechConfig, ) +@pytest.fixture +def httpx_transport(monkeypatch: pytest.MonkeyPatch) -> Iterator[None]: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + litellm.in_memory_llm_clients_cache.flush_cache() + yield + litellm.in_memory_llm_clients_cache.flush_cache() + + +@pytest.mark.asyncio +async def test_async_speech_polls_the_runwayml_task_with_the_caller_key_and_returns_its_audio( + respx_mock: respx.MockRouter, httpx_transport: None +) -> None: + respx_mock.post("https://api.dev.runwayml.com/v1/text_to_speech").mock( + return_value=httpx.Response(200, json={"id": "task_1", "status": "PENDING"}) + ) + poll: Final = respx_mock.get("https://api.dev.runwayml.com/v1/tasks/task_1").mock( + return_value=httpx.Response( + 200, json={"id": "task_1", "status": "SUCCEEDED", "output": ["https://storage.example.com/a1.mp3"]} + ) + ) + respx_mock.get("https://storage.example.com/a1.mp3").mock( + return_value=httpx.Response(200, content=b"ID3runway-audio") + ) + + speech: Final = await litellm.aspeech( + model="runwayml/eleven_multilingual_v2", input="Hello.", voice="alloy", api_key="rw-key" + ) + + assert speech.content == b"ID3runway-audio" + assert poll.calls.last.request.headers["authorization"] == "Bearer rw-key" + + def test_openai_voice_mapping_to_runwayml(): """ Test that OpenAI voice names are correctly mapped to RunwayML preset IDs diff --git a/tests/unit/llms/test_get_complete_url_trailing_slash.py b/tests/unit/llms/test_get_complete_url_trailing_slash.py index 43a74e56028..e35e21c67f9 100644 --- a/tests/unit/llms/test_get_complete_url_trailing_slash.py +++ b/tests/unit/llms/test_get_complete_url_trailing_slash.py @@ -137,6 +137,7 @@ API_BASE_IS_THE_FULL_URL: Final[Mapping[str, str]] = MappingProxyType( "brave.search.transformation.BraveSearchConfig", "cohere.chat.v2_transformation.CohereV2ChatConfig", "cohere.embed.transformation.CohereEmbeddingConfig", + "dashscope.image_edit.transformation.DashScopeImageEditConfig", "dashscope.image_generation.transformation.DashScopeImageGenerationConfig", "dataforseo.search.transformation.DataForSEOSearchConfig", "datarobot.chat.transformation.DataRobotConfig", diff --git a/tests/unit/test_dashscope_image_generation.py b/tests/unit/test_dashscope_image_generation.py index 9c65399350c..0f35136bac3 100644 --- a/tests/unit/test_dashscope_image_generation.py +++ b/tests/unit/test_dashscope_image_generation.py @@ -14,6 +14,7 @@ import respx from pydantic import ValidationError import litellm +from litellm.cost_calculator import default_image_cost_calculator from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.dashscope.common_utils import ( DASHSCOPE_DEFAULT_ORIGIN, @@ -23,7 +24,8 @@ from litellm.llms.dashscope.image_generation.transformation import ( DEFAULT_API_BASE, DashScopeImageGenerationConfig, ) -from litellm.types.utils import ImageResponse +from litellm.litellm_core_utils.llm_cost_calc.utils import CostCalculatorUtils +from litellm.types.utils import ImageResponse, ModelInfo from litellm.utils import get_llm_provider # --------------------------------------------------------------------------- @@ -614,6 +616,16 @@ def test_transform_response_without_image_content_has_no_images( assert _transform_response(payload).data == [] +@pytest.mark.parametrize("usage", [None, "n/a", ["input_image_count", 2]]) +def test_transform_response_returns_the_images_whatever_shape_usage_has(usage: object): + response = _transform_response( + {"output": {"choices": [{"message": {"content": [{"image": "https://a.example/1.png"}]}}]}, "usage": usage} + ) + + assert [image.url for image in response.data] == ["https://a.example/1.png"] + assert "billable_input_image_count" not in response.hidden_params + + def test_transform_response_skips_content_items_without_an_image(): response = _transform_response( { @@ -672,3 +684,50 @@ def test_transform_response_rejects_malformed_payloads_without_echoing_them( _transform_response(payload) assert "input_value" not in str(exc_info.value) + + +@pytest.mark.parametrize( + ("model", "usage"), + ( + ("qwen-image-2.0", {"width": 2048, "height": 1536, "image_count": 1}), + ( + "qwen-image-3.0", + {"output_width": 2048, "output_height": 1536, "output_image_count": 1, "input_image_count": 0}, + ), + ), +) +def test_a_per_pixel_deployment_bills_the_size_dashscope_rendered_when_the_caller_sent_none( + model: str, usage: dict[str, int] +): + response = _transform_response( + { + "output": {"choices": [{"message": {"content": [{"image": "https://a.example/1.png"}]}}]}, + "usage": usage, + } + ) + + cost = CostCalculatorUtils.route_image_generation_cost_calculator( + model=model, + completion_response=response, + custom_llm_provider="dashscope", + call_type="image_generation", + model_info=ModelInfo(input_cost_per_pixel=1e-08), + ) + + assert cost == pytest.approx(2048 * 1536 * 1e-08) + + +@pytest.mark.parametrize("model", ("qwen-image-2.0", "qwen-image-2.0-pro", "qwen-image-3.0")) +@pytest.mark.usefixtures("local_model_cost_map") +def test_a_shared_row_bills_a_generated_image_the_same_on_the_default_calculator_released_proxies_use(model: str): + dashscope_cost: Final = CostCalculatorUtils.route_image_generation_cost_calculator( + model=model, + completion_response=ImageResponse(data=[{"url": "https://a.example/1.png"}]), + custom_llm_provider="dashscope", + call_type="image_generation", + ) + + assert dashscope_cost > 0 + assert default_image_cost_calculator(model=f"dashscope/{model}", custom_llm_provider="dashscope") == pytest.approx( + dashscope_cost + ) From 60601ca99505da4be5cd27fb80524b14d204f340 Mon Sep 17 00:00:00 2001 From: "Juan V." Date: Fri, 30 Oct 2026 01:52:43 +0100 Subject: [PATCH 12/17] feat(pass-through): add eu/jev decision API pass-through with spend tracking (#46875) * feat(pass-through): add eu/jev decision API pass-through with spend tracking Adds a /eu_jev/{endpoint:path} pass-through route for eu/jev, an EU-hosted decision API that speaks the same POST /v1/systemone contract as TypeSafe. It reads EU_JEV_API_KEY and EU_JEV_API_BASE (defaults to https://jev.bevel.software), books spend through the shared decision logging handler, and prices eu_jev/eu-jeff-1.0.0 and eu_jev/eu-jeff-latest from the cost map * fix(pass-through): default eu_jev to its canonical eu-jev.com origin * fix(pass-through): reject custom_body on the eu_jev route A key limited to other models could send an eu/jev request through custom_body, because auth checks the outer body while the shared pass-through forwards custom_body upstream. The route now answers 400 before anything is forwarded, the same way validate_oss_request guards the laya and bespoke routes * fix(pass-through): give the eu_jev route request and 200 response schemas The OpenAPI docs gate flagged POST /eu_jev/{endpoint} for OA001 and OA003. The route now documents the eu/jev System One request body and its 200 answer, usage and meta shape, taken from the eu/jev OpenAPI document. The lazy OpenAPI snapshot and the dashboard API types are regenerated with python -m litellm.proxy._lazy_openapi_snapshot and npm run gen:api * fix(pass-through): reject streaming on the eu_jev route A body with stream=true switched the route to streaming, which skips the decision spend tracking. Reject it with a 400, as validate_oss_request does for laya and bespoke. * fix(pass-through): type the eu_jev request body like the other decision routes * fix(pass-through): check eu_jev decisions against the model eu/jev runs Serve only POST /eu_jev/v1/systemone, eu/jev's one endpoint, resolve an omitted model and the legacy names to eu-jeff-latest for key model checks and pricing, check scopes under eu_jev/ like Laya and Bespoke, and answer 503 when EU_JEV_API_KEY is unset * fix(pass-through): apply eu_jev key limits, register eu_jev, price unlisted releases Pre-call hooks and guardrails now see the body model as eu_jev/, so model_rpm_limit and model_tpm_limit set on eu_jev/eu-jeff-latest apply, and upstream still gets the client's exact body. eu_jev is a registered decisions provider, so access groups, config deployments, /v1/systemone and /v1/decisions accept it, and the Admin UI can add it. A release without its own cost map row is priced as eu-jeff-latest instead of $0 * fix(eu_jev): use the pass-through deployment's api_base and accept provider-prefixed models * fix(eu_jev): declare /v1/systemone on the eu/jev cost map entries so the generated endpoint table keeps it * fix(eu_jev): take pass-through credentials only from proxy-wide deployments The eu/jev route now ignores team-scoped deployments when it picks the upstream key and base, so one team's deployment can no longer redirect another team's calls, and a base ending in /v1 is not doubled * fix(eu_jev): let a configured pass-through at /eu_jev/v1/systemone keep serving it Scope the mapped pass-through and gateway allowlist entries to the decision route instead of the whole /eu_jev prefix, so custom routes under /eu_jev keep their auth, metering, and gateway behavior from before this PR * fix(eu_jev): treat /eu_jev/v1/systemone as a provider route only while the built-in route serves it Drop the decision path from the mapped pass-through list and classify it with one check that asks the pass-through registry, so an operator's own pass-through at that path keeps its auth, metering, LLM-route status and litellm_user_api_key handling from before this PR. The built-in route now yields on a per-request root path and on any configured method, and the gateway keeps only the exact decision path * fix(eu_jev): find a configured route at the decision path with or without a trailing slash, and stop a deleted one forwarding * test(eu_jev): annotate new test variables with Final * fix(proxy): keep eu/jev pass-through routing stable during updates and method mismatches Swap an updated pass-through's routes with no await between removal and re-registration, so the built-in eu/jev route never sees the path as unconfigured mid-update. The built-in eu/jev handler no longer takes a configured route's target, and the built-in route also yields on a method-only match, so GET and PUT on the no-slash form of a slash route redirect again. * test(pass-through): annotate the route-swap test's locals as Final * fix(pass-through): keep an updated auth route's LLM API route checks on the updating worker * fix(eu_jev): leave a null model out of the upstream decision request A JSON model of null is limited and counted as eu-jeff-latest, so send the request upstream without a model field, the way an omitted model goes, since eu/jev answers a null model with 400 Unknown model * fix(pass-through): keep route checks on a path another auth endpoint still serves after an update Forward a non-string eu/jev model unchanged so eu/jev rejects it, and drop only a null or missing one * test(eu_jev): keep the non-string model test in the eu/jev route test file so a CI shard runs it * refactor(eu_jev): build the scoped pass-through body as a new dict instead of updating it in place --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- README.md | 1 + gateway/routes/allowlist.py | 1 + litellm/__init__.py | 3 + litellm/_lazy_imports_registry.py | 2 + litellm/llms/eu_jev/__init__.py | 0 litellm/llms/eu_jev/common_utils.py | 34 ++ .../llms/eu_jev/decisions/transformation.py | 9 + .../pass_through/typesafe_logging_handler.py | 13 +- ...odel_prices_and_context_window_backup.json | 22 + .../provider_endpoints_support_backup.json | 26 + litellm/proxy/_lazy_features.py | 1 + litellm/proxy/_lazy_openapi_snapshot.json | 521 ++++++++++++++++++ litellm/proxy/auth/auth_utils.py | 9 +- litellm/proxy/auth/route_checks.py | 15 +- .../billable_request_metrics_middleware.py | 3 + .../llm_passthrough_endpoints.py | 82 +++ .../pass_through_endpoints.py | 108 +++- .../passthrough_endpoint_router.py | 68 ++- .../pass_through_endpoints/success_handler.py | 2 +- .../provider_create_fields.json | 28 + litellm/types/passthrough_endpoints/eu_jev.py | 100 ++++ litellm/types/utils.py | 1 + litellm/utils.py | 3 +- model_prices_and_context_window.json | 22 + provider_endpoints_support.json | 26 + tests/unit/decisions/test_main.py | 1 + tests/unit/proxy/auth/test_auth_utils.py | 34 ++ tests/unit/proxy/auth/test_route_checks.py | 35 ++ tests/unit/proxy/conftest.py | 40 ++ ...est_billable_request_metrics_middleware.py | 8 + ...st_typesafe_passthrough_logging_handler.py | 91 +++ .../test_llm_pass_through_endpoints.py | 385 ++++++++++++- .../test_pass_through_endpoints.py | 183 ++++++ .../test_passthrough_auth_default.py | 13 + .../test_passthrough_endpoint_router.py | 44 ++ .../public_endpoints/test_public_endpoints.py | 13 +- tests/unit/proxy/test_component_allowlists.py | 16 + ui/litellm-dashboard/src/lib/http/schema.d.ts | 208 +++++++ 38 files changed, 2136 insertions(+), 35 deletions(-) create mode 100644 litellm/llms/eu_jev/__init__.py create mode 100644 litellm/llms/eu_jev/common_utils.py create mode 100644 litellm/llms/eu_jev/decisions/transformation.py create mode 100644 litellm/types/passthrough_endpoints/eu_jev.py diff --git a/README.md b/README.md index 5a796257654..531a50e1aca 100644 --- a/README.md +++ b/README.md @@ -398,6 +398,7 @@ Set `LITELLM_PROXY_API_BASE` and `LITELLM_PROXY_API_KEY` and every model call th | [Eden AI (`edenai`)](https://docs.litellm.ai/docs/providers/edenai) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | | | | | [ElevenLabs (`elevenlabs`)](https://docs.litellm.ai/docs/providers/elevenlabs) | ✅ | ✅ | ✅ | | | ✅ | ✅ | | | | | [Empower (`empower`)](https://docs.litellm.ai/docs/providers/empower) | ✅ | ✅ | ✅ | | | | | | | | +| [eu/jev Decisions API (`eu_jev`)](https://docs.litellm.ai/docs/pass_through/eu_jev) | | | | | | | | | | | | [Fal AI (`fal_ai`)](https://docs.litellm.ai/docs/providers/fal_ai) | ✅ | ✅ | ✅ | | ✅ | | | | | | | [Featherless AI (`featherless_ai`)](https://docs.litellm.ai/docs/providers/featherless_ai) | ✅ | ✅ | ✅ | | | | | | | | | [Fireworks AI (`fireworks_ai`)](https://docs.litellm.ai/docs/providers/fireworks_ai) | ✅ | ✅ | ✅ | | | | | | | | diff --git a/gateway/routes/allowlist.py b/gateway/routes/allowlist.py index cc87a2ed16b..c2740ac1d41 100644 --- a/gateway/routes/allowlist.py +++ b/gateway/routes/allowlist.py @@ -142,6 +142,7 @@ GATEWAY_EXACT_PATHS: frozenset[str] = frozenset( "/test", "/debug/memory/summary", "/api/event_logging/batch", + "/eu_jev/v1/systemone", } ) diff --git a/litellm/__init__.py b/litellm/__init__.py index 3e8cddac12c..084a4c516ce 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -1783,6 +1783,9 @@ if TYPE_CHECKING: from .llms.bespoke.decisions.transformation import ( BespokeDecisionsConfig as BespokeDecisionsConfig, ) + from .llms.eu_jev.decisions.transformation import ( + EuJevDecisionsConfig as EuJevDecisionsConfig, + ) from .llms.openai.decisions.transformation import ( OpenAIDecisionsConfig as OpenAIDecisionsConfig, ) diff --git a/litellm/_lazy_imports_registry.py b/litellm/_lazy_imports_registry.py index 599234bab1a..7e81602fe77 100644 --- a/litellm/_lazy_imports_registry.py +++ b/litellm/_lazy_imports_registry.py @@ -174,6 +174,7 @@ LLM_CONFIG_NAMES: Final = ( "OllamaDecisionsConfig", "LayaDecisionsConfig", "BespokeDecisionsConfig", + "EuJevDecisionsConfig", "OpenAIDecisionsConfig", "AzureDecisionsConfig", "AzureAIDecisionsConfig", @@ -753,6 +754,7 @@ _LLM_CONFIGS_IMPORT_MAP: Final = { "OllamaDecisionsConfig": (".llms.ollama.decisions.transformation", "OllamaDecisionsConfig"), "LayaDecisionsConfig": (".llms.laya.decisions.transformation", "LayaDecisionsConfig"), "BespokeDecisionsConfig": (".llms.bespoke.decisions.transformation", "BespokeDecisionsConfig"), + "EuJevDecisionsConfig": (".llms.eu_jev.decisions.transformation", "EuJevDecisionsConfig"), "OpenAIDecisionsConfig": (".llms.openai.decisions.transformation", "OpenAIDecisionsConfig"), "AzureDecisionsConfig": (".llms.azure.decisions.transformation", "AzureDecisionsConfig"), "AzureAIDecisionsConfig": (".llms.azure_ai.decisions.transformation", "AzureAIDecisionsConfig"), diff --git a/litellm/llms/eu_jev/__init__.py b/litellm/llms/eu_jev/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/litellm/llms/eu_jev/common_utils.py b/litellm/llms/eu_jev/common_utils.py new file mode 100644 index 00000000000..18385ef8e5d --- /dev/null +++ b/litellm/llms/eu_jev/common_utils.py @@ -0,0 +1,34 @@ +from collections.abc import Mapping +from types import MappingProxyType +from typing import Final + +EU_JEV_LATEST_MODEL: Final = "eu-jeff-latest" +_EU_JEV_CURRENT_RELEASE_NAMES: Final = frozenset((EU_JEV_LATEST_MODEL, "jeff-latest", "jeff-1.0.0")) +_EU_JEV_PREFIX: Final = "eu_jev/" + + +def eu_jev_registry_model(requested_model: str | None) -> str: + release: Final = requested_model.removeprefix(_EU_JEV_PREFIX) if requested_model is not None else None + if release is None or release in _EU_JEV_CURRENT_RELEASE_NAMES: + return EU_JEV_LATEST_MODEL + return release + + +def eu_jev_scoped_model(decision_body: Mapping[str, object]) -> str: + requested_model: Final = decision_body.get("model") + return f"{_EU_JEV_PREFIX}{eu_jev_registry_model(requested_model if isinstance(requested_model, str) else None)}" + + +def eu_jev_upstream_body(hook_body: Mapping[str, object], client_body: Mapping[str, object]) -> Mapping[str, object]: + hook_model: Final = hook_body.get("model") + without_model: Final = {key: value for key, value in hook_body.items() if key != "model"} + if hook_model == eu_jev_scoped_model(client_body): + client_model: Final = client_body.get("model") + if client_model is None: + return MappingProxyType(without_model) + if isinstance(client_model, str): + return MappingProxyType({**without_model, "model": client_model.removeprefix(_EU_JEV_PREFIX)}) + return MappingProxyType({**without_model, "model": client_model}) + if isinstance(hook_model, str): + return MappingProxyType({**without_model, "model": hook_model.removeprefix(_EU_JEV_PREFIX)}) + return hook_body diff --git a/litellm/llms/eu_jev/decisions/transformation.py b/litellm/llms/eu_jev/decisions/transformation.py new file mode 100644 index 00000000000..18ef391b040 --- /dev/null +++ b/litellm/llms/eu_jev/decisions/transformation.py @@ -0,0 +1,9 @@ +from litellm.llms.base_llm.decisions.transformation import BaseDecisionsConfig + + +class EuJevDecisionsConfig(BaseDecisionsConfig): + api_key_env = ("EU_JEV_API_KEY",) + api_base_env = ("EU_JEV_API_BASE",) + + def get_default_api_base(self) -> str | None: + return "https://eu-jev.com" diff --git a/litellm/llms/pass_through/typesafe_logging_handler.py b/litellm/llms/pass_through/typesafe_logging_handler.py index 397ec408ae9..9eae57774c2 100644 --- a/litellm/llms/pass_through/typesafe_logging_handler.py +++ b/litellm/llms/pass_through/typesafe_logging_handler.py @@ -11,6 +11,7 @@ from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging from litellm.litellm_core_utils.litellm_logging import ( get_standard_logging_object_payload, # pyright: ignore[reportUnknownVariableType] # legacy helper has an untyped signature ) +from litellm.llms.eu_jev.common_utils import EU_JEV_LATEST_MODEL, eu_jev_registry_model from litellm.llms.laya.common_utils import laya_response_model from litellm.types.llms.base import LiteLLMBaseModel from litellm.types.utils import ModelResponse, StandardPassThroughResponseObject, Usage @@ -82,13 +83,21 @@ class TypeSafePassthroughLoggingHandler: response_model: Final = ( laya_response_model(response_body, request_model) if custom_llm_provider == "laya" else response.model ) - logged_model: Final = response_model or request_model or "unknown" + registry_request_model: Final = ( + eu_jev_registry_model(request_model) if custom_llm_provider == "eu_jev" else request_model + ) + logged_model: Final = response_model or registry_request_model or "unknown" model_name: Final = f"{custom_llm_provider}/{logged_model}" usage: Final = response.usage or _TypeSafeUsage() input_tokens: Final = usage.input_tokens output_tokens: Final = usage.output_tokens + candidate_models: Final = ( + (response_model, registry_request_model, EU_JEV_LATEST_MODEL) + if custom_llm_provider == "eu_jev" + else (response_model, registry_request_model) + ) candidate_model_keys: Final = tuple( - f"{custom_llm_provider}/{model}" for model in (response_model, request_model) if model is not None + f"{custom_llm_provider}/{model}" for model in candidate_models if model is not None ) pricing: Final = _pricing_for(candidate_model_keys) response_cost: Final = ( diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 50cfc6007ca..b5d09d6ff4f 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -72383,6 +72383,28 @@ "source": "https://docs.typesafe.ai/models", "supports_box_questions": true }, + "eu_jev/eu-jeff-1.0.0": { + "input_cost_per_token": 4.15e-08, + "litellm_provider": "eu_jev", + "max_input_tokens": 32768, + "mode": "evaluation", + "output_cost_per_token": 0.0, + "source": "https://eu-jev.com/docs", + "supported_endpoints": [ + "/v1/systemone" + ] + }, + "eu_jev/eu-jeff-latest": { + "input_cost_per_token": 4.15e-08, + "litellm_provider": "eu_jev", + "max_input_tokens": 32768, + "mode": "evaluation", + "output_cost_per_token": 0.0, + "source": "https://eu-jev.com/docs", + "supported_endpoints": [ + "/v1/systemone" + ] + }, "wandb/deepseek-ai/DeepSeek-V4.1-Flash": { "cache_read_input_token_cost": 3e-08, "input_cost_per_token": 2e-07, diff --git a/litellm/provider_endpoints_support_backup.json b/litellm/provider_endpoints_support_backup.json index 069ecfb82b2..18edb2cd7dc 100644 --- a/litellm/provider_endpoints_support_backup.json +++ b/litellm/provider_endpoints_support_backup.json @@ -2241,6 +2241,32 @@ "generateContent": false } }, + "eu_jev": { + "display_name": "eu/jev (`eu_jev`)", + "url": "https://docs.litellm.ai/docs/pass_through/eu_jev", + "endpoints": { + "chat_completions": false, + "responses": false, + "text_completion": false, + "embeddings": false, + "image_generations": false, + "image_edits": false, + "audio_transcriptions": false, + "audio_speech": false, + "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": true, + "a2a": false, + "generateContent": false + } + }, "laya": { "display_name": "Laya (`laya`)", "url": "https://docs.litellm.ai/docs/auto_router/decision_classifiers", diff --git a/litellm/proxy/_lazy_features.py b/litellm/proxy/_lazy_features.py index db01475154b..dc6575f1e1b 100644 --- a/litellm/proxy/_lazy_features.py +++ b/litellm/proxy/_lazy_features.py @@ -238,6 +238,7 @@ LAZY_FEATURES: Final[tuple[LazyFeature, ...]] = ( "/tinyfish/", "/transcribe", "/typesafe/", + "/eu_jev/", "/laya/", "/bespoke/", "/openrouter/", diff --git a/litellm/proxy/_lazy_openapi_snapshot.json b/litellm/proxy/_lazy_openapi_snapshot.json index a9694634de7..bc64a4c7ef0 100644 --- a/litellm/proxy/_lazy_openapi_snapshot.json +++ b/litellm/proxy/_lazy_openapi_snapshot.json @@ -23170,6 +23170,39 @@ "title": "CachedTokensDetails", "type": "object" }, + "ChoiceAnswer": { + "additionalProperties": true, + "properties": { + "choice": { + "title": "Choice", + "type": "string" + }, + "confidence": { + "title": "Confidence", + "type": "number" + }, + "probabilities": { + "additionalProperties": { + "type": "number" + }, + "title": "Probabilities", + "type": "object" + }, + "type": { + "const": "choice", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type", + "choice", + "confidence", + "probabilities" + ], + "title": "ChoiceAnswer", + "type": "object" + }, "Click": { "additionalProperties": true, "description": "A click action.", @@ -23934,6 +23967,112 @@ "title": "ErrorResponse", "type": "object" }, + "EuJevDecisionResponse": { + "properties": { + "answers": { + "additionalProperties": { + "discriminator": { + "mapping": { + "choice": "#/components/schemas/ChoiceAnswer", + "noul": "#/components/schemas/NoulAnswer", + "score": "#/components/schemas/ScoreAnswer" + }, + "propertyName": "type" + }, + "oneOf": [ + { + "$ref": "#/components/schemas/ChoiceAnswer" + }, + { + "$ref": "#/components/schemas/NoulAnswer" + }, + { + "$ref": "#/components/schemas/ScoreAnswer" + } + ] + }, + "description": "One answer per question, under the same names", + "title": "Answers", + "type": "object" + }, + "meta": { + "$ref": "#/components/schemas/EuJevMeta" + }, + "model": { + "description": "The release that answered, such as eu-jeff-1.0.0", + "title": "Model", + "type": "string" + }, + "usage": { + "$ref": "#/components/schemas/EuJevUsage" + } + }, + "required": [ + "model", + "answers", + "usage", + "meta" + ], + "title": "EuJevDecisionResponse", + "type": "object" + }, + "EuJevMeta": { + "properties": { + "cost_eur": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "The exact charge in euros, on authenticated requests", + "title": "Cost Eur" + }, + "latency_ms": { + "description": "Server-side time for token quoting, credit reservation and inference", + "title": "Latency Ms", + "type": "integer" + }, + "mode": { + "const": "live", + "title": "Mode", + "type": "string" + }, + "request_id": { + "title": "Request Id", + "type": "string" + } + }, + "required": [ + "request_id", + "mode", + "latency_ms" + ], + "title": "EuJevMeta", + "type": "object" + }, + "EuJevUsage": { + "properties": { + "input_tokens": { + "minimum": 0.0, + "title": "Input Tokens", + "type": "integer" + }, + "output_tokens": { + "minimum": 0.0, + "title": "Output Tokens", + "type": "integer" + } + }, + "required": [ + "input_tokens", + "output_tokens" + ], + "title": "EuJevUsage", + "type": "object" + }, "FileSearchTool": { "additionalProperties": true, "description": "A tool that searches for relevant content from uploaded files.\n\nLearn more about the [file search tool](https://platform.openai.com/docs/guides/tools-file-search).", @@ -25594,6 +25733,26 @@ "title": "NamespaceTool", "type": "object" }, + "NoulAnswer": { + "additionalProperties": true, + "properties": { + "noul": { + "title": "Noul", + "type": "number" + }, + "type": { + "const": "noul", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type", + "noul" + ], + "title": "NoulAnswer", + "type": "object" + }, "OperationCreateFile": { "additionalProperties": true, "description": "Instruction describing how to create a file via the apply_patch tool.", @@ -28866,6 +29025,59 @@ "title": "Result", "type": "object" }, + "ScoreAnswer": { + "additionalProperties": true, + "properties": { + "confidence": { + "title": "Confidence", + "type": "number" + }, + "legend": { + "additionalProperties": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "items": {}, + "type": "array" + } + ] + }, + "title": "Legend", + "type": "object" + }, + "probabilities": { + "additionalProperties": { + "type": "number" + }, + "title": "Probabilities", + "type": "object" + }, + "score": { + "title": "Score", + "type": "number" + }, + "type": { + "const": "score", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type", + "score", + "confidence", + "legend", + "probabilities" + ], + "title": "ScoreAnswer", + "type": "object" + }, "Screenshot": { "additionalProperties": true, "description": "A screenshot action.", @@ -31758,6 +31970,315 @@ ] } }, + "/eu_jev/v1/systemone": { + "post": { + "description": "Forward a request to eu/jev, an EU-hosted decision API, with `EU_JEV_API_KEY`.\n\n`POST /eu_jev/v1/systemone` sends a `state` and one to eight named `questions` (`choice`, `noul` or `score`)\nand returns one answer per question name, with spend booked from the reported `usage`. `custom_body` and\n`stream` are rejected with a 400. A pass-through endpoint configured at this path keeps serving it.\n\n[Docs](https://docs.litellm.ai/docs/pass_through/eu_jev)", + "operationId": "eu_jev_proxy_route_eu_jev_v1_systemone_post", + "requestBody": { + "content": { + "application/json": { + "example": { + "model": "eu-jeff-latest", + "questions": { + "frustration": { + "criteria": [ + "calm", + "mildly annoyed", + "angry" + ], + "instructions": "How frustrated is the customer?", + "type": "score" + }, + "is_refund_request": { + "instructions": "Is the customer asking for a refund?", + "type": "noul" + }, + "team": { + "criteria": { + "billing": "Payments, invoices, and refunds", + "support": "Technical issues and bugs" + }, + "instructions": "Which team should handle this?", + "type": "choice" + } + }, + "state": "I was charged twice. Can I get a refund?" + }, + "schema": { + "properties": { + "model": { + "default": "eu-jeff-latest", + "description": "eu-jeff-latest follows the current release, eu-jeff-1.0.0 pins it", + "title": "Model", + "type": "string" + }, + "questions": { + "additionalProperties": { + "discriminator": { + "propertyName": "type" + }, + "oneOf": [ + { + "properties": { + "criteria": { + "additionalProperties": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "items": {}, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "Free text, or a JSON object, array or null that eu/jev serializes to text before inference" + }, + "description": "Two to 64 options, option name to description. A null description uses the option name", + "maxProperties": 64, + "minProperties": 2, + "title": "Criteria", + "type": "object" + }, + "instructions": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "items": {}, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "Free text, or a JSON object, array or null that eu/jev serializes to text before inference", + "title": "Instructions" + }, + "type": { + "const": "choice", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type", + "instructions", + "criteria" + ], + "title": "EuJevChoiceQuestion", + "type": "object" + }, + { + "properties": { + "criteria": { + "anyOf": [ + { + "additionalProperties": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "items": {}, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "propertyNames": { + "enum": [ + "true", + "false" + ] + }, + "type": "object" + }, + { + "type": "null" + } + ], + "description": "Optional descriptions of what true and false mean", + "title": "Criteria" + }, + "instructions": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "items": {}, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "Free text, or a JSON object, array or null that eu/jev serializes to text before inference", + "title": "Instructions" + }, + "type": { + "const": "noul", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type", + "instructions" + ], + "title": "EuJevNoulQuestion", + "type": "object" + }, + { + "properties": { + "criteria": { + "description": "Two to 10 level descriptions, ordered from low to high", + "items": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "items": {}, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "Free text, or a JSON object, array or null that eu/jev serializes to text before inference" + }, + "maxItems": 10, + "minItems": 2, + "title": "Criteria", + "type": "array" + }, + "instructions": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "items": {}, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "Free text, or a JSON object, array or null that eu/jev serializes to text before inference", + "title": "Instructions" + }, + "type": { + "const": "score", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type", + "instructions", + "criteria" + ], + "title": "EuJevScoreQuestion", + "type": "object" + } + ] + }, + "description": "One to eight questions, keyed by names of up to 80 characters", + "maxProperties": 8, + "minProperties": 1, + "propertyNames": { + "maxLength": 80, + "minLength": 1 + }, + "title": "Questions", + "type": "object" + }, + "state": { + "anyOf": [ + { + "maxLength": 100000, + "minLength": 1, + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "items": {}, + "type": "array" + } + ], + "description": "The content to evaluate, a string of up to 100,000 characters or a JSON object or array", + "title": "State" + } + }, + "required": [ + "state", + "questions" + ], + "title": "EuJevDecisionRequest", + "type": "object" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/EuJevDecisionResponse" + } + } + }, + "description": "Successful Response" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Eu Jev Proxy Route", + "tags": [ + "llm_passthrough" + ] + } + }, "/fal_ai/{endpoint}": { "delete": { "operationId": "fal_ai_proxy_route_fal_ai__endpoint__delete", diff --git a/litellm/proxy/auth/auth_utils.py b/litellm/proxy/auth/auth_utils.py index 33bec02e6bc..9350c9c935c 100644 --- a/litellm/proxy/auth/auth_utils.py +++ b/litellm/proxy/auth/auth_utils.py @@ -33,6 +33,7 @@ from litellm.litellm_core_utils.url_utils import ( from litellm.llms.azure.passthrough.transformation import azure_router_model_in_endpoint from litellm.llms.nvidia_nim.passthrough.transformation import nvidia_nim_model_group_in_path from litellm.proxy._types import * +from litellm.proxy.auth.route_checks import RouteChecks from litellm.proxy.common_utils.http_parsing_utils import extract_nested_form_metadata from litellm.types.passthrough_endpoints.pass_through_endpoints import ( LITELLM_PASS_THROUGH_ENDPOINT_MARKER, @@ -884,7 +885,6 @@ def route_in_additonal_public_routes(current_route: str): public_routes: ["LiteLLMRoutes.public_routes", "/spend/calculate", "/api/*"] ``` """ - from litellm.proxy.auth.route_checks import RouteChecks from litellm.proxy.proxy_server import general_settings, premium_user try: @@ -2175,6 +2175,13 @@ def _extract_model_candidates_from_request( except ValueError as exc: raise HTTPException(status_code=400, detail=str(exc)) from exc return _dedupe_model_candidates((f"{provider}/{decision_model}",)) + if RouteChecks.is_builtin_eu_jev_decision_route(route=route): + from litellm.llms.eu_jev.common_utils import eu_jev_registry_model + + requested_model: Final = TypeAdapter(Mapping[str, object]).validate_python(request_data).get("model") + if requested_model is not None and not isinstance(requested_model, str): + raise HTTPException(status_code=400, detail="eu_jev model must be a string") + return [f"eu_jev/{eu_jev_registry_model(requested_model)}"] if route == "/cost/predict-cache": prediction_models: Final = _cache_prediction_model_candidates(request_data, llm_router, team_id) # pyright: ignore[reportUnknownArgumentType] # the typed reader validates each deployment ID from this legacy payload return _dedupe_model_candidates(prediction_models) diff --git a/litellm/proxy/auth/route_checks.py b/litellm/proxy/auth/route_checks.py index 74e4c4c6d11..56be44600fd 100644 --- a/litellm/proxy/auth/route_checks.py +++ b/litellm/proxy/auth/route_checks.py @@ -73,6 +73,7 @@ _PROXY_ADMIN_VIEW_ONLY_BLOCKED_ROUTES: Final = frozenset( _PROXY_ADMIN_VIEW_ONLY_BLOCKED_KEY_SUFFIXES: Final = ("/regenerate", "/reset_spend") _LLM_API_ROUTE_GROUPS: Final = frozenset(("llm_api_routes", "llm_api_routes_without_mcp")) +_EU_JEV_DECISION_ROUTE: Final = "/eu_jev/v1/systemone" _AUTH_ENFORCED_PASS_THROUGH_ROUTE_GROUPS: Final = frozenset(("openai_routes", *_LLM_API_ROUTE_GROUPS)) # Writes that only touch the caller's own credential row (BYOK connections and @@ -503,7 +504,19 @@ class RouteChecks: for _llm_passthrough_route in LiteLLMRoutes.mapped_pass_through_routes.value: if route == _llm_passthrough_route or route.startswith(_llm_passthrough_route + "/"): return True - return False + return RouteChecks.is_builtin_eu_jev_decision_route(route=route) + + @staticmethod + def is_builtin_eu_jev_decision_route(route: str) -> bool: + if route.rstrip("/") != _EU_JEV_DECISION_ROUTE: + return False + from litellm.proxy.pass_through_endpoints.pass_through_endpoints import ( # noqa: PLC0415 # pass_through_endpoints imports the auth stack + InitPassThroughEndpointHelpers, + ) + + return not InitPassThroughEndpointHelpers.has_configured_route_ignoring_trailing_slash( + route=_EU_JEV_DECISION_ROUTE + ) @staticmethod def _is_get_mcp_server_discovery_route(route: str, request: Request | None) -> bool: diff --git a/litellm/proxy/middleware/billable_request_metrics_middleware.py b/litellm/proxy/middleware/billable_request_metrics_middleware.py index 96c3276efac..fd78b6b58da 100644 --- a/litellm/proxy/middleware/billable_request_metrics_middleware.py +++ b/litellm/proxy/middleware/billable_request_metrics_middleware.py @@ -25,6 +25,7 @@ from starlette.types import ASGIApp, Message, Receive, Scope, Send from litellm._logging import verbose_proxy_logger from litellm.proxy._types import LiteLLMRoutes +from litellm.proxy.auth.route_checks import RouteChecks class BillableCategory(str, Enum): @@ -116,6 +117,8 @@ def _classify_llm_route(path: str) -> str | None: suffix_match = next((suffix for suffix in _LLM_ROUTE_SUFFIXES if path == suffix or path.endswith(suffix)), None) if suffix_match is not None: return suffix_match + if RouteChecks.is_builtin_eu_jev_decision_route(route=path): + return path # Deep passthrough paths only: the bare prefix itself is not an inference call. return next((prefix for prefix in _PASSTHROUGH_PREFIXES if path.startswith(f"{prefix}/")), None) diff --git a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py index 0a09743217c..21078dfc848 100644 --- a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py @@ -91,6 +91,7 @@ from litellm.proxy.common_request_processing import ( open_sse_before_first_byte, resolve_litellm_call_id, ) +from litellm.proxy.common_utils.custom_openapi_spec import inline_request_body from litellm.proxy.common_utils.http_parsing_utils import ( # noqa: F401 # legacy module exports _read_request_body, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export _safe_get_request_headers, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export @@ -119,6 +120,7 @@ from litellm.proxy.pass_through_endpoints.injected_stream_usage import ( ) from litellm.proxy.pass_through_endpoints.pass_through_endpoints import ( HttpPassThroughEndpointHelpers, + YieldsToConfiguredPassThroughRoute, create_pass_through_route, create_websocket_passthrough_route, websocket_passthrough_request, @@ -132,6 +134,7 @@ from litellm.proxy.vector_store_endpoints.utils import ( is_allowed_to_call_vector_store_endpoint, ) from litellm.secret_managers.main import get_secret_str, normalize_nonempty_secret_str, str_to_bool +from litellm.types.passthrough_endpoints.eu_jev import EuJevDecisionRequest, EuJevDecisionResponse from litellm.types.passthrough_endpoints.pass_through_endpoints import ( LITELLM_PASS_THROUGH_CUSTOM_BODY_STATE_KEY, LITELLM_PASS_THROUGH_DEPLOYMENT_MODEL_INFO_STATE_KEY, @@ -714,6 +717,85 @@ def typesafe_allowlist_model_names() -> tuple[str, ...]: return ("typesafe/*", *jev_models_typesafe_serves) +_EU_JEV_DECISION_REQUEST_BODY: Final = inline_request_body( + EuJevDecisionRequest, + { + "model": "eu-jeff-latest", + "state": "I was charged twice. Can I get a refund?", + "questions": { + "team": { + "type": "choice", + "instructions": "Which team should handle this?", + "criteria": {"billing": "Payments, invoices, and refunds", "support": "Technical issues and bugs"}, + }, + "is_refund_request": {"type": "noul", "instructions": "Is the customer asking for a refund?"}, + "frustration": { + "type": "score", + "instructions": "How frustrated is the customer?", + "criteria": ["calm", "mildly annoyed", "angry"], + }, + }, + }, +) + + +eu_jev_router: Final = APIRouter(route_class=YieldsToConfiguredPassThroughRoute) + + +@eu_jev_router.post( + "/eu_jev/v1/systemone", + tags=["eu/jev Pass-through", "pass-through"], + responses={200: {"model": EuJevDecisionResponse}}, + openapi_extra={"requestBody": _EU_JEV_DECISION_REQUEST_BODY}, +) +async def eu_jev_proxy_route( + request: Request, + fastapi_response: Response, + user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)], +) -> Response: + """ + Forward a request to eu/jev, an EU-hosted decision API, with `EU_JEV_API_KEY`. + + `POST /eu_jev/v1/systemone` sends a `state` and one to eight named `questions` (`choice`, `noul` or `score`) + and returns one answer per question name, with spend booked from the reported `usage`. `custom_body` and + `stream` are rejected with a 400. A pass-through endpoint configured at this path keeps serving it. + + [Docs](https://docs.litellm.ai/docs/pass_through/eu_jev) + """ + body: Final = TypeAdapter(dict[str, object]).validate_python(await read_request_body(request)) + if "custom_body" in body: + raise HTTPException(status_code=400, detail="custom_body is not supported for eu_jev requests") + if body.get("stream"): + raise HTTPException(status_code=400, detail="Streaming is not supported for eu_jev requests") + eu_jev_credentials: Final = passthrough_endpoint_router.get_proxy_wide_credentials(custom_llm_provider="eu_jev") + if eu_jev_credentials is None: + raise HTTPException(status_code=503, detail="eu_jev is not configured, set EU_JEV_API_KEY on the proxy") + base_url: Final = httpx.URL( + (eu_jev_credentials.api_base or get_secret_str("EU_JEV_API_BASE") or "https://eu-jev.com") + .rstrip("/") + .removesuffix("/v1") + ) + updated_url: Final = base_url.copy_with( + path=HttpPassThroughEndpointHelpers.join_base_and_endpoint_path(base_url, "/v1/systemone"), + ) + endpoint_func: Final = create_pass_through_route( + endpoint="v1/systemone", + target=str(updated_url), + custom_headers={ + "Authorization": f"Bearer {eu_jev_credentials.api_key}", + "Content-Type": "application/json", + }, + custom_llm_provider="eu_jev", + is_streaming_request=False, + ) + return TypeAdapter(Response, config=ConfigDict(arbitrary_types_allowed=True)).validate_python( + await endpoint_func(request, fastapi_response, user_api_key_dict) + ) + + +router.include_router(eu_jev_router) + + @router.post( "/laya/v1/systemone", tags=["Laya Pass-through", "pass-through"], diff --git a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py index 23dffb4a2bb..bda2bfb9e04 100644 --- a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py @@ -26,9 +26,11 @@ from fastapi import ( status, ) from fastapi.responses import StreamingResponse +from fastapi.routing import APIRoute from pydantic import TypeAdapter from starlette.datastructures import UploadFile as StarletteUploadFile -from starlette.routing import BaseRoute, Route +from starlette.routing import BaseRoute, Match, Route +from starlette.types import Scope from starlette.websockets import WebSocketState from typing_extensions import ReadOnly from websockets.asyncio.client import connect @@ -73,6 +75,7 @@ from litellm.llms.base_llm.managed_resources.utils import ( resolve_passthrough_managed_id_provider, ) from litellm.llms.custom_httpx.http_handler import get_async_httpx_client +from litellm.llms.eu_jev.common_utils import eu_jev_scoped_model, eu_jev_upstream_body from litellm.llms.oss_decision import validate_oss_request from litellm.passthrough import BasePassthroughUtils from litellm.proxy._lazy_features import lazy_owned_routes @@ -90,6 +93,7 @@ from litellm.proxy.auth.auth_utils import ( get_request_route, request_dispatched_to_pass_through_endpoint, ) +from litellm.proxy.auth.route_checks import RouteChecks from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.common_request_processing import ( ProxyBaseLLMRequestProcessing, @@ -433,9 +437,9 @@ class HttpPassThroughEndpointHelpers(BasePassthroughUtils): parsed_url: Final = urlparse(url) if custom_llm_provider == "chatgpt": return EndpointType.OPENAI - if custom_llm_provider in ("typesafe", "laya", "bespoke") and parsed_url.path.removesuffix("/").endswith( - "/v1/systemone" - ): + if custom_llm_provider in ("typesafe", "eu_jev", "laya", "bespoke") and parsed_url.path.removesuffix( + "/" + ).endswith("/v1/systemone"): return EndpointType.DECISIONS if ( ("generateContent") in url @@ -1137,6 +1141,16 @@ def _unfiltered_headers(headers: Mapping[str, str]) -> Mapping[str, str]: return headers +def _eu_jev_client_body(custom_llm_provider: str | None, body: Mapping[str, object]) -> Mapping[str, object] | None: + return MappingProxyType(dict(body)) if custom_llm_provider == "eu_jev" else None + + +def _eu_jev_scoped_model_override(eu_jev_client_body: Mapping[str, object] | None) -> Mapping[str, str]: + if eu_jev_client_body is None: + return MappingProxyType({}) + return MappingProxyType({"model": eu_jev_scoped_model(eu_jev_client_body)}) + + async def pass_through_request( request: Request, target: str, @@ -1255,7 +1269,12 @@ async def pass_through_request( strip_client_guardrail_information(client_body) if not _key_or_team_allows_client_pricing_override(user_api_key_dict): _strip_client_pricing_overrides(client_body) - _parsed_body = client_body + eu_jev_client_body: Final = _eu_jev_client_body(custom_llm_provider, client_body) + scoped_client_body: Final = { + **client_body, + **_eu_jev_scoped_model_override(eu_jev_client_body), + } + _parsed_body = scoped_client_body if custom_llm_provider in ("laya", "bespoke"): decision_request: Final = TypeAdapter(Mapping[str, object]).validate_python(_parsed_body) checkpoint: Final = validate_oss_request(custom_llm_provider, decision_request) @@ -1339,6 +1358,12 @@ async def pass_through_request( ) _ = validate_oss_request(custom_llm_provider, decision_body) _parsed_body = TypeAdapter(dict[str, object]).validate_python(decision_body) + if eu_jev_client_body is not None: + _parsed_body = TypeAdapter(dict[str, object]).validate_python( # rebind-ok: body is rebound per stage + eu_jev_upstream_body( + TypeAdapter(Mapping[str, object]).validate_python(_parsed_body), eu_jev_client_body + ) + ) resolved_timeout: Final = resolve_pass_through_request_timeout(timeout) async_client_obj: Final = get_async_httpx_client( llm_provider=httpxSpecialProvider.PassThroughEndpoint, @@ -2240,17 +2265,23 @@ def create_pass_through_route( stream, ) = await _parse_request_data_by_content_type(request) - if not InitPassThroughEndpointHelpers.is_registered_pass_through_route(route=path): + if not InitPassThroughEndpointHelpers.is_registered_pass_through_route(route=path) or ( + custom_llm_provider is None and RouteChecks.is_builtin_eu_jev_decision_route(route=path) + ): raise HTTPException( status_code=404, detail=f"Pass-through endpoint {endpoint} not found. This could have been deleted or not yet added to the proxy.", ) - passthrough_params: Final = InitPassThroughEndpointHelpers.get_registered_pass_through_route( - route=path, method=request.method + serves_configured_route: Final = custom_llm_provider != "eu_jev" + passthrough_params: Final = ( + InitPassThroughEndpointHelpers.get_registered_pass_through_route(route=path, method=request.method) + if serves_configured_route + else None ) if ( - passthrough_params is None + serves_configured_route + and passthrough_params is None and InitPassThroughEndpointHelpers.get_registered_pass_through_route(route=path) is not None ): raise HTTPException( @@ -3392,6 +3423,37 @@ class InitPassThroughEndpointHelpers: del _registered_pass_through_routes[key] verbose_proxy_logger.debug("Removed pass-through route from registry: %s", key) + @staticmethod + def _llm_api_routes_of_route(route_info: Mapping[str, object]) -> tuple[str, ...]: + path: Final = route_info.get("path") + if not isinstance(path, str): + return () + if route_info.get("type") == "subpath": + return (path, path.rstrip("/") + "/*") + return (path,) + + @staticmethod + def llm_api_routes_of_endpoint(endpoint_id: str) -> frozenset[str]: + route_infos: Final = ( + route_info + for route_info in _registered_pass_through_routes.values() + if route_info["endpoint_id"] == endpoint_id + ) + return frozenset(chain.from_iterable(map(InitPassThroughEndpointHelpers._llm_api_routes_of_route, route_infos))) + + @staticmethod + def auth_enforced_llm_api_routes() -> frozenset[str]: + route_infos: Final = ( + route_info for route_info in _registered_pass_through_routes.values() if route_info.get("auth") is True + ) + return frozenset(chain.from_iterable(map(InitPassThroughEndpointHelpers._llm_api_routes_of_route, route_infos))) + + @staticmethod + def drop_llm_api_routes(routes: frozenset[str]) -> None: + openai_routes: Final = LiteLLMRoutes.openai_routes.value + for route in routes.intersection(openai_routes): + openai_routes.remove(route) + @staticmethod def clear_all_pass_through_routes(): """Clear all pass-through routes from the registry""" @@ -3436,6 +3498,8 @@ class InitPassThroughEndpointHelpers: return True comparison_route: Final = InitPassThroughEndpointHelpers._route_for_registry_lookup(route) + if RouteChecks.is_builtin_eu_jev_decision_route(route=comparison_route): + return True # Fast path: check if any registered route key contains this path # Keys are in format: "{endpoint_id}:exact:{path}:{methods}" or "{endpoint_id}:subpath:{path}:{methods}" @@ -3512,6 +3576,25 @@ class InitPassThroughEndpointHelpers: return None + @staticmethod + def has_configured_route_ignoring_trailing_slash(route: str) -> bool: + bare_route: Final = route.rstrip("/") + return any( + InitPassThroughEndpointHelpers.get_registered_pass_through_route(route=form) is not None + for form in (bare_route, f"{bare_route}/") + ) + + +class YieldsToConfiguredPassThroughRoute(APIRoute): + def matches(self, scope: Scope) -> tuple[Match, Scope]: + match, child_scope = super().matches(scope) + if match is Match.NONE: + return match, child_scope + configured: Final = InitPassThroughEndpointHelpers.has_configured_route_ignoring_trailing_slash( + route=get_request_route(Request(scope)) + ) + return (Match.NONE, {}) if configured else (match, child_scope) + def _get_combined_pass_through_endpoints( pass_through_endpoints: list[dict] | list[PassThroughGenericEndpoint], @@ -3971,9 +4054,7 @@ async def update_pass_through_endpoints( # Update the list pass_through_endpoint_data[endpoint_index] = endpoint_dict - - # Remove old routes from registry before they get re-registered - InitPassThroughEndpointHelpers.remove_endpoint_routes(endpoint_id) + replaced_llm_api_routes: Final = InitPassThroughEndpointHelpers.llm_api_routes_of_endpoint(endpoint_id) ## Update db updated_data: Final = ConfigFieldUpdate( @@ -4023,6 +4104,9 @@ async def update_pass_through_endpoints( auth=updated_endpoint.auth, timeout=updated_endpoint.timeout, ) + InitPassThroughEndpointHelpers.drop_llm_api_routes( + replaced_llm_api_routes - InitPassThroughEndpointHelpers.auth_enforced_llm_api_routes() + ) return PassThroughEndpointResponse(endpoints=[updated_endpoint] if updated_endpoint else []) diff --git a/litellm/proxy/pass_through_endpoints/passthrough_endpoint_router.py b/litellm/proxy/pass_through_endpoints/passthrough_endpoint_router.py index 7092055bf6f..3b49ca1450a 100644 --- a/litellm/proxy/pass_through_endpoints/passthrough_endpoint_router.py +++ b/litellm/proxy/pass_through_endpoints/passthrough_endpoint_router.py @@ -1,6 +1,7 @@ import json import os -from collections.abc import Callable +from collections.abc import Callable, Mapping +from dataclasses import dataclass from typing import TYPE_CHECKING, Final from fastapi import HTTPException @@ -11,6 +12,7 @@ from litellm.integrations.vector_store_integrations.vector_store_pre_call_hook i LiteLLM_ManagedVectorStore, ) from litellm.litellm_core_utils.credential_accessor import CredentialAccessor +from litellm.llms.azure.ptu_capacity import is_str_keyed_mapping from litellm.secret_managers.main import ( GOOGLE_OIDC_SECRET_PREFIX, UnresolvableSecretReference, @@ -25,12 +27,23 @@ if TYPE_CHECKING: from litellm.router import Router +@dataclass(frozen=True, slots=True) +class _PassThroughCredentials: + api_key: str + api_base: str | None + + def _get_proxy_llm_router() -> "Router | None": from litellm.proxy.proxy_server import llm_router return llm_router +def _is_team_deployment(deployment: Mapping[str, object]) -> bool: + model_info: Final = deployment.get("model_info") + return is_str_keyed_mapping(model_info) and bool(model_info.get("team_id")) + + def _get_str_value(values: dict[str, object] | None, key: str) -> str | None: value: Final = values.get(key) if values is not None else None return value if isinstance(value, str) else None @@ -84,12 +97,13 @@ class PassthroughEndpointRouter: custom_llm_provider: str, region_name: str | None, ) -> str | None: - deployment_api_key: Final = self._get_deployment_api_key( + deployment_credentials: Final = self._get_deployment_credentials( custom_llm_provider=custom_llm_provider, region_name=region_name, + include_team_deployments=True, ) - if deployment_api_key is not None: - return _resolve_oidc_reference(deployment_api_key) + if deployment_credentials is not None: + return _resolve_oidc_reference(deployment_credentials.api_key) verbose_router_logger.debug( "No pass-through deployment credentials found for %s, looking for env variable", custom_llm_provider ) @@ -98,29 +112,56 @@ class PassthroughEndpointRouter: ) return _resolve_oidc_reference(get_secret_str(_env_variable_name)) + def get_proxy_wide_credentials(self, custom_llm_provider: str) -> _PassThroughCredentials | None: + deployment_credentials: Final = self._get_deployment_credentials( + custom_llm_provider=custom_llm_provider, + region_name=None, + include_team_deployments=False, + ) + api_key: Final = _resolve_oidc_reference( + deployment_credentials.api_key + if deployment_credentials is not None + else get_secret_str( + self._get_default_env_variable_name_passthrough_endpoint(custom_llm_provider=custom_llm_provider) + ) + ) + if not api_key: + return None + return _PassThroughCredentials( + api_key=api_key, + api_base=deployment_credentials.api_base if deployment_credentials is not None else None, + ) + def has_credentials(self, custom_llm_provider: str) -> bool: - if self._get_deployment_api_key(custom_llm_provider=custom_llm_provider, region_name=None) is not None: + if ( + self._get_deployment_credentials( + custom_llm_provider=custom_llm_provider, region_name=None, include_team_deployments=True + ) + is not None + ): return True env_variable_name: Final = self._get_default_env_variable_name_passthrough_endpoint( custom_llm_provider=custom_llm_provider, ) return bool(os.environ.get(env_variable_name)) - def _get_deployment_api_key( + def _get_deployment_credentials( self, custom_llm_provider: str, region_name: str | None, - ) -> str | None: + include_team_deployments: bool, + ) -> _PassThroughCredentials | None: llm_router: Final = self.llm_router_getter() if llm_router is None: return None deployments: Final = llm_router.registry().deployments() or () return next( ( - api_key + credentials for deployment in deployments - if ( - api_key := self._resolve_matching_deployment_api_key( + if (include_team_deployments or not _is_team_deployment(deployment)) + and ( + credentials := self._resolve_matching_deployment_credentials( litellm_params=deployment["litellm_params"], custom_llm_provider=custom_llm_provider, region_name=region_name, @@ -131,12 +172,12 @@ class PassthroughEndpointRouter: None, ) - def _resolve_matching_deployment_api_key( + def _resolve_matching_deployment_credentials( self, litellm_params: LiteLLMParamsTypedDict, custom_llm_provider: str, region_name: str | None, - ) -> str | None: + ) -> _PassThroughCredentials | None: if litellm_params.get("use_in_pass_through") is not True: return None if self._get_deployment_provider(litellm_params) != custom_llm_provider: @@ -152,7 +193,8 @@ class PassthroughEndpointRouter: ) if deployment_region != region_name: return None - return _get_str_value(credential_values, "api_key") or litellm_params.get("api_key") + api_key: Final = _get_str_value(credential_values, "api_key") or litellm_params.get("api_key") + return _PassThroughCredentials(api_key=api_key, api_base=api_base) if api_key is not None else None def _get_deployment_provider(self, litellm_params: LiteLLMParamsTypedDict) -> str | None: model: Final = litellm_params.get("model") diff --git a/litellm/proxy/pass_through_endpoints/success_handler.py b/litellm/proxy/pass_through_endpoints/success_handler.py index bafdd9e197d..1f4e7b6df95 100644 --- a/litellm/proxy/pass_through_endpoints/success_handler.py +++ b/litellm/proxy/pass_through_endpoints/success_handler.py @@ -375,7 +375,7 @@ class PassThroughEndpointLogging: kwargs = transcribe_handler_result["kwargs"] # rebind-ok: elif-chain contract elif ( self.is_typesafe_route(custom_llm_provider) - or custom_llm_provider in ("laya", "bespoke") + or custom_llm_provider in ("eu_jev", "laya", "bespoke") or self.is_openrouter_decisions_route(url_route, custom_llm_provider) ): from .llm_provider_handlers.typesafe_passthrough_logging_handler import ( diff --git a/litellm/proxy/public_endpoints/provider_create_fields.json b/litellm/proxy/public_endpoints/provider_create_fields.json index 04e6c5d8ba9..d8ff32c3819 100644 --- a/litellm/proxy/public_endpoints/provider_create_fields.json +++ b/litellm/proxy/public_endpoints/provider_create_fields.json @@ -1553,6 +1553,34 @@ ], "default_model_placeholder": "gpt-3.5-turbo" }, + { + "provider": "EU_JEV", + "provider_display_name": "eu/jev", + "litellm_provider": "eu_jev", + "credential_fields": [ + { + "key": "api_base", + "label": "API Base", + "placeholder": "https://eu-jev.com", + "tooltip": null, + "required": false, + "field_type": "text", + "options": null, + "default_value": null + }, + { + "key": "api_key", + "label": "API Key", + "placeholder": null, + "tooltip": null, + "required": true, + "field_type": "password", + "options": null, + "default_value": null + } + ], + "default_model_placeholder": "eu_jev/eu-jeff-latest" + }, { "provider": "FalAI", "provider_display_name": "Fal AI", diff --git a/litellm/types/passthrough_endpoints/eu_jev.py b/litellm/types/passthrough_endpoints/eu_jev.py new file mode 100644 index 00000000000..3b81d68deea --- /dev/null +++ b/litellm/types/passthrough_endpoints/eu_jev.py @@ -0,0 +1,100 @@ +from collections.abc import Mapping, Sequence +from typing import Annotated, Literal, TypeAlias + +from pydantic import ConfigDict, Field + +from litellm.types.decisions import ChoiceAnswer, DecisionsJSON, NoulAnswer, NoulCriteria, ScoreAnswer +from litellm.types.llms.base import LiteLLMPydanticObjectBase + +EuJevEntry: TypeAlias = Annotated[ + DecisionsJSON | None, + Field(description="Free text, or a JSON object, array or null that eu/jev serializes to text before inference"), +] + + +class EuJevChoiceQuestion(LiteLLMPydanticObjectBase): + type: Literal["choice"] + instructions: EuJevEntry + criteria: Annotated[ + Mapping[str, EuJevEntry], + Field( + min_length=2, + max_length=64, + description="Two to 64 options, option name to description. A null description uses the option name", + ), + ] + + model_config = ConfigDict(frozen=True) + + +class EuJevNoulQuestion(LiteLLMPydanticObjectBase): + type: Literal["noul"] + instructions: EuJevEntry + criteria: Annotated[NoulCriteria | None, Field(description="Optional descriptions of what true and false mean")] = ( + None + ) + + model_config = ConfigDict(frozen=True) + + +class EuJevScoreQuestion(LiteLLMPydanticObjectBase): + type: Literal["score"] + instructions: EuJevEntry + criteria: Annotated[ + Sequence[EuJevEntry], + Field(min_length=2, max_length=10, description="Two to 10 level descriptions, ordered from low to high"), + ] + + model_config = ConfigDict(frozen=True) + + +EuJevQuestion: TypeAlias = Annotated[ + EuJevChoiceQuestion | EuJevNoulQuestion | EuJevScoreQuestion, + Field(discriminator="type"), +] + + +class EuJevDecisionRequest(LiteLLMPydanticObjectBase): + model: Annotated[str, Field(description="eu-jeff-latest follows the current release, eu-jeff-1.0.0 pins it")] = ( + "eu-jeff-latest" + ) + state: Annotated[ + Annotated[str, Field(min_length=1, max_length=100_000)] | Mapping[str, object] | Sequence[object], + Field(description="The content to evaluate, a string of up to 100,000 characters or a JSON object or array"), + ] + questions: Annotated[ + Mapping[Annotated[str, Field(min_length=1, max_length=80)], EuJevQuestion], + Field(min_length=1, max_length=8, description="One to eight questions, keyed by names of up to 80 characters"), + ] + + model_config = ConfigDict(frozen=True) + + +EuJevAnswer: TypeAlias = Annotated[ChoiceAnswer | NoulAnswer | ScoreAnswer, Field(discriminator="type")] + + +class EuJevUsage(LiteLLMPydanticObjectBase): + input_tokens: Annotated[int, Field(ge=0)] + output_tokens: Annotated[int, Field(ge=0)] + + model_config = ConfigDict(frozen=True) + + +class EuJevMeta(LiteLLMPydanticObjectBase): + request_id: str + mode: Literal["live"] + latency_ms: Annotated[ + int, Field(description="Server-side time for token quoting, credit reservation and inference") + ] + cost_eur: Annotated[str | None, Field(description="The exact charge in euros, on authenticated requests")] = None + + model_config = ConfigDict(frozen=True) + + +class EuJevDecisionResponse(LiteLLMPydanticObjectBase): + model: Annotated[str, Field(description="The release that answered, such as eu-jeff-1.0.0")] + answers: Annotated[Mapping[str, EuJevAnswer], Field(description="One answer per question, under the same names")] + usage: EuJevUsage + meta: EuJevMeta + + model_config = ConfigDict(frozen=True) diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 1e59ec0c2cb..1055ffe7b2f 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -4489,6 +4489,7 @@ class LlmProviders(str, Enum): STRANDS_DECIDER = "strands_decider" LAYA = "laya" BESPOKE = "bespoke" + EU_JEV = "eu_jev" MISTRAL = "mistral" MILVUS = "milvus" GROQ = "groq" diff --git a/litellm/utils.py b/litellm/utils.py index 904d154f8df..549590e1cf5 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -9968,11 +9968,12 @@ class ProviderConfigManager: return litellm.LayaDecisionsConfig() if provider == LlmProviders.BESPOKE: return litellm.BespokeDecisionsConfig() - if provider in (LlmProviders.OPENAI, LlmProviders.AZURE, LlmProviders.AZURE_AI): + if provider in (LlmProviders.OPENAI, LlmProviders.AZURE, LlmProviders.AZURE_AI, LlmProviders.EU_JEV): configs: Final = { LlmProviders.OPENAI: litellm.OpenAIDecisionsConfig, LlmProviders.AZURE: litellm.AzureDecisionsConfig, LlmProviders.AZURE_AI: litellm.AzureAIDecisionsConfig, + LlmProviders.EU_JEV: litellm.EuJevDecisionsConfig, } return configs[provider]() return None diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 50cfc6007ca..b5d09d6ff4f 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -72383,6 +72383,28 @@ "source": "https://docs.typesafe.ai/models", "supports_box_questions": true }, + "eu_jev/eu-jeff-1.0.0": { + "input_cost_per_token": 4.15e-08, + "litellm_provider": "eu_jev", + "max_input_tokens": 32768, + "mode": "evaluation", + "output_cost_per_token": 0.0, + "source": "https://eu-jev.com/docs", + "supported_endpoints": [ + "/v1/systemone" + ] + }, + "eu_jev/eu-jeff-latest": { + "input_cost_per_token": 4.15e-08, + "litellm_provider": "eu_jev", + "max_input_tokens": 32768, + "mode": "evaluation", + "output_cost_per_token": 0.0, + "source": "https://eu-jev.com/docs", + "supported_endpoints": [ + "/v1/systemone" + ] + }, "wandb/deepseek-ai/DeepSeek-V4.1-Flash": { "cache_read_input_token_cost": 3e-08, "input_cost_per_token": 2e-07, diff --git a/provider_endpoints_support.json b/provider_endpoints_support.json index 069ecfb82b2..18edb2cd7dc 100644 --- a/provider_endpoints_support.json +++ b/provider_endpoints_support.json @@ -2241,6 +2241,32 @@ "generateContent": false } }, + "eu_jev": { + "display_name": "eu/jev (`eu_jev`)", + "url": "https://docs.litellm.ai/docs/pass_through/eu_jev", + "endpoints": { + "chat_completions": false, + "responses": false, + "text_completion": false, + "embeddings": false, + "image_generations": false, + "image_edits": false, + "audio_transcriptions": false, + "audio_speech": false, + "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": true, + "a2a": false, + "generateContent": false + } + }, "laya": { "display_name": "Laya (`laya`)", "url": "https://docs.litellm.ai/docs/auto_router/decision_classifiers", diff --git a/tests/unit/decisions/test_main.py b/tests/unit/decisions/test_main.py index e5501310556..0c357837d31 100644 --- a/tests/unit/decisions/test_main.py +++ b/tests/unit/decisions/test_main.py @@ -123,6 +123,7 @@ _PROVIDERS: Final[tuple[tuple[str, str, str, str], ...]] = ( ), ("typesafe", "typesafe/jev-1.13", "https://api.typesafe.ai/v1/systemone", "jev-1.13"), ("nace", "nace/drex-v2.0", "https://console.nace.ai/v1/systemone", "drex-v2.0"), + ("eu_jev", "eu_jev/eu-jeff-latest", "https://eu-jev.com/v1/systemone", "eu-jeff-latest"), ( "openrouter", "openrouter/typesafe/jev-1.13", diff --git a/tests/unit/proxy/auth/test_auth_utils.py b/tests/unit/proxy/auth/test_auth_utils.py index de37b710f36..8612351eda6 100644 --- a/tests/unit/proxy/auth/test_auth_utils.py +++ b/tests/unit/proxy/auth/test_auth_utils.py @@ -1028,6 +1028,40 @@ def test_oss_native_model_cannot_implicitly_select_an_unauthorized_checkpoint(pr assert denied.value.status_code == 400 +# eu-jev.com/docs (read 2026-10-21): an omitted model means eu-jeff-latest, and the legacy names +# jeff-latest and jeff-1.0.0 answer with the current release. +@pytest.mark.parametrize( + ("request_data", "permission_model"), + [ + ({}, "eu_jev/eu-jeff-latest"), + ({"model": None}, "eu_jev/eu-jeff-latest"), + ({"model": "eu-jeff-latest"}, "eu_jev/eu-jeff-latest"), + ({"model": "jeff-latest"}, "eu_jev/eu-jeff-latest"), + ({"model": "jeff-1.0.0"}, "eu_jev/eu-jeff-latest"), + ({"model": "eu-jeff-1.0.0"}, "eu_jev/eu-jeff-1.0.0"), + ({"model": "eu_jev/eu-jeff-1.0.0"}, "eu_jev/eu-jeff-1.0.0"), + ({"model": "eu_jev/jeff-latest"}, "eu_jev/eu-jeff-latest"), + ], +) +@pytest.mark.parametrize("suffix", ["", "/"]) +def test_eu_jev_decision_is_checked_against_the_model_eu_jev_runs( + request_data: dict[str, object], permission_model: str, suffix: str +) -> None: + assert get_model_from_request(request_data=request_data, route=f"/eu_jev/v1/systemone{suffix}") == permission_model + + +@pytest.mark.parametrize("model", [["eu-jeff-latest"], 7, {"name": "eu-jeff-latest"}]) +def test_eu_jev_decision_rejects_a_model_that_is_not_a_string(model: object) -> None: + with pytest.raises(HTTPException) as denied: + get_model_from_request(request_data={"model": model}, route="/eu_jev/v1/systemone") + assert denied.value.status_code == 400 + + +@pytest.mark.usefixtures("configured_eu_jev_route") +def test_a_pass_through_configured_at_the_eu_jev_decision_path_keeps_the_requested_model() -> None: + assert get_model_from_request(request_data={"model": "jeff-latest"}, route="/eu_jev/v1/systemone") == "jeff-latest" + + def test_laya_model_normalization_does_not_change_other_provider_routes() -> None: assert get_model_from_request(request_data={"model": "jev-latest"}, route="/typesafe/v1/systemone") == "jev-latest" assert get_model_from_request(request_data={}, route="/laya/health") is None diff --git a/tests/unit/proxy/auth/test_route_checks.py b/tests/unit/proxy/auth/test_route_checks.py index b2c67b4e009..529f639999d 100644 --- a/tests/unit/proxy/auth/test_route_checks.py +++ b/tests/unit/proxy/auth/test_route_checks.py @@ -662,6 +662,41 @@ def test_virtual_key_llm_api_routes_allows_model_info(route): assert result is True +@pytest.mark.parametrize( + ("route", "is_llm_route"), + [ + ("/eu_jev/v1/systemone", True), + ("/eu_jev/v1/systemone/extra", False), + ("/eu_jev/v1/systemone2", False), + ("/eu_jev/custom", False), + ], +) +def test_only_the_built_in_eu_jev_decision_path_is_an_llm_route(route: str, is_llm_route: bool) -> None: + assert RouteChecks.is_llm_api_route(route=route) is is_llm_route + + +@pytest.mark.usefixtures("configured_eu_jev_route") +def test_a_pass_through_configured_at_the_eu_jev_decision_path_is_not_an_llm_route() -> None: + assert RouteChecks.is_llm_api_route(route="/eu_jev/v1/systemone") is False + + +@pytest.mark.usefixtures("configured_eu_jev_route_with_trailing_slash") +@pytest.mark.parametrize("route", ["/eu_jev/v1/systemone", "/eu_jev/v1/systemone/"]) +def test_a_pass_through_configured_at_the_eu_jev_decision_path_with_a_trailing_slash_is_not_an_llm_route( + route: str, +) -> None: + assert RouteChecks.is_llm_api_route(route=route) is False + + +@pytest.mark.parametrize("allowed_route", ["llm_api_routes", "llm_api_routes_without_mcp"]) +def test_an_llm_api_key_can_call_the_built_in_eu_jev_decision_route(allowed_route: str) -> None: + valid_token: Final = UserAPIKeyAuth(user_id="test_user", allowed_routes=[allowed_route]) + + assert RouteChecks.is_virtual_key_allowed_to_call_route( + route="/eu_jev/v1/systemone", valid_token=valid_token, request=_mock_request("POST") + ) + + @pytest.mark.parametrize("route", ["/model/info", "/v1/model/info", "/model_group/info"]) def test_model_info_not_classified_as_llm_api(route): """Membership in `llm_api_routes` must not promote /model/info to an diff --git a/tests/unit/proxy/conftest.py b/tests/unit/proxy/conftest.py index e13eddef43b..298d32def7d 100644 --- a/tests/unit/proxy/conftest.py +++ b/tests/unit/proxy/conftest.py @@ -8,17 +8,23 @@ import os import tempfile import warnings from collections.abc import Awaitable, Callable, Iterator +from contextlib import contextmanager from typing import Dict, Final, Optional from unittest.mock import AsyncMock, MagicMock, patch import pytest import yaml +from fastapi import FastAPI from fastapi.testclient import TestClient from prisma.errors import ClientNotConnectedError import litellm import litellm.proxy.proxy_server from litellm._logging import verbose_proxy_logger +from litellm.proxy.pass_through_endpoints.pass_through_endpoints import ( + InitPassThroughEndpointHelpers, + _registered_pass_through_routes, +) from litellm._service_logger import ServiceTypes from litellm.integrations.otel.model.payloads import ServiceSpanData from litellm.integrations.otel.model.spans import service_span_name @@ -239,6 +245,40 @@ def secret_vault_factory() -> type[FakeSecretVault]: return FakeSecretVault +@contextmanager +def _eu_jev_route_configured_at(path: str) -> Iterator[None]: + saved_registry: Final = dict(_registered_pass_through_routes) + InitPassThroughEndpointHelpers.add_exact_path_route( + app=FastAPI(), + path=path, + target="https://operator.example/decide", + custom_headers=None, + forward_headers=None, + merge_query_params=None, + dependencies=None, + cost_per_request=None, + endpoint_id="operator-eu-jev", + methods=["POST"], + ) + try: + yield + finally: + _registered_pass_through_routes.clear() + _registered_pass_through_routes.update(saved_registry) + + +@pytest.fixture +def configured_eu_jev_route() -> Iterator[None]: + with _eu_jev_route_configured_at("/eu_jev/v1/systemone"): + yield + + +@pytest.fixture +def configured_eu_jev_route_with_trailing_slash() -> Iterator[None]: + with _eu_jev_route_configured_at("/eu_jev/v1/systemone/"): + yield + + @pytest.fixture def httpx_transport(monkeypatch: pytest.MonkeyPatch) -> Iterator[None]: monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) diff --git a/tests/unit/proxy/middleware/test_billable_request_metrics_middleware.py b/tests/unit/proxy/middleware/test_billable_request_metrics_middleware.py index 8c81a2c9b36..d2e961ad158 100644 --- a/tests/unit/proxy/middleware/test_billable_request_metrics_middleware.py +++ b/tests/unit/proxy/middleware/test_billable_request_metrics_middleware.py @@ -123,6 +123,7 @@ def test_is_pure_asgi_not_base_http_middleware(): ("/azure_speech/speechtotext/v3.2/transcriptions", (BillableCategory.LLM, "/azure_speech")), ("/transcribe", (BillableCategory.LLM, "/transcribe")), ("/transcribe/StartTranscriptionJob", (BillableCategory.LLM, "/transcribe")), + ("/eu_jev/v1/systemone", (BillableCategory.LLM, "/eu_jev/v1/systemone")), ("/mcp", (BillableCategory.MCP, "/mcp")), ("/mcp/", (BillableCategory.MCP, "/mcp")), ("/mcp/tools/list", (BillableCategory.MCP, "/mcp")), @@ -165,6 +166,8 @@ def test_classify_billable(path: str, expected: Tuple[BillableCategory, str]): "/langfuse/api/public/ingestion", # a bare provider prefix is not an inference call "/bedrock", + "/eu_jev/v1/systemone/extra", + "/eu_jev/v1/systemone2", "/v1/mcp", "/v1/mcp/tools", "/v1/mcp/server", @@ -184,6 +187,11 @@ def test_classify_non_billable_returns_none(path: str): assert classify_billable_request(path) is None +@pytest.mark.usefixtures("configured_eu_jev_route") +def test_a_pass_through_configured_at_the_eu_jev_decision_path_is_not_billed() -> None: + assert classify_billable_request("/eu_jev/v1/systemone") is None + + @pytest.mark.parametrize( "path", [ diff --git a/tests/unit/proxy/pass_through_endpoints/llm_provider_handlers/test_typesafe_passthrough_logging_handler.py b/tests/unit/proxy/pass_through_endpoints/llm_provider_handlers/test_typesafe_passthrough_logging_handler.py index 79209a32f28..4011bc0294a 100644 --- a/tests/unit/proxy/pass_through_endpoints/llm_provider_handlers/test_typesafe_passthrough_logging_handler.py +++ b/tests/unit/proxy/pass_through_endpoints/llm_provider_handlers/test_typesafe_passthrough_logging_handler.py @@ -366,3 +366,94 @@ def test_success_handler_skips_typesafe_pricing_for_non_decisions_openrouter_rou assert normalized["standard_logging_response_object"] is None assert "combined_usage_object" not in normalized["kwargs"] assert normalized["kwargs"].get("model") != "openrouter/typesafe/jev-1.13-20260917" + + +def test_eu_jev_decision_is_priced_from_its_registry_row(): + model_cost: Final = litellm.model_cost["eu_jev/eu-jeff-1.0.0"] + response: Final = TypeSafePassthroughLoggingHandler.typesafe_passthrough_handler( + httpx_response=_response(), + response_body={"model": "eu-jeff-1.0.0", "usage": {"input_tokens": 64, "output_tokens": 0}}, + logging_obj=_logging_obj(), + url_route="https://eu-jev.com/v1/systemone", + result='{"answers": {}}', + start_time=datetime.now(), + end_time=datetime.now(), + cache_hit=False, + request_body={"model": "eu-jeff-latest"}, + custom_llm_provider="eu_jev", + ) + + assert response["kwargs"]["model"] == "eu_jev/eu-jeff-1.0.0" + assert response["kwargs"]["custom_llm_provider"] == "eu_jev" + assert response["kwargs"]["response_cost"] == pytest.approx(64 * model_cost["input_cost_per_token"]) + assert response["kwargs"]["response_cost"] > 0 + assert response["kwargs"]["combined_usage_object"].prompt_tokens == 64 + assert response["kwargs"]["combined_usage_object"].completion_tokens == 0 + + +# eu-jev.com/docs (read 2026-10-21): an omitted model and the legacy names answer with the current release, +# which a future eu/jev release reports under a version the registry does not list yet, and a caller may pin it. +@pytest.mark.parametrize( + "request_body", [{}, {"model": "jeff-latest"}, {"model": "jeff-1.0.0"}, {"model": "eu-jeff-99.0.0"}] +) +def test_eu_jev_release_without_its_own_row_is_priced_as_the_latest_alias(request_body: dict[str, object]) -> None: + unlisted_release: Final = "eu-jeff-99.0.0" + assert f"eu_jev/{unlisted_release}" not in litellm.model_cost + response: Final = TypeSafePassthroughLoggingHandler.typesafe_passthrough_handler( + httpx_response=_response(), + response_body={"model": unlisted_release, "usage": {"input_tokens": 64, "output_tokens": 0}}, + logging_obj=_logging_obj(), + url_route="https://eu-jev.com/v1/systemone", + result='{"answers": {}}', + start_time=datetime.now(), + end_time=datetime.now(), + cache_hit=False, + request_body=request_body, + custom_llm_provider="eu_jev", + ) + + latest_cost: Final = litellm.model_cost["eu_jev/eu-jeff-latest"]["input_cost_per_token"] + assert response["kwargs"]["model"] == f"eu_jev/{unlisted_release}" + assert latest_cost > 0 + assert response["kwargs"]["response_cost"] == pytest.approx(64 * latest_cost) + + +def test_eu_jev_pinned_release_is_not_priced_as_the_latest_alias(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setitem( + litellm.model_cost, + "eu_jev/eu-jeff-latest", + {**litellm.model_cost["eu_jev/eu-jeff-latest"], "input_cost_per_token": 1.0}, + ) + response: Final = TypeSafePassthroughLoggingHandler.typesafe_passthrough_handler( + httpx_response=_response(), + response_body={"model": "eu-jeff-1.0.0", "usage": {"input_tokens": 64, "output_tokens": 0}}, + logging_obj=_logging_obj(), + url_route="https://eu-jev.com/v1/systemone", + result='{"answers": {}}', + start_time=datetime.now(), + end_time=datetime.now(), + cache_hit=False, + request_body={"model": "eu-jeff-1.0.0"}, + custom_llm_provider="eu_jev", + ) + + pinned_cost: Final = litellm.model_cost["eu_jev/eu-jeff-1.0.0"]["input_cost_per_token"] + assert response["kwargs"]["response_cost"] == pytest.approx(64 * pinned_cost) + + +def test_success_handler_dispatches_eu_jev_to_the_shared_handler(): + normalized: Final = PassThroughEndpointLogging().normalize_llm_passthrough_logging_payload( + httpx_response=_response(), + response_body={"model": "eu-jeff-1.0.0", "usage": {"input_tokens": 64, "output_tokens": 0}}, + request_body={"model": "eu-jeff-latest"}, + logging_obj=_logging_obj(), + url_route="https://eu-jev.com/v1/systemone", + result="{}", + start_time=datetime.now(), + end_time=datetime.now(), + cache_hit=False, + custom_llm_provider="eu_jev", + ) + + assert normalized["kwargs"]["custom_llm_provider"] == "eu_jev" + assert normalized["kwargs"]["model"] == "eu_jev/eu-jeff-1.0.0" diff --git a/tests/unit/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py b/tests/unit/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py index 8d330cbd749..806d88b2467 100644 --- a/tests/unit/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py +++ b/tests/unit/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py @@ -15,7 +15,7 @@ from urllib.parse import parse_qs import httpx import pytest import respx -from fastapi import HTTPException, Request, Response +from fastapi import FastAPI, HTTPException, Request, Response from fastapi.routing import APIRoute from fastapi.responses import StreamingResponse from fastapi.testclient import TestClient @@ -52,6 +52,7 @@ from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import ( bedrock_proxy_route, create_pass_through_route, cursor_proxy_route, + eu_jev_proxy_route, fal_ai_proxy_route, get_azure_ai_search_index_from_endpoint, get_vertex_base_url, @@ -74,6 +75,7 @@ from litellm.proxy.auth.handle_jwt import JWTHandler from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.types.guardrails import GuardrailEventHooks from litellm.types.passthrough_endpoints.vertex_ai import VertexPassThroughCredentials +from litellm.llms.eu_jev.common_utils import eu_jev_scoped_model, eu_jev_upstream_body def _assert_bedrock_processing_data_classification(data: dict[str, object], is_streaming: bool) -> None: @@ -8601,6 +8603,387 @@ class TestTypeSafePassthroughRoute: ) +class _RecordsPreCallModel(CustomLogger): + def __init__(self, switch_to: str | None = None) -> None: + super().__init__() + self.switch_to: Final = switch_to + self.models: Final[list[object]] = [] + + async def async_pre_call_hook( + self, user_api_key_dict: UserAPIKeyAuth, cache: DualCache, data: dict, call_type: str + ) -> dict | None: + self.models.append(data.get("model")) + return None if self.switch_to is None else {**data, "model": self.switch_to} + + +class TestEuJevPassthroughRoute: + @pytest.fixture + def client(self, monkeypatch: pytest.MonkeyPatch) -> Iterator[TestClient]: + from litellm.proxy.proxy_server import app + + monkeypatch.setenv("EU_JEV_API_KEY", "eu-jev-test-key") + monkeypatch.setenv("EU_JEV_API_BASE", "https://eu-jev.example/base") + monkeypatch.delenv("SERVER_ROOT_PATH", raising=False) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + litellm.in_memory_llm_clients_cache.flush_cache() + monkeypatch.setitem(app.dependency_overrides, user_api_key_auth, lambda: UserAPIKeyAuth(api_key="sk-virtual")) + yield TestClient(app) + + def test_forwards_a_decision_upstream_with_the_eu_jev_key(self, client: TestClient) -> None: + body: Final = {"model": "eu-jeff-latest", "state": "x"} + with respx.mock(assert_all_called=True) as upstream: + route: Final = upstream.post("https://eu-jev.example/base/v1/systemone").mock( + return_value=httpx.Response(200, json={"id": "upstream_123"}) + ) + response: Final = client.post("/eu_jev/v1/systemone", json=body) + + assert (response.status_code, response.json()) == (200, {"id": "upstream_123"}) + sent: Final = route.calls.last.request + assert sent.headers["authorization"] == "Bearer eu-jev-test-key" + assert json.loads(sent.content) == body + + @pytest.mark.parametrize( + ("body", "hook_model"), + [ + ({"state": "x"}, "eu_jev/eu-jeff-latest"), + ({"model": "jeff-latest", "state": "x"}, "eu_jev/eu-jeff-latest"), + ({"model": "eu-jeff-1.0.0", "state": "x"}, "eu_jev/eu-jeff-1.0.0"), + ], + ) + def test_pre_call_hooks_see_the_model_key_limits_name_while_upstream_gets_the_client_body( + self, client: TestClient, monkeypatch: pytest.MonkeyPatch, body: dict[str, str], hook_model: str + ) -> None: + recorder: Final = _RecordsPreCallModel() + monkeypatch.setattr(litellm, "callbacks", [recorder]) + with respx.mock(assert_all_called=True) as upstream: + route: Final = upstream.post("https://eu-jev.example/base/v1/systemone").respond(200, json={"answers": {}}) + response: Final = client.post("/eu_jev/v1/systemone", json=body) + + assert response.status_code == 200 + assert recorder.models == [hook_model] + assert json.loads(route.calls.last.request.content) == body + + @pytest.mark.parametrize("model", [7, ["eu-jeff-latest"], False]) + def test_a_non_string_client_model_reaches_upstream_unchanged(self, model: int | list[str] | bool) -> None: + client_body: Final = {"model": model, "state": "x"} + hook_body: Final = {"model": eu_jev_scoped_model(client_body), "state": "x"} + + assert eu_jev_upstream_body(hook_body, client_body) == client_body + + def test_a_null_model_is_limited_as_the_latest_release_and_left_out_upstream( + self, client: TestClient, monkeypatch: pytest.MonkeyPatch + ) -> None: + recorder: Final = _RecordsPreCallModel() + monkeypatch.setattr(litellm, "callbacks", [recorder]) + with respx.mock(assert_all_called=True) as upstream: + route: Final = upstream.post("https://eu-jev.example/base/v1/systemone").respond(200, json={"answers": {}}) + response: Final = client.post("/eu_jev/v1/systemone", json={"model": None, "state": "x"}) + + assert response.status_code == 200 + assert recorder.models == ["eu_jev/eu-jeff-latest"] + assert json.loads(route.calls.last.request.content) == {"state": "x"} + + def test_forwards_the_release_a_pre_call_hook_switched_to( + self, client: TestClient, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.setattr(litellm, "callbacks", [_RecordsPreCallModel(switch_to="eu_jev/eu-jeff-1.0.0")]) + with respx.mock(assert_all_called=True) as upstream: + route: Final = upstream.post("https://eu-jev.example/base/v1/systemone").respond(200, json={"answers": {}}) + response: Final = client.post("/eu_jev/v1/systemone", json={"model": "jeff-latest", "state": "x"}) + + assert response.status_code == 200 + assert json.loads(route.calls.last.request.content) == {"model": "eu-jeff-1.0.0", "state": "x"} + + def test_a_provider_prefixed_model_is_limited_once_and_forwarded_without_the_prefix( + self, client: TestClient, monkeypatch: pytest.MonkeyPatch + ) -> None: + recorder: Final = _RecordsPreCallModel() + monkeypatch.setattr(litellm, "callbacks", [recorder]) + with respx.mock(assert_all_called=True) as upstream: + route: Final = upstream.post("https://eu-jev.example/base/v1/systemone").respond(200, json={"answers": {}}) + response: Final = client.post("/eu_jev/v1/systemone", json={"model": "eu_jev/eu-jeff-1.0.0", "state": "x"}) + + assert response.status_code == 200 + assert recorder.models == ["eu_jev/eu-jeff-1.0.0"] + assert json.loads(route.calls.last.request.content) == {"model": "eu-jeff-1.0.0", "state": "x"} + + def test_a_pass_through_deployment_key_goes_to_that_deployment_base( + self, client: TestClient, monkeypatch: pytest.MonkeyPatch + ) -> None: + from litellm.proxy import proxy_server + + deployment_router: Final = litellm.Router( + model_list=[ + { + "model_name": "eu-jev-friendly", + "litellm_params": { + "model": "eu_jev/eu-jeff-latest", + "api_key": "eu-jev-deployment-key", + "api_base": "https://eu-jev-deployment.example/v2", + "use_in_pass_through": True, + }, + } + ] + ) + monkeypatch.setattr(proxy_server, "llm_router", deployment_router) + with respx.mock(assert_all_called=True) as upstream: + route: Final = upstream.post("https://eu-jev-deployment.example/v2/v1/systemone").respond( + 200, json={"answers": {}} + ) + response: Final = client.post("/eu_jev/v1/systemone", json={"state": "x"}) + + assert response.status_code == 200 + assert route.calls.last.request.headers["authorization"] == "Bearer eu-jev-deployment-key" + + def test_a_team_deployment_never_redirects_another_team_s_calls( + self, client: TestClient, monkeypatch: pytest.MonkeyPatch + ) -> None: + from litellm.proxy import proxy_server + + team_router: Final = litellm.Router( + model_list=[ + { + "model_name": "team-a-eu-jev", + "litellm_params": { + "model": "eu_jev/eu-jeff-latest", + "api_key": "team-a-own-key", + "api_base": "https://team-a-server.example", + "use_in_pass_through": True, + }, + "model_info": {"team_id": "team-a"}, + } + ] + ) + monkeypatch.setattr(proxy_server, "llm_router", team_router) + with respx.mock(assert_all_called=False) as upstream: + team_server: Final = upstream.route(host="team-a-server.example").respond(200, json={"answers": {}}) + eu_jev: Final = upstream.post("https://eu-jev.example/base/v1/systemone").respond( + 200, json={"answers": {}} + ) + response: Final = client.post("/eu_jev/v1/systemone", json={"state": "team b data"}) + + assert response.status_code == 200 + assert not team_server.called + assert eu_jev.calls.last.request.headers["authorization"] == "Bearer eu-jev-test-key" + + def test_a_base_ending_in_v1_is_not_doubled(self, client: TestClient, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("EU_JEV_API_BASE", "https://eu-jev.example/base/v1/") + with respx.mock(assert_all_called=True) as upstream: + route: Final = upstream.post("https://eu-jev.example/base/v1/systemone").respond(200, json={"answers": {}}) + response: Final = client.post("/eu_jev/v1/systemone", json={"state": "x"}) + + assert response.status_code == 200 + assert route.called + + def test_forwards_query_params_to_the_decision_endpoint(self, client: TestClient) -> None: + with respx.mock(assert_all_called=True) as upstream: + route: Final = upstream.post("https://eu-jev.example/base/v1/systemone", params={"trace": "yes"}).respond( + 200, json={"answers": {}} + ) + response: Final = client.post("/eu_jev/v1/systemone?trace=yes", json={"state": "x"}) + + assert response.status_code == 200 + assert route.called + + def test_uses_the_hosted_api_when_base_is_unset(self, client: TestClient, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("EU_JEV_API_BASE") + with respx.mock(assert_all_called=True) as upstream: + route: Final = upstream.post("https://eu-jev.com/v1/systemone").respond(200, json={"answers": {}}) + response: Final = client.post("/eu_jev/v1/systemone", json={"state": "x"}) + + assert response.status_code == 200 + assert route.called + + def test_answers_503_without_calling_upstream_when_the_key_is_unset( + self, client: TestClient, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.delenv("EU_JEV_API_KEY") + with respx.mock(assert_all_called=False) as upstream: + route: Final = upstream.route(host="eu-jev.example").respond(200, json={"answers": {}}) + response: Final = client.post("/eu_jev/v1/systemone", json={"state": "x"}) + + assert response.status_code == 503 + assert "EU_JEV_API_KEY" in response.text + assert not route.called + + @pytest.mark.parametrize( + ("method", "path"), + [("GET", "/eu_jev/api/account/keys"), ("DELETE", "/eu_jev/v1/keys/abc"), ("GET", "/eu_jev/v1/systemone")], + ) + def test_forwards_nothing_but_a_decision(self, client: TestClient, method: str, path: str) -> None: + with respx.mock(assert_all_called=False) as upstream: + route: Final = upstream.route(host="eu-jev.example").respond(200, json={}) + response: Final = client.request(method, path) + + assert response.status_code in (404, 405) + assert not route.called + + def test_rejects_custom_body_that_would_replace_the_authorized_request(self, client: TestClient) -> None: + smuggled: Final = {"model": "eu-jeff-latest", "state": "refund", "questions": {"q": {"type": "noul"}}} + with respx.mock(assert_all_called=False) as upstream: + route: Final = upstream.post("https://eu-jev.example/base/v1/systemone").respond(200, json={"answers": {}}) + response: Final = client.post("/eu_jev/v1/systemone", json={"model": "gpt-4o", "custom_body": smuggled}) + + assert response.status_code == 400 + assert "custom_body" in response.text + assert not route.called + + def test_rejects_streaming_that_would_skip_spend_tracking(self, client: TestClient) -> None: + with respx.mock(assert_all_called=False) as upstream: + route: Final = upstream.post("https://eu-jev.example/base/v1/systemone").respond(200, json={"answers": {}}) + response: Final = client.post( + "/eu_jev/v1/systemone", json={"model": "eu-jeff-latest", "state": "x", "stream": True} + ) + + assert response.status_code == 400 + assert "Streaming" in response.text + assert not route.called + + @pytest.fixture + def operator_app(self, monkeypatch: pytest.MonkeyPatch) -> Iterator[Callable[[list[str], str], FastAPI]]: + from litellm.proxy._lazy_features import LAZY_FEATURES, attach_lazy_features, reserve_lazy_slot + from litellm.proxy.pass_through_endpoints.pass_through_endpoints import ( + InitPassThroughEndpointHelpers, + _registered_pass_through_routes, + ) + + monkeypatch.delenv("EU_JEV_API_KEY", raising=False) + monkeypatch.delenv("LITELLM_DISABLE_LAZY_ROUTES", raising=False) + monkeypatch.delenv("SERVER_ROOT_PATH", raising=False) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + litellm.in_memory_llm_clients_cache.flush_cache() + features: Final = tuple(feature for feature in LAZY_FEATURES if feature.name == "llm_passthrough") + + def build(methods: list[str], path: str) -> FastAPI: + app: Final = FastAPI() + reserve_lazy_slot(app, "llm_passthrough", features=features) + attach_lazy_features(app, features) + monkeypatch.setitem( + app.dependency_overrides, user_api_key_auth, lambda: UserAPIKeyAuth(api_key="sk-virtual") + ) + InitPassThroughEndpointHelpers.add_exact_path_route( + app=app, + path=path, + target="https://operator.example/decide", + custom_headers={"x-operator": "on"}, + forward_headers=None, + merge_query_params=None, + dependencies=None, + cost_per_request=None, + endpoint_id="operator-eu-jev", + methods=methods, + ) + return app + + saved_registry: Final = dict(_registered_pass_through_routes) + try: + yield build + finally: + _registered_pass_through_routes.clear() + _registered_pass_through_routes.update(saved_registry) + + @pytest.mark.parametrize("root_path", ["", "/tenant-a"]) + def test_a_pass_through_endpoint_configured_at_the_same_path_keeps_serving_it( + self, operator_app: Callable[[list[str], str], FastAPI], root_path: str + ) -> None: + with ( + respx.mock(assert_all_called=True) as upstream, + TestClient(operator_app(["POST"], "/eu_jev/v1/systemone"), root_path=root_path) as client, + ): + route: Final = upstream.post("https://operator.example/decide").respond(200, json={"operator": True}) + response: Final = client.post(f"{root_path}/eu_jev/v1/systemone", json={"state": "x", "stream": True}) + + assert (response.status_code, response.json()) == (200, {"operator": True}) + assert route.calls.last.request.headers["x-operator"] == "on" + + def test_a_post_to_a_get_only_pass_through_at_the_same_path_is_still_refused( + self, operator_app: Callable[[list[str], str], FastAPI] + ) -> None: + with TestClient(operator_app(["GET"], "/eu_jev/v1/systemone")) as client: + response: Final = client.post("/eu_jev/v1/systemone", json={"state": "x"}) + + assert response.status_code == 405 + + def test_a_pass_through_configured_with_a_trailing_slash_still_gets_the_path_without_one( + self, operator_app: Callable[[list[str], str], FastAPI] + ) -> None: + with ( + respx.mock(assert_all_called=True) as upstream, + TestClient(operator_app(["POST"], "/eu_jev/v1/systemone/")) as client, + ): + route: Final = upstream.post("https://operator.example/decide").respond(200, json={"operator": True}) + response: Final = client.post("/eu_jev/v1/systemone", json={"state": "x"}) + + assert (response.status_code, response.json()) == (200, {"operator": True}) + assert route.calls.last.request.headers["x-operator"] == "on" + + @pytest.mark.parametrize("method", ["GET", "PUT"]) + def test_a_pass_through_configured_with_a_trailing_slash_still_gets_other_methods_without_one( + self, operator_app: Callable[[list[str], str], FastAPI], method: str + ) -> None: + with ( + respx.mock(assert_all_called=True) as upstream, + TestClient(operator_app(["GET", "POST", "PUT"], "/eu_jev/v1/systemone/")) as client, + ): + route: Final = upstream.route(method=method, url="https://operator.example/decide").respond( + 200, json={"operator": True} + ) + response: Final = client.request(method, "/eu_jev/v1/systemone", json={"state": "x"}) + + assert (response.status_code, response.json()) == (200, {"operator": True}) + assert route.calls.last.request.headers["x-operator"] == "on" + + def test_a_request_the_built_in_route_took_is_not_forwarded_with_a_pass_through_added_meanwhile( + self, operator_app: Callable[[list[str], str], FastAPI], monkeypatch: pytest.MonkeyPatch + ) -> None: + from litellm.proxy.pass_through_endpoints.pass_through_endpoints import InitPassThroughEndpointHelpers + + monkeypatch.setenv("EU_JEV_API_KEY", "eu-jev-test-key") + monkeypatch.setenv("EU_JEV_API_BASE", "https://eu-jev.example/base") + app: Final = operator_app(["POST"], "/eu_jev/v1/systemone") + InitPassThroughEndpointHelpers.remove_endpoint_routes("operator-eu-jev") + + def add_the_pass_through_after_routing() -> UserAPIKeyAuth: + InitPassThroughEndpointHelpers.add_exact_path_route( + app=app, + path="/eu_jev/v1/systemone", + target="https://operator.example/decide", + custom_headers={"x-operator": "on"}, + forward_headers=None, + merge_query_params=None, + dependencies=None, + cost_per_request=None, + endpoint_id="operator-eu-jev", + methods=["POST"], + ) + return UserAPIKeyAuth(api_key="sk-virtual") + + monkeypatch.setitem(app.dependency_overrides, user_api_key_auth, add_the_pass_through_after_routing) + with respx.mock(assert_all_called=False) as upstream, TestClient(app) as client: + built_in: Final = upstream.post("https://eu-jev.example/base/v1/systemone").respond(200, json={"answers": {}}) + operator: Final = upstream.route(host="operator.example").respond(200, json={"operator": True}) + response: Final = client.post("/eu_jev/v1/systemone", json={"model": "eu-jeff-latest", "state": "x"}) + + assert (response.status_code, response.json()) == (200, {"answers": {}}) + assert built_in.calls.last.request.headers["authorization"] == "Bearer eu-jev-test-key" + assert not operator.called + + def test_a_deleted_pass_through_at_the_same_path_stops_forwarding_when_built_in_routes_come_first( + self, operator_app: Callable[[list[str], str], FastAPI], monkeypatch: pytest.MonkeyPatch + ) -> None: + from litellm.proxy.pass_through_endpoints.pass_through_endpoints import InitPassThroughEndpointHelpers + + monkeypatch.setenv("LITELLM_BUILTIN_PASS_THROUGH_ROUTES_FIRST", "true") + app: Final = operator_app(["GET", "PUT"], "/eu_jev/v1/systemone") + InitPassThroughEndpointHelpers.remove_endpoint_routes("operator-eu-jev") + with respx.mock(assert_all_called=False) as upstream, TestClient(app) as client: + operator: Final = upstream.route(host="operator.example").respond(200, json={"operator": True}) + responses: Final = (client.get("/eu_jev/v1/systemone"), client.put("/eu_jev/v1/systemone", json={})) + + assert [response.status_code for response in responses] == [404, 404] + assert not operator.called + + @pytest.mark.parametrize("provider", ["laya", "bespoke"]) class TestOssDecisionPassthroughRoute: @pytest.fixture diff --git a/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py b/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py index 72e5d13661f..7b62853b0e9 100644 --- a/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py +++ b/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py @@ -82,6 +82,7 @@ PROTOCOL_CONSTRAINED_PASS_THROUGH_ROUTES: Final = MappingProxyType( "/tinyfish/{endpoint:path}": frozenset({"GET", "POST"}), "/laya/v1/systemone": frozenset({"POST"}), "/bespoke/v1/systemone": frozenset({"POST"}), + "/eu_jev/v1/systemone": frozenset({"POST"}), } ) @@ -2977,6 +2978,56 @@ async def test_update_pass_through_endpoint_preserves_auth_false(): assert RouteChecks.is_auth_enforced_pass_through_route(route="/public-passthrough", method="POST") is False +@pytest.mark.asyncio +async def test_update_pass_through_endpoint_keeps_serving_the_old_route_until_the_new_one_is_registered(): + from litellm.proxy._types import ConfigFieldInfo, PassThroughGenericEndpoint, UserAPIKeyAuth + from litellm.proxy.pass_through_endpoints.pass_through_endpoints import ( + InitPassThroughEndpointHelpers, + update_pass_through_endpoints, + ) + + def registered_target() -> object: + registered: Final = InitPassThroughEndpointHelpers.get_registered_pass_through_route(route="/operator") or {} + return registered.get("passthrough_params", {}).get("target") + + targets_during_db_write: Final[list[object]] = [] + + async def record_registered_target(**_: object) -> None: + targets_during_db_write.append(registered_target()) + + with ( + patch("litellm.proxy.proxy_server.get_config_general_settings") as mock_get_config, + patch("litellm.proxy.proxy_server.update_config_general_settings", side_effect=record_registered_target), + patch("litellm.proxy.pass_through_endpoints.pass_through_endpoints._registered_pass_through_routes", {}), + ): + InitPassThroughEndpointHelpers.add_exact_path_route( + app=FastAPI(), + path="/operator", + target="http://old.example/api", + custom_headers=None, + forward_headers=None, + merge_query_params=None, + dependencies=None, + cost_per_request=None, + endpoint_id="operator-route", + methods=["POST"], + ) + mock_get_config.return_value = ConfigFieldInfo( + field_name="pass_through_endpoints", + field_value=[ + {"id": "operator-route", "path": "/operator", "target": "http://old.example/api", "methods": ["POST"]} + ], + ) + await update_pass_through_endpoints( + endpoint_id="operator-route", + data=PassThroughGenericEndpoint(path="/operator", target="http://new.example/api", methods=["POST"]), + request=MagicMock(spec=Request), + user_api_key_dict=MagicMock(spec=UserAPIKeyAuth), + ) + + assert (targets_during_db_write, registered_target()) == (["http://old.example/api"], "http://new.example/api") + + @pytest.mark.asyncio async def test_update_pass_through_endpoint_not_found(): """ @@ -10548,6 +10599,138 @@ async def test_direct_passthrough_keeps_mapped_baggage_tags_outside_upstream_bod assert await read_request_body(request) == {"model": "qa-model"} +@pytest.mark.parametrize( + ("update", "expected"), + [ + ({"path": "/moved-to"}, (True, True, False, False)), + ({"path": "/moved-from"}, (True, True, True, True)), + ({"path": "/moved-from", "auth": False}, (False, False, False, False)), + ], +) +@pytest.mark.asyncio +async def test_update_pass_through_endpoint_keeps_llm_api_route_checks_on_the_updated_path( + monkeypatch: pytest.MonkeyPatch, + update: dict[str, str | bool], + expected: tuple[bool, bool, bool, bool], +) -> None: + from litellm.proxy import proxy_server + from litellm.proxy._types import ( + ConfigFieldInfo, + ConfigFieldUpdate, + LiteLLMRoutes, + PassThroughGenericEndpoint, + UserAPIKeyAuth, + ) + from litellm.proxy.auth.route_checks import RouteChecks + from litellm.proxy.pass_through_endpoints.pass_through_endpoints import update_pass_through_endpoints + + async def serve_stored_endpoints(data: ConfigFieldUpdate, **_: object) -> None: + await initialize_pass_through_endpoints(data.field_value) + + stored_endpoint: Final = { + "id": "moved-route", + "path": "/moved-from", + "target": "http://example.com/api", + "auth": True, + "include_subpath": True, + } + saved_registry: Final = dict(_registered_pass_through_routes) + saved_openai_routes: Final = tuple(LiteLLMRoutes.openai_routes.value) + try: + _registered_pass_through_routes.clear() + monkeypatch.setattr(proxy_server, "app", FastAPI()) + monkeypatch.setattr(proxy_server, "premium_user", True) + monkeypatch.setattr(proxy_server, "config_passthrough_endpoints", None) + await initialize_pass_through_endpoints([stored_endpoint]) + with ( + patch( + "litellm.proxy.proxy_server.get_config_general_settings", + return_value=ConfigFieldInfo(field_name="pass_through_endpoints", field_value=[stored_endpoint]), + ), + patch("litellm.proxy.proxy_server.update_config_general_settings", side_effect=serve_stored_endpoints), + ): + await update_pass_through_endpoints( + endpoint_id="moved-route", + data=PassThroughGenericEndpoint.model_validate({"target": "http://example.com/api", **update}), + request=MagicMock(spec=Request), + user_api_key_dict=MagicMock(spec=UserAPIKeyAuth), + ) + + updated_path: Final = str(update["path"]) + assert ( + RouteChecks.is_llm_api_route(route=updated_path), + f"{updated_path}/*" in LiteLLMRoutes.openai_routes.value, + RouteChecks.is_llm_api_route(route="/moved-from"), + "/moved-from/*" in LiteLLMRoutes.openai_routes.value, + ) == expected + finally: + _registered_pass_through_routes.clear() + _registered_pass_through_routes.update(saved_registry) + LiteLLMRoutes.openai_routes.value[:] = saved_openai_routes + + +@pytest.mark.asyncio +async def test_update_pass_through_endpoint_keeps_llm_api_route_checks_on_a_path_another_endpoint_shares( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from litellm.proxy import proxy_server + from litellm.proxy._types import ( + ConfigFieldInfo, + ConfigFieldUpdate, + LiteLLMRoutes, + PassThroughGenericEndpoint, + UserAPIKeyAuth, + ) + from litellm.proxy.auth.route_checks import RouteChecks + from litellm.proxy.pass_through_endpoints.pass_through_endpoints import update_pass_through_endpoints + + async def serve_stored_endpoints(data: ConfigFieldUpdate, **_: object) -> None: + await initialize_pass_through_endpoints(data.field_value) + + stored_endpoints: Final = [ + { + "id": f"shared-{method.lower()}", + "path": "/shared", + "target": "http://example.com/api", + "auth": True, + "methods": [method], + } + for method in ("GET", "POST") + ] + saved_registry: Final = dict(_registered_pass_through_routes) + saved_openai_routes: Final = tuple(LiteLLMRoutes.openai_routes.value) + try: + _registered_pass_through_routes.clear() + monkeypatch.setattr(proxy_server, "app", FastAPI()) + monkeypatch.setattr(proxy_server, "premium_user", True) + monkeypatch.setattr(proxy_server, "config_passthrough_endpoints", None) + await initialize_pass_through_endpoints(stored_endpoints) + with ( + patch( + "litellm.proxy.proxy_server.get_config_general_settings", + return_value=ConfigFieldInfo(field_name="pass_through_endpoints", field_value=stored_endpoints), + ), + patch("litellm.proxy.proxy_server.update_config_general_settings", side_effect=serve_stored_endpoints), + ): + await update_pass_through_endpoints( + endpoint_id="shared-get", + data=PassThroughGenericEndpoint.model_validate( + {"target": "http://example.com/api", "path": "/moved", "auth": True, "methods": ["GET"]} + ), + request=MagicMock(spec=Request), + user_api_key_dict=MagicMock(spec=UserAPIKeyAuth), + ) + + assert (RouteChecks.is_llm_api_route(route="/shared"), RouteChecks.is_llm_api_route(route="/moved")) == ( + True, + True, + ) + finally: + _registered_pass_through_routes.clear() + _registered_pass_through_routes.update(saved_registry) + LiteLLMRoutes.openai_routes.value[:] = saved_openai_routes + + @pytest.mark.asyncio @pytest.mark.parametrize( ("flat_cost", "upstream_cost", "expected"), diff --git a/tests/unit/proxy/pass_through_endpoints/test_passthrough_auth_default.py b/tests/unit/proxy/pass_through_endpoints/test_passthrough_auth_default.py index 0e337195bb8..7771ff0274e 100644 --- a/tests/unit/proxy/pass_through_endpoints/test_passthrough_auth_default.py +++ b/tests/unit/proxy/pass_through_endpoints/test_passthrough_auth_default.py @@ -26,6 +26,7 @@ import pytest from fastapi import FastAPI from fastapi.routing import APIRoute from fastapi.testclient import TestClient +from starlette.requests import Request from litellm.proxy._types import PassThroughGenericEndpoint, ProxyException @@ -137,3 +138,15 @@ async def test_runtime_check_explicit_auth_false_still_skips_validation(): ) assert isinstance(result, UserAPIKeyAuth) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("route", ["/eu_jev/v1/systemone", "/eu_jev/v1/systemone/extra", "/eu_jev/v1/systemone2"]) +async def test_no_eu_jev_route_reads_the_litellm_user_api_key_header(route: str) -> None: + request: Final = Request({"type": "http", "headers": [(b"litellm_user_api_key", b"sk-from-header")]}) + + result: Final = await check_api_key_for_custom_headers_or_pass_through_endpoints( + request=request, route=route, pass_through_endpoints=None, api_key="sk-from-bearer" + ) + + assert result == "sk-from-bearer" diff --git a/tests/unit/proxy/pass_through_endpoints/test_passthrough_endpoint_router.py b/tests/unit/proxy/pass_through_endpoints/test_passthrough_endpoint_router.py index ab4f3f7a675..dd48a3e55ef 100644 --- a/tests/unit/proxy/pass_through_endpoints/test_passthrough_endpoint_router.py +++ b/tests/unit/proxy/pass_through_endpoints/test_passthrough_endpoint_router.py @@ -1,6 +1,7 @@ import asyncio import os import unittest +from typing import Final from unittest.mock import patch import pytest @@ -60,6 +61,49 @@ def test_credential_loaded_after_deployment_registration_still_resolves(): ) +def test_proxy_wide_credentials_pair_a_deployment_key_with_that_deployment_base() -> None: + llm_router: Final = litellm.Router( + model_list=[ + _flagged_deployment("openai/gpt-4o", api_base="https://keyless.example/v1"), + _flagged_deployment("openai/gpt-4o-mini", api_key="sk-deployment", api_base="https://keyed.example/v1"), + ] + ) + credentials: Final = _passthrough_router(llm_router).get_proxy_wide_credentials(custom_llm_provider="openai") + + assert credentials is not None + assert (credentials.api_key, credentials.api_base) == ("sk-deployment", "https://keyed.example/v1") + + +def test_proxy_wide_credentials_never_come_from_a_team_deployment(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("OPENAI_API_KEY", "sk-proxy-env") + team_deployment: Final = { + **_flagged_deployment("openai/gpt-4o", api_key="sk-team-a", api_base="https://team-a.example/v1"), + "model_info": {"team_id": "team-a"}, + } + passthrough_router: Final = _passthrough_router(litellm.Router(model_list=[team_deployment])) + + credentials: Final = passthrough_router.get_proxy_wide_credentials(custom_llm_provider="openai") + + assert credentials is not None + assert (credentials.api_key, credentials.api_base) == ("sk-proxy-env", None) + + +def test_credentials_for_a_provider_route_still_include_a_team_deployment_key() -> None: + team_deployment: Final = { + **_flagged_deployment("openai/gpt-4o", api_key="sk-team-a"), + "model_info": {"team_id": "team-a"}, + } + passthrough_router: Final = _passthrough_router(litellm.Router(model_list=[team_deployment])) + + assert passthrough_router.get_credentials(custom_llm_provider="openai", region_name=None) == "sk-team-a" + + +def test_proxy_wide_credentials_are_missing_without_a_deployment_or_env_key() -> None: + passthrough_router: Final = _passthrough_router(litellm.Router(model_list=[])) + + assert passthrough_router.get_proxy_wide_credentials(custom_llm_provider="anthropic") is None + + def test_credential_rotation_is_reflected_without_deployment_update(): CredentialAccessor.upsert_credentials([_credential("cred_openai", "sk-before-rotation")]) llm_router = litellm.Router( diff --git a/tests/unit/proxy/public_endpoints/test_public_endpoints.py b/tests/unit/proxy/public_endpoints/test_public_endpoints.py index da608dbca0e..48672c7a305 100644 --- a/tests/unit/proxy/public_endpoints/test_public_endpoints.py +++ b/tests/unit/proxy/public_endpoints/test_public_endpoints.py @@ -481,9 +481,16 @@ def _decisions_provider_entry(provider: str) -> ProviderCreateInfo: return entry -@pytest.mark.parametrize("display_name,provider", (("TypeSafe", LlmProviders.TYPESAFE), ("Nace", LlmProviders.NACE))) -def test_decisions_provider_fields(display_name: str, provider: LlmProviders) -> None: - entry: Final = _decisions_provider_entry(display_name) +@pytest.mark.parametrize( + ("provider_key", "display_name", "provider"), + ( + ("TypeSafe", "TypeSafe", LlmProviders.TYPESAFE), + ("Nace", "Nace", LlmProviders.NACE), + ("EU_JEV", "eu/jev", LlmProviders.EU_JEV), + ), +) +def test_decisions_provider_fields(provider_key: str, display_name: str, provider: LlmProviders) -> None: + entry: Final = _decisions_provider_entry(provider_key) assert entry.provider_display_name == display_name assert entry.litellm_provider == provider.value diff --git a/tests/unit/proxy/test_component_allowlists.py b/tests/unit/proxy/test_component_allowlists.py index eb058d01770..4c0b3ad913b 100644 --- a/tests/unit/proxy/test_component_allowlists.py +++ b/tests/unit/proxy/test_component_allowlists.py @@ -397,6 +397,22 @@ def test_gateway_keeps_memory_summary_and_trims_the_other_debug_routes(): assert not _is_gateway_route(debug_memory_routes[path]), f"{path} must not be served by the gateway" +@pytest.mark.parametrize( + ("path", "served"), + [ + ("/eu_jev/v1/systemone", True), + ("/eu_jev/v1/systemone2", False), + ("/eu_jev/v1/systemone/{subpath:path}", False), + ("/eu_jev/custom", False), + ], +) +def test_gateway_serves_the_eu_jev_decision_route_and_no_other_eu_jev_path(path: str, served: bool) -> None: + async def endpoint() -> None: + return None + + assert _is_gateway_route(Route(path, endpoint, methods=["POST"])) is served + + def test_every_app_mount_is_assigned_to_a_component(): """Every Mount on the proxy app must be consciously assigned to a component. diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 6c4c5ba3b11..d542880f868 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -5362,6 +5362,32 @@ export interface paths { patch: operations["assemblyai_proxy_route_eu_assemblyai__endpoint__patch"]; trace?: never; }; + "/eu_jev/v1/systemone": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + /** + * Eu Jev Proxy Route + * @description Forward a request to eu/jev, an EU-hosted decision API, with `EU_JEV_API_KEY`. + * + * `POST /eu_jev/v1/systemone` sends a `state` and one to eight named `questions` (`choice`, `noul` or `score`) + * and returns one answer per question name, with spend booked from the reported `usage`. `custom_body` and + * `stream` are rejected with a 400. A pass-through endpoint configured at this path keeps serving it. + * + * [Docs](https://docs.litellm.ai/docs/pass_through/eu_jev) + */ + post: operations["eu_jev_proxy_route_eu_jev_v1_systemone_post"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/extraction": { parameters: { query?: never; @@ -32287,6 +32313,50 @@ export interface components { [key: string]: unknown; }; }; + /** EuJevDecisionResponse */ + EuJevDecisionResponse: { + /** + * Answers + * @description One answer per question, under the same names + */ + answers: { + [key: string]: components["schemas"]["ChoiceAnswer"] | components["schemas"]["NoulAnswer"] | components["schemas"]["ScoreAnswer"]; + }; + meta: components["schemas"]["EuJevMeta"]; + /** + * Model + * @description The release that answered, such as eu-jeff-1.0.0 + */ + model: string; + usage: components["schemas"]["EuJevUsage"]; + }; + /** EuJevMeta */ + EuJevMeta: { + /** + * Cost Eur + * @description The exact charge in euros, on authenticated requests + */ + cost_eur?: string | null; + /** + * Latency Ms + * @description Server-side time for token quoting, credit reservation and inference + */ + latency_ms: number; + /** + * Mode + * @constant + */ + mode: "live"; + /** Request Id */ + request_id: string; + }; + /** EuJevUsage */ + EuJevUsage: { + /** Input Tokens */ + input_tokens: number; + /** Output Tokens */ + output_tokens: number; + }; /** * Eval * @description Represents an evaluation from the OpenAI Evals API @@ -59476,6 +59546,144 @@ export interface operations { }; }; }; + eu_jev_proxy_route_eu_jev_v1_systemone_post: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody: { + content: { + /** + * @example { + * "model": "eu-jeff-latest", + * "questions": { + * "frustration": { + * "criteria": [ + * "calm", + * "mildly annoyed", + * "angry" + * ], + * "instructions": "How frustrated is the customer?", + * "type": "score" + * }, + * "is_refund_request": { + * "instructions": "Is the customer asking for a refund?", + * "type": "noul" + * }, + * "team": { + * "criteria": { + * "billing": "Payments, invoices, and refunds", + * "support": "Technical issues and bugs" + * }, + * "instructions": "Which team should handle this?", + * "type": "choice" + * } + * }, + * "state": "I was charged twice. Can I get a refund?" + * } + */ + "application/json": { + /** + * Model + * @description eu-jeff-latest follows the current release, eu-jeff-1.0.0 pins it + * @default eu-jeff-latest + */ + model?: string; + /** + * Questions + * @description One to eight questions, keyed by names of up to 80 characters + */ + questions: { + [key: string]: { + /** + * Criteria + * @description Two to 64 options, option name to description. A null description uses the option name + */ + criteria: { + [key: string]: string | { + [key: string]: unknown; + } | unknown[] | null; + }; + /** + * Instructions + * @description Free text, or a JSON object, array or null that eu/jev serializes to text before inference + */ + instructions: string | { + [key: string]: unknown; + } | unknown[] | null; + /** + * Type + * @constant + */ + type: "choice"; + } | { + /** + * Criteria + * @description Optional descriptions of what true and false mean + */ + criteria?: { + [key: string]: string | { + [key: string]: unknown; + } | unknown[] | null; + } | null; + /** + * Instructions + * @description Free text, or a JSON object, array or null that eu/jev serializes to text before inference + */ + instructions: string | { + [key: string]: unknown; + } | unknown[] | null; + /** + * Type + * @constant + */ + type: "noul"; + } | { + /** + * Criteria + * @description Two to 10 level descriptions, ordered from low to high + */ + criteria: (string | { + [key: string]: unknown; + } | unknown[] | null)[]; + /** + * Instructions + * @description Free text, or a JSON object, array or null that eu/jev serializes to text before inference + */ + instructions: string | { + [key: string]: unknown; + } | unknown[] | null; + /** + * Type + * @constant + */ + type: "score"; + }; + }; + /** + * State + * @description The content to evaluate, a string of up to 100,000 characters or a JSON object or array + */ + state: string | { + [key: string]: unknown; + } | unknown[]; + }; + }; + }; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["EuJevDecisionResponse"]; + }; + }; + }; + }; extraction_extraction_post: { parameters: { query?: never; From 24afa12253ce5493a27acfd4799b6d143615f88b Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Thu, 29 Oct 2026 18:02:31 -0700 Subject: [PATCH 13/17] chore(caching)!: remove the disk cache backend (#48114) * chore(caching)!: remove the disk cache backend Drop the disk cache type and its diskcache dependency, along with the caching install extra that existed only to pull it in. diskcache has not had a release since August 2023 Cache now raises a ValueError listing the supported types when it gets a type it does not know. Before, an unknown type built a Cache with no backend, which only failed on first use BREAKING CHANGE: cache_params type "disk" and the litellm[caching] extra are gone. A proxy configured with type disk now refuses to start. Use local or redis instead * test(proxy): expect the unsupported cache type error from _init_cache The test pinned the old fallthrough, where an unknown type left the cache without a backend and the proxy hit an AttributeError reading it * chore(deps): drop the diskcache scan ignore main extended diskcache is no longer in the lock, so the ignore matches nothing * fix(caching): ignore a leftover disk_cache_dir instead of crashing redis and s3 Dropping the disk_cache_dir parameter sent the key into **kwargs, which the redis and s3 branches hand to their clients, so a config that moved from the disk cache to redis or s3 and kept the key failed with a TypeError. Accept the key again, ignore it, and log a warning The unsupported type test now checks the whole supported list, and two proxy docstrings no longer name disk as a cache backend * fix(caching): warn about a leftover disk_cache_dir only once the cache is built The warning ran before the type check, so a config with type disk logged that the key was ignored and then failed on the type anyway. It now logs only after a supported backend is built. The test also checks that a cache without the key logs nothing --- .github/workflows/test-unit.yml | 8 +- license_cache.json | 1 - litellm/caching/Readme.md | 8 +- litellm/caching/__init__.py | 1 - litellm/caching/caching.py | 19 +-- litellm/caching/caching_handler.py | 2 +- litellm/caching/disk_cache.py | 92 ---------- litellm/proxy/proxy_server.py | 4 +- litellm/router.py | 2 +- litellm/types/caching.py | 1 - osv-scanner.toml | 5 - pyproject.toml | 1 - tests/unit/caching/test_caching.py | 25 +++ tests/unit/caching/test_disk_cache.py | 158 ------------------ .../proxy/proxy_server/test_proxy_config.py | 2 +- uv.lock | 15 +- 16 files changed, 47 insertions(+), 297 deletions(-) delete mode 100644 litellm/caching/disk_cache.py delete mode 100644 tests/unit/caching/test_disk_cache.py diff --git a/.github/workflows/test-unit.yml b/.github/workflows/test-unit.yml index 509271cbf54..a5013155eae 100644 --- a/.github/workflows/test-unit.yml +++ b/.github/workflows/test-unit.yml @@ -1003,9 +1003,9 @@ jobs: run: | diff -u model_prices_and_context_window.json litellm/model_prices_and_context_window_backup.json if [ -z "$RUST_BRIDGE_ARTIFACT" ]; then - .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra caching --extra extra_proxy --extra proxy-runtime --extra utils + .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra extra_proxy --extra proxy-runtime --extra utils else - .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra caching --extra extra_proxy --extra proxy-runtime --extra utils --no-install-project + .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra extra_proxy --extra proxy-runtime --extra utils --no-install-project uv pip install --no-deps --python .venv/bin/python rust-bridge-dist/*.whl cp rust-bridge-dist/litellm/rust_bridge/_native.abi3.so litellm/rust_bridge/_native.abi3.so uv run --no-sync python -c "import importlib.metadata; import litellm.rust_bridge._native; print(importlib.metadata.version('litellm'))" @@ -1451,9 +1451,9 @@ jobs: RUST_BRIDGE_ARTIFACT: ${{ needs.rust-bridge.outputs.artifact }} run: | if [ -z "$RUST_BRIDGE_ARTIFACT" ]; then - .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra caching --extra extra_proxy --extra proxy-runtime --extra utils + .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra extra_proxy --extra proxy-runtime --extra utils else - .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra caching --extra extra_proxy --extra proxy-runtime --extra utils --no-install-project + .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra extra_proxy --extra proxy-runtime --extra utils --no-install-project uv pip install --no-deps --python .venv/bin/python rust-bridge-dist/*.whl cp rust-bridge-dist/litellm/rust_bridge/_native.abi3.so litellm/rust_bridge/_native.abi3.so fi diff --git a/license_cache.json b/license_cache.json index dc061b48f4f..cd8a54d72a0 100644 --- a/license_cache.json +++ b/license_cache.json @@ -11,7 +11,6 @@ "azure-keyvault-secrets:4.10.0": "MIT License", "google-cloud-kms:2.24.2": "Apache 2.0", "numpydoc:1.8.0": "Copyright (C) 2008-2023 Stefan van der Walt , Pauli Virtanen Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ", - "diskcache:5.6.3": "Apache 2.0", "mlflow:3.9.0": "Copyright 2018 Databricks, Inc. All rights reserved.\n \n \t\t\t\tApache License\n Version 2.0, January 2004\n http://www.apache.org/licenses/\n \n TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION\n \n 1. Definitions.\n \n \"License\" shall mean the terms and conditions for use, reproduction,\n and distribution as defined by Sections 1 through 9 of this document.\n \n \"Licensor\" shall mean the copyright owner or entity authorized by\n the copyright owner that is granting the License.\n \n \"Legal Entity\" shall mean the union of the acting entity and all\n other entities that control, are controlled by, or are under common\n control with that entity. For the purposes of this definition,\n \"control\" means (i) the power, direct or indirect, to cause the\n direction or management of such entity, whether by contract or\n otherwise, or (ii) ownership of fifty percent (50%) or more of the\n outstanding shares, or (iii) beneficial ownership of such entity.\n \n \"You\" (or \"Your\") shall mean an individual or Legal Entity\n exercising permissions granted by this License.\n \n \"Source\" form shall mean the preferred form for making modifications,\n including but not limited to software source code, documentation\n source, and configuration files.\n \n \"Object\" form shall mean any form resulting from mechanical\n transformation or translation of a Source form, including but\n not limited to compiled object code, generated documentation,\n and conversions to other media types.\n \n \"Work\" shall mean the work of authorship, whether in Source or\n Object form, made available under the License, as indicated by a\n copyright notice that is included in or attached to the work\n (an example is provided in the Appendix below).\n \n \"Derivative Works\" shall mean any work, whether in Source or Object\n form, that is based on (or derived from) the Work and for which the\n editorial revisions, annotations, elaborations, or other modifications\n represent, as a whole, an original work of authorship. For the purposes\n of this License, Derivative Works shall not include works that remain\n separable from, or merely link (or bind by name) to the interfaces of,\n the Work and Derivative Works thereof.\n \n \"Contribution\" shall mean any work of authorship, including\n the original version of the Work and any modifications or additions\n to that Work or Derivative Works thereof, that is intentionally\n submitted to Licensor for inclusion in the Work by the copyright owner\n or by an individual or Legal Entity authorized to submit on behalf of\n the copyright owner. For the purposes of this definition, \"submitted\"\n means any form of electronic, verbal, or written communication sent\n to the Licensor or its representatives, including but not limited to\n communication on electronic mailing lists, source code control systems,\n and issue tracking systems that are managed by, or on behalf of, the\n Licensor for the purpose of discussing and improving the Work, but\n excluding communication that is conspicuously marked or otherwise\n designated in writing by the copyright owner as \"Not a Contribution.\"\n \n \"Contributor\" shall mean Licensor and any individual or Legal Entity\n on behalf of whom a Contribution has been received by Licensor and\n subsequently incorporated within the Work.\n \n 2. Grant of Copyright License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n copyright license to reproduce, prepare Derivative Works of,\n publicly display, publicly perform, sublicense, and distribute the\n Work and such Derivative Works in Source or Object form.\n \n 3. Grant of Patent License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n (except as stated in this section) patent license to make, have made,\n use, offer to sell, sell, import, and otherwise transfer the Work,\n where such license applies only to those patent claims licensable\n by such Contributor that are necessarily infringed by their\n Contribution(s) alone or by combination of their Contribution(s)\n with the Work to which such Contribution(s) was submitted. If You\n institute patent litigation against any entity (including a\n cross-claim or counterclaim in a lawsuit) alleging that the Work\n or a Contribution incorporated within the Work constitutes direct\n or contributory patent infringement, then any patent licenses\n granted to You under this License for that Work shall terminate\n as of the date such litigation is filed.\n \n 4. Redistribution. You may reproduce and distribute copies of the\n Work or Derivative Works thereof in any medium, with or without\n modifications, and in Source or Object form, provided that You\n meet the following conditions:\n \n (a) You must give any other recipients of the Work or\n Derivative Works a copy of this License; and\n \n (b) You must cause any modified files to carry prominent notices\n stating that You changed the files; and\n \n (c) You must retain, in the Source form of any Derivative Works\n that You distribute, all copyright, patent, trademark, and\n attribution notices from the Source form of the Work,\n excluding those notices that do not pertain to any part of\n the Derivative Works; and\n \n (d) If the Work includes a \"NOTICE\" text file as part of its\n distribution, then any Derivative Works that You distribute must\n include a readable copy of the attribution notices contained\n within such NOTICE file, excluding those notices that do not\n pertain to any part of the Derivative Works, in at least one\n of the following places: within a NOTICE text file distributed\n as part of the Derivative Works; within the Source form or\n documentation, if provided along with the Derivative Works; or,\n within a display generated by the Derivative Works, if and\n wherever such third-party notices normally appear. The contents\n of the NOTICE file are for informational purposes only and\n do not modify the License. You may add Your own attribution\n notices within Derivative Works that You distribute, alongside\n or as an addendum to the NOTICE text from the Work, provided\n that such additional attribution notices cannot be construed\n as modifying the License.\n \n You may add Your own copyright statement to Your modifications and\n may provide additional or different license terms and conditions\n for use, reproduction, or distribution of Your modifications, or\n for any such Derivative Works as a whole, provided Your use,\n reproduction, and distribution of the Work otherwise complies with\n the conditions stated in this License.\n \n 5. Submission of Contributions. Unless You explicitly state otherwise,\n any Contribution intentionally submitted for inclusion in the Work\n by You to the Licensor shall be under the terms and conditions of\n this License, without any additional terms or conditions.\n Notwithstanding the above, nothing herein shall supersede or modify\n the terms of any separate license agreement you may have executed\n with Licensor regarding such Contributions.\n \n 6. Trademarks. This License does not grant permission to use the trade\n names, trademarks, service marks, or product names of the Licensor,\n except as required for reasonable and customary use in describing the\n origin of the Work and reproducing the content of the NOTICE file.\n \n 7. Disclaimer of Warranty. Unless required by applicable law or\n agreed to in writing, Licensor provides the Work (and each\n Contributor provides its Contributions) on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or\n implied, including, without limitation, any warranties or conditions\n of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A\n PARTICULAR PURPOSE. You are solely responsible for determining the\n appropriateness of using or redistributing the Work and assume any\n risks associated with Your exercise of permissions under this License.\n \n 8. Limitation of Liability. In no event and under no legal theory,\n whether in tort (including negligence), contract, or otherwise,\n unless required by applicable law (such as deliberate and grossly\n negligent acts) or agreed to in writing, shall any Contributor be\n liable to You for damages, including any direct, indirect, special,\n incidental, or consequential damages of any character arising as a\n result of this License or out of the use or inability to use the\n Work (including but not limited to damages for loss of goodwill,\n work stoppage, computer failure or malfunction, or any and all\n other commercial damages or losses), even if such Contributor\n has been advised of the possibility of such damages.\n \n 9. Accepting Warranty or Additional Liability. While redistributing\n the Work or Derivative Works thereof, You may choose to offer,\n and charge a fee for, acceptance of support, warranty, indemnity,\n or other liability obligations and/or rights consistent with this\n License. However, in accepting such obligations, You may act only\n on Your own behalf and on Your sole responsibility, not on behalf\n of any other Contributor, and only if You agree to indemnify,\n defend, and hold each Contributor harmless for any liability\n incurred by, or claims asserted against, such Contributor by reason\n of your accepting any such warranty or additional liability.\n \n END OF TERMS AND CONDITIONS\n APPENDIX: How to apply the Apache License to your work.\n \n To apply the Apache License to your work, attach the following\n boilerplate notice, with the fields enclosed by brackets \"[]\"\n replaced with your own identifying information. (Don't include\n the brackets!) The text should be enclosed in the appropriate\n comment syntax for the file format. We also recommend that a\n file or class name and description of purpose be included on the\n same \"printed page\" as the copyright notice for easier\n identification within third-party archives.\n \n Copyright [yyyy] [name of copyright owner]\n \n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n \n http://www.apache.org/licenses/LICENSE-2.0\n \n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n ", "diff-cover:9.7.2": "Apache-2.0", "flake8:7.3.0": "MIT", diff --git a/litellm/caching/Readme.md b/litellm/caching/Readme.md index 1d920219830..e91a5fec888 100644 --- a/litellm/caching/Readme.md +++ b/litellm/caching/Readme.md @@ -8,10 +8,9 @@ The following caching mechanisms are supported: 2. **RedisSemanticCache** 3. **QdrantSemanticCache** 4. **InMemoryCache** -5. **DiskCache** -6. **S3Cache** -7. **AzureBlobCache** -8. **DualCache** (updates both Redis and an in-memory cache simultaneously) +5. **S3Cache** +6. **AzureBlobCache** +7. **DualCache** (updates both Redis and an in-memory cache simultaneously) ## Folder Structure @@ -20,7 +19,6 @@ litellm/caching/ ├── base_cache.py ├── caching.py ├── caching_handler.py -├── disk_cache.py ├── dual_cache.py ├── in_memory_cache.py ├── qdrant_semantic_cache.py diff --git a/litellm/caching/__init__.py b/litellm/caching/__init__.py index 87f4f7a7c63..db79a9cafee 100644 --- a/litellm/caching/__init__.py +++ b/litellm/caching/__init__.py @@ -1,6 +1,5 @@ from .azure_blob_cache import AzureBlobCache from .caching import Cache, LiteLLMCacheType -from .disk_cache import DiskCache from .dual_cache import DualCache from .gcs_cache import GCSCache from .in_memory_cache import InMemoryCache diff --git a/litellm/caching/caching.py b/litellm/caching/caching.py index 1ced13445de..d875ac09d40 100644 --- a/litellm/caching/caching.py +++ b/litellm/caching/caching.py @@ -31,7 +31,6 @@ from litellm.types.utils import EmbeddingResponse, is_litellm_owned_kwarg from .azure_blob_cache import AzureBlobCache from .base_cache import BaseCache -from .disk_cache import DiskCache from .dual_cache import DualCache # noqa: F401 # re-exported, callers import DualCache from litellm.caching.caching from .gcs_cache import GCSCache from .in_memory_cache import InMemoryCache @@ -153,7 +152,7 @@ class Cache: Initializes the cache based on the given type. Args: - type (str, optional): The type of cache to initialize. Can be "local", "redis", "redis-semantic", "qdrant-semantic", "s3" or "disk". Defaults to "local". + type (str, optional): The type of cache to initialize. Can be "local", "redis", "redis-semantic", "qdrant-semantic" or "s3". Defaults to "local". # Redis Cache Args host (str, optional): The host address for the Redis cache. Required if type is "redis". @@ -174,9 +173,6 @@ class Cache: semantic_cache_scope (str, optional): "key" isolates semantic-cache buckets per key/team/org. "end_user" additionally isolates per end user (falls back to the key scope when the request carries no end-user id). Defaults to "key". max_messages (int, optional): Requests with more `messages` (or Responses API `input` items) than this are neither looked up nor stored, so long agent conversations never serve or create a cache entry. None disables the limit. Defaults to 4. - # Disk Cache Args - disk_cache_dir (str, optional): The directory for the disk cache. Defaults to None. - # S3 Cache Args s3_bucket_name (str, optional): The bucket name for the s3 cache. Defaults to None. s3_region_name (str, optional): The region name for the s3 cache. Defaults to None. @@ -306,8 +302,11 @@ class Cache: account_url=azure_account_url, container=azure_blob_container, ) - elif type == LiteLLMCacheType.DISK: - self.cache = DiskCache(disk_cache_dir=disk_cache_dir) + else: + supported_types: Final = ", ".join(member.value for member in LiteLLMCacheType) + raise ValueError(f"Unsupported cache type {type!r}. Supported types: {supported_types}") + if disk_cache_dir is not None: + verbose_logger.warning("Ignoring disk_cache_dir, the disk cache backend was removed") if "cache" not in litellm.input_callback: litellm.input_callback.append("cache") if "cache" not in litellm.success_callback: @@ -1044,7 +1043,7 @@ def enable_cache( Enable cache with the specified configuration. Args: - type (Optional[Literal["local", "redis", "s3", "disk"]]): The type of cache to enable. Defaults to "local". + type (Optional[Literal["local", "redis", "s3"]]): The type of cache to enable. Defaults to "local". host (Optional[str]): The host address of the cache server. Defaults to None. port (Optional[str]): The port number of the cache server. Defaults to None. password (Optional[str]): The password for the cache server. Defaults to None. @@ -1056,7 +1055,7 @@ def enable_cache( None Raises: - None + ValueError: If no cache is set yet and the cache type is unsupported. """ print_verbose("LiteLLM: Enabling Cache") if "cache" not in litellm.input_callback: @@ -1091,7 +1090,7 @@ def update_cache( Update the cache for LiteLLM. Args: - type (Optional[Literal["local", "redis", "s3", "disk"]]): The type of cache. Defaults to "local". + type (Optional[Literal["local", "redis", "s3"]]): The type of cache. Defaults to "local". host (Optional[str]): The host of the cache. Defaults to None. port (Optional[str]): The port of the cache. Defaults to None. password (Optional[str]): The password for the cache. Defaults to None. diff --git a/litellm/caching/caching_handler.py b/litellm/caching/caching_handler.py index 364ce9450cd..1d80d6952a4 100644 --- a/litellm/caching/caching_handler.py +++ b/litellm/caching/caching_handler.py @@ -9,7 +9,7 @@ This file is a wrapper around caching.py This class is used to handle caching logic specific for LLM API requests (completion / embedding / text_completion / transcription etc) -It utilizes the (RedisCache, s3Cache, RedisSemanticCache, QdrantSemanticCache, InMemoryCache, DiskCache) based on what the user has setup +It utilizes the (RedisCache, s3Cache, RedisSemanticCache, QdrantSemanticCache, InMemoryCache) based on what the user has setup In each method it will call the appropriate method from caching.py """ diff --git a/litellm/caching/disk_cache.py b/litellm/caching/disk_cache.py deleted file mode 100644 index 9470f23570a..00000000000 --- a/litellm/caching/disk_cache.py +++ /dev/null @@ -1,92 +0,0 @@ -import json -from typing import TYPE_CHECKING, Any, Final - -from pydantic import TypeAdapter - -from .base_cache import BaseCache - -if TYPE_CHECKING: - from opentelemetry.trace import Span as _Span - - Span = _Span -else: - Span = Any - -_DECODED_DISK_VALUE: Final = TypeAdapter(object) - - -class DiskCache(BaseCache): - def __init__(self, disk_cache_dir: str | None = None): - try: - import diskcache as dc - except ModuleNotFoundError as e: - raise ModuleNotFoundError("Please install litellm with `litellm[caching]` to use disk caching.") from e - - # if users don't provider one, use the default litellm cache - if disk_cache_dir is None: - self.disk_cache = dc.Cache(".litellm_cache") - else: - self.disk_cache = dc.Cache(disk_cache_dir) - - def set_cache(self, key, value, **kwargs): - if "ttl" in kwargs: - self.disk_cache.set(key, value, expire=kwargs["ttl"]) - else: - self.disk_cache.set(key, value) - - async def async_set_cache(self, key, value, **kwargs): - self.set_cache(key=key, value=value, **kwargs) - - async def async_set_cache_pipeline(self, cache_list, **kwargs): - for cache_key, cache_value in cache_list: - if "ttl" in kwargs: - self.set_cache(key=cache_key, value=cache_value, ttl=kwargs["ttl"]) - else: - self.set_cache(key=cache_key, value=cache_value) - - def get_cache(self, key, **kwargs): - original_cached_response: Final = self.disk_cache.get(key) - if original_cached_response: - try: - cached_response = _DECODED_DISK_VALUE.validate_python(json.loads(original_cached_response)) - except Exception: - cached_response = original_cached_response - return cached_response - return None - - def batch_get_cache(self, keys: list, **kwargs): - return_val: Final = [] - for k in keys: - val = self.get_cache(key=k, **kwargs) - return_val.append(val) - return return_val - - def increment_cache(self, key, value: int, **kwargs) -> int: - with self.disk_cache.transact(): - cached_value: Final = self.get_cache(key=key) - init_value: Final = cached_value if isinstance(cached_value, int) else 0 - new_value: Final = init_value + value - self.set_cache(key, new_value, **kwargs) - return new_value - - async def async_get_cache(self, key, **kwargs): - return self.get_cache(key=key, **kwargs) - - async def async_batch_get_cache(self, keys: list, **kwargs): - return_val: Final = [] - for k in keys: - val = self.get_cache(key=k, **kwargs) - return_val.append(val) - return return_val - - async def async_increment(self, key, value: int, **kwargs) -> int: - return self.increment_cache(key=key, value=value, **kwargs) - - def flush_cache(self): - self.disk_cache.clear() - - async def disconnect(self): - pass - - def delete_cache(self, key): - self.disk_cache.pop(key) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index e6d157a7f81..6d92e8733a3 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -5190,7 +5190,7 @@ def _build_redis_usage_cache_from_environment() -> RedisCache | None: Lets the proxy's coordination Redis (cross-pod tpm/rpm rate limits, spend tracking, pod lock manager) run when the response-cache backend is not a - plain Redis KV cache (e.g. a semantic cache, disk, or s3). + plain Redis KV cache (e.g. a semantic cache or s3). Returns None when the environment carries no connection target (host, url, cluster nodes, or sentinel nodes). @@ -10974,7 +10974,7 @@ class ProxyStartupEvent: """ Builds a standalone Redis cache from REDIS_* environment variables so use_redis_transaction_buffer can run when the proxy cache backend is not - Redis (e.g. disk, s3). + Redis (e.g. s3). Returns None when the buffer is disabled, or when no Redis host or url is set in the environment. diff --git a/litellm/router.py b/litellm/router.py index 77958686957..f2e68424c8d 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -1617,7 +1617,7 @@ class Router(DeprecatedRouterMembers): self._core_deployment_names: list[str] = [] # names of models under litellm_params. ex. azure/chatgpt-v-2 self._core_deployment_latency_map = {} ### CACHING ### - cache_type: Literal["local", "redis", "redis-semantic", "s3", "disk"] = "local" # default to an in-memory cache + cache_type: Literal["local", "redis", "redis-semantic", "s3"] = "local" # default to an in-memory cache redis_cache = None cache_config: Final[dict[str, Any]] = {} diff --git a/litellm/types/caching.py b/litellm/types/caching.py index ced2065a674..fb5ab3b4cea 100644 --- a/litellm/types/caching.py +++ b/litellm/types/caching.py @@ -13,7 +13,6 @@ class LiteLLMCacheType(str, Enum): REDIS_SEMANTIC = "redis-semantic" VALKEY_SEMANTIC = "valkey-semantic" S3 = "s3" - DISK = "disk" QDRANT_SEMANTIC = "qdrant-semantic" AZURE_BLOB = "azure-blob" GCS = "gcs" diff --git a/osv-scanner.toml b/osv-scanner.toml index 32a4f65fdf9..45fb7635088 100644 --- a/osv-scanner.toml +++ b/osv-scanner.toml @@ -78,11 +78,6 @@ id = "GHSA-r4j5-9xpp-3985" ignoreUntil = 2026-11-27 reason = "tornado 6.6 (published 2026-10-21) is the only fix and requires Python >= 3.11, so the Python 3.10 resolution stays on 6.5.10 while 3.11+ locks 6.6; litellm starts no tornado server or client, since semantic-router and aurelio-sdk only declare it and tenacity and sentry-sdk import it only to wrap tornado code when it is present, so tornado's chunked body reading is never reached; remove this entry once 3.10 is dropped" -[[IgnoredVulns]] -id = "GHSA-w8v5-vhqr-4h9v" -ignoreUntil = 2026-11-27 -reason = "diskcache still has no fixed release published (checked 2026-10-28), and it is reached only through the optional caching extra; remove this entry once one exists" - [[IgnoredVulns]] id = "GHSA-45c3-73f7-pv4m" ignoreUntil = 2026-11-27 diff --git a/pyproject.toml b/pyproject.toml index 6e4732b4f7a..bff00a2dbf1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -130,7 +130,6 @@ utils = [ # Not in Docker or PyPI proxy extra. "numpydoc>=1.8.0,<2.0", ] -caching = ["diskcache>=5.6.3,<6.0"] mcp = ["mcp>=2.3.0,<3", "httpx2>=2.10.0,<3", "pydantic>=2.12.0,<3"] # Driver for the MongoDB Atlas vector store; Atlas Vector Search has no HTTP query API. # The floor is 4.9 because that is the release AsyncMongoClient landed in. diff --git a/tests/unit/caching/test_caching.py b/tests/unit/caching/test_caching.py index 91083921130..c4727cbeff0 100644 --- a/tests/unit/caching/test_caching.py +++ b/tests/unit/caching/test_caching.py @@ -250,6 +250,15 @@ def test_semantic_cache_key_reads_tenant_identity_from_litellm_metadata(): assert key_a == key_a_in_litellm_params +def test_cache_rejects_unsupported_type() -> None: + cache_params: Final = {"type": "not-a-cache"} + with pytest.raises( + ValueError, match=re.escape("Unsupported cache type 'not-a-cache'. Supported types: ") + ) as raised: + Cache(**cache_params) + assert str(raised.value).split("Supported types: ")[1].split(", ") == [member.value for member in LiteLLMCacheType] + + def test_semantic_cache_scope_rejects_unknown_value(): with pytest.raises(ValueError, match="'team' is not a valid SemanticCacheScope"): _semantic_cache(semantic_cache_scope="team") @@ -1230,6 +1239,22 @@ def test_custom_redis_cache_params(redis_client_without_network: None) -> None: assert cache.cache.redis_client.connection_pool.connection_kwargs["db"] == 4 +def test_redis_cache_ignores_a_leftover_disk_cache_dir( + redis_client_without_network: None, caplog: pytest.LogCaptureFixture +) -> None: + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + Cache(type=LiteLLMCacheType.REDIS, host="127.0.0.1", port="6379") + assert "disk_cache_dir" not in caplog.text + + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + cache: Final = Cache( + type=LiteLLMCacheType.REDIS, host="127.0.0.1", port="6379", disk_cache_dir="/tmp/litellm-cache" + ) + + assert isinstance(cache.cache, RedisCache) + assert "disk_cache_dir" in caplog.text + + def test_custom_redis_cache_with_key(monkeypatch: pytest.MonkeyPatch) -> None: cache: Final = Cache(type=LiteLLMCacheType.LOCAL) monkeypatch.setattr(litellm, "cache", cache) diff --git a/tests/unit/caching/test_disk_cache.py b/tests/unit/caching/test_disk_cache.py deleted file mode 100644 index 243f194cc7a..00000000000 --- a/tests/unit/caching/test_disk_cache.py +++ /dev/null @@ -1,158 +0,0 @@ -import threading -import time -from concurrent.futures import ThreadPoolExecutor - -import pytest - -pytest.importorskip("diskcache") - -from litellm.caching.disk_cache import DiskCache -from tests._wait_helpers import await_until - - -class _SlowInt(int): - def __add__(self, value: int) -> "_SlowInt": - time.sleep(0.05) - return _SlowInt(int(self) + value) - - -@pytest.fixture -def cache(tmp_path): - return DiskCache(disk_cache_dir=str(tmp_path)) - - -def test_increment_cache_starts_from_zero_when_key_missing(cache): - assert cache.increment_cache("counter", 3) == 3 - assert cache.get_cache("counter") == 3 - - -def test_increment_cache_adds_to_existing_int(cache): - cache.set_cache("counter", 7) - assert cache.increment_cache("counter", 5) == 12 - assert cache.get_cache("counter") == 12 - - -def test_increment_cache_treats_non_int_cached_value_as_zero(cache): - cache.set_cache("counter", "not-a-number") - assert cache.increment_cache("counter", 4) == 4 - assert cache.get_cache("counter") == 4 - - -def test_increment_cache_is_atomic_under_thread_concurrency(cache): - seed = 1000 - cache.set_cache("counter", _SlowInt(seed)) - thread_count = 8 - barrier = threading.Barrier(thread_count) - - def increment(_: int) -> int: - barrier.wait() - return cache.increment_cache("counter", 1) - - with ThreadPoolExecutor(max_workers=thread_count) as executor: - tuple(executor.map(increment, range(thread_count))) - - assert cache.get_cache("counter") == seed + thread_count - - -async def test_async_increment_starts_from_zero_when_key_missing(cache): - assert await cache.async_increment("counter", 2) == 2 - - -async def test_async_increment_adds_to_existing_int(cache): - await cache.async_set_cache("counter", 10) - assert await cache.async_increment("counter", 5) == 15 - - -async def test_async_increment_treats_non_int_cached_value_as_zero(cache): - await cache.async_set_cache("counter", "corrupt") - assert await cache.async_increment("counter", 9) == 9 - - -@pytest.fixture -def disk_cache(tmp_path): - import litellm - from litellm.caching import Cache - - litellm.cache = Cache(type="disk", disk_cache_dir=str(tmp_path)) - yield litellm.cache - litellm.cache = None - - -COMPLETION_MESSAGES = [{"role": "user", "content": "write a one sentence poem about: disk-cache-unit-key"}] - - -def test_completion_is_served_from_disk_cache_until_params_change(disk_cache): - import litellm - - first = litellm.completion( - "gpt-3.5-turbo", messages=COMPLETION_MESSAGES, caching=True, max_tokens=20, mock_response="first" - ) - repeat = litellm.completion( - "gpt-3.5-turbo", messages=COMPLETION_MESSAGES, caching=True, max_tokens=20, mock_response="second" - ) - changed = litellm.completion( - "gpt-3.5-turbo", messages=COMPLETION_MESSAGES, caching=True, temperature=0.5, mock_response="third" - ) - - assert (repeat.choices[0].message.content, repeat.id, repeat.created) == ( - first.choices[0].message.content, - first.id, - first.created, - ) - assert changed.choices[0].message.content == "third" - - -async def test_async_completion_is_served_from_disk_cache_until_params_change(disk_cache): - import litellm - - first = await litellm.acompletion( - "gpt-3.5-turbo", messages=COMPLETION_MESSAGES, caching=True, max_tokens=20, mock_response="first" - ) - await await_until(lambda: len(disk_cache.cache.disk_cache) == 1, message="the first completion is cached") - repeat = await litellm.acompletion( - "gpt-3.5-turbo", messages=COMPLETION_MESSAGES, caching=True, max_tokens=20, mock_response="second" - ) - changed = await litellm.acompletion( - "gpt-3.5-turbo", messages=COMPLETION_MESSAGES, caching=True, temperature=0.5, mock_response="third" - ) - - assert (repeat.choices[0].message.content, repeat.id, repeat.created) == ( - first.choices[0].message.content, - first.id, - first.created, - ) - assert changed.choices[0].message.content == "third" - - -async def test_embedding_is_served_from_disk_cache_until_params_change(disk_cache, respx_mock, monkeypatch): - import httpx - - import litellm - - monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) - input_text = ["hello disk-embedding-unit-key"] - embeddings_route = respx_mock.post("https://api.openai.com/v1/embeddings").mock( - return_value=httpx.Response( - 200, - json={ - "object": "list", - "data": [{"object": "embedding", "index": 0, "embedding": [0.1, 0.2, 0.3]}], - "model": "text-embedding-ada-002", - "usage": {"prompt_tokens": 2, "total_tokens": 2}, - }, - ) - ) - - await litellm.aembedding("openai/text-embedding-ada-002", input=input_text, caching=True, api_key="sk-unit") - await await_until(lambda: len(disk_cache.cache.disk_cache) == 1, message="the first embedding is cached") - cached = await litellm.aembedding( - "openai/text-embedding-ada-002", input=input_text, caching=True, api_key="sk-unit" - ) - assert cached._hidden_params["cache_hit"] is True - assert cached.data[0]["embedding"] == [0.1, 0.2, 0.3] - assert embeddings_route.call_count == 1 - - await litellm.aembedding( - "openai/text-embedding-ada-002", input=input_text, caching=True, user="charlie", api_key="sk-unit" - ) - assert embeddings_route.call_count == 2 diff --git a/tests/unit/proxy/proxy_server/test_proxy_config.py b/tests/unit/proxy/proxy_server/test_proxy_config.py index 6bd1468cfc1..49f9e99cf48 100644 --- a/tests/unit/proxy/proxy_server/test_proxy_config.py +++ b/tests/unit/proxy/proxy_server/test_proxy_config.py @@ -1659,7 +1659,7 @@ def test_ProxyConfig__init_cache_sets_litellm_cache(monkeypatch): def test_ProxyConfig__init_cache_invalid_params_raises(): pc = ProxyConfig() - with pytest.raises(AttributeError): + with pytest.raises(ValueError, match="Unsupported cache type 'this-cache-type-does-not-exist'"): pc._init_cache(cache_params={"type": "this-cache-type-does-not-exist"}) diff --git a/uv.lock b/uv.lock index b3e8ddc492c..f36fd3495a9 100644 --- a/uv.lock +++ b/uv.lock @@ -1955,15 +1955,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/1e/77/dc8c558f7593132cf8fefec57c4f60c83b16941c574ac5f619abb3ae7933/dill-0.4.1-py3-none-any.whl", hash = "sha256:1e1ce33e978ae97fcfcff5638477032b801c46c7c65cf717f95fbc2248f79a9d", size = 120019, upload-time = "2026-01-19T02:36:55.663Z" }, ] -[[package]] -name = "diskcache" -version = "5.6.3" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/3f/21/1c1ffc1a039ddcc459db43cc108658f32c57d271d7289a2794e401d0fdb6/diskcache-5.6.3.tar.gz", hash = "sha256:2c3a3fa2743d8535d832ec61c2054a1641f41775aa7c556758a109941e33e4fc", size = 67916, upload-time = "2023-08-31T06:12:00.316Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/3f/27/4570e78fc0bf5ea0ca45eb1de3818a23787af9b390c0b0a0033a1b8236f9/diskcache-5.6.3-py3-none-any.whl", hash = "sha256:5e31b2d5fbad117cc363ebaf6b689474db18a1f6438bc82358b024abd4c2ca19", size = 45550, upload-time = "2023-08-31T06:11:58.822Z" }, -] - [[package]] name = "distro" version = "1.9.0" @@ -4995,9 +4986,6 @@ dependencies = [ bedrock-realtime = [ { name = "aws-sdk-bedrock-runtime", extra = ["awscrt"], marker = "python_full_version >= '3.12'" }, ] -caching = [ - { name = "diskcache" }, -] cli = [ { name = "inquirerpy" }, { name = "keyring" }, @@ -5250,7 +5238,6 @@ requires-dist = [ { name = "cryptography", marker = "extra == 'proxy'", specifier = ">=49.0.0,<51.0" }, { name = "ddtrace", marker = "extra == 'proxy-runtime'", specifier = ">=4.8.2,<5.0" }, { name = "detect-secrets", marker = "extra == 'proxy-runtime'", specifier = ">=1.5.0,<2.0" }, - { name = "diskcache", marker = "extra == 'caching'", specifier = ">=5.6.3,<6.0" }, { name = "expression", marker = "extra == 'proxy'", specifier = ">=5.6.0,<6.0" }, { name = "fastapi", marker = "extra == 'proxy'", specifier = ">=0.136.3,<1.0" }, { name = "fastapi-sso", marker = "extra == 'proxy'", specifier = ">=0.19.0,<1.0" }, @@ -5343,7 +5330,7 @@ requires-dist = [ { name = "uvloop", marker = "sys_platform != 'win32' and extra == 'proxy'", specifier = ">=0.22.1,<1.0" }, { name = "websockets", marker = "extra == 'proxy'", specifier = ">=15.0.1,<16.0" }, ] -provides-extras = ["proxy", "cli", "extra-proxy", "utils", "caching", "mcp", "saml", "semantic-router", "mlflow", "grpc", "stt-vertex-chirp", "stt-nvidia-riva", "google", "bedrock-realtime", "proxy-runtime"] +provides-extras = ["proxy", "cli", "extra-proxy", "utils", "mcp", "saml", "semantic-router", "mlflow", "grpc", "stt-vertex-chirp", "stt-nvidia-riva", "google", "bedrock-realtime", "proxy-runtime"] [package.metadata.requires-dev] admin-mcp = [{ name = "litellm-admin-mcp", marker = "python_full_version >= '3.12'", url = "https://github.com/BerriAI/liteadmin-mcp/archive/d35ec9c19c117d4c50cc1eccf6ce1296aac25a1a.tar.gz" }] From d76db470f9e1b3bf3916f95fe94008abf08a2cf0 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Thu, 29 Oct 2026 18:06:28 -0700 Subject: [PATCH 14/17] test: update integration tests left stale by recent product changes (#49143) * test: update integration tests left stale by recent product changes * test: expect proxy Messages calls to stay on Python with Rust enabled --- ...uto_router_databricks_classifier_writes.py | 23 ++++- .../integration/mcp/test_mcp_access_matrix.py | 31 ++++-- tests/integration/mcp/test_mcp_credentials.py | 96 ++++++++++++------- .../integration/mcp/test_mcp_llm_endpoints.py | 35 ++++--- tests/integration/mcp/test_mcp_oauth_flows.py | 56 +++++++---- tests/integration/mcp/test_mcp_resilience.py | 66 +++++++++---- tests/integration/mcp/test_mcp_toolsets.py | 23 ++++- .../observability/test_guardrail_effects.py | 9 +- .../test_databricks_decisions_wire.py | 40 ++++++-- .../test_decisions_openai_format_wire.py | 13 ++- ...test_idle_upstream_keepalive_edges_wire.py | 5 +- ...t_idle_upstream_keepalive_override_wire.py | 2 +- ...e_upstream_reconnect_rust_messages_wire.py | 56 ----------- .../providers/test_inferenceapis_wire.py | 10 +- ...nses_bridge_tool_search_undeferred_wire.py | 33 +++++-- .../test_responses_bridge_tool_search_wire.py | 8 +- ...st_rust_anthropic_safeguards_beta_chaos.py | 2 +- ...est_rust_anthropic_safeguards_beta_wire.py | 18 ++-- .../test_vertex_credentials_source_wire.py | 4 +- .../golden/daily_activity_team_paginated.json | 29 ------ .../golden/daily_activity_user_paginated.json | 29 ------ .../integration/spend/test_shutdown_flush.py | 16 +++- .../chat_completions/bases/openai.py | 7 ++ .../chat_completions/cross_model/openai.py | 63 ++++++++++-- 24 files changed, 415 insertions(+), 259 deletions(-) delete mode 100644 tests/integration/providers/test_idle_upstream_reconnect_rust_messages_wire.py diff --git a/tests/integration/management/test_auto_router_databricks_classifier_writes.py b/tests/integration/management/test_auto_router_databricks_classifier_writes.py index af486d90a7e..d17ce947671 100644 --- a/tests/integration/management/test_auto_router_databricks_classifier_writes.py +++ b/tests/integration/management/test_auto_router_databricks_classifier_writes.py @@ -3,6 +3,7 @@ import json import threading import uuid from collections.abc import Callable, Mapping, Sequence +from pathlib import Path from queue import SimpleQueue from typing import Final @@ -64,6 +65,21 @@ _ROWS_QUERY: Final = ( ) +def _number(value: JsonValue) -> float: + assert isinstance(value, (int, float)) and not isinstance(value, bool), value + return float(value) + + +def _endpoint_cost() -> float: + prices: Final = object_value( + json.loads(Path("model_prices_and_context_window.json").read_text())[f"databricks/{_ENDPOINT}"] + ) + usage: Final = object_value(_ENDPOINT_ANSWER["usage"]) + return _number(usage["input_tokens"]) * _number(prices["input_cost_per_token"]) + _number( + usage["output_tokens"] + ) * _number(prices["output_cost_per_token"]) + + def _classifier(request: Request) -> Reply: answer: Final = _ENDPOINT_ANSWER if request.target.endswith("/invocations") else _AI_DECIDE_ANSWER return Reply(body=json.dumps(answer).encode()) @@ -221,11 +237,14 @@ def test_validate_names_each_databricks_classifier_misconfiguration(gateway: Gat @pytest.mark.parametrize( ("model", "target", "cost"), - [(_MODEL, _ROUTE, {}), (_ENDPOINT, f"/{_ENDPOINT}/invocations", {"classifier_cost": 0.0})], + [ + (_MODEL, _ROUTE, {}), + (_ENDPOINT, f"/{_ENDPOINT}/invocations", {"classifier_cost": pytest.approx(_endpoint_cost())}), + ], ids=["ai_decide", "serving_endpoint"], ) def test_test_routing_classifies_through_databricks_without_calling_the_tier_it_picked( - gateway: Gateway, model: str, target: str, cost: dict[str, JsonValue] + gateway: Gateway, model: str, target: str, cost: dict[str, object] ) -> None: prompt: Final = f"design a distributed cache {uuid.uuid4().hex}" with ( diff --git a/tests/integration/mcp/test_mcp_access_matrix.py b/tests/integration/mcp/test_mcp_access_matrix.py index e59f3e52a11..762fbee384d 100644 --- a/tests/integration/mcp/test_mcp_access_matrix.py +++ b/tests/integration/mcp/test_mcp_access_matrix.py @@ -181,7 +181,6 @@ def test_same_tool_name_on_two_servers_routes_by_prefix(gateway: Gateway) -> Non _PROBE: Final = "catalog-probe" _ECHO: Final = "catalog-echo" -_UNLISTED: Final = "" _GUARDRAIL_CODE: Final = ( "def apply_guardrail(inputs, request_data, input_type):\n" f' if "{_PROBE}" not in list(inputs.get("texts") or []):\n' @@ -217,13 +216,26 @@ def _echoed_description(outcome: Outcome) -> str: return found.group(1) +def _add_described_anew_per_fetch(described: str) -> ScriptedTool: + return ScriptedTool("add", lambda _: text_result("3"), description=lambda _: f"{described} {uuid.uuid4().hex}") + + +def _fetched(described: str) -> re.Pattern[str]: + return re.compile(rf"{re.escape(described)} [0-9a-f]{{32}}") + + +def _listed_description(listing: Outcome, described: str) -> str: + found: Final = _fetched(described).search(listing.raw) + assert found is not None, listing.raw + return found.group(0) + + @pytest.mark.parametrize("subject", SUBJECTS) def test_each_subjects_call_is_evaluated_only_against_the_catalog_its_own_listing_served( echo_rig: Gateway, subject: Subject ) -> None: described: Final = "Adds under grant " + uuid.uuid4().hex[:8] - tool: Final = ScriptedTool("add", lambda _: text_result("3"), description=described) - with scripted_peer(tool) as peer, echo_rig.scenario() as scenario: + with scripted_peer(_add_described_anew_per_fetch(described)) as peer, echo_rig.scenario() as scenario: group: Final = "grp" + uuid.uuid4().hex[:8] alias: Final = "cat" + uuid.uuid4().hex[:8] identity: Final = register_mcp(scenario, peer, alias, mcp_access_groups=[group]) @@ -235,8 +247,10 @@ def test_each_subjects_call_is_evaluated_only_against_the_catalog_its_own_listin cold: Final = _echoed_description(reach.call(f"{alias}-add", {"probe": _PROBE})) listed: Final = reach.list_tools() assert listed.ok and f"{alias}-add" in listed.tools, listed.raw + served: Final = _listed_description(listed, described) warm: Final = _echoed_description(reach.call(f"{alias}-add", {"probe": _PROBE})) - assert (cold, warm) == (_UNLISTED, described), (cold, warm) + assert _fetched(described).fullmatch(cold) and cold != served, (cold, served) + assert warm == served, (cold, served, warm) assert tool_calls(peer.drain()) == (), "a blocked probe reached the peer" @@ -244,8 +258,7 @@ def test_end_users_of_one_key_share_its_catalog_slot_because_the_identity_exclud echo_rig: Gateway, ) -> None: described: Final = "Adds for end users " + uuid.uuid4().hex[:8] - tool: Final = ScriptedTool("add", lambda _: text_result("3"), description=described) - with scripted_peer(tool) as peer, echo_rig.scenario() as scenario: + with scripted_peer(_add_described_anew_per_fetch(described)) as peer, echo_rig.scenario() as scenario: alias: Final = "eu" + uuid.uuid4().hex[:8] identity: Final = register_mcp(scenario, peer, alias) granted: Final = grant(scenario, "end_user", (identity,), (identity,)) @@ -254,10 +267,12 @@ def test_end_users_of_one_key_share_its_catalog_slot_because_the_identity_exclud echo_rig, granted.key, "mcp", alias, {"x-litellm-end-user-id": "integration-" + uuid.uuid4().hex[:10]} ) assert second.initialize().ok - assert _echoed_description(second.call(f"{alias}-add", {"probe": _PROBE})) == _UNLISTED + cold: Final = _echoed_description(second.call(f"{alias}-add", {"probe": _PROBE})) listed: Final = first.list_tools() assert listed.ok and f"{alias}-add" in listed.tools, listed.raw - assert _echoed_description(second.call(f"{alias}-add", {"probe": _PROBE})) == described, ( + served: Final = _listed_description(listed, described) + assert _fetched(described).fullmatch(cold) and cold != served, (cold, served) + assert _echoed_description(second.call(f"{alias}-add", {"probe": _PROBE})) == served, ( "the end-user header is intentionally not part of the catalog identity: one key, one slot" ) assert tool_calls(peer.drain()) == (), "a blocked probe reached the peer" diff --git a/tests/integration/mcp/test_mcp_credentials.py b/tests/integration/mcp/test_mcp_credentials.py index 63f1b0024c0..08dd5234b68 100644 --- a/tests/integration/mcp/test_mcp_credentials.py +++ b/tests/integration/mcp/test_mcp_credentials.py @@ -448,7 +448,6 @@ def test_deprecated_string_x_mcp_auth_lists_a_byok_server_for_a_key_without_a_us _PROBE: Final = "catalog-probe" _ECHO: Final = "catalog-echo" -_UNLISTED: Final = "" _GUARDRAIL_CODE: Final = ( "def apply_guardrail(inputs, request_data, input_type):\n" f' if "{_PROBE}" not in list(inputs.get("texts") or []):\n' @@ -488,6 +487,28 @@ _PROBE_ARGUMENTS: Final = {"probe": _PROBE} _HEADERS: Final = TypeAdapter(dict[str, str]) +def _add_described_anew_per_fetch_for(header: str) -> ScriptedTool: + return ScriptedTool( + "add", + lambda _: text_result("3"), + description=lambda headers: f"Adds for {headers.get(header, 'nobody')} at {uuid.uuid4().hex}", + ) + + +def _fetched_for(credential: str) -> re.Pattern[str]: + return re.compile(rf"Adds for {re.escape(credential)} at [0-9a-f]{{32}}") + + +def _listed_description(listing: Outcome, credential: str) -> str: + found: Final = _fetched_for(credential).search(listing.raw) + assert found is not None, listing.raw + return found.group(0) + + +def _sent_header(request: dict[str, object], name: str) -> str | None: + return _HEADERS.validate_python(request["headers"]).get(name) + + def _store_byok_credential(scenario: Scenario, identity: str, key: str, secret: str) -> None: stored: Final = scenario.gateway.client.post( f"/v1/mcp/server/{identity}/user-credential", json={"credential": secret}, headers={"x-litellm-api-key": key} @@ -499,7 +520,7 @@ def _store_byok_credential(scenario: Scenario, identity: str, key: str, secret: def test_rotating_the_credential_drops_the_callers_listing_until_it_lists_again(echo_rig: Gateway) -> None: - with mcp_peer() as peer, echo_rig.scenario() as scenario: + with scripted_peer(_add_described_anew_per_fetch_for("authorization")) as peer, echo_rig.scenario() as scenario: first: Final = "cred-" + uuid.uuid4().hex second: Final = "cred-" + uuid.uuid4().hex alias: Final = "rot" + uuid.uuid4().hex[:8] @@ -508,53 +529,62 @@ def test_rotating_the_credential_drops_the_callers_listing_until_it_lists_again( ) key: Final = scenario.key(object_permission={"mcp_servers": [identity]}) caller: Final = McpCaller(echo_rig, key, "mcp", alias) - assert caller.list_tools().ok - assert _echoed_description(caller.call(f"{alias}-add", _PROBE_ARGUMENTS)) == "Add two integers" + listed: Final = caller.list_tools() + assert listed.ok, listed.raw + served: Final = _listed_description(listed, f"Bearer {first}") + assert _echoed_description(caller.call(f"{alias}-add", _PROBE_ARGUMENTS)) == served rotated: Final = echo_rig.request( "PUT", "/v1/mcp/server", {"server_id": identity, "credentials": {"auth_value": second}} ) assert rotated.status_code == 202, rotated.text eventually( - lambda: _echoed_description(caller.call(f"{alias}-add", _PROBE_ARGUMENTS)), lambda seen: seen == _UNLISTED + lambda: _echoed_description(caller.call(f"{alias}-add", _PROBE_ARGUMENTS)), + lambda seen: _fetched_for(f"Bearer {second}").fullmatch(seen) is not None, ) peer.drain() - assert caller.list_tools().ok - assert _echoed_description(caller.call(f"{alias}-add", _PROBE_ARGUMENTS)) == "Add two integers" + relisting: Final = caller.list_tools() + assert relisting.ok, relisting.raw + served_again: Final = _listed_description(relisting, f"Bearer {second}") + assert _echoed_description(caller.call(f"{alias}-add", _PROBE_ARGUMENTS)) == served_again relisted: Final = _listings(peer) assert len(relisted) == 1, relisted - assert _header(relisted[0], b"authorization") == f"Bearer {second}".encode(), relisted[0]["headers"] + assert _sent_header(relisted[0], "authorization") == f"Bearer {second}", relisted[0]["headers"] def test_byok_callers_are_evaluated_against_their_own_listing_and_the_stored_secret_never_keys_the_slot( echo_rig: Gateway, ) -> None: - with mcp_peer() as peer, echo_rig.scenario() as scenario: + with scripted_peer(_add_described_anew_per_fetch_for("x-api-key")) as peer, echo_rig.scenario() as scenario: alias: Final = "byok" + uuid.uuid4().hex[:8] identity: Final = register_mcp(scenario, peer, alias, auth_type="api_key", is_byok=True) owner_key: Final = scenario.key(user_id=scenario.user(), object_permission={"mcp_servers": [identity]}) stranger_key: Final = scenario.key(user_id=scenario.user(), object_permission={"mcp_servers": [identity]}) owner_secret: Final = "byok-" + uuid.uuid4().hex + stranger_secret: Final = "byok-" + uuid.uuid4().hex replacement: Final = "byok-" + uuid.uuid4().hex _store_byok_credential(scenario, identity, owner_key, owner_secret) - _store_byok_credential(scenario, identity, stranger_key, "byok-" + uuid.uuid4().hex) + _store_byok_credential(scenario, identity, stranger_key, stranger_secret) owner: Final = McpCaller(echo_rig, owner_key, "mcp", alias) stranger: Final = McpCaller(echo_rig, stranger_key, "mcp", alias) peer.drain() - assert owner.list_tools().ok + listed: Final = owner.list_tools() + assert listed.ok, listed.raw listings: Final = _listings(peer) - assert [_header(item, b"x-api-key") for item in listings] == [owner_secret.encode()], listings + assert [_sent_header(item, "x-api-key") for item in listings] == [owner_secret], listings + served: Final = _listed_description(listed, owner_secret) own: Final = _echoed_description(owner.call(f"{alias}-add", _PROBE_ARGUMENTS)) other: Final = _echoed_description(stranger.call(f"{alias}-add", _PROBE_ARGUMENTS)) - assert (own, other) == ("Add two integers", _UNLISTED), (own, other) + assert own == served, (own, served) + assert _fetched_for(stranger_secret).fullmatch(other), (other, served) _store_byok_credential(scenario, identity, owner_key, replacement) - assert _echoed_description(owner.call(f"{alias}-add", _PROBE_ARGUMENTS)) == "Add two integers", ( - "the slot is keyed by the client-supplied header, never by the stored credential" - ) sent: Final = eventually( lambda: (owner.call(f"{alias}-add", ADD).ok, tool_calls(peer.drain())), - lambda value: any(_header(call, b"x-api-key") == replacement.encode() for call in value[1]), + lambda value: any(_sent_header(call, "x-api-key") == replacement for call in value[1]), ) assert sent[0], sent + assert _echoed_description(owner.call(f"{alias}-add", _PROBE_ARGUMENTS)) == served, ( + "the slot is keyed by the client-supplied header, never by the stored credential" + ) def test_callers_with_different_server_scoped_auth_headers_are_evaluated_against_their_own_listings( @@ -585,12 +615,7 @@ def test_callers_with_different_server_scoped_auth_headers_are_evaluated_against def test_deprecated_string_x_mcp_auth_callers_on_a_user_less_key_own_separate_listings(echo_rig: Gateway) -> None: - tool: Final = ScriptedTool( - "add", - lambda _: text_result("3"), - description=lambda headers: "Adds for " + headers.get("authorization", "nobody"), - ) - with scripted_peer(tool) as peer, echo_rig.scenario() as scenario: + with scripted_peer(_add_described_anew_per_fetch_for("authorization")) as peer, echo_rig.scenario() as scenario: alias: Final = "legacy" + uuid.uuid4().hex[:8] identity: Final = register_mcp(scenario, peer, alias, auth_type="bearer_token") key: Final = scenario.key(object_permission={"mcp_servers": [identity]}) @@ -598,28 +623,33 @@ def test_deprecated_string_x_mcp_auth_callers_on_a_user_less_key_own_separate_li second_token: Final = "second-" + uuid.uuid4().hex first: Final = McpCaller(echo_rig, key, "mcp", alias, {"x-mcp-auth": f"Bearer {first_token}"}) second: Final = McpCaller(echo_rig, key, "mcp", alias, {"x-mcp-auth": f"Bearer {second_token}"}) - assert _echoed_description(first.call(f"{alias}-add", _PROBE_ARGUMENTS)) == _UNLISTED + first_cold: Final = _echoed_description(first.call(f"{alias}-add", _PROBE_ARGUMENTS)) + assert _fetched_for(f"Bearer {first_token}").fullmatch(first_cold), first_cold peer.drain() - assert first.list_tools().ok + first_listing: Final = first.list_tools() + assert first_listing.ok, first_listing.raw listings: Final = _listings(peer) assert len(listings) == 1, listings - listed_with: Final = _HEADERS.validate_python(listings[0]["headers"]) - assert listed_with.get("authorization") == f"Bearer {first_token}", listed_with + assert _sent_header(listings[0], "authorization") == f"Bearer {first_token}", listings[0]["headers"] + first_served: Final = _listed_description(first_listing, f"Bearer {first_token}") warm: Final = eventually( lambda: _echoed_description(first.call(f"{alias}-add", _PROBE_ARGUMENTS)), - lambda seen: seen != _UNLISTED, + lambda seen: seen == first_served, ) - assert warm == f"Adds for Bearer {first_token}", warm - assert _echoed_description(second.call(f"{alias}-add", _PROBE_ARGUMENTS)) == _UNLISTED - assert second.list_tools().ok + assert warm == first_served != first_cold, (first_cold, first_served, warm) + second_cold: Final = _echoed_description(second.call(f"{alias}-add", _PROBE_ARGUMENTS)) + assert _fetched_for(f"Bearer {second_token}").fullmatch(second_cold), (second_cold, first_served) + second_listing: Final = second.list_tools() + assert second_listing.ok, second_listing.raw + second_served: Final = _listed_description(second_listing, f"Bearer {second_token}") seen: Final = eventually( lambda: ( _echoed_description(first.call(f"{alias}-add", _PROBE_ARGUMENTS)), _echoed_description(second.call(f"{alias}-add", _PROBE_ARGUMENTS)), ), - lambda pair: _UNLISTED not in pair, + lambda pair: pair == (first_served, second_served), ) - assert seen == (f"Adds for Bearer {first_token}", f"Adds for Bearer {second_token}"), seen + assert seen == (first_served, second_served), seen assert tool_calls(peer.drain()) == (), "a blocked probe reached the peer" diff --git a/tests/integration/mcp/test_mcp_llm_endpoints.py b/tests/integration/mcp/test_mcp_llm_endpoints.py index f751d1f8f4c..4b967802893 100644 --- a/tests/integration/mcp/test_mcp_llm_endpoints.py +++ b/tests/integration/mcp/test_mcp_llm_endpoints.py @@ -1,5 +1,6 @@ import asyncio import json +import re import uuid from collections.abc import Callable, Generator, Iterator, Mapping, Sequence from concurrent.futures import ThreadPoolExecutor @@ -1408,10 +1409,6 @@ REPORT: Final[tuple[str, dict[str, JsonValue]]] = ( "Write one report", {"type": "object", "properties": {"query": {"type": "string"}, "format": {}}}, ) -COLD: Final[tuple[str, dict[str, JsonValue]]] = ( - "", - {"type": "object", "properties": {}, "additionalProperties": False}, -) RELOAD_FAST: Final = {"PROXY_CONFIG_RELOAD_INTERVAL_SECONDS": "5"} CALL_ID: Final = "x-litellm-call-id" T = TypeVar("T") @@ -1577,12 +1574,14 @@ def _echoed(value: JsonValue) -> Echo: @contextmanager -def _bridge_rig(hooked: Hooked, bridge: Bridge) -> Generator[BridgeRig, None, None]: +def _bridge_rig( + hooked: Hooked, bridge: Bridge, lookup_description: str | Callable[[Mapping[str, str]], str] = LOOKUP[0] +) -> Generator[BridgeRig, None, None]: alias: Final = "brg" + uuid.uuid4().hex[:8] lookup: Final = ScriptedTool( "lookup", lambda params: text_result("found:" + json.dumps(JSON_OBJECT.validate_python(params)["arguments"])), - description=LOOKUP[0], + description=lookup_description, input_schema=LOOKUP[1], ) report: Final = ScriptedTool( @@ -1753,23 +1752,35 @@ def test_bridge_hook_sees_the_definition_of_the_tool_filtered_for_that_request( assert _spend_row(key, found.call_id)["status"] == "success" +def _fetched_lookup(description: JsonValue) -> bool: + return isinstance(description, str) and re.fullmatch(rf"{LOOKUP[0]} [0-9a-f]{{32}}", description) is not None + + @pytest.mark.parametrize("bridge", BRIDGES) def test_direct_call_after_bridge_only_discovery_stays_cold(hooked: Hooked, bridge: Bridge) -> None: - with _bridge_rig(hooked, bridge) as rig: + with _bridge_rig(hooked, bridge, lambda _: f"{LOOKUP[0]} {uuid.uuid4().hex}") as rig: key: Final = _bridge_key(rig) workers: Final = _both_workers(rig.hooked.proxy) bridged: Final = _ask(rig, key, HOOK_PROBE, [rig.mcp("lookup")], False, "sync") - assert _echoed(bridged.text) == _served(LOOKUP), bridged + discovered: Final = _echoed(bridged.text) + offered: Final = rig.upstream_by_prompt() + assert _fetched_lookup(discovered[0]) and discovered[1] == _served(LOOKUP)[1], bridged + assert set(offered) == {HOOK_PROBE} and all( + definitions == ((rig.tool("lookup"), *discovered),) for definitions in offered[HOOK_PROBE] + ), (discovered, offered) direct: Final = eventually( lambda: tuple( _on_worker(rig.hooked.proxy, lambda client: _direct_probe(rig, key, "lookup", client)) for _ in range(6) ), lambda seen: frozenset(pid for pid, _ in seen) == workers, ) - assert all(echo == COLD for _, echo in direct) and frozenset(pid for pid, _ in direct) == workers, ( - direct, - workers, - ) + assert ( + all( + _fetched_lookup(description) and description != discovered[0] and parameters == discovered[1] + for _, (description, parameters) in direct + ) + and frozenset(pid for pid, _ in direct) == workers + ), (direct, discovered, workers) assert rig.peer_calls() == () diff --git a/tests/integration/mcp/test_mcp_oauth_flows.py b/tests/integration/mcp/test_mcp_oauth_flows.py index 90a462472b2..0e174ac37aa 100644 --- a/tests/integration/mcp/test_mcp_oauth_flows.py +++ b/tests/integration/mcp/test_mcp_oauth_flows.py @@ -19,7 +19,7 @@ import jwt import pytest import yaml from cryptography.hazmat.primitives.asymmetric.rsa import RSAPrivateKey, generate_private_key -from integration._support.client import Gateway, eventually, gateway_from_environment +from integration._support.client import Gateway, Scenario, eventually, gateway_from_environment from integration._support.database import read_rows from integration._support.mcp import ( ENTRY_POINTS, @@ -786,13 +786,20 @@ def test_dcr_bridge_relays_client_registration_and_advertises_gateway_endpoints( assert "S256" in issuer.json()["code_challenge_methods_supported"] -def _ui_session_cookie(gateway: Gateway, user_id: str) -> dict[str, str]: - claims: Final = {"user_id": user_id, "login_method": "username_password", "exp": int(time.time()) + 600} - return {"token": jwt.encode(claims, gateway.key, algorithm="HS256")} +def _ui_session_cookie(scenario: Scenario, user_id: str) -> dict[str, str]: + session_key: Final = scenario.key(user_id=user_id, duration="10m") + claims: Final = { + "user_id": user_id, + "key": session_key, + "login_method": "username_password", + "exp": int(time.time()) + 600, + } + return {"token": jwt.encode(claims, scenario.gateway.key, algorithm="HS256")} def _gateway_session_sign_in( gateway: Gateway, + scenario: Scenario, user_id: str, resource: str | None = None, *, @@ -806,7 +813,7 @@ def _gateway_session_sign_in( assert registered.status_code in (200, 201), registered.text client_id: Final = str(_response_object(registered)["client_id"]) pkce: Final = _Pkce(secrets.token_urlsafe(48)) - cookies: Final = _ui_session_cookie(gateway, user_id) + cookies: Final = _ui_session_cookie(scenario, user_id) started: Final = gateway.client.get( authorization_endpoint, params={ @@ -843,8 +850,8 @@ def _gateway_session_sign_in( return _response_object(issued), client_id -def _gateway_session_bearer(gateway: Gateway, user_id: str, resource: str | None = None) -> str: - issued, _ = _gateway_session_sign_in(gateway, user_id, resource) +def _gateway_session_bearer(gateway: Gateway, scenario: Scenario, user_id: str, resource: str | None = None) -> str: + issued, _ = _gateway_session_sign_in(gateway, scenario, user_id, resource) return _issued_token(issued) @@ -887,7 +894,7 @@ def test_spec_client_discovers_the_aggregate_authorization_server_and_signs_in_t assert registered.status_code in (200, 201), registered.text client_id: Final = str(_response_object(registered)["client_id"]) pkce: Final = _Pkce(secrets.token_urlsafe(48)) - cookies: Final = _ui_session_cookie(gateway, member) + cookies: Final = _ui_session_cookie(scenario, member) started: Final = gateway.client.get( authorization_endpoint, params={ @@ -997,7 +1004,7 @@ def test_per_server_resource_scopes_the_session_bearer_to_the_named_server(gatew assert registered.status_code in (200, 201), registered.text client_id: Final = str(_response_object(registered)["client_id"]) pkce: Final = _Pkce(secrets.token_urlsafe(48)) - cookies: Final = _ui_session_cookie(gateway, member) + cookies: Final = _ui_session_cookie(scenario, member) started: Final = gateway.client.get( "/authorize/mcp-session", params={ @@ -1110,7 +1117,7 @@ def test_rotated_session_refresh_token_cannot_be_replayed_after_revocation(gatew alias: Final = "rt6" + uuid.uuid4().hex[:10] identity: Final = register_mcp(scenario, peer, alias) member: Final = scenario.member(scenario.team(object_permission={"mcp_servers": [identity]})) - issued, client_id = _gateway_session_sign_in(gateway, member) + issued, client_id = _gateway_session_sign_in(gateway, scenario, member) r1: Final = str(issued["refresh_token"]) a1: Final = str(issued["access_token"]) assert r1.startswith("llm_srefresh_"), r1 @@ -1207,7 +1214,7 @@ def test_lite_logout_revokes_the_cli_refresh_token_sent_the_way_the_cli_sends_it assert registered.status_code in (200, 201), registered.text client_id: Final = str(_response_object(registered)["client_id"]) pkce: Final = _Pkce(secrets.token_urlsafe(64)) - cookies: Final = _ui_session_cookie(gateway, member) + cookies: Final = _ui_session_cookie(scenario, member) consent: Final = gateway.client.get( str(contract["authorization_endpoint"]), params={ @@ -1362,6 +1369,7 @@ def test_gateway_session_bearer_of_a_team_member_is_served_the_team_toolset_on_i token_endpoint: Final = str(authorization_body["token_endpoint"]) issued, client_id = _gateway_session_sign_in( gateway, + scenario, member, resource, registration_endpoint=str(authorization_body["registration_endpoint"]), @@ -1413,7 +1421,7 @@ def test_resource_scoped_session_bearer_opens_a_team_toolset_inside_its_server_a inside_id: Final = create_toolset(scenario, ((inside_server, "add"),), toolset_name=inside_name) outside_id: Final = create_toolset(scenario, ((outside_server, "add"),), toolset_name=outside_name) member: Final = scenario.member(scenario.team(object_permission={"mcp_toolsets": [inside_id, outside_id]})) - bearer: Final = _gateway_session_bearer(gateway, member, resource=f"{_base(gateway)}/{inside}/mcp") + bearer: Final = _gateway_session_bearer(gateway, scenario, member, resource=f"{_base(gateway)}/{inside}/mcp") assert bearer.startswith("llm_session_"), bearer[:16] listed: Final = _toolset_rpc(gateway, bearer, inside_name, "tools/list", {}) assert listed.tools == (f"{inside}-add",), listed.raw @@ -1430,7 +1438,6 @@ def test_resource_scoped_session_bearer_opens_a_team_toolset_inside_its_server_a _PROBE: Final = "catalog-probe" _ECHO: Final = "catalog-echo" -_UNLISTED: Final = "" _GUARDRAIL_CODE: Final = ( "def apply_guardrail(inputs, request_data, input_type):\n" f' if "{_PROBE}" not in list(inputs.get("texts") or []):\n' @@ -1466,8 +1473,22 @@ def _echoed_description(outcome: Outcome) -> str: return found.group(1) +def _add_described_for_the_exchanged_subject(auth: AuthorizationServer) -> ScriptedTool: + def described(headers: Mapping[str, str]) -> str: + token: Final = headers.get("authorization", "").removeprefix("Bearer ") + with auth.lock: + grant: Final = auth.access_tokens.get(token, {}) + return "Add for " + grant.get("subject", "nobody") + + return ScriptedTool("add", lambda _: text_result("must not run"), description=described) + + def test_token_exchange_callers_with_different_subject_tokens_own_separate_listings(echo_rig: Gateway) -> None: - with mcp_peer() as peer, oauth_server() as auth, echo_rig.scenario() as scenario: + with ( + oauth_server() as auth, + scripted_peer(_add_described_for_the_exchanged_subject(auth)) as peer, + echo_rig.scenario() as scenario, + ): alias: Final = "te" + uuid.uuid4().hex[:8] identity: Final = register_mcp( scenario, @@ -1479,19 +1500,20 @@ def test_token_exchange_callers_with_different_subject_tokens_own_separate_listi ) key: Final = scenario.key(object_permission={"mcp_servers": [identity]}) first_subject: Final = "subject-" + uuid.uuid4().hex + second_subject: Final = "subject-" + uuid.uuid4().hex first: Final = McpCaller(echo_rig, key, "mcp", alias, {"Authorization": f"Bearer {first_subject}"}) - second: Final = McpCaller(echo_rig, key, "mcp", alias, {"Authorization": "Bearer subject-" + uuid.uuid4().hex}) + second: Final = McpCaller(echo_rig, key, "mcp", alias, {"Authorization": f"Bearer {second_subject}"}) auth.drain() assert first.list_tools().ok assert [request["subject_token"] for request in auth.token_requests()] == [first_subject] probe: Final = {"probe": _PROBE} own: Final = _echoed_description(first.call(f"{alias}-add", probe)) other: Final = _echoed_description(second.call(f"{alias}-add", probe)) - assert (own, other) == ("Add two integers", _UNLISTED), ( + assert (own, other) == (f"Add for exchanged:{first_subject}", f"Add for exchanged:{second_subject}"), ( "the caller bearer is part of the identity on a token-exchange server: one subject, one slot" ) assert second.list_tools().ok - assert _echoed_description(second.call(f"{alias}-add", probe)) == "Add two integers" + assert _echoed_description(second.call(f"{alias}-add", probe)) == f"Add for exchanged:{second_subject}" assert tool_calls(peer.drain()) == (), "a blocked probe reached the peer" diff --git a/tests/integration/mcp/test_mcp_resilience.py b/tests/integration/mcp/test_mcp_resilience.py index c335012d787..145994f1b29 100644 --- a/tests/integration/mcp/test_mcp_resilience.py +++ b/tests/integration/mcp/test_mcp_resilience.py @@ -1,4 +1,5 @@ import itertools +import re import uuid from collections.abc import Mapping from concurrent.futures import ThreadPoolExecutor @@ -37,6 +38,7 @@ pytestmark: Final = pytest.mark.shares_worker_state _ECHO: Final = "catalog-echo:" _PROBE: Final = "catalog-probe" _DESCRIPTION: Final = "Look up one record" +_FETCHED: Final = re.compile(rf"{_DESCRIPTION} [0-9a-f]{{32}}") _GUARDRAIL_CODE: Final = ( "def apply_guardrail(inputs, request_data, input_type):\n" ' texts = list(inputs.get("texts") or [])\n' @@ -69,6 +71,20 @@ class _RpcReply(BaseModel): result: _Result +class _ListedTool(BaseModel): + name: str + description: str | None = None + + +class _ListedTools(BaseModel): + tools: tuple[_ListedTool, ...] + + +class _ListReply(BaseModel): + id: int + result: _ListedTools + + class _SessionsReport(BaseModel): worker_pid: int @@ -207,9 +223,13 @@ def test_peer_restart_on_the_same_url_is_picked_up_without_gateway_restart(gatew assert len(tool_calls(replacement.drain())) >= 1 -def _rpc_reply(raw: str) -> _RpcReply: +def _rpc_payload(raw: str) -> str: data: Final = tuple(line[5:].strip() for line in raw.splitlines() if line.startswith("data:")) - return _RpcReply.model_validate_json(data[-1] if data else raw) + return data[-1] if data else raw + + +def _rpc_reply(raw: str) -> _RpcReply: + return _RpcReply.model_validate_json(_rpc_payload(raw)) def _call_params(call: Mapping[str, object]) -> Mapping[str, object]: @@ -220,9 +240,18 @@ def _call_nonce(call: Mapping[str, object]) -> str: return _STRINGS.validate_python(_call_params(call)["arguments"])["nonce"] -def _listed(caller: McpCaller, name: str) -> None: +def _listed(caller: McpCaller, name: str) -> Outcome: listing: Final = eventually(caller.list_tools, lambda outcome: name in outcome.tools, seconds=45) assert listing.error is None, (caller.gateway.client.base_url, listing.raw) + return listing + + +def _listed_description(caller: McpCaller, name: str) -> str: + listing: Final = _listed(caller, name) + tools: Final = _ListReply.model_validate_json(_rpc_payload(listing.raw)).result.tools + described: Final = next(tool.description for tool in tools if tool.name == name) + assert described is not None and _FETCHED.fullmatch(described), listing.raw + return described @dataclass(frozen=True, slots=True) @@ -251,15 +280,18 @@ def _probed_description(worker: _Worker, name: str) -> str | None: return _ECHOED.validate_json(carrier.split(_ECHO, 1)[1])["description"] -def _catalog_is_cold(worker: _Worker, name: str) -> bool: - return not _probed_description(worker, name) +def _catalog_is_cold(worker: _Worker, name: str, listings: tuple[str, ...]) -> bool: + probed: Final = _probed_description(worker, name) + return probed is not None and _FETCHED.fullmatch(probed) is not None and probed not in listings @pytest.mark.timeout(600) def test_worker_restart_cools_its_listed_catalog_while_the_sibling_worker_keeps_serving( gateway: Gateway, echo_config: Path, tmp_path: Path ) -> None: - tool: Final = ScriptedTool("lookup", lambda _: text_result("found"), description=_DESCRIPTION) + tool: Final = ScriptedTool( + "lookup", lambda _: text_result("found"), description=lambda _: f"{_DESCRIPTION} {uuid.uuid4().hex}" + ) with scripted_peer(tool) as peer, gateway.scenario() as scenario: alias: Final = "cold" + uuid.uuid4().hex[:8] identity: Final = register_mcp(scenario, peer, alias) @@ -272,21 +304,23 @@ def test_worker_restart_cools_its_listed_catalog_while_the_sibling_worker_keeps_ assert first.pid != sibling.pid _served(first, name, "first-unlisted") _served(sibling, name, "sibling-unlisted") - assert _catalog_is_cold(first, name) and _catalog_is_cold(sibling, name) - _listed(first.caller, name) - assert _probed_description(first, name) == _DESCRIPTION - assert _catalog_is_cold(sibling, name), "a listing on one worker warmed its sibling" - _listed(sibling.caller, name) - assert _probed_description(sibling, name) == _DESCRIPTION + assert _catalog_is_cold(first, name, ()) and _catalog_is_cold(sibling, name, ()) + first_listing: Final = _listed_description(first.caller, name) + assert _probed_description(first, name) == first_listing + assert _catalog_is_cold(sibling, name, (first_listing,)), "a listing on one worker warmed its sibling" + sibling_listing: Final = _listed_description(sibling.caller, name) + assert _probed_description(sibling, name) == sibling_listing _served(sibling, name, "sibling-alone") with owned_proxy_process(gateway, tmp_path / "restarted", {}, config=echo_config) as restarted_proxy: restarted: Final = _worker(restarted_proxy.gateway, key, alias) assert restarted.pid not in (first.pid, sibling.pid) - assert _catalog_is_cold(restarted, name), "a restarted worker kept the old process's catalog" - assert _probed_description(sibling, name) == _DESCRIPTION + assert _catalog_is_cold(restarted, name, (first_listing, sibling_listing)), ( + "a restarted worker kept the old process's catalog" + ) + assert _probed_description(sibling, name) == sibling_listing _served(restarted, name, "restarted-unlisted") - _listed(restarted.caller, name) - assert _probed_description(restarted, name) == _DESCRIPTION + restarted_listing: Final = _listed_description(restarted.caller, name) + assert _probed_description(restarted, name) == restarted_listing calls: Final = tool_calls(peer.drain()) assert [_call_nonce(call) for call in calls] == [ "first-unlisted", diff --git a/tests/integration/mcp/test_mcp_toolsets.py b/tests/integration/mcp/test_mcp_toolsets.py index f275bd0e427..fc040810970 100644 --- a/tests/integration/mcp/test_mcp_toolsets.py +++ b/tests/integration/mcp/test_mcp_toolsets.py @@ -538,7 +538,6 @@ def test_owned_proxy_loss_retains_exit_status_and_first_http_failure(gateway: Ga _PROBE: Final = "catalog-probe" _ECHO: Final = "catalog-echo" -_UNLISTED: Final = "" _GUARDRAIL_CODE: Final = ( "def apply_guardrail(inputs, request_data, input_type):\n" f' if "{_PROBE}" not in list(inputs.get("texts") or []):\n' @@ -574,9 +573,21 @@ def _echoed_description(outcome: Outcome) -> str: return found.group(1) +def _fetched(described: str) -> re.Pattern[str]: + return re.compile(rf"{re.escape(described)} [0-9a-f]{{32}}") + + +def _listed_description(listing: Outcome, described: str) -> str: + found: Final = _fetched(described).search(listing.raw) + assert found is not None, listing.raw + return found.group(0) + + def test_a_team_keys_toolset_route_listing_feeds_its_own_calls_but_not_a_team_mates(echo_rig: Gateway) -> None: described: Final = "Adds for the team " + uuid.uuid4().hex[:8] - tool: Final = ScriptedTool("add", lambda _: text_result("9"), description=described) + tool: Final = ScriptedTool( + "add", lambda _: text_result("9"), description=lambda _: f"{described} {uuid.uuid4().hex}" + ) with scripted_peer(tool) as peer, echo_rig.scenario() as scenario: alias: Final = "lit6029echo" + uuid.uuid4().hex[:6] server_id: Final = register_mcp(scenario, peer, alias) @@ -587,10 +598,14 @@ def test_a_team_keys_toolset_route_listing_feeds_its_own_calls_but_not_a_team_ma _assert_team_grants_only(echo_rig, team_id, key, granted_id) listed: Final = _toolset_rpc(echo_rig, _bearer(key), granted_name, "tools/list", {}) assert listed.ok and listed.tools == (f"{alias}-add",), listed.raw + served: Final = _listed_description(listed, described) probe: Final[dict[str, object]] = {"name": f"{alias}-add", "arguments": {"probe": _PROBE}} own: Final = _echoed_description(_toolset_rpc(echo_rig, _bearer(key), granted_name, "tools/call", probe)) mate: Final = _echoed_description(_toolset_rpc(echo_rig, _bearer(team_mate), granted_name, "tools/call", probe)) - assert (own, mate) == (described, _UNLISTED), ( - "the slot is keyed by the hashed key, so a team-mate that never listed is handed nothing" + assert own == served, (own, served) + assert _fetched(described).fullmatch(mate) and mate != served, ( + "the slot is keyed by the hashed key, so a team-mate that never listed is not handed the key's listing", + mate, + served, ) assert tool_calls(peer.drain()) == (), "a blocked probe reached the peer" diff --git a/tests/integration/observability/test_guardrail_effects.py b/tests/integration/observability/test_guardrail_effects.py index 537eb2bbf69..4f5c04bddcc 100644 --- a/tests/integration/observability/test_guardrail_effects.py +++ b/tests/integration/observability/test_guardrail_effects.py @@ -1085,7 +1085,8 @@ def test_request_selected_mcp_guardrail_blocks_direct_and_virtual_calls(gateway: mcp_peer() as peer, candidate.scenario() as scenario, ): - identity = register_mcp(scenario, peer, "guardrail" + uuid.uuid4().hex) + server = "guardrail" + uuid.uuid4().hex + identity = register_mcp(scenario, peer, server) permission = {"mcp_servers": [identity], "mcp_tool_search_enabled": True} key = scenario.key(object_permission=permission) key_selected = scenario.key(object_permission=permission, guardrails=[guardrail]) @@ -1116,8 +1117,10 @@ def test_request_selected_mcp_guardrail_blocks_direct_and_virtual_calls(gateway: ) calls = tuple(item for item in peer.drain() if item["body"].get("method") == "tools/call") if expected is None: - assert response.status_code == 400, response.text - assert "integration resolved add denied" in response.text, response.text + assert response.status_code == 403, response.text + assert response.json() == { + "detail": {"error": f"Tool add is unavailable after MCP guardrail checks for server {server}."} + }, response.text assert calls == (), "pre-call denial must prevent upstream execution" else: assert response.status_code == 200, response.text diff --git a/tests/integration/providers/test_databricks_decisions_wire.py b/tests/integration/providers/test_databricks_decisions_wire.py index f105092e9a3..7b54895adaf 100644 --- a/tests/integration/providers/test_databricks_decisions_wire.py +++ b/tests/integration/providers/test_databricks_decisions_wire.py @@ -1,3 +1,4 @@ +import json import threading import time import uuid @@ -67,7 +68,12 @@ _NON_BARE_NAMES: Final[tuple[tuple[str, str], ...]] = ( ) _DATABRICKS_ERRORS: Final[tuple[tuple[str, str, int, dict[str, JsonValue]], ...]] = ( (_MODEL, _ROUTE, 400, {"error_code": "BAD_REQUEST", "message": "Invalid input: questions must not be empty"}), - (_MODEL, _ROUTE, 403, {"error_code": "PERMISSION_DENIED", "message": "ai_decide is not enabled for this workspace"}), + ( + _MODEL, + _ROUTE, + 403, + {"error_code": "PERMISSION_DENIED", "message": "ai_decide is not enabled for this workspace"}, + ), ( _ENDPOINT, f"/{_ENDPOINT}/invocations", @@ -89,6 +95,15 @@ def _number(value: JsonValue) -> float: return float(value) +def _endpoint_spend() -> float: + prices: Final = object_value( + json.loads(Path("model_prices_and_context_window.json").read_text())[f"databricks/{_ENDPOINT}"] + ) + return _number(_USAGE["input_tokens"]) * _number(prices["input_cost_per_token"]) + _number( + _USAGE["output_tokens"] + ) * _number(prices["output_cost_per_token"]) + + def _register(scenario: Scenario, body: dict[str, JsonValue], *, status: int = 200) -> ScenarioHandle: handle: Final = register_scenario( f"databricks-{uuid.uuid4().hex[:12]}", JsonResponse(content_type="application/json", body=body, status=status) @@ -300,9 +315,14 @@ def test_an_openai_format_request_at_v1_decisions_reaches_the_serving_endpoint_a row["model_group"], row["api_base"], _number(row["spend"]), - ) == ("success", "adecisions", "databricks", deployment, f"{handle.api_base()}/{_ENDPOINT}/invocations", 0.0), ( - row - ) + ) == ( + "success", + "adecisions", + "databricks", + deployment, + f"{handle.api_base()}/{_ENDPOINT}/invocations", + pytest.approx(_endpoint_spend(), rel=1e-9), + ), row def test_an_openai_format_request_at_v1_decisions_reaches_the_ai_decide_route_without_a_model_field( @@ -381,6 +401,7 @@ def test_a_noul_probability_reaches_the_caller_as_the_system_one_noul_answer(gat def test_a_wildcard_deployment_sends_each_request_to_the_named_endpoint_and_bills_each_once(gateway: Gateway) -> None: + expected_spend: Final = _endpoint_spend() with gateway.scenario() as scenario: handle: Final = _register(scenario, _ANSWER_BODY) prefix: Final = _wildcard_deployment(scenario, api_base=handle.api_base(), api_key=_API_KEY) @@ -408,9 +429,14 @@ def test_a_wildcard_deployment_sends_each_request_to_the_named_endpoint_and_bill row["model_group"], row["api_base"], _number(row["spend"]), - ) == ("success", "asystemone", "databricks", model, f"{handle.api_base()}/{_ENDPOINT}/invocations", 0.0), ( - row - ) + ) == ( + "success", + "asystemone", + "databricks", + model, + f"{handle.api_base()}/{_ENDPOINT}/invocations", + pytest.approx(expected_spend, rel=1e-9), + ), row def test_a_wildcard_deployment_sends_each_ai_decide_request_to_the_route_and_logs_each_once(gateway: Gateway) -> None: diff --git a/tests/integration/providers/test_decisions_openai_format_wire.py b/tests/integration/providers/test_decisions_openai_format_wire.py index 2f82561d1b1..0c40fe06199 100644 --- a/tests/integration/providers/test_decisions_openai_format_wire.py +++ b/tests/integration/providers/test_decisions_openai_format_wire.py @@ -187,7 +187,16 @@ _PROVIDERS: Final = ( "cloudflare/@cf/cloudflare/clef", takes_images=True, ), - _Provider("hosted_vllm", "hosted_vllm/Qwen/Qwen3-0.6B", "/v1/systemone", "Qwen/Qwen3-0.6B", None, False, None), + _Provider( + "hosted_vllm", + "hosted_vllm/Qwen/Qwen3-0.6B", + "/v1/systemone", + "Qwen/Qwen3-0.6B", + None, + False, + None, + takes_images=True, + ), _Provider( "databricks", "databricks/ai_decide", @@ -205,7 +214,7 @@ _PROVIDERS: Final = ( "databricks-openjev-qwen35-4b", _API_KEY, False, - None, + "databricks/databricks-openjev-qwen35-4b", ), _Provider( "azure_ai", "azure_ai/decision-1", "/providers/microsoft/v1/systemone", "decision-1", _API_KEY, False, None diff --git a/tests/integration/providers/test_idle_upstream_keepalive_edges_wire.py b/tests/integration/providers/test_idle_upstream_keepalive_edges_wire.py index fc27f0f6e1e..a5628d7764c 100644 --- a/tests/integration/providers/test_idle_upstream_keepalive_edges_wire.py +++ b/tests/integration/providers/test_idle_upstream_keepalive_edges_wire.py @@ -33,15 +33,14 @@ pytestmark: Final = pytest.mark.timeout(int(2 * graceful_stop_seconds() + 120 + REUSE_ROWS: Final = ( Row("reuse-chat-openai-sdk-sync", openai_deployment, chat_sdk), - Row("reuse-messages-rust-anthropic-sdk-sync", anthropic_deployment, messages_sdk, rust=True), + Row("reuse-messages-anthropic-sdk-sync", anthropic_deployment, messages_sdk), ) QUIET_ROWS: Final = ( Row("quiet-chat-openai-sdk-async-stream", openai_deployment, chat_sdk_stream, respond=quiet_stream_answer), Row( - "quiet-messages-rust-anthropic-sdk-async-stream", + "quiet-messages-anthropic-sdk-async-stream", anthropic_deployment, messages_sdk_stream, - rust=True, respond=quiet_stream_answer, ), ) diff --git a/tests/integration/providers/test_idle_upstream_keepalive_override_wire.py b/tests/integration/providers/test_idle_upstream_keepalive_override_wire.py index 8b5a69d729e..ddcf1ed7741 100644 --- a/tests/integration/providers/test_idle_upstream_keepalive_override_wire.py +++ b/tests/integration/providers/test_idle_upstream_keepalive_override_wire.py @@ -27,7 +27,7 @@ pytestmark: Final = pytest.mark.timeout(int(2 * graceful_stop_seconds() + 120 + ROWS: Final = ( Row("override-chat-openai-sdk-sync", openai_deployment, chat_sdk), - Row("override-messages-rust-anthropic-sdk-sync", anthropic_deployment, messages_sdk, rust=True), + Row("override-messages-anthropic-sdk-sync", anthropic_deployment, messages_sdk), Row("override-ocr-mistral-rust-bridge", ocr_deployment, ocr, rust=True), ) diff --git a/tests/integration/providers/test_idle_upstream_reconnect_rust_messages_wire.py b/tests/integration/providers/test_idle_upstream_reconnect_rust_messages_wire.py deleted file mode 100644 index 8d92fb609e9..00000000000 --- a/tests/integration/providers/test_idle_upstream_reconnect_rust_messages_wire.py +++ /dev/null @@ -1,56 +0,0 @@ -from collections.abc import Iterator -from typing import Final - -import pytest -from integration._support.client import Gateway -from integration._support.idle_gateway import Drop -from integration._support.process import graceful_stop_seconds -from integration.providers._idle_upstream_support import ( - IDLE_SECONDS, - Row, - StalePools, - answered, - anthropic_deployment, - expected_answers, - landed, - landed_once_undropped, - messages_sdk, - messages_sdk_stream, - served_by_rust, - spend_statuses, - stale_pools, -) - -pytestmark: Final = pytest.mark.timeout(int(2 * graceful_stop_seconds() + 120 + IDLE_SECONDS)) - -ROWS: Final = ( - Row("messages-rust-anthropic-sdk-sync", anthropic_deployment, messages_sdk, rust=True), - Row("messages-rust-anthropic-sdk-async-stream", anthropic_deployment, messages_sdk_stream, rust=True), - Row( - "messages-rust-upstream-resets-stale-connection", - anthropic_deployment, - messages_sdk, - drop=Drop.RESET, - rust=True, - ), -) - - -@pytest.fixture(scope="module") -def pools(session_gateway: Gateway, tmp_path_factory: pytest.TempPathFactory) -> Iterator[StalePools]: - with stale_pools( - session_gateway, tmp_path_factory.mktemp("idle-upstream-rust"), ROWS, {"LITELLM_RUST": "1"} - ) as value: - yield value - - -@pytest.mark.parametrize("name", [row.name for row in ROWS]) -def test_rust_messages_after_upstream_idle_past_keepalive_open_a_fresh_connection_on_each_worker( - pools: StalePools, name: str -) -> None: - row: Final = pools.rows[name] - sents, arrivals = pools.probe(name) - assert answered(sents) == expected_answers(row, sents), sents - assert landed(arrivals) == landed_once_undropped(sents), arrivals - assert served_by_rust(sents) == (row.rust,) * len(sents), sents - assert spend_statuses(sents) == ("success",) * len(sents), sents diff --git a/tests/integration/providers/test_inferenceapis_wire.py b/tests/integration/providers/test_inferenceapis_wire.py index 8288c59ad5d..5f400d05957 100644 --- a/tests/integration/providers/test_inferenceapis_wire.py +++ b/tests/integration/providers/test_inferenceapis_wire.py @@ -30,6 +30,7 @@ _WEATHER_TOOL: Final[JsonValue] = { }, } _COST_MAP_PATH: Final = Path(__file__).resolve().parents[3] / "model_prices_and_context_window.json" +_ENDPOINT_SUPPORT_PATH: Final = Path(__file__).resolve().parents[3] / "litellm/provider_endpoints_support_backup.json" _COST_MAP: Final = TypeAdapter(dict[str, dict[str, JsonValue]]) _JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) _JSON_LIST: Final = TypeAdapter(list[JsonValue]) @@ -512,6 +513,12 @@ def _lists_inferenceapis(endpoint: Mapping[str, JsonValue]) -> bool: return any(object_value(provider)["slug"] == "inferenceapis" for provider in providers) +def _bundled_inferenceapis_endpoints() -> frozenset[str]: + providers: Final = object_value(_JSON_OBJECT.validate_json(_ENDPOINT_SUPPORT_PATH.read_bytes())["providers"]) + flags: Final = object_value(object_value(providers["inferenceapis"])["endpoints"]) + return frozenset(key for key, supported in flags.items() if supported) + + def test_public_routes_list_the_provider_with_its_endpoints_and_fields(gateway: Gateway) -> None: providers: Final = gateway.request("GET", "/public/providers") assert providers.status_code == 200, providers.text @@ -538,7 +545,8 @@ def test_public_routes_list_the_provider_with_its_endpoints_and_fields(gateway: for endpoint in _JSON_LIST.validate_python(_JSON_OBJECT.validate_json(endpoints_response.content)["endpoints"]) ) listed: Final = frozenset(str(endpoint["key"]) for endpoint in endpoints if _lists_inferenceapis(endpoint)) - assert listed == {"chat_completions", "messages", "responses"} + assert {"chat_completions", "messages", "responses"} <= listed, listed + assert listed == _bundled_inferenceapis_endpoints() def test_openai_and_anthropic_sdk_clients_reach_the_vendor(gateway: Gateway) -> None: diff --git a/tests/integration/providers/test_responses_bridge_tool_search_undeferred_wire.py b/tests/integration/providers/test_responses_bridge_tool_search_undeferred_wire.py index a7fc2b379ea..f6037dd4835 100644 --- a/tests/integration/providers/test_responses_bridge_tool_search_undeferred_wire.py +++ b/tests/integration/providers/test_responses_bridge_tool_search_undeferred_wire.py @@ -113,10 +113,8 @@ def _namespace_tool() -> dict[str, JsonValue]: } -def _search_tools(route: Route, search: Search) -> tuple[dict[str, JsonValue], ...]: - if search == "absent": - return () - return (_tool_search(execution="client"),) if route == "anthropic" else (_tool_search(),) +def _search_tools(search: Search) -> tuple[dict[str, JsonValue], ...]: + return () if search == "absent" else (_tool_search(),) def _gemini_frame(marker: str) -> dict[str, JsonValue]: @@ -406,7 +404,6 @@ _UNDEFERRING: Final[tuple[tuple[Route, Search], ...]] = ( _KEEPING: Final[tuple[tuple[Route, Search], ...]] = ( ("unlisted_claude", "absent"), ("anthropic", "absent"), - ("anthropic", "dropped"), ("azure_ai", "absent"), ) @@ -428,7 +425,7 @@ def test_without_a_search_tool_on_the_wire_every_tool_arrives_without_defer_load call: Final = _Call(route, stream, uuid.uuid4().hex) with wire_server(_vendor(route)) as wire, gateway.scenario() as scenario: model: Final = _deployment(scenario, route, wire.url) - tools: Final = (*_search_tools(route, search), _function_tool(), _namespace_tool()) + tools: Final = (*_search_tools(search), _function_tool(), _namespace_tool()) response: Final = _raw(gateway, _responses_body(model, call.marker, tools, stream)) _assert_served(response, call.marker, stream) request, body = _only_request(wire) @@ -442,7 +439,7 @@ def test_a_claude_route_sent_no_server_search_tool_keeps_each_tools_own_defer_lo call: Final = _Call(route, stream, uuid.uuid4().hex) with wire_server(_vendor(route)) as wire, gateway.scenario() as scenario: model: Final = _deployment(scenario, route, wire.url) - tools: Final = (*_search_tools(route, search), _function_tool(), _namespace_tool(), _loaded_tool()) + tools: Final = (*_search_tools(search), _function_tool(), _namespace_tool(), _loaded_tool()) response: Final = _raw(gateway, _responses_body(model, call.marker, tools, stream)) _assert_served(response, call.marker, stream) request, body = _only_request(wire) @@ -456,6 +453,26 @@ def test_a_claude_route_sent_no_server_search_tool_keeps_each_tools_own_defer_lo ], sent +@pytest.mark.parametrize("stream", [False, True], ids=["sync", "stream"]) +def test_a_client_search_tool_on_anthropic_goes_out_as_a_function_and_holds_back_the_deferred_tools( + gateway: Gateway, stream: bool +) -> None: + marker: Final = uuid.uuid4().hex + with wire_server(_claude) as wire, gateway.scenario() as scenario: + model: Final = _deployment(scenario, "anthropic", wire.url) + tools: Final = (_tool_search(execution="client"), _function_tool(), _namespace_tool(), _loaded_tool()) + response: Final = _raw(gateway, _responses_body(model, marker, tools, stream)) + _assert_served(response, marker, stream) + request, body = _only_request(wire) + _assert_target("anthropic", stream, request, body) + sent: Final = _ITEMS.validate_python(body["tools"]) + assert [(tool["name"], tool.get("defer_loading")) for tool in sent] == [ + ("tool_search", None), + ("send_email", None), + ], sent + assert "defer_loading" not in request.body.decode(), body + + def test_a_false_defer_loading_never_reaches_an_openai_compatible_wire_as_a_key(gateway: Gateway) -> None: call: Final = _Call("mistral", False, uuid.uuid4().hex) with wire_server(_mistral) as wire, gateway.scenario() as scenario: @@ -682,7 +699,7 @@ def test_a_streamed_answer_closes_its_message_item_with_every_streamed_delta(gat async def _send(client: httpx.AsyncClient, key: str, model: str, call: _Call) -> _Served: - tools: Final = (*_search_tools(call.route, "dropped"), _function_tool(), _namespace_tool()) + tools: Final = (*_search_tools("dropped"), _function_tool(), _namespace_tool()) body: Final = _responses_body(model, call.marker, tools, call.stream) async with client.stream( "POST", "/v1/responses", json=body, headers={"Authorization": f"Bearer {key}"} diff --git a/tests/integration/providers/test_responses_bridge_tool_search_wire.py b/tests/integration/providers/test_responses_bridge_tool_search_wire.py index 03cd4ce9dc4..57d9655b7d5 100644 --- a/tests/integration/providers/test_responses_bridge_tool_search_wire.py +++ b/tests/integration/providers/test_responses_bridge_tool_search_wire.py @@ -835,15 +835,17 @@ def test_a_claude_the_cost_map_does_not_list_gets_the_tool_dropped_on_the_anthro _assert_dropped_on_wire(body, _DEFERRED_NAMES) -def test_client_execution_drops_the_tool_search_tool(gateway: Gateway) -> None: +def test_client_execution_offers_tool_search_as_a_function_and_holds_back_the_deferred_tools(gateway: Gateway) -> None: marker: Final = uuid.uuid4().hex with wire_server(_Vendor().respond) as wire, gateway.scenario() as scenario: model: Final = _deployment(scenario, "anthropic", wire.url, gateway.upstream_url) tools: Final = (_tool_search(execution="client"), _function_tool(defer_loading=False), _namespace_tool()) answer: Final = _raw_answer(gateway, _responses_body(model, marker, tools, False)) assert _output_types(answer.payload) == ["message", "function_call"], answer - _, body = _only_request(wire) - _assert_dropped_on_wire(body, _DEFERRED_NAMES) + request, body = _only_request(wire) + assert request.target == "/v1/messages", request.target + _assert_dropped_on_wire(body, ("tool_search", "get_weather")) + assert "defer_loading" not in request.body.decode(), body @pytest.mark.parametrize("drop_params", [False, True], ids=["refused", "dropped"]) diff --git a/tests/integration/providers/test_rust_anthropic_safeguards_beta_chaos.py b/tests/integration/providers/test_rust_anthropic_safeguards_beta_chaos.py index f90d68c1a63..4ddfd16f931 100644 --- a/tests/integration/providers/test_rust_anthropic_safeguards_beta_chaos.py +++ b/tests/integration/providers/test_rust_anthropic_safeguards_beta_chaos.py @@ -45,7 +45,7 @@ def _clients(stack: ExitStack, base_url: str, count: int) -> tuple[httpx.Client, def _post(client: httpx.Client, key: str, marker: str) -> int: body: Final = safeguarded(messages_body(ANTHROPIC_DEPLOYMENT, marker), list(SAFEGUARDS)) response: Final = client.post("/v1/messages", json=body, headers={"Authorization": f"Bearer {key}"}) - assert response.status_code != 200 or rust_routed(response.headers), dict(response.headers) + assert response.status_code != 200 or not rust_routed(response.headers), dict(response.headers) return response.status_code diff --git a/tests/integration/providers/test_rust_anthropic_safeguards_beta_wire.py b/tests/integration/providers/test_rust_anthropic_safeguards_beta_wire.py index c13d4da7dbb..2941f8003ee 100644 --- a/tests/integration/providers/test_rust_anthropic_safeguards_beta_wire.py +++ b/tests/integration/providers/test_rust_anthropic_safeguards_beta_wire.py @@ -155,7 +155,7 @@ def test_sdk_messages_with_safeguards_go_upstream_with_the_dangerous_tool_use_be extra_body={"safeguards": list(SAFEGUARDS), "tools": [dict(BASH_TOOL)]}, ) message: Final = raw.parse() - assert rust_routed(raw.headers), dict(raw.headers) + assert not rust_routed(raw.headers), dict(raw.headers) received: Final = json_object(message.model_dump_json()) assert message.id == f"msg_{marker}", received assert received.get("safeguard_results") == list(SAFEGUARD_RESULTS), received @@ -192,7 +192,7 @@ async def test_sdk_streamed_messages_with_safeguards_go_upstream_with_the_beta( ) as stream: text: Final = "".join([piece async for piece in stream.text_stream]) final: Final = await stream.get_final_message() - assert rust_routed(stream.response.headers), dict(stream.response.headers) + assert not rust_routed(stream.response.headers), dict(stream.response.headers) assert (final.id, text) == (f"msg_{marker}", REPLY_TEXT), (final, text) sent: Final = one_delivery(rig.wire, marker) assert (sent.target, sent.beta, sent.safeguards) == ( @@ -209,7 +209,7 @@ def test_raw_messages_merge_the_safeguards_beta_into_the_client_betas(rig: _Rig, body: Final = safeguarded(messages_body(ANTHROPIC_DEPLOYMENT, marker, stream=stream), list(SAFEGUARDS)) response: Final = _post(rig.gateway, body, beta=CLIENT_BETAS) assert response.status_code == 200, response.text - assert rust_routed(response.headers), dict(response.headers) + assert not rust_routed(response.headers), dict(response.headers) assert f"msg_{marker}" in response.text and "safeguard_results" in response.text, response.text sent: Final = one_delivery(rig.wire, marker) assert (sent.target, sent.beta) == (ANTHROPIC_TARGET, MERGED_BETAS), sent @@ -221,7 +221,7 @@ def test_null_or_absent_safeguards_add_no_beta(rig: _Rig, extra: Mapping[str, Js marker: Final = new_marker() response: Final = _post(rig.gateway, {**messages_body(ANTHROPIC_DEPLOYMENT, marker), **extra}) assert response.status_code == 200, response.text - assert rust_routed(response.headers), dict(response.headers) + assert not rust_routed(response.headers), dict(response.headers) assert f"msg_{marker}" in response.text and "safeguard_results" not in response.text, response.text sent: Final = one_delivery(rig.wire, marker) assert (sent.beta, sent.safeguards) == (None, None), sent @@ -253,7 +253,7 @@ def test_malformed_safeguards_still_add_the_beta_and_relay_the_vendor_verdict( control: Final = new_marker() after: Final = _post(rig.gateway, messages_body(ANTHROPIC_DEPLOYMENT, control)) assert after.status_code == 200, after.text - assert rust_routed(after.headers), dict(after.headers) + assert not rust_routed(after.headers), dict(after.headers) assert one_delivery(rig.wire, control).beta is None assert rig.gateway.request("GET", "/health/liveliness").status_code == 200 @@ -276,7 +276,7 @@ def test_duplicated_safeguards_keys_resolve_to_the_last_value(rig: _Rig, last: s }, ) assert response.status_code == 200, response.text - assert rust_routed(response.headers), dict(response.headers) + assert not rust_routed(response.headers), dict(response.headers) assert ("safeguard_results" in response.text) == (last == "list"), response.text sent: Final = one_delivery(rig.wire, marker) assert sent.beta == (DANGEROUS_TOOL_USE if last == "list" else None), sent @@ -304,7 +304,7 @@ def test_vertex_messages_with_safeguards_keep_the_beta_from_the_shared_helper( body: Final = safeguarded(messages_body(VERTEX_DEPLOYMENT, marker, stream=stream), list(SAFEGUARDS)) response: Final = _post(rig.gateway, {**body, "tools": tools}) assert response.status_code == 200, response.text - assert rust_routed(response.headers), dict(response.headers) + assert not rust_routed(response.headers), dict(response.headers) assert f"msg_{marker}" in response.text and "safeguard_results" in response.text, response.text sent: Final = one_delivery(rig.wire, marker) assert sent.target == (VERTEX_STREAM_TARGET if stream else VERTEX_TARGET), sent @@ -326,7 +326,7 @@ def test_deepseek_messages_forward_safeguards_without_adding_a_beta(rig: _Rig, b beta=beta, ) assert response.status_code == 200, response.text - assert rust_routed(response.headers), dict(response.headers) + assert not rust_routed(response.headers), dict(response.headers) assert f"msg_{marker}" in response.text, response.text sent: Final = one_delivery(rig.wire, marker) assert (sent.target, sent.api_key, sent.safeguards) == (DEEPSEEK_TARGET, DEEPSEEK_KEY, list(SAFEGUARDS)), sent @@ -370,7 +370,7 @@ def _wave(pool: ThreadPoolExecutor, gateway: Gateway, markers: tuple[str, ...], def _assert_served(answer: _Answer) -> None: assert answer.status == 200, answer assert REPLY_TEXT in answer.text, answer - assert answer.rust == (answer.path == "/v1/messages"), answer + assert not answer.rust, answer if answer.path == "/v1/messages": assert f"msg_{answer.marker}" in answer.text and "safeguard_results" in answer.text, answer diff --git a/tests/integration/providers/test_vertex_credentials_source_wire.py b/tests/integration/providers/test_vertex_credentials_source_wire.py index c66bb10f9a9..f3bc86eca19 100644 --- a/tests/integration/providers/test_vertex_credentials_source_wire.py +++ b/tests/integration/providers/test_vertex_credentials_source_wire.py @@ -416,7 +416,7 @@ def vertex_claude_peer(request: Request) -> Reply: @pytest.mark.timeout(2 * graceful_stop_seconds() + 120) -def test_rust_messages_route_loads_a_claude_key_file_and_names_unloadable_credentials( +def test_rust_enabled_messages_route_loads_a_claude_key_file_and_names_unloadable_credentials( gateway: Gateway, tmp_path: Path ) -> None: token_url: Final = gateway.upstream_url.rstrip("/") @@ -452,7 +452,7 @@ def test_rust_messages_route_loads_a_claude_key_file_and_names_unloadable_creden candidate, wire, ask(str(tmp_path / "missing.json"), "rust refuses a missing file"), - "failed to open application credentials file", + FILE_MESSAGE, str(tmp_path), ) refused( diff --git a/tests/integration/spend/golden/daily_activity_team_paginated.json b/tests/integration/spend/golden/daily_activity_team_paginated.json index 4d91b0da209..18ee30a74e6 100644 --- a/tests/integration/spend/golden/daily_activity_team_paginated.json +++ b/tests/integration/spend/golden/daily_activity_team_paginated.json @@ -177,35 +177,6 @@ "endpoints": { "/v1/chat/completions": { "api_key_breakdown": { - "__ptu_flat_cost__": { - "metadata": { - "key_alias": null, - "key_exists": false, - "team_id": null, - "user_email": null, - "user_id": null - }, - "metrics": { - "api_requests": 0, - "autorouter_savings_spend": 0.0, - "cache_creation_input_tokens": 0, - "cache_read_input_tokens": 0, - "completion_tokens": 0, - "compression_saved_tokens": 0, - "compression_savings_spend": 0.0, - "failed_requests": 0, - "flat_cost": 0.0, - "ptu_hours": 0.0, - "gateway_injected_caching_savings_spend": 0.0, - "prompt_caching_savings_spend": 0.0, - "prompt_tokens": 0, - "spend": 1000.0, - "successful_requests": 0, - "timed_requests": 0, - "total_response_time_ms": 0, - "total_tokens": 0 - } - }, "key-a": { "metadata": { "key_alias": "alias-a", diff --git a/tests/integration/spend/golden/daily_activity_user_paginated.json b/tests/integration/spend/golden/daily_activity_user_paginated.json index 00389289ebe..4f7b62b3147 100644 --- a/tests/integration/spend/golden/daily_activity_user_paginated.json +++ b/tests/integration/spend/golden/daily_activity_user_paginated.json @@ -177,35 +177,6 @@ "endpoints": { "/v1/chat/completions": { "api_key_breakdown": { - "__ptu_flat_cost__": { - "metadata": { - "key_alias": null, - "key_exists": false, - "team_id": null, - "user_email": null, - "user_id": null - }, - "metrics": { - "api_requests": 0, - "autorouter_savings_spend": 0.0, - "cache_creation_input_tokens": 0, - "cache_read_input_tokens": 0, - "completion_tokens": 0, - "compression_saved_tokens": 0, - "compression_savings_spend": 0.0, - "failed_requests": 0, - "flat_cost": 0.0, - "ptu_hours": 0.0, - "gateway_injected_caching_savings_spend": 0.0, - "prompt_caching_savings_spend": 0.0, - "prompt_tokens": 0, - "spend": 1000.0, - "successful_requests": 0, - "timed_requests": 0, - "total_response_time_ms": 0, - "total_tokens": 0 - } - }, "key-a": { "metadata": { "key_alias": "alias-a", diff --git a/tests/integration/spend/test_shutdown_flush.py b/tests/integration/spend/test_shutdown_flush.py index 5c2e9cf98da..174403d1873 100644 --- a/tests/integration/spend/test_shutdown_flush.py +++ b/tests/integration/spend/test_shutdown_flush.py @@ -16,7 +16,7 @@ import httpx import psycopg import pytest import yaml -from integration._support.client import Gateway, delete_key_if_present, eventually, string_value +from integration._support.client import Gateway, delete_key_if_present, eventually, object_value, string_value from integration._support.database import read_rows from integration._support.process import OwnedProxy, owned_proxy_process from integration._support.wire import Reply, Request, wire_server @@ -25,7 +25,6 @@ from psycopg import sql REQUESTS_WHILE_BLOCKED: Final = 6 CANCEL_LOG_LINE: Final = "in-flight scheduled job(s) for shutdown" BATCH_DRAINED_LOG_LINE: Final = f"flushed {REQUESTS_WHILE_BLOCKED} daily spend update items from in-memory queue" -MODEL_INSERT_ARRIVED_LOG_LINE: Final = "path=/model/new" COMMIT_DELAY_SECONDS: Final = 15 BURST_REQUESTS: Final = 30 @@ -356,8 +355,15 @@ class _Shutdown: for outcome in pool.map(lambda _: self.chat(), range(requests)): assert outcome is None - def logged(self, line: str, times: int = 1) -> bool: - return self.owned.log.read_text(errors="replace").count(line) >= times + def logged(self, line: str) -> bool: + return line in self.owned.log.read_text(errors="replace") + + def other_requests_in_flight(self) -> int: + response: Final = self.owned.gateway.request("GET", "/health/backlog") + assert response.status_code == 200, response.text + in_flight: Final = object_value(response.json())["in_flight_requests"] + assert isinstance(in_flight, int) + return in_flight - 1 def chat_while_spend_update_is_blocked(self, blocker: psycopg.Connection, table: str) -> None: blocker.execute(f'LOCK TABLE "{table}" IN EXCLUSIVE MODE') @@ -463,7 +469,7 @@ def test_daily_spend_batch_cancelled_while_waiting_for_a_pool_connection_is_writ eventually(lambda: _waiting_on("LiteLLM_ProxyModelTable"), lambda waiting: waiting == 1, seconds=30) shutdown.chat_while_spend_update_is_blocked(memberships, "LiteLLM_TeamMembership") second: Final = shutdown.start_blocked_model_insert() - eventually(lambda: shutdown.logged(MODEL_INSERT_ARRIVED_LOG_LINE, times=2), lambda seen: seen, seconds=30) + eventually(shutdown.other_requests_in_flight, lambda in_flight: in_flight == 2, seconds=30) memberships.rollback() eventually(lambda: _waiting_on("LiteLLM_ProxyModelTable"), lambda waiting: waiting == 2, seconds=30) shutdown.terminate_once(lambda: shutdown.logged(BATCH_DRAINED_LOG_LINE), models.rollback) diff --git a/tests/integration/translation/chat_completions/bases/openai.py b/tests/integration/translation/chat_completions/bases/openai.py index 21886e4c774..25d8121cf31 100644 --- a/tests/integration/translation/chat_completions/bases/openai.py +++ b/tests/integration/translation/chat_completions/bases/openai.py @@ -1,4 +1,5 @@ from typing import Final +from unittest.mock import ANY from integration._support.client import object_value from integration.translation.case import TranslationTestCase @@ -420,6 +421,7 @@ GPT_6_LUNA_TEST_CASE: Final = TranslationTestCase( "summary": [], } ], + "provider_specific_fields": {"phase": "commentary", "openai_responses_replay": ANY}, }, } ], @@ -630,6 +632,7 @@ GPT_5_4_MINI_TEST_CASE: Final = TranslationTestCase( "type": "function", }, ], + "provider_specific_fields": {"phase": "commentary", "openai_responses_replay": ANY}, }, } ], @@ -729,6 +732,7 @@ GPT_6_SOL_TEST_CASE: Final = TranslationTestCase( }, ], "content": None, + "provider_specific_fields": {"openai_responses_replay": ANY}, }, } ], @@ -886,6 +890,7 @@ GPT_6_1_SOL_TEST_CASE: Final = TranslationTestCase( "summary": [], } ], + "provider_specific_fields": {"phase": "commentary", "openai_responses_replay": ANY}, }, } ], @@ -1042,6 +1047,7 @@ GPT_6_ASTRA_TEST_CASE: Final = TranslationTestCase( "summary": [], } ], + "provider_specific_fields": {"phase": "commentary", "openai_responses_replay": ANY}, }, } ], @@ -1201,6 +1207,7 @@ GPT_5_6_TERRA_TEST_CASE: Final = TranslationTestCase( "summary": [], } ], + "provider_specific_fields": {"phase": "commentary", "openai_responses_replay": ANY}, }, } ], diff --git a/tests/integration/translation/chat_completions/cross_model/openai.py b/tests/integration/translation/chat_completions/cross_model/openai.py index 0793daa96b1..46e9277b7cb 100644 --- a/tests/integration/translation/chat_completions/cross_model/openai.py +++ b/tests/integration/translation/chat_completions/cross_model/openai.py @@ -1,5 +1,7 @@ +from collections.abc import Mapping from dataclasses import replace from typing import Final +from unittest.mock import ANY from integration._support.client import list_value, object_value from integration.translation.case import ( @@ -29,6 +31,30 @@ from integration.translation.templates.openai import ( OPENAI_RESPONSES_AGENT_REQUEST, OPENAI_RESPONSES_AGENT_RESPONSE, ) +from pydantic import JsonValue + + +def _without_replay_record(response: Mapping[str, JsonValue]) -> dict[str, JsonValue]: + choices: Final = response["choices"] + assert isinstance(choices, list) and isinstance(choices[0], dict), choices + message: Final = choices[0]["message"] + assert isinstance(message, dict), message + fields: Final = message["provider_specific_fields"] + assert isinstance(fields, dict), fields + kept: Final = {key: value for key, value in fields.items() if key != "openai_responses_replay"} + return { + **response, + "choices": [ + { + **choices[0], + "message": { + **{key: value for key, value in message.items() if key != "provider_specific_fields"}, + **({"provider_specific_fields": kept} if kept else {}), + }, + } + ], + } + GPT_5_MINI_AGENT_FOLLOWUP_TEST_CASE: Final = replace( GPT_5_MINI_TEST_CASE, @@ -559,7 +585,7 @@ GPT_5_4_MINI_AGENT_FOLLOWUP_TEST_CASE: Final = replace( GPT_5_4_MINI_TEST_CASE, scenario="agent_followup", litellm_request=followup_request( - GPT_5_4_MINI_TEST_CASE.litellm_request, GPT_5_4_MINI_TEST_CASE.expected_litellm_response + GPT_5_4_MINI_TEST_CASE.litellm_request, _without_replay_record(GPT_5_4_MINI_TEST_CASE.expected_litellm_response) ), expected_provider_endpoint="/responses", expected_provider_request={ @@ -570,6 +596,7 @@ GPT_5_4_MINI_AGENT_FOLLOWUP_TEST_CASE: Final = replace( { "type": "message", "role": "assistant", + "phase": "commentary", "content": [ { "type": "output_text", @@ -679,6 +706,7 @@ GPT_5_4_MINI_AGENT_FOLLOWUP_TEST_CASE: Final = replace( "caller in `src/worker.py` (`remaining = retries_remaining(3, attempts)`), and " "`tests/test_retry.py` failed with `assert 4 == 2` for `retries_remaining(3, 1)`.", "role": "assistant", + "provider_specific_fields": {"phase": "final_answer", "openai_responses_replay": ANY}, }, } ], @@ -1310,6 +1338,7 @@ GPT_5_6_LUNA_AGENT_TEST_CASE: Final = replace( } ], "content": None, + "provider_specific_fields": {"openai_responses_replay": ANY}, }, } ], @@ -1343,7 +1372,8 @@ GPT_5_6_LUNA_AGENT_FOLLOWUP_TEST_CASE: Final = replace( GPT_5_6_LUNA_AGENT_TEST_CASE, scenario="agent_followup", litellm_request=followup_request( - GPT_5_6_LUNA_AGENT_TEST_CASE.litellm_request, GPT_5_6_LUNA_AGENT_TEST_CASE.expected_litellm_response + GPT_5_6_LUNA_AGENT_TEST_CASE.litellm_request, + _without_replay_record(GPT_5_6_LUNA_AGENT_TEST_CASE.expected_litellm_response), ), expected_provider_endpoint="/responses", expected_provider_request={ @@ -1456,6 +1486,7 @@ GPT_5_6_LUNA_AGENT_FOLLOWUP_TEST_CASE: Final = replace( "of 2 and potentially negative values are not clamped. It is called by " "`src/worker.py:12`, and `tests/test_retry.py::test_retries_remaining` failed.", "role": "assistant", + "provider_specific_fields": {"phase": "final_answer", "openai_responses_replay": ANY}, }, } ], @@ -1819,6 +1850,7 @@ GPT_5_6_SOL_AGENT_TEST_CASE: Final = replace( }, ], "content": None, + "provider_specific_fields": {"openai_responses_replay": ANY}, }, } ], @@ -1852,7 +1884,8 @@ GPT_5_6_SOL_AGENT_FOLLOWUP_TEST_CASE: Final = replace( GPT_5_6_SOL_AGENT_TEST_CASE, scenario="agent_followup", litellm_request=followup_request( - GPT_5_6_SOL_AGENT_TEST_CASE.litellm_request, GPT_5_6_SOL_AGENT_TEST_CASE.expected_litellm_response + GPT_5_6_SOL_AGENT_TEST_CASE.litellm_request, + _without_replay_record(GPT_5_6_SOL_AGENT_TEST_CASE.expected_litellm_response), ), expected_provider_endpoint="/responses", expected_provider_request={ @@ -1961,6 +1994,7 @@ GPT_5_6_SOL_AGENT_FOLLOWUP_TEST_CASE: Final = replace( "`tests/test_retry.py` failed because `retries_remaining(3, 1)` returned `4` " "instead of `2`.", "role": "assistant", + "provider_specific_fields": {"phase": "final_answer", "openai_responses_replay": ANY}, }, } ], @@ -2278,6 +2312,7 @@ GPT_5_6_TERRA_AGENT_TEST_CASE: Final = replace( }, ], "content": None, + "provider_specific_fields": {"openai_responses_replay": ANY}, }, } ], @@ -2311,7 +2346,8 @@ GPT_5_6_TERRA_AGENT_FOLLOWUP_TEST_CASE: Final = replace( GPT_5_6_TERRA_AGENT_TEST_CASE, scenario="agent_followup", litellm_request=followup_request( - GPT_5_6_TERRA_AGENT_TEST_CASE.litellm_request, GPT_5_6_TERRA_AGENT_TEST_CASE.expected_litellm_response + GPT_5_6_TERRA_AGENT_TEST_CASE.litellm_request, + _without_replay_record(GPT_5_6_TERRA_AGENT_TEST_CASE.expected_litellm_response), ), expected_provider_endpoint="/responses", expected_provider_request={ @@ -2420,6 +2456,7 @@ GPT_5_6_TERRA_AGENT_FOLLOWUP_TEST_CASE: Final = replace( "`tests/test_retry.py` fails because `retries_remaining(3, 1)` returns `4` instead " "of `2`.", "role": "assistant", + "provider_specific_fields": {"phase": "final_answer", "openai_responses_replay": ANY}, }, } ], @@ -2737,6 +2774,7 @@ GPT_6_ASTRA_AGENT_TEST_CASE: Final = replace( }, ], "content": None, + "provider_specific_fields": {"openai_responses_replay": ANY}, }, } ], @@ -2769,7 +2807,8 @@ GPT_6_ASTRA_AGENT_FOLLOWUP_TEST_CASE: Final = replace( GPT_6_ASTRA_AGENT_TEST_CASE, scenario="agent_followup", litellm_request=followup_request( - GPT_6_ASTRA_AGENT_TEST_CASE.litellm_request, GPT_6_ASTRA_AGENT_TEST_CASE.expected_litellm_response + GPT_6_ASTRA_AGENT_TEST_CASE.litellm_request, + _without_replay_record(GPT_6_ASTRA_AGENT_TEST_CASE.expected_litellm_response), ), expected_provider_endpoint="/responses", expected_provider_request={ @@ -2878,6 +2917,7 @@ GPT_6_ASTRA_AGENT_FOLLOWUP_TEST_CASE: Final = replace( "and `tests/test_retry.py::test_retries_remaining` failed because it returned 4 " "instead of 2.", "role": "assistant", + "provider_specific_fields": {"phase": "final_answer", "openai_responses_replay": ANY}, }, } ], @@ -3194,6 +3234,7 @@ GPT_6_LUNA_AGENT_TEST_CASE: Final = replace( }, ], "content": None, + "provider_specific_fields": {"openai_responses_replay": ANY}, }, } ], @@ -3227,7 +3268,8 @@ GPT_6_LUNA_AGENT_FOLLOWUP_TEST_CASE: Final = replace( GPT_6_LUNA_AGENT_TEST_CASE, scenario="agent_followup", litellm_request=followup_request( - GPT_6_LUNA_AGENT_TEST_CASE.litellm_request, GPT_6_LUNA_AGENT_TEST_CASE.expected_litellm_response + GPT_6_LUNA_AGENT_TEST_CASE.litellm_request, + _without_replay_record(GPT_6_LUNA_AGENT_TEST_CASE.expected_litellm_response), ), expected_provider_endpoint="/responses", expected_provider_request={ @@ -3336,6 +3378,7 @@ GPT_6_LUNA_AGENT_FOLLOWUP_TEST_CASE: Final = replace( "`tests/test_retry.py` failed because `retries_remaining(3, 1)` returned 4 instead " "of 2.", "role": "assistant", + "provider_specific_fields": {"phase": "final_answer", "openai_responses_replay": ANY}, }, } ], @@ -3587,7 +3630,7 @@ GPT_6_SOL_AGENT_FOLLOWUP_TEST_CASE: Final = replace( GPT_6_SOL_TEST_CASE, scenario="agent_followup", litellm_request=followup_request( - GPT_6_SOL_TEST_CASE.litellm_request, GPT_6_SOL_TEST_CASE.expected_litellm_response + GPT_6_SOL_TEST_CASE.litellm_request, _without_replay_record(GPT_6_SOL_TEST_CASE.expected_litellm_response) ), expected_provider_endpoint="/responses", expected_provider_request={ @@ -3694,6 +3737,7 @@ GPT_6_SOL_AGENT_FOLLOWUP_TEST_CASE: Final = replace( "clamping the result at zero; `src/worker.py` calls it. `tests/test_retry.py` " "failed because `retries_remaining(3, 1)` returned 4 instead of 2.", "role": "assistant", + "provider_specific_fields": {"phase": "final_answer", "openai_responses_replay": ANY}, }, } ], @@ -4007,6 +4051,7 @@ GPT_6_1_SOL_AGENT_TEST_CASE: Final = replace( }, ], "content": None, + "provider_specific_fields": {"openai_responses_replay": ANY}, }, } ], @@ -4039,7 +4084,8 @@ GPT_6_1_SOL_AGENT_FOLLOWUP_TEST_CASE: Final = replace( GPT_6_1_SOL_AGENT_TEST_CASE, scenario="agent_followup", litellm_request=followup_request( - GPT_6_1_SOL_AGENT_TEST_CASE.litellm_request, GPT_6_1_SOL_AGENT_TEST_CASE.expected_litellm_response + GPT_6_1_SOL_AGENT_TEST_CASE.litellm_request, + _without_replay_record(GPT_6_1_SOL_AGENT_TEST_CASE.expected_litellm_response), ), expected_provider_endpoint="/responses", expected_provider_request={ @@ -4148,6 +4194,7 @@ GPT_6_1_SOL_AGENT_FOLLOWUP_TEST_CASE: Final = replace( "`src/worker.py:12`. `tests/test_retry.py::test_retries_remaining` failed because " "it returned 4 instead of 2; no files were changed.", "role": "assistant", + "provider_specific_fields": {"phase": "final_answer", "openai_responses_replay": ANY}, }, } ], From d302096b5a49630a6dc6e34ccc6616f2464267db Mon Sep 17 00:00:00 2001 From: quietjules <349048734+quietjules@users.noreply.github.com> Date: Thu, 29 Oct 2026 18:15:44 -0700 Subject: [PATCH 15/17] feat(lite): offer each model group's reasoning efforts in pi, OpenCode and Codex (#48949) * feat(lite): offer each model group's reasoning efforts in pi, OpenCode and Codex lite pi, lite opencode and lite codex now read supported_reasoning_efforts from /model_group/info and write them into each client's model config. pi gets reasoning and a thinkingLevelMap, OpenCode gets reasoning_options plus one variant per effort, and Codex fallback entries get supported_reasoning_levels with a default level. The proxy-served Codex catalog on /v1/models?client_version= offers the same levels Codex before 0.138.0 fails the whole catalog on an effort it cannot name, so those releases, and an unreadable version, get only the six efforts every release parses. A group with unknown efforts keeps today's output, and a group with none is written as non-reasoning The strength order and nearest_declared_reasoning_effort move to an SDK-free module so the lite CLI, which loads without initializing the SDK, can import the Codex catalog * fix(lite): describe team models by their public name and keep Codex's none default /model_group/info now looks up a team's own deployments when the caller's team serves the requested name, so a team model reports its reasoning levels. Codex fallback entries default to none when the group lists it, and the proxy builds the Codex catalog in a worker thread * fix(lite): round Codex's default level down and describe a team's shared-name model from its own deployments Move the reasoning strength helper to litellm_core_utils so lite and the Codex catalog import it without the SDK or the router, and let make lint-format-check-changed diff the working tree so a staged rename passes. * fix(lite): keep pi's system role on reasoning model groups pi switches its system prompt to the developer role for any model marked reasoning, so an operator's cache_control_injection_points entry for role system stopped matching on Claude groups. lite pi now sets compat.supportsDeveloperRole false on those entries, so pi sends system as it did before. * fix(codex): treat a client_version too long to parse as an old Codex and keep undescribed stock levels A client_version past Python's integer digit limit made the Codex catalog answer 500. It now gets the levels every Codex parses. A stock Codex level with no description validates again, as on base, and lite codex treats undecodable codex --version output as an unknown version * fix(lite): keep REASONING_EFFORT_STRENGTH_ORDER importable from router_utils and skip the Codex sync on undecodable debug models output --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- Makefile | 2 +- .../reasoning_effort_strength.py | 45 +++++ litellm/proxy/client/cli/README.md | 2 +- litellm/proxy/client/cli/commands/agents.py | 113 +++++++++-- litellm/proxy/client/cli/commands/pi.py | 40 +++- .../proxy/common_utils/codex_model_catalog.py | 97 ++++++++-- litellm/proxy/proxy_server.py | 26 ++- litellm/router.py | 19 +- litellm/router_api.py | 2 +- .../reasoning_effort_capability.py | 41 +--- litellm/types/router.py | 5 +- tests/unit/proxy/client/cli/test_agents.py | 179 +++++++++++++++--- tests/unit/proxy/client/cli/test_pi.py | 70 ++++++- .../common_utils/test_codex_model_catalog.py | 134 +++++++++++++ .../proxy_server/test_routes_model_info.py | 105 ++++++++++ .../proxy/proxy_server/test_routes_models.py | 30 +++ .../public_v1/test_model_hub.py | 2 +- tests/unit/proxy/router_entry_point_stubs.py | 2 +- .../test_reasoning_effort_capability.py | 2 +- tests/unit/test_default_branch.py | 11 ++ .../test_router/test_router_entry_points.py | 4 +- ui/litellm-dashboard/src/lib/http/schema.d.ts | 12 +- 22 files changed, 823 insertions(+), 120 deletions(-) create mode 100644 litellm/litellm_core_utils/reasoning_effort_strength.py diff --git a/Makefile b/Makefile index 937873f8835..48df3c8e215 100644 --- a/Makefile +++ b/Makefile @@ -153,7 +153,7 @@ lint-install: # the same set CI's ':(glob)litellm/**/*.py' selects. lint-format-check-changed: $(LINT_DEP_INSTALL) $(LINT_DEP_BASE) @base_ref=$$($(RESOLVE_BASE)) && \ - changed=$$(git diff --name-only --diff-filter=ACMR "$$base_ref...HEAD" -- 'litellm/*.py') && \ + changed=$$(git diff --name-only --diff-filter=ACMR --merge-base "$$base_ref" -- 'litellm/*.py') && \ files=$$(printf '%s\n' "$$changed" | grep -v '^litellm/enterprise/' || true) || exit $$?; \ if [ -z "$$files" ]; then \ echo "No changed litellm Python files to format-check."; \ diff --git a/litellm/litellm_core_utils/reasoning_effort_strength.py b/litellm/litellm_core_utils/reasoning_effort_strength.py new file mode 100644 index 00000000000..f7b51b75a9f --- /dev/null +++ b/litellm/litellm_core_utils/reasoning_effort_strength.py @@ -0,0 +1,45 @@ +"""How strong each reasoning_effort level is, and rounding a requested level onto a declared set. + +Imports nothing from the SDK, so the `lite` CLI, which loads without initializing litellm, can use it. +""" + +from collections.abc import Sequence +from types import MappingProxyType +from typing import Final, Literal, TypeAlias + +from typing_extensions import assert_never + +ReasoningEffortRounding: TypeAlias = Literal["up", "down"] + +REASONING_EFFORT_STRENGTH_ORDER: Final = ("minimal", "low", "medium", "high", "xhigh", "max") +REASONING_EFFORT_STRENGTH_RANK: Final = MappingProxyType( + {effort: rank for rank, effort in enumerate(REASONING_EFFORT_STRENGTH_ORDER)} +) + + +def nearest_declared_reasoning_effort( + requested: str, declared: Sequence[str], rounding: ReasoningEffortRounding = "up" +) -> str: + """Rounding up takes the weakest declared level at least as strong as the request, else the + strongest declared, so the caller gets no less reasoning than it asked for instead of a rejected + call. Rounding down mirrors it: the strongest declared level at most as strong, else the + weakest. none is the off switch rather than a strength, so it is never rounded onto the ladder + and no level is rounded to it: a caller who turned reasoning off must not be billed for it, and + a model that cannot turn it off says so itself. A level outside the strength order is likewise + returned as is for upstream to judge.""" + ranked: Final = sorted( + (effort for effort in declared if effort in REASONING_EFFORT_STRENGTH_RANK), + key=lambda effort: REASONING_EFFORT_STRENGTH_RANK[effort], + ) + if requested in ranked or requested not in REASONING_EFFORT_STRENGTH_RANK or not ranked: + return requested + strength: Final = REASONING_EFFORT_STRENGTH_RANK[requested] + match rounding: + case "up": + return next((effort for effort in ranked if REASONING_EFFORT_STRENGTH_RANK[effort] >= strength), ranked[-1]) + case "down": + return next( + (effort for effort in reversed(ranked) if REASONING_EFFORT_STRENGTH_RANK[effort] <= strength), ranked[0] + ) + case _: + assert_never(rounding) diff --git a/litellm/proxy/client/cli/README.md b/litellm/proxy/client/cli/README.md index 819908f9e70..1fdf8da7d2b 100644 --- a/litellm/proxy/client/cli/README.md +++ b/litellm/proxy/client/cli/README.md @@ -492,7 +492,7 @@ lite codex exec "summarize the repo" Each command resolves your LiteLLM key (logging in via SSO when none is stored and you are at a terminal; otherwise it expects `LITELLM_PROXY_API_KEY` or `--api-key`), checks the key against the proxy so bad credentials fail immediately instead of deep inside the agent, exports the environment variables the agent reads, then replaces itself with the agent process. -The right variables are picked per agent. Claude Code gets `ANTHROPIC_BASE_URL` (the proxy root, so it appends `/v1/messages`) and `ANTHROPIC_AUTH_TOKEN`, with any stray `ANTHROPIC_API_KEY` cleared so the proxy token wins, and `ENABLE_TOOL_SEARCH=true` (unless you already set it) so Claude Code keeps tool search on even though the base URL is a proxy rather than a first-party Anthropic host. It also gets `CLAUDE_CODE_ENABLE_GATEWAY_MODEL_DISCOVERY=1` (again unless you already set it) so Claude Code v2.1.129+ fills its `/model` picker from the proxy's `/v1/models`; Claude Code only lists entries whose id contains `claude` or `anthropic`, so the proxy lists every other group to Claude Code as `claude-router-` and marks a group whose input window reaches 1M with `[1m]`, and a request on such an id is served by the group. Older Claude Code versions ignore the variable. Export it as `0` to turn discovery off. Codex and OpenCode get `OPENAI_BASE_URL` (the proxy plus `/v1`) and `OPENAI_API_KEY`. Codex ignores `OPENAI_BASE_URL`, so it is additionally pointed at the proxy through a custom provider passed as `-c` config overrides (HTTP/SSE Responses transport, since the proxy does not speak the Responses WebSocket protocol). OpenCode additionally gets `OPENCODE_CONFIG_CONTENT` holding a generated `litellm` provider (`@ai-sdk/openai-compatible`, the proxy `/v1` URL, `{env:OPENAI_API_KEY}`) with one model entry per chat model your key can see on `/v1/models`, so its model picker mirrors the proxy without a hand-maintained `opencode.json`; OpenCode merges that over your own config files, and if you already export `OPENCODE_CONFIG_CONTENT` yours is left alone. When the list cannot be fetched, `lite opencode` says so on stderr and launches anyway. Codex gets the same list as a catalog file, `$CODEX_HOME/litellm-models.json` (default `~/.codex/`), passed as `-c model_catalog_json=` so `/model` lists exactly the proxy's chat models; a proxy model the installed Codex already knows (`gpt-5.5`, say) keeps that Codex's own entry, reasoning levels and prompt included, and only its place in the picker comes from the proxy, while a model Codex does not know gets the plain entry Codex uses for an unknown `-m` slug. Before launching, `lite codex` asks the installed Codex for its own list and then has it read the written file back (both through `codex debug models`), and when the fetch, either of those or the write fails (Codex releases older than 0.130 have no such command) it says so on stderr and launches with Codex's built-in catalog, leaving a rejected file in place. +The right variables are picked per agent. Claude Code gets `ANTHROPIC_BASE_URL` (the proxy root, so it appends `/v1/messages`) and `ANTHROPIC_AUTH_TOKEN`, with any stray `ANTHROPIC_API_KEY` cleared so the proxy token wins, and `ENABLE_TOOL_SEARCH=true` (unless you already set it) so Claude Code keeps tool search on even though the base URL is a proxy rather than a first-party Anthropic host. It also gets `CLAUDE_CODE_ENABLE_GATEWAY_MODEL_DISCOVERY=1` (again unless you already set it) so Claude Code v2.1.129+ fills its `/model` picker from the proxy's `/v1/models`; Claude Code only lists entries whose id contains `claude` or `anthropic`, so the proxy lists every other group to Claude Code as `claude-router-` and marks a group whose input window reaches 1M with `[1m]`, and a request on such an id is served by the group. Older Claude Code versions ignore the variable. Export it as `0` to turn discovery off. Codex and OpenCode get `OPENAI_BASE_URL` (the proxy plus `/v1`) and `OPENAI_API_KEY`. Codex ignores `OPENAI_BASE_URL`, so it is additionally pointed at the proxy through a custom provider passed as `-c` config overrides (HTTP/SSE Responses transport, since the proxy does not speak the Responses WebSocket protocol). OpenCode additionally gets `OPENCODE_CONFIG_CONTENT` holding a generated `litellm` provider (`@ai-sdk/openai-compatible`, the proxy `/v1` URL, `{env:OPENAI_API_KEY}`) with one model entry per chat model your key can see on `/v1/models`, so its model picker mirrors the proxy without a hand-maintained `opencode.json`; OpenCode merges that over your own config files, and if you already export `OPENCODE_CONFIG_CONTENT` yours is left alone. When the list cannot be fetched, `lite opencode` says so on stderr and launches anyway. Codex gets the same list as a catalog file, `$CODEX_HOME/litellm-models.json` (default `~/.codex/`), passed as `-c model_catalog_json=` so `/model` lists exactly the proxy's chat models; a proxy model the installed Codex already knows (`gpt-5.5`, say) keeps that Codex's own entry, reasoning levels and prompt included, and only its place in the picker comes from the proxy, while a model Codex does not know gets the plain entry Codex uses for an unknown `-m` slug, plus whichever reasoning levels its model group lists on `/model_group/info` the installed Codex can parse. Before launching, `lite codex` asks the installed Codex for its own list and then has it read the written file back (both through `codex debug models`), and when the fetch, either of those or the write fails (Codex releases older than 0.130 have no such command) it says so on stderr and launches with Codex's built-in catalog, leaving a rejected file in place. pi ignores base-URL environment variables entirely, so `lite pi` (kept out of the `lite --help` command listing for now, but fully functional) wires it up differently: before handoff it fetches the models your key can use from the proxy's `/v1/models` (plus each model's context window and output cap from `/model_group/info`, when available) and syncs them into a `litellm` provider entry in pi's `~/.pi/agent/models.json` (honoring `PI_CODING_AGENT_DIR`), then starts pi on that provider's first model via an injected `--model litellm/`. Only that one provider entry is rewritten; the rest of the file, including any other custom providers, is left alone. The entry references the key as `$LITELLM_PROXY_API_KEY`, which the wrapper exports for the session, so the token itself never lands on disk and plain `pi` outside the wrapper simply shows the litellm models as unavailable. The entry also turns on pi's session header (`x-session-id`, sent by pi 0.80.7 and later), so every chat turn of one pi session shares one session id in the logs and sticks to one deployment when session affinity is on. A custom `pi --session-id` must be at least 8 letters, digits, `_`, or `-` for the proxy to read it. Your own flags come after the injected pin, so `lite pi --model litellm/` wins, and inside the TUI the `/model` picker lists every synced litellm model. diff --git a/litellm/proxy/client/cli/commands/agents.py b/litellm/proxy/client/cli/commands/agents.py index f19fb508c7d..12d97ef6cce 100644 --- a/litellm/proxy/client/cli/commands/agents.py +++ b/litellm/proxy/client/cli/commands/agents.py @@ -11,7 +11,7 @@ from dataclasses import dataclass from itertools import chain from pathlib import Path from types import MappingProxyType -from typing import Final, TypeAlias, TypeVar +from typing import Final, Literal, TypeAlias, TypeVar from urllib.parse import urlsplit import click @@ -41,6 +41,7 @@ from .pi import ( LITELLM_PROXY_API_KEY_ENV, PI_PROVIDER_NAME, ListingFailure, + ModelLimits, PiSyncError, fetch_model_ids, fetch_model_limits, @@ -611,9 +612,27 @@ class _OpenCodeLimit(LiteLLMBaseModel): output: int +class _OpenCodeVariant(LiteLLMBaseModel): + model_config = ConfigDict(frozen=True) + + reasoningEffort: str + + +class _OpenCodeReasoningOption(LiteLLMBaseModel): + model_config = ConfigDict(frozen=True) + + type: Literal["effort"] = "effort" + values: tuple[str, ...] + + class _OpenCodeModel(LiteLLMBaseModel): + model_config = ConfigDict(frozen=True) + name: str limit: _OpenCodeLimit | None = None + reasoning: Literal[True] | None = None + reasoning_options: tuple[_OpenCodeReasoningOption, ...] | None = None + variants: Mapping[str, _OpenCodeVariant] | None = None class _OpenCodeProviderOptions(LiteLLMBaseModel): @@ -632,15 +651,43 @@ class _OpenCodeConfig(LiteLLMBaseModel): provider: Mapping[str, _OpenCodeProvider] -def _opencode_model_entry(model: ListedModel) -> _OpenCodeModel: - if model.max_input_tokens is None or model.max_output_tokens is None: - return _OpenCodeModel(name=model.id) +_NO_GROUPS: Final[Mapping[str, ModelLimits]] = MappingProxyType({}) + + +def _reasoning_efforts(model_id: str, groups: Mapping[str, ModelLimits]) -> tuple[str, ...] | None: + group: Final = groups.get(model_id) + return group.supported_reasoning_efforts if group is not None else None + + +def _opencode_model_entry(model: ListedModel, groups: Mapping[str, ModelLimits]) -> _OpenCodeModel: + """One OpenCode model entry, with the model group's reasoning efforts as its variants when known. + + OpenCode 1.x builds its own variants for a reasoning model and replaces them only through + `reasoning_options`. 2.x ignores `reasoning` and `reasoning_options` and builds its own variants + unless the entry lists some, so the efforts go in all three, and a model with none lists an + empty `variants`.""" + limit: Final = ( + _OpenCodeLimit(context=model.max_input_tokens, output=model.max_output_tokens) + if model.max_input_tokens is not None and model.max_output_tokens is not None + else None + ) + efforts: Final = _reasoning_efforts(model.id, groups) + if efforts is None: + return _OpenCodeModel(name=model.id, limit=limit) + if not efforts: + return _OpenCodeModel(name=model.id, limit=limit, variants=MappingProxyType({})) return _OpenCodeModel( - name=model.id, limit=_OpenCodeLimit(context=model.max_input_tokens, output=model.max_output_tokens) + name=model.id, + limit=limit, + reasoning=True, + reasoning_options=(_OpenCodeReasoningOption(values=efforts),), + variants=MappingProxyType({effort: _OpenCodeVariant(reasoningEffort=effort) for effort in efforts}), ) -def opencode_provider_config(base_url: str, models: Sequence[ListedModel]) -> str: +def opencode_provider_config( + base_url: str, models: Sequence[ListedModel], groups: Mapping[str, ModelLimits] = _NO_GROUPS +) -> str: """OPENCODE_CONFIG_CONTENT declaring the proxy as OpenCode provider `litellm`. One model entry per chat-capable /v1/models row (mode chat, responses, or @@ -648,6 +695,7 @@ def opencode_provider_config(base_url: str, models: Sequence[ListedModel]) -> st is read back through {env:OPENAI_API_KEY}, which build_agent_env exports, so it never lands in the config text. OpenCode merges this inline config over the user's own files, leaving unrelated keys and providers untouched. + `groups` is /model_group/info by model group, where the reasoning efforts come from. """ chat_models: Final = _chat_models(models) provider: Final = _OpenCodeProvider( @@ -657,7 +705,7 @@ def opencode_provider_config(base_url: str, models: Sequence[ListedModel]) -> st baseURL=base_url.rstrip("/") + "/v1", apiKey=f"{{env:{OPENAI_API_KEY_ENV}}}", ), - models=MappingProxyType({m.id: _opencode_model_entry(m) for m in chat_models}), + models=MappingProxyType({m.id: _opencode_model_entry(m, groups) for m in chat_models}), ) config: Final = _OpenCodeConfig(provider=MappingProxyType({OPENCODE_PROVIDER_ID: provider})) return config.model_dump_json(exclude_none=True) @@ -672,7 +720,8 @@ def opencode_model_sync_env( ) -> Mapping[str, str] | ModelSyncSkipped: """Env addition that hands OpenCode the proxy's model list, or why it was skipped. - Fetches /v1/models with the key and packs it into OPENCODE_CONFIG_CONTENT. + Fetches /v1/models with the key, and /model_group/info for each model's reasoning + efforts, and packs them into OPENCODE_CONFIG_CONTENT. An OPENCODE_CONFIG_CONTENT already in the environment is left alone, and a failed fetch is reported rather than raised: OpenCode still launches on the plain OPENAI_* env, just without a synced model list. @@ -682,16 +731,20 @@ def opencode_model_sync_env( listing: Final = _fetch_model_listing(base_url, api_key, get=get) if isinstance(listing, ModelSyncSkipped): return listing - return MappingProxyType({OPENCODE_CONFIG_CONTENT_ENV: opencode_provider_config(base_url, listing)}) + groups: Final = fetch_model_limits(base_url, api_key, get=get) + return MappingProxyType({OPENCODE_CONFIG_CONTENT_ENV: opencode_provider_config(base_url, listing, groups)}) -def _codex_row(model: ListedModel, upstream: ModelUpstream | None) -> CodexCatalogRow: +def _codex_row( + model: ListedModel, upstream: ModelUpstream | None, groups: Mapping[str, ModelLimits] +) -> CodexCatalogRow: return CodexCatalogRow( id=model.id, mode=model.mode, max_input_tokens=model.max_input_tokens, upstream_model=upstream.model if upstream is not None else None, reasoning_effort=upstream.reasoning_effort if upstream is not None else None, + supported_reasoning_efforts=_reasoning_efforts(model.id, groups), ) @@ -700,6 +753,8 @@ def codex_model_catalog( stock: Sequence[CodexStockModel], instructions: str, upstreams: Mapping[str, ModelUpstream], + groups: Mapping[str, ModelLimits] = _NO_GROUPS, + client_version: str | None = None, ) -> str | None: """The `model_catalog_json` body listing the proxy's chat models, or None if there are none. @@ -714,17 +769,19 @@ def codex_model_catalog( deployments pin, and a window no larger than its listed max_input_tokens. A model Codex does not know gets the fallback entry, with the same base instructions Codex itself uses so the agent never - runs without a system prompt. Apart from aliases, the catalog is the one - `/v1/models?client_version=...` serves, built from the installed Codex's own - entries instead of the proxy's vendored ones. + runs without a system prompt, and offers the reasoning efforts `groups` gives its model group + that the installed Codex, at `client_version`, can parse. Apart from aliases, the catalog is + the one `/v1/models?client_version=...` serves, built from the installed Codex's own entries + instead of the proxy's vendored ones. """ - rows: Final = tuple(_codex_row(m, upstreams.get(m.id)) for m in models) + rows: Final = tuple(_codex_row(m, upstreams.get(m.id), groups) for m in models) body: Final = codex_models_response_json( rows, stock=MappingProxyType({m.slug: m for m in stock}), instructions=instructions, byte_limit=None, alias_defaults="deployment", + client_version=client_version, ) return body.json if body.listed else None @@ -771,7 +828,7 @@ def _codex_debug_models( encoding="utf-8", timeout=_CODEX_PREFLIGHT_TIMEOUT_SECONDS, ) - except (OSError, subprocess.TimeoutExpired) as e: + except (OSError, subprocess.TimeoutExpired, UnicodeError) as e: return ModelSyncSkipped(f"`{name} debug models` failed: {e}") if completed.returncode == 0: return completed.stdout @@ -793,6 +850,25 @@ def _stock_codex_models( return ModelSyncSkipped(f"`{name} debug models` printed no model catalog: {e.errors()[0]['msg']}") +def _codex_version( + binary: str, env: Mapping[str, str], *, run: Callable[..., subprocess.CompletedProcess[str]] +) -> str | None: + """The version `codex --version` prints (`codex-cli 0.165.0`), or None when it cannot be read.""" + try: + completed: Final = run( + _windows_command(binary, (binary, "--version")), + env=dict(env), + stdin=subprocess.DEVNULL, + capture_output=True, + encoding="utf-8", + timeout=_CODEX_PREFLIGHT_TIMEOUT_SECONDS, + ) + except (OSError, subprocess.TimeoutExpired, UnicodeError): + return None + words: Final = completed.stdout.split() + return words[-1] if completed.returncode == 0 and words else None + + def codex_model_sync_args( base_env: Mapping[str, str], base_url: str, @@ -811,7 +887,8 @@ def codex_model_sync_args( atomically replaced on every launch. The Codex at `binary` first lists its own models, so the ones the proxy serves keep that Codex's entries, an alias included when every unpaused deployment /v1/model/info lists for it routes - to the same OpenAI model and Codex knows that model, and then + to the same OpenAI model and Codex knows that model, and its version, so a + model it does not know offers only reasoning efforts it can parse, and then reads the file back once before it is handed over. The key never lands in the file. A failed fetch, read, listing, write or read-back is reported rather than raised: Codex still launches with its built-in catalog and takes @@ -834,7 +911,9 @@ def codex_model_sync_args( if isinstance(stock, ModelSyncSkipped): return stock upstreams: Final = _fetch_model_upstreams(base_url, api_key, get=get) - catalog: Final = codex_model_catalog(listing, stock, instructions, upstreams) + groups: Final = fetch_model_limits(base_url, api_key, get=get) + version: Final = _codex_version(binary, base_env, run=run) + catalog: Final = codex_model_catalog(listing, stock, instructions, upstreams, groups, version) if catalog is None: return ModelSyncSkipped(f"{base_url.rstrip('/')}/v1/models lists no chat models") try: diff --git a/litellm/proxy/client/cli/commands/pi.py b/litellm/proxy/client/cli/commands/pi.py index 357b82bdf67..37d485dc1b9 100644 --- a/litellm/proxy/client/cli/commands/pi.py +++ b/litellm/proxy/client/cli/commands/pi.py @@ -50,8 +50,11 @@ class PiSyncError: @dataclass(frozen=True, slots=True) class ModelLimits: + """What /model_group/info says about one model group; `supported_reasoning_efforts` is None when unknown.""" + context_window: int | None max_tokens: int | None + supported_reasoning_efforts: tuple[str, ...] | None = None _NonEmptyString = Annotated[str, StringConstraints(min_length=1)] @@ -76,9 +79,12 @@ class _ModelList(LiteLLMBaseModel): class _ModelGroup(LiteLLMBaseModel): + model_config = ConfigDict(frozen=True) + model_group: str max_input_tokens: float | None = None max_output_tokens: float | None = None + supported_reasoning_efforts: tuple[str, ...] | None = None class _ModelGroupList(LiteLLMBaseModel): @@ -137,8 +143,8 @@ def fetch_model_limits( *, get: Callable[..., requests.Response] = requests.get, ) -> Mapping[str, ModelLimits]: - """Best effort: pi falls back to its own defaults for models without limits, - so an unavailable /model_group/info must not block the launch.""" + """Best effort: pi, OpenCode and Codex fall back to their own defaults for models without + limits or reasoning efforts, so an unavailable /model_group/info must not block the launch.""" url: Final = base_url.rstrip("/") + "/model_group/info" try: resp: Final = get( @@ -156,6 +162,7 @@ def fetch_model_limits( group.model_group: ModelLimits( context_window=int(group.max_input_tokens) if group.max_input_tokens else None, max_tokens=int(group.max_output_tokens) if group.max_output_tokens else None, + supported_reasoning_efforts=group.supported_reasoning_efforts, ) for group in listing.data } @@ -168,6 +175,33 @@ def models_json_path(env: Mapping[str, str]) -> Path: return root / "models.json" +_PI_LADDER_LEVELS: Final = ("minimal", "low", "medium", "high") +_PI_OPT_IN_LEVELS: Final = ("xhigh", "max") +_REASONING_OFF_EFFORT: Final = "none" + + +def _thinking_level_map(efforts: tuple[str, ...]) -> dict[str, JsonValue]: # mutable-ok: JSON object is serialized + """Each level pi offers maps to a value: pi drops a level mapped to null and shows xhigh and max only when set.""" + return { + "off": _REASONING_OFF_EFFORT if _REASONING_OFF_EFFORT in efforts else None, + **{level: level if level in efforts else None for level in _PI_LADDER_LEVELS}, + **{level: level for level in _PI_OPT_IN_LEVELS if level in efforts}, + } + + +def _reasoning_fields(limit: ModelLimits | None) -> Mapping[str, JsonValue]: + efforts: Final = limit.supported_reasoning_efforts if limit else None + if not efforts: + return MappingProxyType({}) + return MappingProxyType( + { + "reasoning": True, + "thinkingLevelMap": _thinking_level_map(efforts), + "compat": {"supportsDeveloperRole": False}, + } + ) + + def _model_entry( model_id: str, limits: Mapping[str, ModelLimits] ) -> dict[str, JsonValue]: # mutable-ok: JSON object is serialized @@ -178,7 +212,7 @@ def _model_entry( output: Final[dict[str, JsonValue]] = ( # mutable-ok: JSON field {"maxTokens": limit.max_tokens} if limit and limit.max_tokens else {} ) - return {"id": model_id, **context, **output} + return {"id": model_id, **context, **output, **_reasoning_fields(limit)} def provider_block( diff --git a/litellm/proxy/common_utils/codex_model_catalog.py b/litellm/proxy/common_utils/codex_model_catalog.py index 63e7178ff99..63fb27d62cf 100644 --- a/litellm/proxy/common_utils/codex_model_catalog.py +++ b/litellm/proxy/common_utils/codex_model_catalog.py @@ -7,7 +7,8 @@ never the OpenAI list shape. This module builds that body from the rows ``/v1/mo lists: a model Codex knows keeps Codex's own entry for it, any other model gets the fallback entry Codex itself uses for an unknown slug, and ``model_info.service_tiers`` becomes the entry's ``service_tiers``, which Codex turns into slash commands (``/ultrafast``) that send ``service_tier`` -upstream. +upstream. A fallback entry offers the model group's ``supported_reasoning_efforts`` as its reasoning +levels, trimmed to the ones the requesting Codex release can parse. The stock entries are Codex 0.159.3's bundled catalog, vendored unchanged from https://github.com/openai/codex/blob/rust-v0.159.3/codex-rs/models-manager/models.json (Apache-2.0) @@ -26,13 +27,16 @@ from pathlib import Path from types import MappingProxyType, NoneType from typing import TYPE_CHECKING, Annotated, Final, Literal, TypeAlias -from pydantic import ConfigDict, StringConstraints, TypeAdapter, ValidationError +from packaging.version import Version +from pydantic import ConfigDict, Field, StringConstraints, TypeAdapter, ValidationError from litellm._logging import verbose_proxy_logger +from litellm.litellm_core_utils.reasoning_effort_strength import nearest_declared_reasoning_effort from litellm.types.llms.base_model import LiteLLMBaseModel if TYPE_CHECKING: from litellm.router import Router + from litellm.router_api import RouterRegistry from litellm.types.proxy.model_listing import ModelInfoResponse CODEX_BASE_INSTRUCTIONS_PATH: Final = Path(__file__).with_name("codex_base_instructions.md") @@ -43,6 +47,10 @@ CODEX_CHAT_MODES: Final = frozenset({"chat", "responses"}) _TierId = Annotated[str, StringConstraints(strip_whitespace=True, min_length=1)] _BODY_PREFIX: Final = '{"models":[' _BODY_SUFFIX: Final = "]}" +_FIRST_CODEX_WITH_CUSTOM_EFFORTS: Final = Version("0.138.0") +_EFFORTS_EVERY_CODEX_PARSES: Final = frozenset({"none", "minimal", "low", "medium", "high", "xhigh"}) +_PREFERRED_DEFAULT_EFFORT: Final = "medium" +_CODEX_UNSET_EFFORT: Final = "none" class CodexServiceTier(LiteLLMBaseModel): @@ -67,6 +75,15 @@ class _ConfiguredTier(LiteLLMBaseModel): _CONFIGURED_TIERS: Final = TypeAdapter(tuple[_TierId | _ConfiguredTier, ...]) +class CodexFallbackReasoningLevel(LiteLLMBaseModel): + """A `ReasoningEffortPreset` as Codex reads it: picking it sends `effort` as the request's reasoning effort.""" + + model_config = ConfigDict(frozen=True) + + effort: str + description: str + + class CodexTruncationPolicy(LiteLLMBaseModel): mode: Literal["bytes"] = "bytes" limit: int = 10_000 @@ -82,10 +99,13 @@ class CodexFallbackModel(LiteLLMBaseModel): same as `codex -m` did. """ + model_config = ConfigDict(frozen=True) + slug: str display_name: str description: NoneType = None - supported_reasoning_levels: tuple[()] = () + default_reasoning_level: str | None = Field(default=None, exclude_if=lambda level: level is None) + supported_reasoning_levels: tuple[CodexFallbackReasoningLevel, ...] = () shell_type: Literal["unified_exec"] = "unified_exec" visibility: Literal["list"] = "list" supported_in_api: Literal[True] = True @@ -153,7 +173,8 @@ class CodexCatalogRow: """What the catalog needs to know about one listed model: its public id, the listing's mode and input limit, the upstream model behind it, the reasoning effort its deployments pin, and the `model_info` values the operator set; `service_tiers` holds one raw `model_info.service_tiers` - value per deployment behind the id, None where a deployment sets none.""" + value per deployment behind the id, None where a deployment sets none. `supported_reasoning_efforts` + is the model group's, None when unknown.""" id: str mode: str | None = None @@ -162,6 +183,7 @@ class CodexCatalogRow: reasoning_effort: str | None = None display_name: str | None = None service_tiers: tuple[object, ...] = () + supported_reasoning_efforts: tuple[str, ...] | None = None search_tool_support: tuple[bool | None, ...] = () @@ -261,8 +283,40 @@ def offered_search_tool(per_deployment: Sequence[bool | None]) -> bool | None: return True if per_deployment and None not in per_deployment else None -def _codex_can_drive(row: CodexCatalogRow) -> bool: - return "*" not in row.id and (row.mode is None or row.mode in CODEX_CHAT_MODES) +def _codex_can_drive(model_id: str, mode: str | None) -> bool: + return "*" not in model_id and (mode is None or mode in CODEX_CHAT_MODES) + + +def _parses_custom_efforts(client_version: str | None) -> bool: + try: + return client_version is not None and Version(client_version) >= _FIRST_CODEX_WITH_CUSTOM_EFFORTS + except ValueError: + return False + + +def codex_reasoning_levels( + efforts: Sequence[str], client_version: str | None +) -> tuple[tuple[CodexFallbackReasoningLevel, ...], str | None]: + """The reasoning levels a fallback entry offers for a model group's efforts, and its default level. + + Codex before 0.138.0 fails the whole catalog on an effort it has no name for, so for it, and for + a version that does not parse, only the efforts every release knows are offered. The default is + set whenever a level is offered, because Codex reads a missing one as `none`: `none` itself when the + group accepts it, so a user who picks no level sends what Codex sent before, else medium, else the + nearest weaker declared level, so picking no level never costs more reasoning than Codex's usual medium, + else the weakest declared level, else the first one.""" + parses_any: Final = _parses_custom_efforts(client_version) + offered: Final = tuple(effort for effort in efforts if parses_any or effort in _EFFORTS_EVERY_CODEX_PARSES) + if not offered: + return (), None + nearest: Final = nearest_declared_reasoning_effort(_PREFERRED_DEFAULT_EFFORT, offered, "down") + levels: Final = tuple( + CodexFallbackReasoningLevel(effort=effort, description=f"Sends reasoning_effort={effort} upstream") + for effort in offered + ) + if _CODEX_UNSET_EFFORT in offered: + return levels, _CODEX_UNSET_EFFORT + return levels, nearest if nearest in offered else offered[0] def _stock_for(row: CodexCatalogRow, stock: Mapping[str, CodexStockModel]) -> CodexStockModel | None: @@ -309,6 +363,7 @@ def _entry( served: frozenset[str], instructions: str, alias_defaults: CodexAliasDefaults, + client_version: str | None, ) -> CodexCatalogEntry: stock_model: Final = _stock_for(row, stock) stock_tiers: Final = MappingProxyType( @@ -317,9 +372,12 @@ def _entry( configured: Final = configured_service_tiers(row.service_tiers, row.id, stock_tiers) search_tool: Final = offered_search_tool(row.search_tool_support) if stock_model is None: + levels, default_level = codex_reasoning_levels(row.supported_reasoning_efforts or (), client_version) return CodexFallbackModel( slug=row.id, display_name=row.display_name if row.display_name is not None else row.id, + default_reasoning_level=default_level, + supported_reasoning_levels=levels, priority=index, service_tiers=configured if configured is not None else (), context_window=row.max_input_tokens, @@ -406,6 +464,7 @@ def codex_models_response_json( instructions: str | None = None, byte_limit: int | None = CODEX_CATALOG_BYTE_LIMIT, alias_defaults: CodexAliasDefaults = "codex", + client_version: str | None = None, ) -> CodexCatalogBody: """Codex's `ModelsResponse` body for `rows`, in listing order, cut to Codex's byte limit. @@ -417,14 +476,15 @@ def codex_models_response_json( keeps every entry. A row listed under another name than the Codex model its `upstream_model` names (an alias) takes that model's entry; `alias_defaults="deployment"` drops the defaults Codex picks for that model in favor of what the alias's deployments pin, while `"codex"` keeps - them, as Codex would use them for the model's own name. + them, as Codex would use them for the model's own name. `client_version` is the requesting Codex's version, which decides the reasoning + levels a fallback entry may offer; None counts as a version too old for custom levels. """ known: Final = bundled_codex_models() if stock is None else stock prompt: Final = codex_base_instructions() if instructions is None else instructions - drivable: Final = tuple(row for row in rows if _codex_can_drive(row)) + drivable: Final = tuple(row for row in rows if _codex_can_drive(row.id, row.mode)) served: Final = frozenset(row.id for row in drivable) entries: Final = tuple( - _entry(index, row, known, served, prompt, alias_defaults) for index, row in enumerate(drivable) + _entry(index, row, known, served, prompt, alias_defaults, client_version) for index, row in enumerate(drivable) ) serialized: Final = tuple(_serialized(entry) for entry in entries) kept: Final = _kept_positions(entries, serialized, byte_limit) @@ -435,6 +495,15 @@ def codex_models_response_json( ) +def _group_reasoning_efforts( + row: ModelInfoResponse, lookup_id: str, registry: RouterRegistry | None, team_id: str | None +) -> tuple[str, ...] | None: + if registry is None or not _codex_can_drive(row["id"], row.get("mode")): + return None + group: Final = registry.group(lookup_id, include_hidden=True, team_id=team_id) + return group.supported_reasoning_efforts if group is not None else None + + def _catalog_row( row: ModelInfoResponse, lookup_id: str, label_id: str, llm_router: Router | None, team_id: str | None ) -> CodexCatalogRow: @@ -446,6 +515,7 @@ def _catalog_row( upstream_model=registry.resolve(lookup_id, kind="upstream", team_id=team_id) if registry is not None else None, display_name=registry.group(label_id, field="display_name") if registry is not None else None, service_tiers=registry.group(lookup_id, field="service_tiers", team_id=team_id) if registry is not None else (), + supported_reasoning_efforts=_group_reasoning_efforts(row, lookup_id, registry, team_id), search_tool_support=( registry.resolve(lookup_id, kind="search_tool_support", team_id=team_id) if registry is not None else () ), @@ -463,7 +533,8 @@ def codex_catalog_rows( team-scoped rows resolve the way the Anthropic listing's display names do, a `model_group_alias` reading its target's deployments under its own name; `team_id` is the requesting key's team, so the upstream model that picks Codex's stock entry and the tiers are read only off the deployments its - requests can route to. Display names are read under `label_entries`' ids when given.""" + requests can route to, and the reasoning efforts are the ones `/model_group/info` reports for the + entry. Display names are read under `label_entries`' ids when given.""" lookup_ids: Final = MappingProxyType(dict(entries)) label_ids: Final = lookup_ids if label_entries is None else MappingProxyType(dict(label_entries)) return tuple( @@ -480,9 +551,13 @@ def codex_model_list_body( llm_router: Router | None, team_id: str | None = None, label_entries: Sequence[tuple[str, str]] | None = None, + *, + client_version: str | None = None, ) -> str: """The `/v1/models?client_version=...` body, logging the models Codex's byte limit left out.""" - body: Final = codex_models_response_json(codex_catalog_rows(rows, entries, llm_router, team_id, label_entries)) + body: Final = codex_models_response_json( + codex_catalog_rows(rows, entries, llm_router, team_id, label_entries), client_version=client_version + ) if body.left_out: verbose_proxy_logger.warning( "Codex model catalog cut at %d bytes, left out: %s. Models offering a service tier are kept first, " diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 6d92e8733a3..1bd221bca98 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -12337,8 +12337,10 @@ async def model_list( - client_version: Sent by Codex CLI (`?client_version=0.159.3`) when it fetches a provider's model catalog. When present, the response is Codex's own catalog shape (`{"models": [...]}`) built from the same listing, with - each model's `model_info.service_tiers` as its service tiers; absent, - the OpenAI shape below is returned + each model's `model_info.service_tiers` as its service tiers and, for a + model that Codex release does not know, the reasoning efforts + `/model_group/info` reports that it can parse; absent, the OpenAI shape + below is returned - include_metadata: Include additional metadata in the response with fallback information - fallback_type: Type of fallbacks to include ("general", "context_window", "content_policy") Defaults to "general" when include_metadata=true @@ -12497,8 +12499,13 @@ async def model_list( admin_listing: Final = cast(Sequence[ModelInfoResponse], model_data) # cast-ok: rows built above if client_version is not None: return Response( - content=codex_model_list_body( - admin_listing, admin_entries, llm_router, team_id or user_api_key_dict.team_id + content=await asyncio.to_thread( + codex_model_list_body, + admin_listing, + admin_entries, + llm_router, + team_id or user_api_key_dict.team_id, + client_version=client_version, ), media_type="application/json", ) @@ -12573,7 +12580,15 @@ async def model_list( listing: Final = cast(Sequence[ModelInfoResponse], model_data) # cast-ok: rows built above if client_version is not None: return Response( - content=codex_model_list_body(listing, entries, llm_router, team_id or user_api_key_dict.team_id, labels), + content=await asyncio.to_thread( + codex_model_list_body, + listing, + entries, + llm_router, + team_id or user_api_key_dict.team_id, + labels, + client_version=client_version, + ), media_type="application/json", ) @@ -18046,6 +18061,7 @@ def get_model_group_info( registry.deployments( resolve_model_group_alias(registry.aliases(), model) or model, source="routed_group", + team_id=team_id, ) ), } diff --git a/litellm/router.py b/litellm/router.py index f2e68424c8d..3a423b79e49 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -12856,7 +12856,10 @@ class Router(DeprecatedRouterMembers): configurable_clientside_auth_params: CONFIGURABLE_CLIENTSIDE_AUTH_PARAMS = None reasoning_efforts_initialized = False reasoning_efforts_unknown = False - model_list: Final = self._core_get_model_list_of_routed_group(model_group) + serves_team_deployments: Final = team_id is not None and bool( + self._team_deployments_named(team_id, model_group) + ) + model_list: Final = self._core_get_model_list_of_routed_group(model_group, team_id) model_group_description: Final[str | None] = next( (d for d in (_configured_model_info(m).get("description") for m in model_list) if isinstance(d, str)), None, @@ -12864,7 +12867,8 @@ class Router(DeprecatedRouterMembers): for model in model_list: is_match = False if ( - "model_name" in model + serves_team_deployments + or "model_name" in model and model["model_name"] == model_group or "model_name" in model and self._core_pattern_router.route(model_group) is not None @@ -14213,17 +14217,22 @@ class Router(DeprecatedRouterMembers): lambda name: self._core_wildcard_route_deployments(name, team_id), ) - def _core_get_model_list_of_routed_group(self, model_group: str) -> list[DeploymentTypedDict]: + def _core_get_model_list_of_routed_group( + self, model_group: str, team_id: str | None = None + ) -> list[DeploymentTypedDict]: """ The deployments a request the router has already resolved to model_group is - served from: the ones named model_group, the routing group of that name, or - the wildcard route matching it when neither exists. + served from: team_id's own deployments under that public name when the team has + any, else the ones named model_group, the routing group of that name, or the + wildcard route matching it when neither exists. Unlike get_model_list, model_group's own model_group_alias entry is not followed. The router resolves an alias exactly once, so a group reached as an alias target is served by its own deployments, never by a second hop: in the chain X -> T -> U a request to X is served from T's deployments. """ + if team_id is not None and self._team_deployments_named(team_id, model_group): + return self._core_get_all_deployments(model_name=model_group, team_id=team_id) own: Final = cast( # cast-ok: the private alias returns what get_all_deployments returns list[DeploymentTypedDict], self._get_all_deployments(model_name=model_group) ) diff --git a/litellm/router_api.py b/litellm/router_api.py index a1cf1805e90..d62999cf261 100644 --- a/litellm/router_api.py +++ b/litellm/router_api.py @@ -419,7 +419,7 @@ class RouterRegistry: if source == "named": return r._core_deployments_named(model_name) if source == "routed_group": - return r._core_get_model_list_of_routed_group(model_name) + return r._core_get_model_list_of_routed_group(model_name, team_id) if source == "wildcard": return r._core_wildcard_route_deployments(model_name, team_id) if source == "routable": diff --git a/litellm/router_utils/reasoning_effort_capability.py b/litellm/router_utils/reasoning_effort_capability.py index 3c3fad93579..3563fcd96ae 100644 --- a/litellm/router_utils/reasoning_effort_capability.py +++ b/litellm/router_utils/reasoning_effort_capability.py @@ -27,20 +27,22 @@ it above. from collections.abc import Mapping, Sequence from dataclasses import dataclass from types import MappingProxyType -from typing import Final, Literal, TypeAlias, get_args - -from typing_extensions import assert_never +from typing import Final, TypeAlias, get_args import litellm from litellm._logging import verbose_logger from litellm.litellm_core_utils.core_helpers import as_str_mapping from litellm.litellm_core_utils.get_supported_openai_params import get_supported_openai_params +from litellm.litellm_core_utils.reasoning_effort_strength import ( + REASONING_EFFORT_STRENGTH_ORDER, + ReasoningEffortRounding, + nearest_declared_reasoning_effort, +) from litellm.litellm_core_utils.reasoning_effort_utils import reasoning_effort_from_thinking from litellm.llms.anthropic.common_utils import AnthropicModelInfo, validated_object_mapping from litellm.types.llms.anthropic import AnthropicThinkingParam from litellm.types.llms.openai import REASONING_EFFORT -ReasoningEffortRounding: TypeAlias = Literal["up", "down"] RequestedReasoningEffort: TypeAlias = str | Mapping[str, object] | None REASONING_EFFORT_ADVERTISEMENT_ORDER: Final[tuple[str, ...]] = get_args(REASONING_EFFORT) @@ -115,35 +117,6 @@ def declared_reasoning_efforts_for_model(model: str, custom_llm_provider: str) - return declared_reasoning_efforts(entry) -REASONING_EFFORT_STRENGTH_ORDER: Final = ("minimal", "low", "medium", "high", "xhigh", "max") -_STRENGTH_RANK: Final = MappingProxyType({effort: rank for rank, effort in enumerate(REASONING_EFFORT_STRENGTH_ORDER)}) - - -def nearest_declared_reasoning_effort( - requested: str, declared: Sequence[str], rounding: ReasoningEffortRounding = "up" -) -> str: - """Rounding up takes the weakest declared level at least as strong as the request, else the - strongest declared, so the caller gets no less reasoning than it asked for instead of a rejected - call. Rounding down mirrors it: the strongest declared level at most as strong, else the - weakest. none is the off switch rather than a strength, so it is never rounded onto the ladder - and no level is rounded to it: a caller who turned reasoning off must not be billed for it, and - a model that cannot turn it off says so itself. A level outside the strength order is likewise - returned as is for upstream to judge.""" - ranked: Final = sorted( - (effort for effort in declared if effort in _STRENGTH_RANK), key=lambda effort: _STRENGTH_RANK[effort] - ) - if requested in ranked or requested not in _STRENGTH_RANK or not ranked: - return requested - strength: Final = _STRENGTH_RANK[requested] - match rounding: - case "up": - return next((effort for effort in ranked if _STRENGTH_RANK[effort] >= strength), ranked[-1]) - case "down": - return next((effort for effort in reversed(ranked) if _STRENGTH_RANK[effort] <= strength), ranked[0]) - case _: - assert_never(rounding) - - _ROUNDING_KEY: Final = "reasoning_effort_rounding" _DEFAULT_EFFORT: Final = "default" _EFFORT_KEY: Final = "effort" @@ -163,7 +136,7 @@ def deployment_reasoning_effort_levels(deployment_model_info: object) -> tuple[s def deployment_rounds_reasoning_effort(deployment_model_info: object) -> bool: declared: Final = deployment_reasoning_effort_levels(deployment_model_info) - return declared is not None and any(level in _STRENGTH_RANK for level in declared) + return declared is not None and any(level in REASONING_EFFORT_STRENGTH_ORDER for level in declared) def with_adaptive_thinking_as_effort( diff --git a/litellm/types/router.py b/litellm/types/router.py index 56dcad4051a..f927ae0475b 100644 --- a/litellm/types/router.py +++ b/litellm/types/router.py @@ -4,9 +4,11 @@ litellm.Router Types - includes RouterConfig, UpdateRouterConfig, ModelInfo etc import datetime import enum +import functools import re from collections.abc import Container, Mapping, Sequence from dataclasses import dataclass, field +from types import MappingProxyType from typing import ( TYPE_CHECKING, Annotated, @@ -989,9 +991,10 @@ class AlertingConfig(LiteLLMBaseModel): alerting_threshold: float | None = 300 +@functools.cache def _resolved_annotations(model_class: type[object]) -> Mapping[str, object]: """Resolve a class's annotations, keeping each resolved annotation opaque.""" - return get_type_hints(model_class) + return MappingProxyType(get_type_hints(model_class)) class ModelGroupInfo(LiteLLMBaseModel): diff --git a/tests/unit/proxy/client/cli/test_agents.py b/tests/unit/proxy/client/cli/test_agents.py index 6334fc1482e..60a19b11d72 100644 --- a/tests/unit/proxy/client/cli/test_agents.py +++ b/tests/unit/proxy/client/cli/test_agents.py @@ -59,6 +59,9 @@ class _FakeResponse: self.status_code = status_code self.content = json.dumps(body).encode() if body is not None else b"" + def json(self): + return json.loads(self.content) + class _Recorder: def __init__(self, returns=None): @@ -160,24 +163,51 @@ _STOCK_CATALOG = json.dumps({"models": list(_STOCK_MODELS.values())}) class _FakeRun: - """A `codex` that prints `stock` from a bare `debug models` and answers a catalog override with `returncode`. + """A `codex` that prints `stock` from a bare `debug models`, `version` from `--version`, and answers a + catalog override with `returncode`. `stock=None` is a Codex with no `debug models` at all: every call answers with `returncode` and `stderr`. + `version=None` is a `--version` that fails, an exception one that cannot run. """ - def __init__(self, returncode=0, stderr="", stock=_STOCK_CATALOG): + def __init__(self, returncode=0, stderr="", stock=_STOCK_CATALOG, version="codex-cli 0.165.0"): self.returncode = returncode self.stderr = stderr self.stock = stock + self.version = version self.calls = [] def __call__(self, args, **kwargs): self.calls.append((args, kwargs)) + if self.stock is not None and "--version" in str(args): + if isinstance(self.version, Exception): + raise self.version + if self.version is None: + return subprocess.CompletedProcess(args, 2, "", "error: unexpected argument '--version'") + return subprocess.CompletedProcess(args, 0, f"{self.version}\n", "") if self.stock is not None and "model_catalog_json=" not in str(args): return subprocess.CompletedProcess(args, 0, self.stock, "") return subprocess.CompletedProcess(args, self.returncode, "", self.stderr) +def _routed_get(captured, listing, groups, model_info=None): + """A proxy answering /model_group/info with `groups` (or `groups` as the status it fails with), + /v1/model/info with `model_info` (raised when it is an exception) and any other URL with `listing`.""" + + def fake_get(url, headers, timeout): + captured.setdefault("urls", []).append(url) + captured["headers"] = headers + if url.endswith("/v1/model/info"): + if isinstance(model_info, Exception): + raise model_info + return model_info if model_info is not None else _FakeResponse(200, {"data": []}) + if not url.endswith("/model_group/info"): + return _FakeResponse(200, listing) + return _FakeResponse(groups) if isinstance(groups, int) else _FakeResponse(200, {"data": list(groups)}) + + return fake_get + + class _FakeJsonResponse: def __init__(self, status_code, payload=None): self.status_code = status_code @@ -534,15 +564,9 @@ class TestOpencodeModelSync: def _listing(*models): return {"object": "list", "data": list(models)} - def _sync(self, listing, base_env=None, base_url="http://localhost:4000/"): + def _sync(self, listing, base_env=None, base_url="http://localhost:4000/", groups=()): captured = {} - - def fake_get(url, headers, timeout): - captured["url"] = url - captured["headers"] = headers - return _FakeResponse(200, listing) - - env = opencode_model_sync_env(base_env or {}, base_url, "sk-key", get=fake_get) + env = opencode_model_sync_env(base_env or {}, base_url, "sk-key", get=_routed_get(captured, listing, groups)) return captured, env def test_declares_proxy_as_litellm_provider_with_listed_models(self): @@ -550,9 +574,9 @@ class TestOpencodeModelSync: {"id": "gpt-5.5", "object": "model", "created": 1, "owned_by": "openai", "mode": "chat"}, {"id": "claude-opus-4-7", "object": "model", "created": 1, "owned_by": "openai"}, ) - captured, env = self._sync(listing) + captured, env = self._sync(listing, groups=({"model_group": "gpt-5.5", "supported_reasoning_efforts": None},)) - assert captured["url"] == "http://localhost:4000/v1/models" + assert captured["urls"] == ["http://localhost:4000/v1/models", "http://localhost:4000/model_group/info"] assert captured["headers"] == {"Authorization": "Bearer sk-key"} config = json.loads(env["OPENCODE_CONFIG_CONTENT"]) provider = config["provider"]["litellm"] @@ -585,6 +609,56 @@ class TestOpencodeModelSync: assert models["gpt-5.5"]["limit"] == {"context": 400000, "output": 128000} assert "limit" not in models["half"] + @pytest.mark.parametrize( + ("efforts", "reasoning"), + [ + ( + ["low", "high", "max"], + { + "reasoning": True, + "reasoning_options": [{"type": "effort", "values": ["low", "high", "max"]}], + "variants": { + "low": {"reasoningEffort": "low"}, + "high": {"reasoningEffort": "high"}, + "max": {"reasoningEffort": "max"}, + }, + }, + ), + ( + ["none", "turbo"], + { + "reasoning": True, + "reasoning_options": [{"type": "effort", "values": ["none", "turbo"]}], + "variants": {"none": {"reasoningEffort": "none"}, "turbo": {"reasoningEffort": "turbo"}}, + }, + ), + ([], {"variants": {}}), + (None, {}), + ], + ids=["declared", "unknown-names", "non-reasoning", "unknown"], + ) + def test_model_group_reasoning_efforts_become_exactly_the_opencode_variants(self, efforts, reasoning): + listing = self._listing( + { + "id": "m", + "object": "model", + "created": 1, + "owned_by": "openai", + "max_input_tokens": 9, + "max_output_tokens": 3, + } + ) + _, env = self._sync(listing, groups=({"model_group": "m", "supported_reasoning_efforts": efforts},)) + models = json.loads(env["OPENCODE_CONFIG_CONTENT"])["provider"]["litellm"]["models"] + assert models == {"m": {"name": "m", "limit": {"context": 9, "output": 3}, **reasoning}} + + def test_an_unavailable_model_group_info_keeps_the_listing_only_config(self): + listing = self._listing({"id": "m", "object": "model", "created": 1, "owned_by": "openai"}) + _, failed = self._sync(listing, groups=500) + _, unknown = self._sync(listing, groups=({"model_group": "m", "supported_reasoning_efforts": None},)) + assert failed == unknown + assert json.loads(failed["OPENCODE_CONFIG_CONTENT"])["provider"]["litellm"]["models"] == {"m": {"name": "m"}} + def test_non_chat_models_are_left_out(self): listing = self._listing( {"id": "chat", "object": "model", "created": 1, "owned_by": "openai", "mode": "chat"}, @@ -671,25 +745,13 @@ class TestCodexModelSync: def _row(model_id, **extra): return {"id": model_id, "object": "model", "created": 1, "owned_by": "openai", **extra} - def _sync(self, listing, codex_home, base_url="http://localhost:4000/", run=None, model_info=None): + def _sync(self, listing, codex_home, base_url="http://localhost:4000/", run=None, groups=(), model_info=None): captured = {} - - def fake_get(url, headers, timeout): - if headers.get("Authorization") != "Bearer sk-key": - return _FakeResponse(401) - if url.endswith("/v1/model/info"): - if isinstance(model_info, Exception): - raise model_info - return model_info if model_info is not None else _FakeResponse(200, {"data": []}) - captured["url"] = url - captured["headers"] = headers - return _FakeResponse(200, listing) - result = codex_model_sync_args( {"CODEX_HOME": str(codex_home)}, base_url, "sk-key", - get=fake_get, + get=_routed_get(captured, listing, groups, model_info), run=_FakeRun() if run is None else run, ) return captured, result @@ -707,7 +769,11 @@ class TestCodexModelSync: listing = self._listing(self._row("gpt-5.5", mode="chat"), self._row("claude-opus-4-7")) captured, result = self._sync(listing, tmp_path / "codex") - assert captured["url"] == "http://localhost:4000/v1/models" + assert captured["urls"] == [ + "http://localhost:4000/v1/models", + "http://localhost:4000/v1/model/info", + "http://localhost:4000/model_group/info", + ] assert captured["headers"] == {"Authorization": "Bearer sk-key"} path = self._catalog_path(result) assert path == str(tmp_path / "codex" / "litellm-models.json") @@ -980,6 +1046,53 @@ class TestCodexModelSync: assert nullable in entry and entry[nullable] is None assert entry["base_instructions"].startswith("You are a coding agent running in the Codex CLI") + @pytest.mark.parametrize( + ("version", "levels"), + [ + ("codex-cli 0.165.0", ["low", "high", "max"]), + ("codex-cli 0.138.0", ["low", "high", "max"]), + ("codex-cli 0.137.0", ["low", "high"]), + ("codex-cli 0.138.0-alpha.3", ["low", "high"]), + ("codex-cli", ["low", "high"]), + (None, ["low", "high"]), + (OSError("codex vanished"), ["low", "high"]), + (UnicodeDecodeError("utf-8", b"\xff", 0, 1, "invalid start byte"), ["low", "high"]), + ], + ids=["current", "first-custom", "pre-custom", "pre-release", "no-number", "failed", "unrunnable", "undecodable"], + ) + def test_unknown_model_offers_the_group_efforts_the_installed_codex_parses(self, tmp_path, version, levels): + _, result = self._sync( + self._listing(self._row("m")), + tmp_path, + run=_FakeRun(version=version), + groups=({"model_group": "m", "supported_reasoning_efforts": ["low", "high", "max"]},), + ) + + assert isinstance(result, ModelSyncArgs) + entry = self._entries(tmp_path)["m"] + assert [level["effort"] for level in entry["supported_reasoning_levels"]] == levels + assert entry["supported_reasoning_levels"][0] == { + "effort": "low", + "description": "Sends reasoning_effort=low upstream", + } + assert entry["default_reasoning_level"] == "low" + + def test_known_models_and_unknown_or_empty_groups_keep_their_entries(self, tmp_path): + listing = self._listing(self._row("gpt-5.5"), self._row("unknown"), self._row("empty")) + groups = ( + {"model_group": "gpt-5.5", "supported_reasoning_efforts": ["max"]}, + {"model_group": "unknown", "supported_reasoning_efforts": None}, + {"model_group": "empty", "supported_reasoning_efforts": []}, + ) + self._sync(listing, tmp_path, groups=groups) + with_groups = self._entries(tmp_path) + self._sync(listing, tmp_path, groups=500) + + assert with_groups == self._entries(tmp_path) + assert with_groups["gpt-5.5"] == {**_STOCK_MODELS["gpt-5.5"], "priority": 0} + assert [with_groups[slug]["supported_reasoning_levels"] for slug in ("unknown", "empty")] == [[], []] + assert all("default_reasoning_level" not in with_groups[slug] for slug in ("unknown", "empty")) + def test_context_window_comes_from_max_input_tokens_for_unknown_models_only(self, tmp_path): listing = self._listing( self._row("big", max_input_tokens=400000), @@ -1094,7 +1207,7 @@ class TestCodexModelSync: run=run, ) assert self._catalog_path(result) == str(tmp_path / "litellm-models.json") - assert len(run.calls) == 2 + assert len(run.calls) == 3 assert all(binary in command for command, _ in run.calls) def test_opencode_dispatch_never_runs_codex(self): @@ -1119,6 +1232,7 @@ class TestCodexModelSync: assert [command for command, _ in run.calls] == [ ("codex", "debug", "models"), + ("codex", "--version"), ("codex", "-c", f"model_catalog_json={json.dumps(path)}", "debug", "models"), ] for _, options in run.calls: @@ -1157,7 +1271,13 @@ class TestCodexModelSync: assert result.reason == "`codex debug models` exited 1: no output" @pytest.mark.parametrize( - "error", [OSError("codex vanished"), subprocess.TimeoutExpired("codex", 10)], ids=["oserror", "timeout"] + "error", + [ + OSError("codex vanished"), + subprocess.TimeoutExpired("codex", 10), + UnicodeDecodeError("utf-8", b"\xff", 0, 1, "invalid start byte"), + ], + ids=["oserror", "timeout", "undecodable"], ) def test_unrunnable_preflight_is_reported_not_raised(self, tmp_path, error): def failing_run(*a, **k): @@ -1183,6 +1303,7 @@ class TestCodexModelSync: doubled = override.replace('"', '""') assert [command for command, _ in run.calls] == [ f'{_CMD_PREFIX}""{shim}" "debug" "models""', + f'{_CMD_PREFIX}""{shim}" "--version""', f'{_CMD_PREFIX}""{shim}" "-c" "{doubled}" "debug" "models""', ] diff --git a/tests/unit/proxy/client/cli/test_pi.py b/tests/unit/proxy/client/cli/test_pi.py index e6669564358..f90febb3178 100644 --- a/tests/unit/proxy/client/cli/test_pi.py +++ b/tests/unit/proxy/client/cli/test_pi.py @@ -152,16 +152,25 @@ class TestFetchModelLimits: 200, { "data": [ - {"model_group": "m-a", "max_input_tokens": 131072, "max_output_tokens": 8192}, + { + "model_group": "m-a", + "max_input_tokens": 131072, + "max_output_tokens": 8192, + "supported_reasoning_efforts": ["low", "high", "max"], + }, {"model_group": "m-b", "max_input_tokens": None, "max_output_tokens": None}, + {"model_group": "m-c", "supported_reasoning_efforts": []}, ] }, ) limits = fetch_model_limits("http://localhost:4000/", "sk-key", get=fake_get) assert captured["url"] == "http://localhost:4000/model_group/info" - assert limits["m-a"] == ModelLimits(context_window=131072, max_tokens=8192) - assert limits["m-b"] == ModelLimits(context_window=None, max_tokens=None) + assert limits["m-a"] == ModelLimits( + context_window=131072, max_tokens=8192, supported_reasoning_efforts=("low", "high", "max") + ) + assert limits["m-b"] == ModelLimits(context_window=None, max_tokens=None, supported_reasoning_efforts=None) + assert limits["m-c"].supported_reasoning_efforts == () def test_non_200_degrades_to_no_limits(self): assert fetch_model_limits("http://localhost:4000", "sk-key", get=lambda *a, **k: _FakeResponse(403)) == {} @@ -219,6 +228,61 @@ class TestProviderBlock: {"id": "m-2"}, ] + @pytest.mark.parametrize( + ("efforts", "thinking_level_map"), + [ + ( + ("low", "medium", "high", "xhigh", "max"), + { + "off": None, + "minimal": None, + "low": "low", + "medium": "medium", + "high": "high", + "xhigh": "xhigh", + "max": "max", + }, + ), + ( + ("none", "low", "high", "max"), + {"off": "none", "minimal": None, "low": "low", "medium": None, "high": "high", "max": "max"}, + ), + ( + ("none", "minimal", "low", "medium", "high"), + {"off": "none", "minimal": "minimal", "low": "low", "medium": "medium", "high": "high"}, + ), + ( + ("high", "turbo"), + {"off": None, "minimal": None, "low": None, "medium": None, "high": "high"}, + ), + ], + ) + def test_reasoning_efforts_become_pi_thinking_levels_and_keep_the_system_role(self, efforts, thinking_level_map): + block = provider_block( + "http://localhost:4000", + ("m-1",), + {"m-1": ModelLimits(context_window=131072, max_tokens=8192, supported_reasoning_efforts=efforts)}, + ) + assert block["models"] == [ + { + "id": "m-1", + "contextWindow": 131072, + "maxTokens": 8192, + "reasoning": True, + "thinkingLevelMap": thinking_level_map, + "compat": {"supportsDeveloperRole": False}, + } + ] + + @pytest.mark.parametrize("efforts", [None, ()]) + def test_unknown_or_no_reasoning_efforts_leave_the_entry_as_it_was(self, efforts): + block = provider_block( + "http://localhost:4000", + ("m-1",), + {"m-1": ModelLimits(context_window=131072, max_tokens=8192, supported_reasoning_efforts=efforts)}, + ) + assert block["models"] == [{"id": "m-1", "contextWindow": 131072, "maxTokens": 8192}] + class TestSyncModelsJson: def test_creates_file_and_parent_dirs(self, tmp_path): diff --git a/tests/unit/proxy/common_utils/test_codex_model_catalog.py b/tests/unit/proxy/common_utils/test_codex_model_catalog.py index 75b3ee70aab..509303d9635 100644 --- a/tests/unit/proxy/common_utils/test_codex_model_catalog.py +++ b/tests/unit/proxy/common_utils/test_codex_model_catalog.py @@ -113,6 +113,94 @@ def test_unknown_model_entry_carries_every_field_codex_deserializes_without_a_de assert (entry["context_window"], entry["base_instructions"], entry["service_tiers"]) == (272000, _PROMPT, []) +_NO_REASONING_BODY = ( + '{"models":[{"slug":"gpt-6-astra","display_name":"gpt-6-astra","description":null,' + '"supported_reasoning_levels":[],"shell_type":"unified_exec","visibility":"list","supported_in_api":true,' + '"priority":0,"service_tiers":[],"availability_nux":null,"upgrade":null,"support_verbosity":false,' + '"supports_reasoning_summaries":false,"supports_parallel_tool_calls":false,"default_verbosity":null,' + '"apply_patch_tool_type":null,"truncation_policy":{"mode":"bytes","limit":10000},' + '"experimental_supported_tools":[],"context_window":272000,"base_instructions":"base prompt",' + '"supports_search_tool":false}]}' +) + + +@pytest.mark.parametrize("efforts", [None, ()], ids=["unknown", "non-reasoning"]) +@pytest.mark.parametrize("client_version", ["0.165.0", "0.120.0", None]) +def test_unknown_or_no_reasoning_efforts_keep_the_fallback_entry_byte_for_byte(efforts, client_version): + """The body an unknown model got before reasoning efforts reached the catalog.""" + assert ( + _body(_row("gpt-6-astra", supported_reasoning_efforts=efforts), client_version=client_version).json + == _NO_REASONING_BODY + ) + + +_ABSENT = object() + + +@pytest.mark.parametrize( + ("efforts", "client_version", "levels", "default"), + [ + (("low", "high", "max"), "0.165.0", ["low", "high", "max"], "low"), + (("high", "xhigh", "max"), "0.165.0", ["high", "xhigh", "max"], "high"), + ( + ("none", "low", "medium", "high", "xhigh", "max"), + "0.165.0", + ["none", "low", "medium", "high", "xhigh", "max"], + "none", + ), + (("low", "medium", "high"), "0.165.0", ["low", "medium", "high"], "medium"), + (("none", "low", "max"), "0.137.0", ["none", "low"], "none"), + (("none",), "0.165.0", ["none"], "none"), + (("low", "turbo"), "0.165.0", ["low", "turbo"], "low"), + (("low", "high", "max"), "0.138.0", ["low", "high", "max"], "low"), + (("low", "high", "max"), "0.137.9", ["low", "high"], "low"), + (("low", "high", "max"), "0.138.0-alpha.2", ["low", "high"], "low"), + (("minimal", "max"), "not-a-version", ["minimal"], "minimal"), + pytest.param(("minimal", "max"), "9" * 5000, ["minimal"], "minimal", id="version-past-int-digit-limit"), + (("low", "turbo"), None, ["low"], "low"), + (("max",), "0.137.0", [], _ABSENT), + ], +) +def test_unknown_model_offers_the_group_efforts_its_codex_release_parses(efforts, client_version, levels, default): + (entry,) = _models(_row("gpt-6-astra", supported_reasoning_efforts=efforts), client_version=client_version) + + assert [level["effort"] for level in entry["supported_reasoning_levels"]] == levels + assert all( + level["description"] == f"Sends reasoning_effort={level['effort']} upstream" + for level in entry["supported_reasoning_levels"] + ) + assert entry.get("default_reasoning_level", _ABSENT) == default + + +@pytest.mark.parametrize( + "levels", + [[{"effort": "medium", "description": "Codex's own medium"}], [{"effort": "medium"}]], + ids=["described", "undescribed"], +) +def test_a_model_codex_knows_keeps_its_own_reasoning_levels(levels): + stock = CodexStockModel.model_validate( + { + "slug": "gpt-5.5", + "display_name": "GPT-5.5", + "priority": 3, + "visibility": "list", + "default_reasoning_level": "medium", + "supported_reasoning_levels": levels, + } + ) + + (entry,) = json.loads( + codex_models_response_json( + (_row("gpt-5.5", supported_reasoning_efforts=("low", "max")),), + stock={"gpt-5.5": stock}, + instructions=_PROMPT, + client_version="0.165.0", + ).json + )["models"] + + assert (entry["default_reasoning_level"], entry["supported_reasoning_levels"]) == ("medium", levels) + + def test_known_model_keeps_codex_stock_entry_listed_under_the_served_id(): (entry,) = _models(_row("gpt-daybreak")) @@ -507,6 +595,9 @@ def test_catalog_rows_read_the_router_by_each_entry_lookup_id(): upstream_model="openai/gpt-5.5", display_name="Team 5.5", service_tiers=(["ultrafast"], None), + supported_reasoning_efforts=router.get_model_group_info( + "model_name_team-1_c0ffee" + ).supported_reasoning_efforts, search_tool_support=(None, None), ), CodexCatalogRow( @@ -538,6 +629,7 @@ def test_catalog_rows_read_an_alias_off_its_target_under_the_alias_name(): max_input_tokens=9, upstream_model="openai/gpt-6-astra", service_tiers=(["ultrafast"],), + supported_reasoning_efforts=router.get_model_group_info("gpt-6-astra").supported_reasoning_efforts, search_tool_support=(None,), ) (entry,) = json.loads(codex_model_list_body(listing, (("gpt-6", "gpt-6"),), router))["models"] @@ -615,6 +707,48 @@ def test_model_list_body_takes_stock_metadata_off_the_deployment_the_key_team_ro assert [entry["slug"] for entry in served("team-2")] == ["coding-model", "coding"] +def test_model_list_body_offers_what_model_group_info_reports_for_each_listed_model(): + """Declared, unknown and non-reasoning groups through a real router, a hidden alias read like its + target, and the requesting Codex's version deciding which declared levels it is offered.""" + vllm = {"model": "hosted_vllm/qa-local-coder", "api_base": "https://vllm.example/v1"} + router = litellm.Router( + model_list=[ + { + "model_name": "declared", + "litellm_params": vllm, + "model_info": {"supports_reasoning": True, "reasoning_effort_levels": ["low", "high", "max"]}, + }, + {"model_name": "unknown", "litellm_params": vllm}, + {"model_name": "non-reasoning", "litellm_params": vllm, "model_info": {"supports_reasoning": False}}, + ], + model_group_alias={"hidden-declared": {"model": "declared", "hidden": True}}, + ) + names = ("declared", "unknown", "non-reasoning", "hidden-declared") + listing = tuple( + {"id": name, "object": "model", "created": 0, "owned_by": "openai", "mode": "chat"} for name in names + ) + entries = tuple((name, name) for name in names) + + def offered(client_version): + body = codex_model_list_body(listing, entries, router, client_version=client_version) + return { + entry["slug"]: ( + [level["effort"] for level in entry["supported_reasoning_levels"]], + entry.get("default_reasoning_level"), + ) + for entry in json.loads(body)["models"] + } + + assert router.get_model_group_info("declared").supported_reasoning_efforts == ("low", "high", "max") + assert offered("0.165.0") == { + "declared": (["low", "high", "max"], "low"), + "unknown": ([], None), + "non-reasoning": ([], None), + "hidden-declared": (["low", "high", "max"], "low"), + } + assert offered("0.137.0")["declared"] == (["low", "high"], "low") + + def test_catalog_rows_without_a_router_carry_only_the_listing(): listing = ({"id": "plain", "object": "model", "created": 0, "owned_by": "openai", "mode": "chat"},) diff --git a/tests/unit/proxy/proxy_server/test_routes_model_info.py b/tests/unit/proxy/proxy_server/test_routes_model_info.py index 067b7f0e862..86364d93ed0 100644 --- a/tests/unit/proxy/proxy_server/test_routes_model_info.py +++ b/tests/unit/proxy/proxy_server/test_routes_model_info.py @@ -989,6 +989,111 @@ def test_model_group_info_lists_batch_only_to_the_team_that_owns_the_batch_deplo assert ("/v1/batch" in (groups["gpt-mixed"]["supported_endpoints"] or ())) is lists_batch +@pytest.mark.parametrize(("team_id", "efforts"), [(_BYOK_TEAM_ID, ["low", "high", "max"]), ("team-other", None)]) +def test_model_group_info_describes_a_team_model_under_the_public_name_its_team_calls( + client, auth_as, monkeypatch, team_id: str, efforts: list[str] | None +): + """A team's own model is listed under its public name, so its row must describe the team's deployment + behind that name instead of a placeholder, and only for the team that owns it.""" + from litellm.proxy._types import LitellmUserRoles + from litellm.proxy.agent_endpoints import model_list_helpers + + router: Final = litellm.Router( + model_list=[ + { + "model_name": _BYOK_INTERNAL_NAME, + "litellm_params": {"model": "hosted_vllm/qa-local-coder", "api_base": "https://vllm.example/v1"}, + "model_info": { + "team_id": _BYOK_TEAM_ID, + "team_public_model_name": _BYOK_PUBLIC_NAME, + "supports_reasoning": True, + "reasoning_effort_levels": ["low", "high", "max"], + }, + } + ] + ) + + async def append_agents_to_model_group(*, model_groups, user_api_key_dict): + return model_groups + + monkeypatch.setattr(proxy_server, "llm_router", router) + monkeypatch.setattr(proxy_server, "llm_model_list", router.model_list) + monkeypatch.setattr(proxy_server, "user_model", None) + monkeypatch.setattr(proxy_server, "general_settings", {}) + monkeypatch.setattr(proxy_server, "prisma_client", None) + monkeypatch.setattr( + model_list_helpers, "append_agents_to_model_group", AsyncMock(side_effect=append_agents_to_model_group) + ) + + with auth_as(LitellmUserRoles.INTERNAL_USER, user_id=None, team_id=team_id, team_models=[_BYOK_PUBLIC_NAME]): + response: Final = client.get("/model_group/info") + + assert response.status_code == 200 + groups: Final = {group["model_group"]: group for group in response.json()["data"]} + assert groups[_BYOK_PUBLIC_NAME]["supported_reasoning_efforts"] == efforts + + +@pytest.mark.parametrize( + ("team_id", "providers", "efforts", "custom_tags"), + [ + (_BYOK_TEAM_ID, ["fireworks_ai"], ["low", "high", "max"], ["team-tag"]), + ("team-other", ["openai"], [], ["global-tag"]), + ], +) +def test_model_group_info_describes_the_deployments_a_shared_name_routes_the_team_to( + client, auth_as, monkeypatch, team_id: str, providers: list[str], efforts: list[str], custom_tags: list[str] +): + """A team model whose public name is also a global model's name routes the team's requests to the + team's deployment, so the team's row must describe that deployment in every field, while another + team still sees the global model.""" + from litellm.proxy._types import LitellmUserRoles + from litellm.proxy.agent_endpoints import model_list_helpers + + router: Final = litellm.Router( + model_list=[ + { + "model_name": _BYOK_PUBLIC_NAME, + "litellm_params": {"model": "openai/gpt-4.1-mini"}, + "model_info": {"supports_reasoning": False, "custom_tags": ["global-tag"]}, + }, + { + "model_name": _BYOK_INTERNAL_NAME, + "litellm_params": {"model": "fireworks_ai/accounts/fireworks/models/kimi-k3"}, + "model_info": { + "team_id": _BYOK_TEAM_ID, + "team_public_model_name": _BYOK_PUBLIC_NAME, + "supports_reasoning": True, + "reasoning_effort_levels": ["low", "high", "max"], + "custom_tags": ["team-tag"], + }, + }, + ] + ) + + async def append_agents_to_model_group(*, model_groups, user_api_key_dict): + return model_groups + + monkeypatch.setattr(proxy_server, "llm_router", router) + monkeypatch.setattr(proxy_server, "llm_model_list", router.model_list) + monkeypatch.setattr(proxy_server, "user_model", None) + monkeypatch.setattr(proxy_server, "general_settings", {}) + monkeypatch.setattr(proxy_server, "prisma_client", None) + monkeypatch.setattr( + model_list_helpers, "append_agents_to_model_group", AsyncMock(side_effect=append_agents_to_model_group) + ) + + with auth_as(LitellmUserRoles.INTERNAL_USER, user_id=None, team_id=team_id, team_models=[_BYOK_PUBLIC_NAME]): + response: Final = client.get("/model_group/info") + + assert response.status_code == 200 + group: Final = {group["model_group"]: group for group in response.json()["data"]}[_BYOK_PUBLIC_NAME] + assert (group["providers"], group["supported_reasoning_efforts"], group["custom_tags"]) == ( + providers, + efforts, + custom_tags, + ) + + # --------------------------------------------------------------------------- # GET /v2/model/info?exclude_auto_routers # --------------------------------------------------------------------------- diff --git a/tests/unit/proxy/proxy_server/test_routes_models.py b/tests/unit/proxy/proxy_server/test_routes_models.py index b91994b2d04..ffd23fa3a6f 100644 --- a/tests/unit/proxy/proxy_server/test_routes_models.py +++ b/tests/unit/proxy/proxy_server/test_routes_models.py @@ -52,6 +52,7 @@ def patched_models(monkeypatch): router.get_routable_upstream_model = MagicMock(return_value="gpt-4") router.get_configured_display_name = MagicMock(return_value=None) router.get_configured_service_tiers = MagicMock(return_value=()) + router.get_model_group_info = MagicMock(return_value=None) router.get_search_tool_support = MagicMock(return_value=()) route_entry_points_to_stubs(router) @@ -451,6 +452,35 @@ def test_codex_format_carries_configured_service_tiers(client, auth_as, patched_ assert all("service_tiers" not in m for m in openai_response.json()["data"]) +@pytest.mark.parametrize("params", [{}, {"scope": "expand"}]) +def test_codex_reasoning_levels_are_the_ones_the_requesting_codex_parses(client, auth_as, monkeypatch, params): + """Both listing paths hand the route's ``client_version`` to the catalog: Codex before 0.138.0 cannot + parse max, so it is offered there only to a newer release.""" + declared = {"supports_reasoning": True, "reasoning_effort_levels": ["low", "max"]} + vllm = {"model": "hosted_vllm/qa-local-coder", "api_base": "https://vllm.example/v1"} + router = litellm.Router( + model_list=[ + {"model_name": name, "litellm_params": vllm, "model_info": declared} for name in ("gpt-4", "claude-sonnet") + ] + ) + monkeypatch.setattr(proxy_server, "llm_router", router) + monkeypatch.setattr(proxy_server, "llm_model_list", router.model_list) + monkeypatch.setattr(proxy_server, "user_model", None) + monkeypatch.setattr(proxy_server, "general_settings", {}) + monkeypatch.setattr(proxy_server, "prisma_client", None) + + with auth_as(): + current = client.get("/v1/models", params={**params, "client_version": "0.165.0"}) + older = client.get("/v1/models", params={**params, "client_version": "0.137.0"}) + + assert [[level["effort"] for level in m["supported_reasoning_levels"]] for m in current.json()["models"]] == [ + ["low", "max"] + ] * 2 + assert [[level["effort"] for level in m["supported_reasoning_levels"]] for m in older.json()["models"]] == [ + ["low"] + ] * 2 + + @pytest.mark.parametrize("params", [{}, {"scope": "expand"}]) def test_codex_service_tiers_are_read_for_the_key_team(client, auth_as, patched_models, params): """A tier and the upstream model that picks Codex's stock entry are read off the deployments the diff --git a/tests/unit/proxy/public_endpoints/public_v1/test_model_hub.py b/tests/unit/proxy/public_endpoints/public_v1/test_model_hub.py index 11ea33e61c7..143f5bfb83b 100644 --- a/tests/unit/proxy/public_endpoints/public_v1/test_model_hub.py +++ b/tests/unit/proxy/public_endpoints/public_v1/test_model_hub.py @@ -30,7 +30,7 @@ class _FakeRouter: def get_model_group_info(self, model_group: str, team_id: str | None = None) -> ModelGroupInfo | None: return self.infos.get(model_group) - def get_model_list_of_routed_group(self, model_group: str) -> list[DeploymentTypedDict]: + def get_model_list_of_routed_group(self, model_group: str, team_id: str | None = None) -> list[DeploymentTypedDict]: return [] diff --git a/tests/unit/proxy/router_entry_point_stubs.py b/tests/unit/proxy/router_entry_point_stubs.py index afc79f36caa..41acd487b67 100644 --- a/tests/unit/proxy/router_entry_point_stubs.py +++ b/tests/unit/proxy/router_entry_point_stubs.py @@ -50,7 +50,7 @@ def route_entry_points_to_stubs(router: _RouterDouble) -> _RouterDouble: if source == "routing_groups": return router.get_model_list_from_routing_groups(model_name=model_name) if source == "routed_group": - return router.get_model_list_of_routed_group(model_name) + return router.get_model_list_of_routed_group(model_name, **_given(team_id=team_id)) if source == "routable": return router.get_routable_deployments( model_name, *(() if request is None else (request,)), **_given(call_type=call_type) diff --git a/tests/unit/router_utils/test_reasoning_effort_capability.py b/tests/unit/router_utils/test_reasoning_effort_capability.py index 1d8ee338cbb..ce4cb84addf 100644 --- a/tests/unit/router_utils/test_reasoning_effort_capability.py +++ b/tests/unit/router_utils/test_reasoning_effort_capability.py @@ -8,7 +8,6 @@ import pytest import litellm from litellm._logging import JsonFormatter, verbose_logger from litellm.router_utils.reasoning_effort_capability import ( - REASONING_EFFORT_STRENGTH_ORDER, RequestedReasoningEffort, deployment_is_catalog_mapped, intersect_supported_reasoning_efforts, @@ -17,6 +16,7 @@ from litellm.router_utils.reasoning_effort_capability import ( resolve_supported_reasoning_efforts, thinking_budget_effort, ) +from litellm.litellm_core_utils.reasoning_effort_strength import REASONING_EFFORT_STRENGTH_ORDER class TestDeploymentIsCatalogMapped: diff --git a/tests/unit/test_default_branch.py b/tests/unit/test_default_branch.py index bb04ea2a0fa..577b1126976 100644 --- a/tests/unit/test_default_branch.py +++ b/tests/unit/test_default_branch.py @@ -119,6 +119,17 @@ def test_explicit_base_works_without_remote_access( assert "No changed litellm Python files" in checked.stdout +def test_format_check_reads_staged_renames(remote_and_clone: tuple[Path, Path]) -> None: + _, repo = remote_and_clone + (repo / "litellm" / "before_move.py").write_text("value = 2\n") + _commit(repo, "add module") + _git(repo, "mv", "litellm/before_move.py", "litellm/after_move.py") + checked: Final = _make(repo, "lint-format-check-changed", "BASE_REF=origin/release_branch", "UV_RUN=echo") + assert checked.returncode == 0, checked.stdout + checked.stderr + assert "litellm/after_move.py" in checked.stdout + assert "before_move.py" not in checked.stdout + + def _freshness(repo: Path, *args: str) -> subprocess.CompletedProcess[str]: return subprocess.run( [ diff --git a/tests/unit/test_router/test_router_entry_points.py b/tests/unit/test_router/test_router_entry_points.py index 2df5cbc9457..2af7db930c6 100644 --- a/tests/unit/test_router/test_router_entry_points.py +++ b/tests/unit/test_router/test_router_entry_points.py @@ -592,9 +592,9 @@ FORWARDS = [ ((), {"model_name": "gpt"}), ), ( - lambda r: r.registry().deployments("gpt", source="routed_group"), + lambda r: r.registry().deployments("gpt", source="routed_group", team_id="t"), "_core_get_model_list_of_routed_group", - (("gpt",), {}), + (("gpt", "t"), {}), ), ( lambda r: r.registry().deployments("gpt", source="wildcard", team_id="t"), diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index d542880f868..90a0597cea5 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -10547,8 +10547,10 @@ export interface paths { * - client_version: Sent by Codex CLI (`?client_version=0.159.3`) when it fetches a * provider's model catalog. When present, the response is Codex's own * catalog shape (`{"models": [...]}`) built from the same listing, with - * each model's `model_info.service_tiers` as its service tiers; absent, - * the OpenAI shape below is returned + * each model's `model_info.service_tiers` as its service tiers and, for a + * model that Codex release does not know, the reasoning efforts + * `/model_group/info` reports that it can parse; absent, the OpenAI shape + * below is returned * - include_metadata: Include additional metadata in the response with fallback information * - fallback_type: Type of fallbacks to include ("general", "context_window", "content_policy") * Defaults to "general" when include_metadata=true @@ -21465,8 +21467,10 @@ export interface paths { * - client_version: Sent by Codex CLI (`?client_version=0.159.3`) when it fetches a * provider's model catalog. When present, the response is Codex's own * catalog shape (`{"models": [...]}`) built from the same listing, with - * each model's `model_info.service_tiers` as its service tiers; absent, - * the OpenAI shape below is returned + * each model's `model_info.service_tiers` as its service tiers and, for a + * model that Codex release does not know, the reasoning efforts + * `/model_group/info` reports that it can parse; absent, the OpenAI shape + * below is returned * - include_metadata: Include additional metadata in the response with fallback information * - fallback_type: Type of fallbacks to include ("general", "context_window", "content_policy") * Defaults to "general" when include_metadata=true From dacaaad72ed25663ad7f40941c48f653a45289be Mon Sep 17 00:00:00 2001 From: tin-berri Date: Thu, 29 Oct 2026 18:20:47 -0700 Subject: [PATCH 16/17] build: allow filelock 4.x and sync package metadata (#49137) * build: allow filelock 4.x filelock 3.x is the only line the current cap permits, and three CVEs (CVE-2026-112940, CVE-2026-112913, CVE-2026-112907) are fixed only in 4.2.1+. The cap means every downstream that depends on litellm is held on a vulnerable filelock with no way out, since pip has no override. litellm uses FileLock(path, timeout=..., mode=...) and Timeout, in litellm/proxy/client/cli/commands/configure_profiles.py and claude_settings.py. Both keyword arguments still exist in 4.2.4 and Timeout is still exported, so the two call sites are unaffected. The CVEs are in the SQLite-backed ReadWriteLock / AsyncReadWriteLock, which litellm does not use. <5 rather than a floor bump because litellm supports Python 3.10 and filelock 4.x requires >=3.11: widening the cap lets 3.11+ resolve 4.x while 3.10 keeps a 3.x filelock automatically, through filelock's own requires_python. * build: sync core filelock bound and dependency lock --------- Co-authored-by: Speculator55005 <50082482+fas89@users.noreply.github.com> --- packaging/litellm-core/pyproject.toml | 2 +- pyproject.toml | 2 +- uv.lock | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packaging/litellm-core/pyproject.toml b/packaging/litellm-core/pyproject.toml index db50589a675..1141e69c0aa 100644 --- a/packaging/litellm-core/pyproject.toml +++ b/packaging/litellm-core/pyproject.toml @@ -11,7 +11,7 @@ authors = [ ] dependencies = [ "fastuuid>=0.14.0,<1.0; python_version < '3.15'", - "filelock>=3.16.1,<4.0", + "filelock>=3.16.1,<5", "httpx[http2]>=0.28.0,<1.0", "openai>=2.20.0,<4.0.0", "python-dateutil>=2.8.2,<3.0", diff --git a/pyproject.toml b/pyproject.toml index bff00a2dbf1..de1abeddd7e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,7 +15,7 @@ dependencies = [ # When changing a floor, verify it installs + imports on every supported # Python with: `uv pip install --resolution=lowest-direct .` "fastuuid>=0.14.0,<1.0; python_version < '3.15'", - "filelock>=3.16.1,<4.0", + "filelock>=3.16.1,<5", "httpx[http2]>=0.28.0,<1.0", "openai>=2.20.0,<4.0.0", "python-dotenv>=1.0.0,<2.0", diff --git a/uv.lock b/uv.lock index f36fd3495a9..2ba97d15327 100644 --- a/uv.lock +++ b/uv.lock @@ -5242,7 +5242,7 @@ requires-dist = [ { name = "fastapi", marker = "extra == 'proxy'", specifier = ">=0.136.3,<1.0" }, { name = "fastapi-sso", marker = "extra == 'proxy'", specifier = ">=0.19.0,<1.0" }, { name = "fastuuid", marker = "python_full_version < '3.15'", specifier = ">=0.14.0,<1.0" }, - { name = "filelock", specifier = ">=3.16.1,<4.0" }, + { name = "filelock", specifier = ">=3.16.1,<5" }, { name = "google-cloud-aiplatform", marker = "extra == 'google'", specifier = ">=1.133.0,<2.0" }, { name = "google-cloud-aiplatform", marker = "extra == 'proxy-runtime'", specifier = ">=1.133.0,<2.0" }, { name = "google-cloud-iam", marker = "extra == 'extra-proxy'", specifier = ">=2.19.1,<3.0" }, From 636e70708984527aa2a4b89fdeb16886f7cff278 Mon Sep 17 00:00:00 2001 From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 29 Oct 2026 18:20:52 -0700 Subject: [PATCH 17/17] fix(proxy)!: stop answering with the literal string "None" as error type and param on the proxy-server, auth, spend, and key health routes (#39540) * fix(proxy): stop answering with the literal string "None" on the proxy-server, auth, and health routes The error objects these routes return defaulted `type` and `param` to the four-character string "None", so a client's error handler matched no known OpenAI type and fell into its generic branch. Route them through the shared openai_error_payload helpers instead, so `type` comes from the carried type or the HTTP status and `param` serializes as JSON null. * fix(proxy): stop answering with the literal string "None" on the spend and management routes The error objects these routes return defaulted `type` and `param` to the four-character string "None", so a client's error handler matched no known OpenAI type and fell into its generic branch. Route them through the shared openai_error_payload helpers instead, so `type` comes from the carried type or the HTTP status and `param` serializes as JSON null. * test(proxy): pin the mid-stream failure frame and the DB outage body to a JSON null param * fix(proxy): type upstream timeouts as timeout_error and update integration asserts for the null param A 408 with no carried type now answers timeout_error instead of invalid_request_error, so a client branching on type keeps retrying it. Three integration suites still compared auth rejections against param="None". * fix(proxy): keep a 408 on main's invalid_request_error type and pin the transcription file cap's 400 * fix(proxy): type a route failure by the status it answers with after the tag-routing and no-deployment remap * fix(proxy): keep auth_error on proxy-raised 401/403 route failures and ban the string None as an error type or param * fix(proxy): type a hook's 401 or 403 on completions, moderations and transcription by its status, as chat does Only a status the proxy remapped into 401 or 403, such as a tag-routing denial, keeps auth_error. * test(proxy): build the transcription file-cap test on a real router so it holds after the router refactor --- .github/workflows/test-linting.yml | 6 + litellm/proxy/auth/auth_exception_handler.py | 7 +- litellm/proxy/auth/handle_jwt.py | 2 +- litellm/proxy/auth/route_checks.py | 2 +- litellm/proxy/auth/user_api_key_auth.py | 12 +- .../common_utils/openai_error_payload.py | 31 ++++- litellm/proxy/db/exception_handler.py | 2 +- .../key_management_endpoints.py | 6 +- .../team_callback_endpoints.py | 11 +- litellm/proxy/proxy_server.py | 106 ++++++++--------- .../spend_management_endpoints.py | 49 ++++---- .../check_no_none_string_error_fields.py | 71 ++++++++++++ .../test_check_no_none_string_error_fields.py | 38 +++++++ .../test_guardrail_submissions.py | 2 +- .../test_jwt_key_mapping_lifecycle.py | 4 +- .../authorization/test_memory_ownership.py | 2 +- .../authorization/test_team_admin_gate.py | 2 +- .../test_unknown_key_generic_401.py | 8 +- .../test_user_self_update_guard.py | 2 +- .../authorization/test_warmed_policy.py | 2 +- .../authorization/test_workflow_ownership.py | 2 +- .../management/test_health_routing.py | 2 +- .../management/test_scim_admin_group.py | 2 +- .../test_user_delete_revokes_keys.py | 2 +- ...st_arize_otel_v2_openinference_sad_edge.py | 2 +- .../test_presidio_entity_masking.py | 2 +- .../security/test_config_overrides.py | 6 +- .../spend/test_global_spend_reset.py | 2 +- .../proxy/auth/test_auth_exception_handler.py | 14 +++ tests/unit/proxy/auth/test_handle_jwt.py | 1 + .../unit/proxy/auth/test_user_api_key_auth.py | 27 +++++ .../test_internal_user_endpoints.py | 2 +- .../test_key_management_endpoints.py | 17 +++ .../test_team_endpoints.py | 2 +- .../proxy_server/test_exception_handlers.py | 2 +- .../proxy/proxy_server/test_routes_audio.py | 107 +++++++++++++++++- .../proxy_server/test_routes_completions.py | 38 +++++++ .../proxy_server/test_streaming_helpers.py | 35 +++++- .../test_spend_management_endpoints.py | 16 +++ ...test_proxy_server_endpoints_and_startup.py | 90 +++++++++++++++ 40 files changed, 606 insertions(+), 130 deletions(-) create mode 100644 tests/code_coverage_tests/check_no_none_string_error_fields.py create mode 100644 tests/code_coverage_tests/test_check_no_none_string_error_fields.py diff --git a/.github/workflows/test-linting.yml b/.github/workflows/test-linting.yml index bd7343ea222..88c66f0011d 100644 --- a/.github/workflows/test-linting.yml +++ b/.github/workflows/test-linting.yml @@ -630,6 +630,12 @@ jobs: - name: check_migrations_lock_timeout run: uv run --no-sync python ./tests/code_coverage_tests/check_migrations_lock_timeout.py + - name: check_no_none_string_error_fields + run: uv run --no-sync python ./tests/code_coverage_tests/check_no_none_string_error_fields.py + + - name: test_check_no_none_string_error_fields + run: uv run --no-sync pytest -q --noconftest -p no:cacheprovider -c /dev/null tests/code_coverage_tests/test_check_no_none_string_error_fields.py + - name: check_no_publicly_known_master_key run: uv run --no-sync python ./tests/code_coverage_tests/check_no_publicly_known_master_key.py diff --git a/litellm/proxy/auth/auth_exception_handler.py b/litellm/proxy/auth/auth_exception_handler.py index ed743764fce..2ec2274db2b 100644 --- a/litellm/proxy/auth/auth_exception_handler.py +++ b/litellm/proxy/auth/auth_exception_handler.py @@ -30,6 +30,7 @@ from litellm.proxy.auth.auth_utils import ( # noqa: F401 # legacy module expor ) from litellm.proxy.auth.model_access_denied import ModelAccessDeniedHTTPException from litellm.proxy.auth.resolvers.exceptions import KeyNotFoundError +from litellm.proxy.common_utils.openai_error_payload import openai_error_param from litellm.proxy.db.exception_handler import PrismaDBExceptionHandler from litellm.types.services import ServiceTypes @@ -61,14 +62,14 @@ def _as_proxy_exception(e: Exception) -> ProxyException: message=str(e.detail), internal_message=e.internal_message, type=ProxyErrorTypes.auth_error, - param="None", + param=None, code=e.status_code, ) if isinstance(e, HTTPException): return ProxyException( message=getattr(e, "detail", f"Authentication Error({e})"), type=ProxyErrorTypes.auth_error, - param=getattr(e, "param", "None"), + param=openai_error_param(e), code=getattr(e, "status_code", status.HTTP_401_UNAUTHORIZED), ) if isinstance(e, ProxyException): @@ -78,7 +79,7 @@ def _as_proxy_exception(e: Exception) -> ProxyException: return ProxyException( message="Authentication Error, " + str(e), type=ProxyErrorTypes.auth_error, - param=getattr(e, "param", "None"), + param=openai_error_param(e), code=status.HTTP_401_UNAUTHORIZED, ) diff --git a/litellm/proxy/auth/handle_jwt.py b/litellm/proxy/auth/handle_jwt.py index 9d800a1fd63..7bd7b1226b1 100644 --- a/litellm/proxy/auth/handle_jwt.py +++ b/litellm/proxy/auth/handle_jwt.py @@ -225,7 +225,7 @@ def jwks_unavailable_exception(error: JWKSUnreachableError) -> ProxyException: f"unreachable, so the JWT signature could not be verified. Please retry shortly. Error: {error}" ), type=ProxyErrorTypes.auth_provider_unavailable, - param="None", + param=None, code=status.HTTP_503_SERVICE_UNAVAILABLE, ) diff --git a/litellm/proxy/auth/route_checks.py b/litellm/proxy/auth/route_checks.py index 56be44600fd..105acb237af 100644 --- a/litellm/proxy/auth/route_checks.py +++ b/litellm/proxy/auth/route_checks.py @@ -302,7 +302,7 @@ class RouteChecks: f"keys/users/teams. Route={route}. Your role={user_role}. Your user_id={masked_user_id}" ), type=ProxyErrorTypes.auth_error, - param="None", + param=None, code=status.HTTP_401_UNAUTHORIZED, ) diff --git a/litellm/proxy/auth/user_api_key_auth.py b/litellm/proxy/auth/user_api_key_auth.py index 2b6471952fd..67410e867d6 100644 --- a/litellm/proxy/auth/user_api_key_auth.py +++ b/litellm/proxy/auth/user_api_key_auth.py @@ -2586,7 +2586,7 @@ async def user_api_key_auth_builder( raise ProxyException( message="Authentication Error, No api key passed in.", type=ProxyErrorTypes.auth_error, - param="None", + param=None, code=status.HTTP_401_UNAUTHORIZED, ) elif api_key == "": @@ -2594,7 +2594,7 @@ async def user_api_key_auth_builder( raise ProxyException( message="Authentication Error, Malformed API Key passed in. Ensure Key has `Bearer ` prefix.", type=ProxyErrorTypes.auth_error, - param="None", + param=None, code=status.HTTP_401_UNAUTHORIZED, ) @@ -2835,7 +2835,7 @@ async def user_api_key_auth_builder( raise ProxyException( message=f"Authentication Error, Tried to access route={route}, which is only for MASTER KEY", type=ProxyErrorTypes.auth_error, - param="None", + param=None, code=status.HTTP_401_UNAUTHORIZED, ) @@ -2968,7 +2968,7 @@ async def validate_resolved_virtual_key( # noqa: C901 # Preserve ordering of e raise ProxyException( message="Authentication Error, Key is blocked. Update via `/key/unblock` if you're an admin.", type=ProxyErrorTypes.auth_error, - param="None", + param=None, code=status.HTTP_401_UNAUTHORIZED, ) await enforce_key_and_fallback_model_access( @@ -3019,7 +3019,7 @@ async def validate_resolved_virtual_key( # noqa: C901 # Preserve ordering of e "Keys owned by this user cannot be used." ), type=ProxyErrorTypes.auth_error, - param="None", + param=None, code=status.HTTP_401_UNAUTHORIZED, ) @@ -3325,7 +3325,7 @@ async def validate_resolved_virtual_key( # noqa: C901 # Preserve ordering of e raise ProxyException( message="Authentication Error, Invalid proxy server token passed", type=ProxyErrorTypes.auth_error, - param="None", + param=None, code=status.HTTP_401_UNAUTHORIZED, ) if valid_token_dict is not None: diff --git a/litellm/proxy/common_utils/openai_error_payload.py b/litellm/proxy/common_utils/openai_error_payload.py index f92fbeef9a1..53b2a05d40f 100644 --- a/litellm/proxy/common_utils/openai_error_payload.py +++ b/litellm/proxy/common_utils/openai_error_payload.py @@ -147,8 +147,37 @@ def _is_failover_refusal(exc: Exception) -> bool: return any(refusal in message for refusal in _FAILOVER_REFUSALS) +def answered_status_code(message: str, status_code: int) -> int: + return int(proxy_exception_status_code(message, str(status_code))) + + +def route_failure_proxy_exception( + exc: Exception, + message: str, + default_status_code: int, + headers: dict[str, str] | None = None, # mutable-ok: ProxyException.headers + openai_code: str | None = None, +) -> ProxyException: + raised_status: Final = error_status_code(exc, default_status_code) + answered_status: Final = answered_status_code(message, raised_status) + return ProxyException( + message=message, + type=_route_failure_error_type(exc, raised_status, answered_status), + param=openai_error_param(exc), + headers=headers, + openai_code=openai_code, + code=answered_status, + ) + + +def _route_failure_error_type(exc: Exception, raised_status: int, answered_status: int) -> str: + if answered_status != raised_status and answered_status in _AUTH_FAILURE_STATUS_CODES: + return ProxyErrorTypes.auth_error.value + return openai_error_type(exc, answered_status) + + def _openai_shaped_proxy_exception(exc: Exception, message: str, status_code: int) -> ProxyException: - answered_status: Final = int(proxy_exception_status_code(message, str(status_code))) + answered_status: Final = answered_status_code(message, status_code) return ProxyException( message=message, type=_catch_all_error_type(exc, answered_status), diff --git a/litellm/proxy/db/exception_handler.py b/litellm/proxy/db/exception_handler.py index 3f7ea334a99..baf53604ea5 100644 --- a/litellm/proxy/db/exception_handler.py +++ b/litellm/proxy/db/exception_handler.py @@ -435,7 +435,7 @@ class PrismaDBExceptionHandler: return ProxyException( message=PrismaDBExceptionHandler.database_unavailable_message(e), type=ProxyErrorTypes.no_db_connection, - param="None", + param=None, code=HTTPStatus.SERVICE_UNAVAILABLE.value, ) diff --git a/litellm/proxy/management_endpoints/key_management_endpoints.py b/litellm/proxy/management_endpoints/key_management_endpoints.py index 69f28b6d388..a6814b593ca 100644 --- a/litellm/proxy/management_endpoints/key_management_endpoints.py +++ b/litellm/proxy/management_endpoints/key_management_endpoints.py @@ -75,7 +75,7 @@ from litellm.proxy.common_utils.config_sync_pubsub import ( coordination_redis_cache, publish_config_change, ) -from litellm.proxy.common_utils.openai_error_payload import proxy_exception_for +from litellm.proxy.common_utils.openai_error_payload import openai_error_param, proxy_exception_for from litellm.proxy.common_utils.rbac_utils import check_org_admin_can_generate_keys from litellm.proxy.common_utils.timezone_utils import get_budget_reset_time from litellm.proxy.common_utils.user_api_key_cache import AUTH_OBJECTS_TARGET, UserApiKeyCache @@ -4959,7 +4959,7 @@ async def info_key_fn( raise ProxyException( message="Internal server error from /key/info. Details are in the proxy server logs.", type="internal_error", - param=getattr(e, "param", "None"), + param=openai_error_param(e), code=status.HTTP_500_INTERNAL_SERVER_ERROR, ) @@ -8156,7 +8156,7 @@ async def key_health( raise ProxyException( message=f"Key health check failed: {e}", type=ProxyErrorTypes.internal_server_error, - param=getattr(e, "param", "None"), + param=openai_error_param(e), code=status.HTTP_500_INTERNAL_SERVER_ERROR, ) diff --git a/litellm/proxy/management_endpoints/team_callback_endpoints.py b/litellm/proxy/management_endpoints/team_callback_endpoints.py index d1e1c4bd25e..5b778392664 100644 --- a/litellm/proxy/management_endpoints/team_callback_endpoints.py +++ b/litellm/proxy/management_endpoints/team_callback_endpoints.py @@ -46,6 +46,7 @@ from litellm.proxy.common_utils.callback_utils import ( # noqa: F401 # legacy encrypt_callback_vars, is_sensitive_callback_key, ) +from litellm.proxy.common_utils.openai_error_payload import openai_error_param from litellm.proxy.litellm_pre_call_utils import ( # noqa: F401 # legacy module exports _get_validated_callback_metadata, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export convert_key_logging_metadata_to_callback, @@ -463,7 +464,7 @@ async def add_team_callbacks( raise ProxyException( message="Internal Server Error, " + str(e), type=ProxyErrorTypes.internal_server_error.value, - param=getattr(e, "param", "None"), + param=openai_error_param(e), code=status.HTTP_500_INTERNAL_SERVER_ERROR, ) @@ -606,7 +607,7 @@ async def delete_team_callback( raise ProxyException( message="Internal Server Error, " + str(e), type=ProxyErrorTypes.internal_server_error.value, - param=getattr(e, "param", "None"), + param=openai_error_param(e), code=status.HTTP_500_INTERNAL_SERVER_ERROR, ) else: @@ -747,7 +748,7 @@ async def disable_team_logging( raise ProxyException( message="Internal Server Error, " + str(e), type=ProxyErrorTypes.internal_server_error.value, - param=getattr(e, "param", "None"), + param=openai_error_param(e), code=status.HTTP_500_INTERNAL_SERVER_ERROR, ) @@ -838,7 +839,7 @@ async def get_team_callbacks( raise ProxyException( message=getattr(e, "detail", f"Internal Server Error({e})"), type=ProxyErrorTypes.internal_server_error.value, - param=getattr(e, "param", "None"), + param=openai_error_param(e), code=getattr(e, "status_code", status.HTTP_500_INTERNAL_SERVER_ERROR), ) elif isinstance(e, ProxyException): @@ -846,6 +847,6 @@ async def get_team_callbacks( raise ProxyException( message="Internal Server Error, " + str(e), type=ProxyErrorTypes.internal_server_error.value, - param=getattr(e, "param", "None"), + param=openai_error_param(e), code=status.HTTP_500_INTERNAL_SERVER_ERROR, ) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 1bd221bca98..b0ccf74c63b 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -529,9 +529,12 @@ from litellm.proxy.common_utils.openai_endpoint_utils import ( ) from litellm.proxy.common_utils.openai_error_payload import ( LITELLM_CALL_ID_HEADER, + attribute_of, headers_with_litellm_call_id, litellm_call_id_headers, + openai_error_param, proxy_exception_for, + route_failure_proxy_exception, with_litellm_call_id, ) from litellm.proxy.common_utils.periodic_reload_schedule import ( @@ -9898,11 +9901,8 @@ async def async_assistants_data_generator(response, user_api_key_dict: UserAPIKe # Including it in the SSE response leaks internal details to clients. error_msg: Final = str(e) - proxy_exception: Final = ProxyException( - message=getattr(e, "message", error_msg), - type=getattr(e, "type", "None"), - param=getattr(e, "param", "None"), - code=getattr(e, "status_code", 500), + proxy_exception: Final = route_failure_proxy_exception( + e, str(attribute_of(e, "message", error_msg)), status.HTTP_500_INTERNAL_SERVER_ERROR ) error_returned: Final = json.dumps({"error": proxy_exception.to_dict()}) yield f"data: {error_returned}\n\n" @@ -10712,11 +10712,10 @@ async def async_data_generator( { "error": sse_error_payload(client_error)[1] if isinstance(client_error, HTTPException) - else ProxyException( - message=getattr(client_error, "message", str(client_error)), - type=getattr(client_error, "type", "None"), - param=getattr(client_error, "param", "None"), - code=getattr(client_error, "status_code", 500), + else route_failure_proxy_exception( + client_error, + str(attribute_of(client_error, "message", client_error)), + status.HTTP_500_INTERNAL_SERVER_ERROR, ).to_dict() }, default=str, @@ -13096,14 +13095,12 @@ async def completion( log_llm_api_exception(e, litellm_call_id) if isinstance(e, ProxyException): raise with_litellm_call_id(e, litellm_call_id) - error_msg: Final = f"{e}" - raise ProxyException( - message=getattr(e, "message", error_msg), - type=getattr(e, "type", "None"), - param=getattr(e, "param", "None"), + raise route_failure_proxy_exception( + e, + str(attribute_of(e, "message", e)), + status.HTTP_500_INTERNAL_SERVER_ERROR, headers=litellm_call_id_headers(litellm_call_id), openai_code=getattr(e, "code", None), - code=getattr(e, "status_code", 500), ) @@ -13324,22 +13321,18 @@ async def moderations( if isinstance(e, ProxyException): raise with_litellm_call_id(e, litellm_call_id) if isinstance(e, HTTPException): - raise ProxyException( - message=getattr(e, "message", str(e)), - type=getattr(e, "type", "None"), - param=getattr(e, "param", "None"), + raise route_failure_proxy_exception( + e, + str(attribute_of(e, "message", e)), + status.HTTP_400_BAD_REQUEST, headers=litellm_call_id_headers(litellm_call_id), - code=getattr(e, "status_code", status.HTTP_400_BAD_REQUEST), - ) - else: - error_msg: Final = f"{e}" - raise ProxyException( - message=getattr(e, "message", error_msg), - type=getattr(e, "type", "None"), - param=getattr(e, "param", "None"), - headers=litellm_call_id_headers(litellm_call_id), - code=getattr(e, "status_code", 500), ) + raise route_failure_proxy_exception( + e, + str(attribute_of(e, "message", e)), + status.HTTP_500_INTERNAL_SERVER_ERROR, + headers=litellm_call_id_headers(litellm_call_id), + ) async def _audio_speech_chunk_generator( @@ -13477,13 +13470,12 @@ async def audio_speech( detail=e.detail, headers=headers_with_litellm_call_id(e.headers, litellm_call_id), ) - raise ProxyException( - message=getattr(e, "message", f"{e}"), - type=getattr(e, "type", "None"), - param=getattr(e, "param", "None"), + raise route_failure_proxy_exception( + e, + str(attribute_of(e, "message", e)), + status.HTTP_500_INTERNAL_SERVER_ERROR, headers=litellm_call_id_headers(litellm_call_id), openai_code=getattr(e, "code", None), - code=getattr(e, "status_code", 500), ) @@ -13623,23 +13615,19 @@ async def audio_transcriptions( ) log_llm_api_exception(e, litellm_call_id) if isinstance(e, HTTPException): - raise ProxyException( - message=getattr(e, "message", str(e.detail)), - type=getattr(e, "type", "None"), - param=getattr(e, "param", "None"), + raise route_failure_proxy_exception( + e, + str(attribute_of(e, "message", e.detail)), + status.HTTP_400_BAD_REQUEST, headers=litellm_call_id_headers(litellm_call_id), - code=getattr(e, "status_code", status.HTTP_400_BAD_REQUEST), - ) - else: - error_msg: Final = f"{e}" - raise ProxyException( - message=getattr(e, "message", error_msg), - type=getattr(e, "type", "None"), - param=getattr(e, "param", "None"), - headers=litellm_call_id_headers(litellm_call_id), - openai_code=getattr(e, "code", None), - code=getattr(e, "status_code", 500), ) + raise route_failure_proxy_exception( + e, + str(attribute_of(e, "message", e)), + status.HTTP_500_INTERNAL_SERVER_ERROR, + headers=litellm_call_id_headers(litellm_call_id), + openai_code=getattr(e, "code", None), + ) ###################################################################### @@ -17239,7 +17227,7 @@ async def model_streaming_metrics( raise ProxyException( message=CommonProxyErrors.db_not_connected_error.value, type="internal_error", - param="None", + param=None, code=status.HTTP_500_INTERNAL_SERVER_ERROR, ) @@ -17373,7 +17361,7 @@ async def model_metrics( raise ProxyException( message="Prisma Client is not initialized", type="internal_error", - param="None", + param=None, code=status.HTTP_500_INTERNAL_SERVER_ERROR, ) startTime = startTime or datetime.now() - timedelta(days=DAYS_IN_A_MONTH) @@ -17488,7 +17476,7 @@ async def model_metrics_slow_responses( raise ProxyException( message="Prisma Client is not initialized", type="internal_error", - param="None", + param=None, code=status.HTTP_500_INTERNAL_SERVER_ERROR, ) if api_key is None or api_key == "undefined": @@ -17577,7 +17565,7 @@ async def model_metrics_exceptions( raise ProxyException( message="Prisma Client is not initialized", type="internal_error", - param="None", + param=None, code=status.HTTP_500_INTERNAL_SERVER_ERROR, ) @@ -18791,7 +18779,7 @@ async def login_v2(request: Request): raise ProxyException( message=getattr(e, "detail", str(e)), type=ProxyErrorTypes.auth_error, - param=getattr(e, "param", "None"), + param=openai_error_param(e), code=getattr(e, "status_code", status.HTTP_500_INTERNAL_SERVER_ERROR), ) else: @@ -18799,7 +18787,7 @@ async def login_v2(request: Request): raise ProxyException( message=error_msg, type=ProxyErrorTypes.auth_error, - param="None", + param=None, code=status.HTTP_500_INTERNAL_SERVER_ERROR, ) @@ -18867,7 +18855,7 @@ async def login_v3(request: Request): raise ProxyException( message=getattr(e, "detail", str(e)), type=ProxyErrorTypes.auth_error, - param=getattr(e, "param", "None"), + param=openai_error_param(e), code=getattr(e, "status_code", status.HTTP_500_INTERNAL_SERVER_ERROR), ) else: @@ -18875,7 +18863,7 @@ async def login_v3(request: Request): raise ProxyException( message=error_msg, type=ProxyErrorTypes.auth_error, - param="None", + param=None, code=status.HTTP_500_INTERNAL_SERVER_ERROR, ) @@ -18940,7 +18928,7 @@ async def login_v3_exchange(request: Request): raise ProxyException( message=str(e), type=ProxyErrorTypes.auth_error, - param="None", + param=None, code=status.HTTP_500_INTERNAL_SERVER_ERROR, ) diff --git a/litellm/proxy/spend_tracking/spend_management_endpoints.py b/litellm/proxy/spend_tracking/spend_management_endpoints.py index 38dcc3b06df..89200792c0b 100644 --- a/litellm/proxy/spend_tracking/spend_management_endpoints.py +++ b/litellm/proxy/spend_tracking/spend_management_endpoints.py @@ -50,6 +50,7 @@ from litellm.proxy.auth.authorization_dependencies import ( LogTeamLookupDependency, ) from litellm.proxy.auth.user_api_key_auth import user_api_key_auth +from litellm.proxy.common_utils.openai_error_payload import error_status_code, openai_error_param, openai_error_type from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup from litellm.proxy.spend_tracking.spend_capture_rate import ( ProviderBillingCredentialMissing, @@ -636,7 +637,7 @@ async def view_spend_tags( raise ProxyException( message=getattr(e, "detail", f"/spend/tags Error({e})"), type="internal_error", - param=getattr(e, "param", "None"), + param=openai_error_param(e), code=getattr(e, "status_code", status.HTTP_500_INTERNAL_SERVER_ERROR), ) elif isinstance(e, ProxyException): @@ -644,7 +645,7 @@ async def view_spend_tags( raise ProxyException( message="/spend/tags Error" + str(e), type="internal_error", - param=getattr(e, "param", "None"), + param=openai_error_param(e), code=status.HTTP_500_INTERNAL_SERVER_ERROR, ) @@ -2142,7 +2143,7 @@ async def global_get_all_tag_names(): raise ProxyException( message=getattr(e, "detail", f"/spend/all_tag_names Error({e})"), type="internal_error", - param=getattr(e, "param", "None"), + param=openai_error_param(e), code=getattr(e, "status_code", status.HTTP_500_INTERNAL_SERVER_ERROR), ) elif isinstance(e, ProxyException): @@ -2150,7 +2151,7 @@ async def global_get_all_tag_names(): raise ProxyException( message="/spend/all_tag_names Error" + str(e), type="internal_error", - param=getattr(e, "param", "None"), + param=openai_error_param(e), code=status.HTTP_500_INTERNAL_SERVER_ERROR, ) @@ -2224,7 +2225,7 @@ async def global_view_spend_tags( raise ProxyException( message=getattr(e, "detail", f"/spend/tags Error({error_str})"), type="internal_error", - param=getattr(e, "param", "None"), + param=openai_error_param(e), code=getattr(e, "status_code", status.HTTP_500_INTERNAL_SERVER_ERROR), ) elif isinstance(e, ProxyException): @@ -2232,7 +2233,7 @@ async def global_view_spend_tags( raise ProxyException( message="/spend/tags Error" + error_str, type="internal_error", - param=getattr(e, "param", "None"), + param=openai_error_param(e), code=status.HTTP_500_INTERNAL_SERVER_ERROR, ) @@ -2447,9 +2448,9 @@ async def calculate_spend(request: SpendCalculateRequest): if isinstance(e, HTTPException): raise ProxyException( message=getattr(e, "detail", str(e)), - type=getattr(e, "type", "None"), - param=getattr(e, "param", "None"), - code=getattr(e, "status_code", status.HTTP_400_BAD_REQUEST), + type=openai_error_type(e, error_status_code(e, status.HTTP_400_BAD_REQUEST)), + param=openai_error_param(e), + code=error_status_code(e, status.HTTP_400_BAD_REQUEST), ) if isinstance(e, litellm.exceptions.ModelNotMappedError): raise ProxyException( @@ -2461,9 +2462,9 @@ async def calculate_spend(request: SpendCalculateRequest): error_msg: Final = f"{e}" raise ProxyException( message=getattr(e, "message", error_msg), - type=getattr(e, "type", "None"), - param=getattr(e, "param", "None"), - code=getattr(e, "status_code", 500), + type=openai_error_type(e, error_status_code(e, 500)), + param=openai_error_param(e), + code=error_status_code(e, 500), ) @@ -2633,7 +2634,7 @@ async def ui_view_spend_logs( raise ProxyException( message="Prisma Client is not initialized", type="internal_error", - param="None", + param=None, code=status.HTTP_401_UNAUTHORIZED, ) @@ -2700,7 +2701,7 @@ async def ui_view_spend_logs( raise ProxyException( message="Start date and end date are required", type="bad_request", - param="None", + param=None, code=status.HTTP_400_BAD_REQUEST, ) formats: Final = ["%Y-%m-%d %H:%M:%S", "%Y-%m-%d"] if is_v2 else ["%Y-%m-%d %H:%M:%S"] @@ -3681,7 +3682,7 @@ async def view_spend_logs( raise ProxyException( message=getattr(e, "detail", f"/spend/logs Error({e})"), type="internal_error", - param=getattr(e, "param", "None"), + param=openai_error_param(e), code=getattr(e, "status_code", status.HTTP_500_INTERNAL_SERVER_ERROR), ) elif isinstance(e, ProxyException): @@ -3689,7 +3690,7 @@ async def view_spend_logs( raise ProxyException( message="/spend/logs Error" + str(e), type="internal_error", - param=getattr(e, "param", "None"), + param=openai_error_param(e), code=status.HTTP_500_INTERNAL_SERVER_ERROR, ) @@ -3716,7 +3717,7 @@ async def global_spend_reset(): raise ProxyException( message="Prisma Client is not initialized", type="internal_error", - param="None", + param=None, code=status.HTTP_401_UNAUTHORIZED, ) @@ -3747,7 +3748,7 @@ async def global_spend_refresh(): raise ProxyException( message="Prisma Client is not initialized", type="internal_error", - param="None", + param=None, code=status.HTTP_401_UNAUTHORIZED, ) @@ -3818,7 +3819,7 @@ async def global_spend_for_internal_user( raise ProxyException( message="Prisma Client is not initialized", type="internal_error", - param="None", + param=None, code=status.HTTP_500_INTERNAL_SERVER_ERROR, ) try: @@ -3879,7 +3880,7 @@ async def global_spend_logs( raise ProxyException( message="Prisma Client is not initialized", type="internal_error", - param="None", + param=None, code=status.HTTP_500_INTERNAL_SERVER_ERROR, ) @@ -3923,7 +3924,7 @@ async def global_spend_logs( raise ProxyException( message=getattr(e, "detail", f"/global/spend/logs Error({error_str})"), type="internal_error", - param=getattr(e, "param", "None"), + param=openai_error_param(e), code=getattr(e, "status_code", status.HTTP_500_INTERNAL_SERVER_ERROR), ) elif isinstance(e, ProxyException): @@ -3931,7 +3932,7 @@ async def global_spend_logs( raise ProxyException( message="/global/spend/logs Error" + error_str, type="internal_error", - param=getattr(e, "param", "None"), + param=openai_error_param(e), code=status.HTTP_500_INTERNAL_SERVER_ERROR, ) @@ -3971,7 +3972,7 @@ async def global_spend(): raise ProxyException( message=getattr(e, "detail", f"/global/spend Error({error_str})"), type="internal_error", - param=getattr(e, "param", "None"), + param=openai_error_param(e), code=getattr(e, "status_code", status.HTTP_500_INTERNAL_SERVER_ERROR), ) elif isinstance(e, ProxyException): @@ -3979,7 +3980,7 @@ async def global_spend(): raise ProxyException( message="/global/spend Error" + error_str, type="internal_error", - param=getattr(e, "param", "None"), + param=openai_error_param(e), code=status.HTTP_500_INTERNAL_SERVER_ERROR, ) diff --git a/tests/code_coverage_tests/check_no_none_string_error_fields.py b/tests/code_coverage_tests/check_no_none_string_error_fields.py new file mode 100644 index 00000000000..3a75b56fc8d --- /dev/null +++ b/tests/code_coverage_tests/check_no_none_string_error_fields.py @@ -0,0 +1,71 @@ +from __future__ import annotations + +import ast +import sys +from collections.abc import Iterator +from itertools import chain +from pathlib import Path +from typing import Final + +ERROR_FIELDS: Final = frozenset({"type", "param"}) +SOURCE_ROOT: Final = Path(__file__).resolve().parents[2] / "litellm" + + +def _is_none_string(node: ast.expr | None) -> bool: + return isinstance(node, ast.Constant) and node.value == "None" + + +def _names_an_error_field(node: ast.expr | None) -> bool: + return isinstance(node, ast.Constant) and node.value in ERROR_FIELDS + + +def _keyword_defaults_to_none_string(call: ast.Call) -> bool: + return any(keyword.arg in ERROR_FIELDS and _is_none_string(keyword.value) for keyword in call.keywords) + + +def _getattr_defaults_to_none_string(call: ast.Call) -> bool: + return ( + isinstance(call.func, ast.Name) + and call.func.id == "getattr" + and len(call.args) == 3 + and _names_an_error_field(call.args[1]) + and _is_none_string(call.args[2]) + ) + + +def _dict_holds_none_string(node: ast.Dict) -> bool: + return any(_names_an_error_field(key) and _is_none_string(value) for key, value in zip(node.keys, node.values)) + + +def _is_violation(node: ast.AST) -> bool: + match node: + case ast.Call(): + return _keyword_defaults_to_none_string(node) or _getattr_defaults_to_none_string(node) + case ast.Dict(): + return _dict_holds_none_string(node) + case _: + return False + + +def violations(source: str, filename: str) -> Iterator[str]: + return (f"{filename}:{node.lineno}" for node in ast.walk(ast.parse(source, filename=filename)) if _is_violation(node)) + + +def _file_violations(path: Path) -> Iterator[str]: + return violations(path.read_text(encoding="utf-8"), str(path.relative_to(SOURCE_ROOT.parent))) + + +def main() -> int: + found: Final = tuple(chain.from_iterable(_file_violations(path) for path in sorted(SOURCE_ROOT.rglob("*.py")))) + if not found: + return 0 + print( + 'An error\'s "type" or "param" is set to the string "None", which OpenAI clients read as a real ' + "error class or field name. Use a named error type and a real None for param instead:" + ) + print("\n".join(found)) + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/code_coverage_tests/test_check_no_none_string_error_fields.py b/tests/code_coverage_tests/test_check_no_none_string_error_fields.py new file mode 100644 index 00000000000..5d2f04b0942 --- /dev/null +++ b/tests/code_coverage_tests/test_check_no_none_string_error_fields.py @@ -0,0 +1,38 @@ +import importlib.util +from pathlib import Path +from typing import Final + +import pytest + +GUARD_PATH: Final = Path(__file__).resolve().parent / "check_no_none_string_error_fields.py" +_spec: Final = importlib.util.spec_from_file_location("check_no_none_string_error_fields", GUARD_PATH) +assert _spec is not None and _spec.loader is not None +guard: Final = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(guard) + + +@pytest.mark.parametrize( + "source", + [ + pytest.param('raise ProxyException(message="m", type="None", param=None, code=500)\n', id="type_keyword"), + pytest.param('raise ProxyException(message="m", type="x", param="None", code=500)\n', id="param_keyword"), + pytest.param('ProxyException(message="m", type=getattr(e, "type", "None"), param=None)\n', id="type_getattr"), + pytest.param('ProxyException(message="m", type="x", param=getattr(e, "param", "None"))\n', id="param_getattr"), + pytest.param('error = {"message": "m", "type": "x", "param": "None"}\n', id="dict_literal"), + ], +) +def test_a_none_string_error_field_is_reported_with_its_line(source: str) -> None: + assert tuple(guard.violations("x = 1\n" + source, "proxy.py")) == ("proxy.py:2",) + + +@pytest.mark.parametrize( + "source", + [ + pytest.param('ProxyException(message="m", type="auth_error", param=None, code=401)\n', id="real_none_param"), + pytest.param('ProxyException(message="None", type="x", param=getattr(e, "param", None))\n', id="none_message"), + pytest.param('label = getattr(e, "provider", "None")\n', id="other_attribute"), + pytest.param('row = {"model": "None"}\n', id="other_dict_key"), + ], +) +def test_a_real_none_or_an_unrelated_none_string_passes(source: str) -> None: + assert tuple(guard.violations(source, "proxy.py")) == () diff --git a/tests/integration/authorization/test_guardrail_submissions.py b/tests/integration/authorization/test_guardrail_submissions.py index f120028f46d..16b9cbf31dc 100644 --- a/tests/integration/authorization/test_guardrail_submissions.py +++ b/tests/integration/authorization/test_guardrail_submissions.py @@ -310,7 +310,7 @@ def test_non_admin_keys_scoped_to_the_submission_routes_get_403_from_the_admin_c f"Your user_id={_mask_user_id(user_id)}" ), type="auth_error", - param="None", + param=None, code="401", ) ) diff --git a/tests/integration/authorization/test_jwt_key_mapping_lifecycle.py b/tests/integration/authorization/test_jwt_key_mapping_lifecycle.py index 715f3731175..da830293145 100644 --- a/tests/integration/authorization/test_jwt_key_mapping_lifecycle.py +++ b/tests/integration/authorization/test_jwt_key_mapping_lifecycle.py @@ -114,7 +114,7 @@ class _ChatRequest(BaseModel): class _ProxyError(BaseModel): message: str type: str - param: str + param: str | None code: str @@ -398,7 +398,7 @@ def _assert_unmapped(response: httpx.Response, client_id: str) -> None: assert response.status_code == 403, response.text error: Final = _ErrorResponse.model_validate_json(response.content) expected: Final = f"JWT Key Mapping: No registered mapping for {_CLIENT_CLAIM}='{client_id}'. Access denied." - assert error.error == _ProxyError(message=expected, type="auth_error", param="None", code="403"), response.text + assert error.error == _ProxyError(message=expected, type="auth_error", param=None, code="403"), response.text def _mapping_info(gateway: Gateway, identity: str) -> _JWTMappingResponse: diff --git a/tests/integration/authorization/test_memory_ownership.py b/tests/integration/authorization/test_memory_ownership.py index bad7c3ccb37..249b74363a2 100644 --- a/tests/integration/authorization/test_memory_ownership.py +++ b/tests/integration/authorization/test_memory_ownership.py @@ -24,7 +24,7 @@ def _admin_only_error(route: str, user_id: str) -> dict[str, JsonValue]: f"keys/users/teams. Route={route}. Your role=internal_user. Your user_id={masked_user_id}" ), "type": "auth_error", - "param": "None", + "param": None, "code": "401", } } diff --git a/tests/integration/authorization/test_team_admin_gate.py b/tests/integration/authorization/test_team_admin_gate.py index eff4060631e..adc374a57d7 100644 --- a/tests/integration/authorization/test_team_admin_gate.py +++ b/tests/integration/authorization/test_team_admin_gate.py @@ -600,7 +600,7 @@ def _admin_only_route(route: str) -> tuple[int, JsonValue]: "message": "Authentication Error, Only proxy admin can be used to generate, delete, update info for new " f"keys/users/teams. Route={route}. Your role=internal_user. Your user_id=", "type": "auth_error", - "param": "None", + "param": None, "code": "401", } } diff --git a/tests/integration/authorization/test_unknown_key_generic_401.py b/tests/integration/authorization/test_unknown_key_generic_401.py index 8e6ff3fa3ab..025f11d3973 100644 --- a/tests/integration/authorization/test_unknown_key_generic_401.py +++ b/tests/integration/authorization/test_unknown_key_generic_401.py @@ -40,7 +40,7 @@ class _ProxyError(BaseModel): message: str type: str - param: str + param: str | None code: str @@ -260,12 +260,12 @@ def _control_error(response: httpx.Response) -> _ProxyError: def test_requests_without_an_sk_key_keep_their_own_401(gateway: Gateway, peer: Gateway, side: Side) -> None: client: Final = _side(gateway, peer, side).client assert _control_error(client.get("/v1/models")) == _ProxyError( - message="Authentication Error, No api key passed in.", type="auth_error", param="None", code="401" + message="Authentication Error, No api key passed in.", type="auth_error", param=None, code="401" ) assert _control_error(client.get("/v1/models", headers={"Authorization": "Bearer not-a-real-key"})) == _ProxyError( message="LiteLLM Virtual Key expected. Received=not-****-key, expected to start with 'sk-'.", type="auth_error", - param="None", + param=None, code="401", ) @@ -277,7 +277,7 @@ def test_a_blocked_key_keeps_its_own_401_on_both_proxies(gateway: Gateway, peer: expected: Final = _ProxyError( message="Authentication Error, Key is blocked. Update via `/key/unblock` if you're an admin.", type="auth_error", - param="None", + param=None, code="401", ) for candidate in (gateway, peer): diff --git a/tests/integration/authorization/test_user_self_update_guard.py b/tests/integration/authorization/test_user_self_update_guard.py index d058b3c6998..2c495beb77e 100644 --- a/tests/integration/authorization/test_user_self_update_guard.py +++ b/tests/integration/authorization/test_user_self_update_guard.py @@ -65,7 +65,7 @@ def _ui_session_route_error(user_id: str) -> dict[str, JsonValue]: f"Your user_id={masked_user_id}" ), "type": "auth_error", - "param": "None", + "param": None, "code": "401", } } diff --git a/tests/integration/authorization/test_warmed_policy.py b/tests/integration/authorization/test_warmed_policy.py index b4ffc10c3c4..0de4141143e 100644 --- a/tests/integration/authorization/test_warmed_policy.py +++ b/tests/integration/authorization/test_warmed_policy.py @@ -679,7 +679,7 @@ def _expected_scim_write_denial(route: str, user_id: str, user_role: str) -> dic f"keys/users/teams. Route={route}. Your role={user_role}. Your user_id={masked_user_id}" ), "type": "auth_error", - "param": "None", + "param": None, "code": "401", } } diff --git a/tests/integration/authorization/test_workflow_ownership.py b/tests/integration/authorization/test_workflow_ownership.py index 386d4f1ed27..70919760874 100644 --- a/tests/integration/authorization/test_workflow_ownership.py +++ b/tests/integration/authorization/test_workflow_ownership.py @@ -57,7 +57,7 @@ def _route_refusal(response: httpx.Response, user_id: str) -> None: f"keys/users/teams. Route=/v1/workflows/runs. Your role=internal_user. Your user_id={masked_user_id}" ), "type": "auth_error", - "param": "None", + "param": None, "code": "401", } }, response.text diff --git a/tests/integration/management/test_health_routing.py b/tests/integration/management/test_health_routing.py index 3a60efe4342..b74d32380d0 100644 --- a/tests/integration/management/test_health_routing.py +++ b/tests/integration/management/test_health_routing.py @@ -151,7 +151,7 @@ def test_public_readiness_reports_stalled_database_lookups(gateway: Gateway, tmp "Please retry shortly." ), "type": "no_db_connection", - "param": "None", + "param": None, "code": "503", } } diff --git a/tests/integration/management/test_scim_admin_group.py b/tests/integration/management/test_scim_admin_group.py index ffbec93d4a8..92016af1ebb 100644 --- a/tests/integration/management/test_scim_admin_group.py +++ b/tests/integration/management/test_scim_admin_group.py @@ -133,7 +133,7 @@ def _expected_scim_write_denial(route: str, user_id: str, user_role: str) -> dic f"keys/users/teams. Route={route}. Your role={user_role}. Your user_id={masked_user_id}" ), "type": "auth_error", - "param": "None", + "param": None, "code": "401", } } diff --git a/tests/integration/management/test_user_delete_revokes_keys.py b/tests/integration/management/test_user_delete_revokes_keys.py index 324963a0da9..513a80b7319 100644 --- a/tests/integration/management/test_user_delete_revokes_keys.py +++ b/tests/integration/management/test_user_delete_revokes_keys.py @@ -64,7 +64,7 @@ def _blocked_key_error() -> dict[str, JsonValue]: "error": { "message": "Authentication Error, Key is blocked. Update via `/key/unblock` if you're an admin.", "type": "auth_error", - "param": "None", + "param": None, "code": "401", } } diff --git a/tests/integration/observability/test_arize_otel_v2_openinference_sad_edge.py b/tests/integration/observability/test_arize_otel_v2_openinference_sad_edge.py index a05b3f0f045..94e9d46ec8f 100644 --- a/tests/integration/observability/test_arize_otel_v2_openinference_sad_edge.py +++ b/tests/integration/observability/test_arize_otel_v2_openinference_sad_edge.py @@ -177,7 +177,7 @@ def test_arize_otel_v2_d4_unauthenticated_request_has_no_span(gateway: Gateway, "error": { "message": "Authentication Error, No api key passed in.", "type": "auth_error", - "param": "None", + "param": None, "code": "401", } }, body diff --git a/tests/integration/observability/test_presidio_entity_masking.py b/tests/integration/observability/test_presidio_entity_masking.py index ab966873389..ff36e4623f7 100644 --- a/tests/integration/observability/test_presidio_entity_masking.py +++ b/tests/integration/observability/test_presidio_entity_masking.py @@ -325,7 +325,7 @@ def test_long_apply_guardrail_alias_accepts_an_llm_api_scoped_key(gateway: Gatew "error": { "message": "Authentication Error, No api key passed in.", "type": "auth_error", - "param": "None", + "param": None, "code": "401", } }, unauthenticated.text diff --git a/tests/integration/security/test_config_overrides.py b/tests/integration/security/test_config_overrides.py index 72546c54222..70f9226ea79 100644 --- a/tests/integration/security/test_config_overrides.py +++ b/tests/integration/security/test_config_overrides.py @@ -100,7 +100,7 @@ def _assert_non_admin_refusal(response: httpx.Response, route: str, user_id: str f"new keys/users/teams. Route={route}. Your role=internal_user. Your user_id={masked_user_id}" ), "type": "auth_error", - "param": "None", + "param": None, "code": "401", } }, response.text @@ -244,7 +244,7 @@ def test_saved_vault_config_reads_back_masked_and_refuses_non_admins( "message": "user not allowed to access this route, role= proxy_admin_viewer. " "Trying to access: /config_overrides/hashicorp_vault", "type": "auth_error", - "param": "None", + "param": None, "code": "403", } }, viewer_post.text @@ -386,7 +386,7 @@ def test_saved_cyberark_config_reads_back_masked_and_refuses_non_admins( "message": "user not allowed to access this route, role= proxy_admin_viewer. " "Trying to access: /config_overrides/cyberark", "type": "auth_error", - "param": "None", + "param": None, "code": "403", } }, viewer_post.text diff --git a/tests/integration/spend/test_global_spend_reset.py b/tests/integration/spend/test_global_spend_reset.py index c400d90e184..cac66471dea 100644 --- a/tests/integration/spend/test_global_spend_reset.py +++ b/tests/integration/spend/test_global_spend_reset.py @@ -104,7 +104,7 @@ def test_global_spend_reset_zeroes_token_and_team_spend_but_keeps_logs(gateway: "error": { "message": "Authentication Error, Tried to access route=/global/spend/reset, which is only for MASTER KEY", "type": "auth_error", - "param": "None", + "param": None, "code": "401", } }, refused.text diff --git a/tests/unit/proxy/auth/test_auth_exception_handler.py b/tests/unit/proxy/auth/test_auth_exception_handler.py index 92b4c93118f..2523284ec2c 100644 --- a/tests/unit/proxy/auth/test_auth_exception_handler.py +++ b/tests/unit/proxy/auth/test_auth_exception_handler.py @@ -1298,3 +1298,17 @@ def test_as_proxy_exception_keeps_jwt_scope_denial_message_shape(): assert converted.to_dict() == plain.to_dict() assert converted.internal_message == denial.internal_message + + +@pytest.mark.parametrize( + "auth_failure", + [ + pytest.param(HTTPException(status_code=401, detail="Authentication Error, invalid key"), id="http_exception"), + pytest.param(Exception("Invalid proxy server token passed"), id="bare_exception"), + ], +) +def test_auth_failure_error_body_carries_a_json_null_param(auth_failure: Exception) -> None: + body = json.loads(json.dumps({"error": _as_proxy_exception(auth_failure).to_dict()})) + + assert body["error"]["type"] == ProxyErrorTypes.auth_error.value + assert body["error"]["param"] is None diff --git a/tests/unit/proxy/auth/test_handle_jwt.py b/tests/unit/proxy/auth/test_handle_jwt.py index a0ea6ce0403..51cf15996cd 100644 --- a/tests/unit/proxy/auth/test_handle_jwt.py +++ b/tests/unit/proxy/auth/test_handle_jwt.py @@ -4368,6 +4368,7 @@ async def test_removed_signing_key_stops_being_trusted_once_the_stale_window_exp assert exc_info.value.code == "503" assert exc_info.value.type == ProxyErrorTypes.auth_provider_unavailable + assert exc_info.value.param is None @pytest.mark.asyncio diff --git a/tests/unit/proxy/auth/test_user_api_key_auth.py b/tests/unit/proxy/auth/test_user_api_key_auth.py index 117428ee27b..81ea0e3f60b 100644 --- a/tests/unit/proxy/auth/test_user_api_key_auth.py +++ b/tests/unit/proxy/auth/test_user_api_key_auth.py @@ -1310,6 +1310,33 @@ async def test_auth_rejects_wrong_or_empty_present_custom_header( assert exc_info.value.code == "401" +@pytest.mark.asyncio +@pytest.mark.parametrize("api_key", [None, ""], ids=["no_key", "no_bearer_prefix"]) +async def test_auth_rejection_without_a_usable_key_has_a_json_null_param( + api_key: str | None, monkeypatch: pytest.MonkeyPatch +): + from fastapi import Request + + from litellm.proxy import proxy_server + from litellm.proxy._types import ProxyErrorTypes, ProxyException + + monkeypatch.setattr(proxy_server, "master_key", "sk-test-master") + monkeypatch.setattr(proxy_server, "general_settings", {}) + request = Request( + scope={"type": "http", "method": "POST", "path": "/chat/completions", "headers": []}, + receive=AsyncMock(return_value={"type": "http.request", "body": b"", "more_body": False}), + ) + request._url = URL(url="/chat/completions") + + with pytest.raises(ProxyException) as exc_info: + await user_api_key_auth(request=request, api_key=api_key) + + body = json.loads(json.dumps({"error": exc_info.value.to_dict()})) + assert body["error"]["code"] == "401" + assert body["error"]["type"] == ProxyErrorTypes.auth_error.value + assert body["error"]["param"] is None + + @pytest.mark.asyncio async def test_user_api_key_from_query_param(): """Ensure user_api_key_auth reads API key from `key` query parameter.""" diff --git a/tests/unit/proxy/management_endpoints/test_internal_user_endpoints.py b/tests/unit/proxy/management_endpoints/test_internal_user_endpoints.py index d1bbf444d2c..78a0a765fe6 100644 --- a/tests/unit/proxy/management_endpoints/test_internal_user_endpoints.py +++ b/tests/unit/proxy/management_endpoints/test_internal_user_endpoints.py @@ -4970,7 +4970,7 @@ _DB_OUTAGE_503_BODY: Final = { "error": { "message": "Service Unavailable, the authentication database is temporarily unreachable. Please retry shortly.", "type": "no_db_connection", - "param": "None", + "param": None, "code": "503", } } diff --git a/tests/unit/proxy/management_endpoints/test_key_management_endpoints.py b/tests/unit/proxy/management_endpoints/test_key_management_endpoints.py index 859824e4a72..b0ef7e64e60 100644 --- a/tests/unit/proxy/management_endpoints/test_key_management_endpoints.py +++ b/tests/unit/proxy/management_endpoints/test_key_management_endpoints.py @@ -23379,6 +23379,23 @@ def test_key_info_database_not_connected_sanitized_500( proxy_server.app.dependency_overrides.pop(user_api_key_auth, None) +def test_key_health_failure_body_carries_a_json_null_param(): + import litellm.proxy.proxy_server as ps + + app.dependency_overrides[ps.user_api_key_auth] = lambda: UserAPIKeyAuth( + api_key="hashed-key", metadata={"logging": ["not-a-callback-object"]} + ) + try: + response = client.post("/key/health", headers={"Authorization": "Bearer sk-test"}) + finally: + app.dependency_overrides.pop(ps.user_api_key_auth, None) + + assert response.status_code == 500 + error = response.json()["error"] + assert error["type"] == "internal_server_error" + assert error["param"] is None + + @pytest.mark.asyncio async def test_update_key_reports_a_bad_budget_duration_as_a_request_error_not_an_auth_error(monkeypatch: pytest.MonkeyPatch) -> None: from fastapi import Request diff --git a/tests/unit/proxy/management_endpoints/test_team_endpoints.py b/tests/unit/proxy/management_endpoints/test_team_endpoints.py index c6d12ade2a0..e33743bb149 100644 --- a/tests/unit/proxy/management_endpoints/test_team_endpoints.py +++ b/tests/unit/proxy/management_endpoints/test_team_endpoints.py @@ -19095,7 +19095,7 @@ _DB_OUTAGE_503_BODY: Final = { "error": { "message": "Service Unavailable, the authentication database is temporarily unreachable. Please retry shortly.", "type": "no_db_connection", - "param": "None", + "param": None, "code": "503", } } diff --git a/tests/unit/proxy/proxy_server/test_exception_handlers.py b/tests/unit/proxy/proxy_server/test_exception_handlers.py index b34c2d2fb08..0f9d379afa1 100644 --- a/tests/unit/proxy/proxy_server/test_exception_handlers.py +++ b/tests/unit/proxy/proxy_server/test_exception_handlers.py @@ -438,7 +438,7 @@ _DB_OUTAGE_503_BODY: Final = { "error": { "message": "Service Unavailable, the authentication database is temporarily unreachable. Please retry shortly.", "type": "no_db_connection", - "param": "None", + "param": None, "code": "503", } } diff --git a/tests/unit/proxy/proxy_server/test_routes_audio.py b/tests/unit/proxy/proxy_server/test_routes_audio.py index de76c7257cf..59d9490f786 100644 --- a/tests/unit/proxy/proxy_server/test_routes_audio.py +++ b/tests/unit/proxy/proxy_server/test_routes_audio.py @@ -14,9 +14,15 @@ from unittest.mock import AsyncMock, MagicMock import httpx import pytest +from fastapi import HTTPException +from litellm import Router from litellm.proxy import proxy_server +from litellm.proxy._types import ProxyErrorTypes, ProxyException from litellm.types.llms.openai import HttpxBinaryResponseContent +from litellm.types.router import RouterErrors + +_TAG_ROUTING_DENIAL = f"{RouterErrors.no_deployments_with_tag_routing.value}. Passed model=tts-1 and tags=['x']" @pytest.fixture @@ -208,7 +214,8 @@ def test_audio_speech_error(client, auth_as, patched_speech_error, path): with auth_as(): response = client.post(path, json=payload) assert response.status_code == 500 - assert len(response.content) > 0 + error = response.json()["error"] + assert (error["type"], error["param"]) == ("internal_server_error", None) def test_audio_speech_bad_request_maps_to_400(client, auth_as, patched_speech_provider_rejection): @@ -245,6 +252,104 @@ def test_audio_transcription_happy_path(client, auth_as, patched_transcription, } +@pytest.mark.parametrize("path", ["/v1/audio/transcriptions", "/audio/transcriptions"]) +def test_audio_transcription_over_the_file_size_cap_is_rejected_as_a_bad_request( + client, auth_as, patched_transcription, monkeypatch, path +): + from litellm.proxy.common_utils.http_parsing_utils import check_file_size_under_limit + + capped_router = Router( + model_list=[ + {"model_name": "whisper-1", "litellm_params": {"model": "openai/whisper-1", "max_file_size_mb": 0.000001}} + ] + ) + monkeypatch.setattr(proxy_server, "llm_router", capped_router) + monkeypatch.setattr(proxy_server, "premium_user", True) + monkeypatch.setattr(proxy_server, "check_file_size_under_limit", check_file_size_under_limit) + files = {"file": ("audio.mp3", b"\x00\x01\x02", "audio/mpeg")} + with auth_as(): + response = client.post(path, files=files, data={"model": "whisper-1"}) + assert response.status_code == 400 + error = response.json()["error"] + assert "File size is too large" in error["message"] + assert error["param"] == "file" + + +def test_audio_speech_tag_routing_denial_is_typed_as_a_proxy_auth_error( + client, auth_as, patched_speech_error, monkeypatch +): + async def _deny(*args, **kwargs): + raise ValueError(_TAG_ROUTING_DENIAL) + + monkeypatch.setattr(proxy_server, "route_request", _deny) + with auth_as(): + response = client.post("/v1/audio/speech", json={"model": "tts-1", "input": "Hi", "voice": "alloy"}) + assert response.status_code == 401 + error = response.json()["error"] + assert (error["type"], error["param"]) == ("auth_error", None) + + +@pytest.mark.parametrize( + ("failure", "expected_status", "expected_type"), + [ + pytest.param(ValueError(_TAG_ROUTING_DENIAL), 401, "auth_error", id="tag_routing_denial"), + pytest.param( + HTTPException(status_code=403, detail="Blocked by the content policy"), + 403, + "permission_error", + id="hook_policy_block", + ), + ], +) +def test_audio_transcription_failure_is_typed_by_who_rejected_it( + client, auth_as, patched_transcription, monkeypatch, failure, expected_status, expected_type +): + async def _deny(*args, **kwargs): + raise failure + + monkeypatch.setattr(proxy_server, "route_request", _deny) + files = {"file": ("audio.mp3", b"\x00\x01\x02", "audio/mpeg")} + with auth_as(): + response = client.post("/v1/audio/transcriptions", files=files, data={"model": "whisper-1"}) + assert response.status_code == expected_status + error = response.json()["error"] + assert (error["type"], error["param"]) == (expected_type, None) + + +@pytest.mark.parametrize( + "rejection", + [ + pytest.param( + ProxyException( + message="Invalid type for 'metadata': expected an object, but got an integer instead.", + type=ProxyErrorTypes.bad_request_error, + param="metadata", + code=400, + ), + id="bad_metadata", + ), + pytest.param( + ProxyException( + message="tag budgets are unavailable", type=ProxyErrorTypes.no_db_connection, param=None, code=503 + ), + id="tag_budget_store_down", + ), + ], +) +def test_audio_transcription_keeps_the_status_of_a_proxy_rejection( + client, auth_as, patched_transcription, monkeypatch, rejection +): + async def _reject(data, **kwargs): + raise rejection + + monkeypatch.setattr(proxy_server, "add_litellm_data_to_request", _reject) + files = {"file": ("audio.mp3", b"\x00\x01\x02", "audio/mpeg")} + with auth_as(): + response = client.post("/v1/audio/transcriptions", files=files, data={"model": "whisper-1"}) + assert response.status_code == int(rejection.code) + assert response.json()["error"] == rejection.to_dict() + + @pytest.mark.parametrize("path", ["/v1/audio/transcriptions", "/audio/transcriptions"]) def test_audio_transcription_error(client, auth_as, patched_transcription_error, path): """Pins ``POST /v1/audio/transcriptions`` / ``POST /audio/transcriptions`` (error).""" diff --git a/tests/unit/proxy/proxy_server/test_routes_completions.py b/tests/unit/proxy/proxy_server/test_routes_completions.py index b5c60c23c02..2b585357549 100644 --- a/tests/unit/proxy/proxy_server/test_routes_completions.py +++ b/tests/unit/proxy/proxy_server/test_routes_completions.py @@ -12,8 +12,10 @@ from __future__ import annotations from unittest.mock import AsyncMock, MagicMock import pytest +from fastapi import HTTPException from litellm.proxy import common_request_processing, proxy_server +from litellm.types.router import RouterErrors from .conftest import normalize # type: ignore[import-not-found] @@ -124,3 +126,39 @@ def test_completion_pipeline_error(client, auth_as, completion_pipeline_raises, response = client.post(path, json=payload) assert response.status_code == 500 assert response.headers.get("content-type", "").startswith("application/json") + + +@pytest.mark.parametrize( + ("failure", "expected_status", "expected_type"), + [ + pytest.param(ValueError("boom"), 500, "internal_server_error", id="unexpected_failure"), + pytest.param( + HTTPException(status_code=403, detail="Blocked by the content policy"), + 403, + "permission_error", + id="hook_policy_block", + ), + pytest.param(HTTPException(status_code=401, detail="key expired"), 401, "authentication_error", id="hook_401"), + pytest.param( + ValueError(f"{RouterErrors.no_deployments_with_tag_routing.value}. Passed model=gpt-4o and tags=['x']"), + 401, + "auth_error", + id="tag_routing_denial", + ), + ], +) +def test_completion_failure_body_types_the_status_it_answers_with( + client, auth_as, monkeypatch, failure, expected_status, expected_type +): + monkeypatch.setattr(proxy_server, "llm_router", MagicMock()) + monkeypatch.setattr(proxy_server, "proxy_logging_obj", MagicMock(post_call_failure_hook=AsyncMock())) + + async def _raise(self, *args, **kwargs): + raise failure + + monkeypatch.setattr(common_request_processing.ProxyBaseLLMRequestProcessing, "base_process_llm_request", _raise) + with auth_as(): + response = client.post("/v1/completions", json={"model": "gpt-4o", "prompt": "hi"}) + assert response.status_code == expected_status + error = response.json()["error"] + assert (error["type"], error["param"], error["code"]) == (expected_type, None, str(expected_status)) diff --git a/tests/unit/proxy/proxy_server/test_streaming_helpers.py b/tests/unit/proxy/proxy_server/test_streaming_helpers.py index 22ada990fe6..c37475bd539 100644 --- a/tests/unit/proxy/proxy_server/test_streaming_helpers.py +++ b/tests/unit/proxy/proxy_server/test_streaming_helpers.py @@ -32,7 +32,7 @@ import litellm import litellm.proxy.proxy_server as ps from litellm.constants import RETURN_RAW_MODEL_NAME_METADATA_KEY from litellm.integrations.custom_logger import CustomLogger -from litellm.proxy._types import UserAPIKeyAuth +from litellm.proxy._types import ProxyException, UserAPIKeyAuth from litellm.proxy.common_request_processing import create_response from litellm.proxy.proxy_server import ( _apply_streaming_chunk_hooks, @@ -946,6 +946,39 @@ def _error_of(frame: str) -> dict[str, object]: return json.loads(frame.removeprefix("data: "))["error"] +@pytest.mark.asyncio +@pytest.mark.parametrize( + "raised, expected_error", + [ + pytest.param( + ProxyException(message="Budget has been exceeded", type="budget_exceeded", param=None, code=429), + {"message": "Budget has been exceeded", "type": "budget_exceeded", "param": None, "code": "429"}, + id="proxy_exception_keeps_its_status", + ), + pytest.param( + RuntimeError("upstream blew up"), + {"message": "upstream blew up", "type": "internal_server_error", "param": None, "code": "500"}, + id="unexpected_failure", + ), + ], +) +async def test_a_mid_stream_failure_frame_carries_the_status_the_error_was_raised_with( + monkeypatch, raised: Exception, expected_error: dict[str, object] +): + _patch_logging_flags(monkeypatch) + monkeypatch.setattr(ps.proxy_logging_obj, "post_call_failure_hook", AsyncMock(return_value=None)) + + frames: Final = [ + frame + async for frame in async_data_generator( + response=_async_iter_raises(raised), user_api_key_dict=_user_auth(), request_data={} + ) + ] + + error_frames: Final = [frame for frame in frames if isinstance(frame, str) and frame.startswith('data: {"error":')] + assert [_error_of(frame.removesuffix("\n\n")) for frame in error_frames] == [expected_error] + + @pytest.mark.asyncio @pytest.mark.parametrize( "detail,expected_message,expected_fields", diff --git a/tests/unit/proxy/spend_tracking/test_spend_management_endpoints.py b/tests/unit/proxy/spend_tracking/test_spend_management_endpoints.py index 8d53817f51e..6598b94e60f 100644 --- a/tests/unit/proxy/spend_tracking/test_spend_management_endpoints.py +++ b/tests/unit/proxy/spend_tracking/test_spend_management_endpoints.py @@ -8268,3 +8268,19 @@ async def test_global_spend_refresh_runs_the_refresh_on_the_writer_when_a_reader assert [statement for statement in journal.statements if statement[1].startswith("REFRESH")] == [ ("writer", 'REFRESH MATERIALIZED VIEW "MonthlyGlobalSpend";') ] + + +def test_spend_calculate_rejection_body_is_openai_shaped(client): + app.dependency_overrides[ps.user_api_key_auth] = lambda: UserAPIKeyAuth( + user_role=LitellmUserRoles.PROXY_ADMIN, user_id="admin_user", api_key="hashed-k" + ) + try: + response = client.post("/spend/calculate", json={}, headers={"Authorization": "Bearer sk-test"}) + finally: + app.dependency_overrides.pop(ps.user_api_key_auth, None) + + assert response.status_code == 400 + error = response.json()["error"] + assert error["type"] == "invalid_request_error" + assert error["param"] is None + assert error["code"] == "400" diff --git a/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py b/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py index 023ef0b4606..62a90109d26 100644 --- a/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py +++ b/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py @@ -7255,6 +7255,39 @@ async def test_async_data_generator_closes_the_guardrail_chain_before_returning_ assert cleanup_ran == [True] +@pytest.mark.asyncio +async def test_async_data_generator_midstream_failure_frame_is_openai_shaped_with_a_json_null_param(): + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.proxy_server import async_data_generator + + async def chain_failing_after_first_chunk(**_kwargs): + yield {"choices": [{"delta": {"content": "Hello"}}]} + raise ValueError("audit hook could not reach its sink") + + proxy_logging = _guarded_chain_logging(chain_failing_after_first_chunk) + proxy_logging.post_call_failure_hook = AsyncMock(return_value=None) + + with patch("litellm.proxy.proxy_server.proxy_logging_obj", proxy_logging): + frames = [ + frame + async for frame in async_data_generator( + MagicMock(), MagicMock(spec=UserAPIKeyAuth), {"model": "gpt-4o-mini"} + ) + ] + + error_bodies = [json.loads(frame.removeprefix("data: ")) for frame in frames if '"error"' in frame] + assert error_bodies == [ + { + "error": { + "message": "audit hook could not reach its sink", + "type": "internal_server_error", + "param": None, + "code": "500", + } + } + ] + + @pytest.mark.asyncio async def test_async_data_generator_closes_the_guardrail_chain_while_a_keepalive_read_is_pending(): from litellm.proxy._types import UserAPIKeyAuth @@ -14222,6 +14255,63 @@ async def test_moderations_reraises_proxy_exception_unwrapped(): mock_logging.post_call_failure_hook.assert_awaited_once() +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("failure", "expected_type", "expected_code"), + [ + pytest.param(HTTPException(status_code=400, detail="bad input"), "invalid_request_error", "400", id="http_400"), + pytest.param(HTTPException(status_code=429, detail="slow down"), "rate_limit_error", "429", id="http_429"), + pytest.param( + HTTPException(status_code=403, detail="Blocked by the content policy"), + "permission_error", + "403", + id="hook_policy_block", + ), + pytest.param(Exception("provider down"), "internal_server_error", "500", id="bare_exception"), + pytest.param( + ValueError("Not allowed to access model due to tags configuration. Passed model=m and tags=['x']"), + "auth_error", + "401", + id="tag_routing_denial", + ), + ], +) +async def test_moderations_failure_body_is_openai_shaped(failure, expected_type, expected_code): + from litellm.proxy._types import ProxyException + + async def passthrough_add_litellm_data(data, **kwargs): + return data + + request = MagicMock() + request.body = AsyncMock(return_value=b'{"input": "hi"}') + fake_logging = MagicMock() + fake_logging.pre_call_hook = AsyncMock(side_effect=lambda user_api_key_dict, data, call_type: data) + fake_logging.post_call_failure_hook = AsyncMock() + + with ( + patch.object( + proxy_server_module, "add_litellm_data_to_request", new=passthrough_add_litellm_data + ), # test-quality-ok: the route reads this module global, no injection point + patch.object( + proxy_server_module, "route_request", new=AsyncMock(side_effect=failure) + ), # test-quality-ok: fakes the provider failure so the real route's error shaping is observable + patch.object( + proxy_server_module, "proxy_logging_obj", new=fake_logging + ), # test-quality-ok: module global, no injection point + pytest.raises(ProxyException) as raised, + ): + await proxy_server_module.moderations( + request=request, + fastapi_response=MagicMock(), + user_api_key_dict=UserAPIKeyAuth(api_key="sk-test"), + ) + + body = json.loads(json.dumps({"error": raised.value.to_dict()})) + assert body["error"]["type"] == expected_type + assert body["error"]["param"] is None + assert body["error"]["code"] == expected_code + + @pytest.mark.asyncio async def test_moderations_response_carries_litellm_call_id_header(): from fastapi import Response