diff --git a/.github/workflows/test-unit-misc.yml b/.github/workflows/test-unit-misc.yml index 7c3b195f0ad..9afaaaead93 100644 --- a/.github/workflows/test-unit-misc.yml +++ b/.github/workflows/test-unit-misc.yml @@ -27,6 +27,7 @@ jobs: tests/test_litellm/a2a_protocol tests/test_litellm/anthropic_interface tests/test_litellm/completion_extras + tests/test_litellm/compression tests/test_litellm/containers tests/test_litellm/experimental_mcp_client tests/test_litellm/models diff --git a/litellm/compression/compress.py b/litellm/compression/compress.py index 004dd82cbaa..9c5f57bc98f 100644 --- a/litellm/compression/compress.py +++ b/litellm/compression/compress.py @@ -3,6 +3,7 @@ Main compress() function — normalizes input messages, orchestrates BM25/embedd scoring, message stubbing, and retrieval tool injection. """ +from collections.abc import Mapping, Sequence from typing import Any, Dict, List, Optional, Set, Tuple, Union, cast from litellm.caching.dual_cache import DualCache @@ -204,33 +205,21 @@ def _extract_anthropic_tool_exchange_spans( return spans, None -def _get_protected_indices(messages: List[dict]) -> List[int]: +def get_protected_indices(messages: Sequence[Mapping[str, object]]) -> tuple[int, ...]: """ Return indices of messages that must never be compressed: - All system messages - The last user message - The last assistant message + + The last user message is what the model is being asked to act on right now, + so compressing it replaces the live instruction with a marker. Compression + guardrails share this policy; see the Headroom guardrail. """ - protected: List[int] = [] - - last_user_idx = None - last_assistant_idx = None - - for i, msg in enumerate(messages): - role = msg.get("role", "") - if role == "system": - protected.append(i) - elif role == "user": - last_user_idx = i - elif role == "assistant": - last_assistant_idx = i - - if last_user_idx is not None: - protected.append(last_user_idx) - if last_assistant_idx is not None: - protected.append(last_assistant_idx) - - return protected + system_indices = tuple(index for index, msg in enumerate(messages) if msg.get("role", "") == "system") + last_user = tuple(index for index, msg in enumerate(messages) if msg.get("role", "") == "user")[-1:] + last_assistant = tuple(index for index, msg in enumerate(messages) if msg.get("role", "") == "assistant")[-1:] + return system_indices + last_user + last_assistant def _combine_scores( @@ -432,7 +421,7 @@ def compress( combined_scores = bm25_scores # Protected messages are never compressed - protected_indices = _get_protected_indices(normalized_messages) + protected_indices = get_protected_indices(normalized_messages) kept_indices: Set[int] = set(protected_indices) tool_exchange_spans: List[Set[int]] = [] diff --git a/litellm/litellm_core_utils/prompt_templates/factory.py b/litellm/litellm_core_utils/prompt_templates/factory.py index d8ce48f05de..0752bf2d771 100644 --- a/litellm/litellm_core_utils/prompt_templates/factory.py +++ b/litellm/litellm_core_utils/prompt_templates/factory.py @@ -6,7 +6,7 @@ import mimetypes import re import xml.etree.ElementTree as ET from enum import Enum -from collections.abc import Mapping +from collections.abc import Iterator, Mapping, Sequence from typing import Any, Dict, List, Optional, Set, Tuple, TypedDict, Union, cast, overload from jinja2.sandbox import ImmutableSandboxedEnvironment @@ -2210,6 +2210,49 @@ def _is_orphaned_tool_result( return False +def _declared_tool_call_ids(message: Mapping[str, Any]) -> frozenset[str]: + tool_calls = message.get("tool_calls") + if not isinstance(tool_calls, list): + return frozenset() + return frozenset( + str(tool_call["id"]) for tool_call in tool_calls if isinstance(tool_call, Mapping) and tool_call.get("id") + ) + + +def group_tool_exchanges(messages: Sequence[Mapping[str, Any]]) -> tuple[tuple[int, ...], ...]: + """Group message indices into tool exchanges: an assistant row that made + tool calls, together with the tool rows answering the ids it declared. + + Membership is by ``tool_call_id`` ownership rather than adjacency, so a tool + row belonging to some other call opens its own group instead of being swept + into the exchange it happens to sit next to. Every other row is its own + group. Groups stay contiguous and in order, so a caller can convert or + protect them without reordering the conversation. + + Callers need this because an assistant row and the tool rows answering it + are only well-formed together: ``sanitize_messages_for_tool_calling`` reads + an assistant row whose results are missing as an orphaned tool call, and + a tool row whose call is missing as an orphaned result. + """ + return tuple(_iter_tool_exchange_groups(messages)) + + +def _iter_tool_exchange_groups(messages: Sequence[Mapping[str, Any]]) -> Iterator[tuple[int, ...]]: + index = 0 + while index < len(messages): + declared = _declared_tool_call_ids(messages[index]) + end = index + 1 + while ( + declared + and end < len(messages) + and messages[end].get("role") in ("tool", "function") + and str(messages[end].get("tool_call_id")) in declared + ): + end += 1 + yield tuple(range(index, end)) + index = end + + def sanitize_messages_for_tool_calling( messages: List[AllMessageValues], ) -> List[AllMessageValues]: diff --git a/litellm/llms/anthropic/chat/guardrail_translation/handler.py b/litellm/llms/anthropic/chat/guardrail_translation/handler.py index 90f735707bf..a549db94224 100644 --- a/litellm/llms/anthropic/chat/guardrail_translation/handler.py +++ b/litellm/llms/anthropic/chat/guardrail_translation/handler.py @@ -361,14 +361,34 @@ class AnthropicMessagesHandler(BaseTranslation): @staticmethod def _write_back_structured_messages(data: dict, structured_messages: list) -> None: - """Convert compressed structured_messages back to Anthropic format and write to data.""" + """Convert compressed structured_messages back to Anthropic format and write to data. + + ``anthropic_messages_pt`` merges every run of consecutive user/tool rows + into a single message, so a turn carrying only tool results and the user + turn that follows it come back fused, and the request the model sees no + longer has the boundaries the client sent. Converting a row at a time + would keep them apart but breaks tool pairing: an assistant row whose + tool results sit outside its own call reads as an orphaned tool call, + and under ``modify_params`` the sanitizer answers it with a synthetic + "tool execution skipped" result and drops the real one. Converting each + assistant row together with the tool rows that answer it, and every + other row on its own, satisfies both. + """ from litellm.litellm_core_utils.prompt_templates.factory import ( anthropic_messages_pt, + group_tool_exchanges, ) model = str(data.get("model") or "") non_system = [m for m in structured_messages if m.get("role") != "system"] - converted = anthropic_messages_pt(messages=non_system, model=model, llm_provider="anthropic") + groups = tuple([non_system[index] for index in group] for group in group_tool_exchanges(non_system)) or ( + non_system, + ) + converted = [ + message + for group in groups + for message in anthropic_messages_pt(messages=group, model=model, llm_provider="anthropic") + ] for msg in converted: content = msg.get("content") if isinstance(content, list): diff --git a/litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py b/litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py index 423cda5eea2..5d8ac8d678f 100644 --- a/litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py +++ b/litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py @@ -11,6 +11,7 @@ from typing_extensions import assert_never import litellm from litellm._logging import verbose_logger from litellm.proxy._experimental.mcp_server.oauth_utils import ( + get_passthrough_resource_metadata_url, get_request_base_url, well_known_root_suffix, ) @@ -152,52 +153,83 @@ def _is_mcp_admitted_user_subject(user_api_key_auth: UserAPIKeyAuth | None) -> b return user_api_key_auth is not None and user_api_key_auth.mcp_admitted_user_subject is True -def _is_aggregate_mcp_scope(route: str, mcp_servers: list[str] | None) -> bool: - """True when a request targets the aggregate ``/mcp`` endpoint rather than any named - server. Named targets arrive either through ``x-mcp-servers`` (``mcp_servers``) or a - path segment (``/mcp/{server}`` / ``/{server}/mcp``); the aggregate scope has neither. - The gateway-DCR session arm and challenge fire only here, so a per-server flow is never - affected.""" - if mcp_servers: - return False - return len(MCPRequestHandler._extract_target_server_names_from_path(route)) == 0 +def _gateway_dcr_challenge_target( + route: str, + mcp_servers: list[str] | None, + client_ip: str | None, +) -> str | None: + """The single path-named server this request targets, iff it resolves to a + gateway-managed oauth2 server — the one per-server shape the gateway's own keyless + DCR flow serves end to end, so the 401 challenge may advertise the per-server + protected-resource metadata (whose ``authorization_servers`` names the gateway). + + Multi-server CSV paths, header/path mismatches, unknown names, and every + client-forwarded or delegated mode return ``None``: those cells keep their existing + challenge (or absence of one), and a challenge is never emitted for a name the + public discovery routes would 404, so this reveals exactly the server set the + per-server protected-resource metadata already reveals.""" + from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + global_mcp_server_manager, + ) + + targets = _parse_mcp_server_names_from_path(route, mcp_servers) + if targets is None: + return None + server = global_mcp_server_manager.get_mcp_server_by_name(targets[0], client_ip=client_ip) + if server is None or not server.is_gateway_managed_oauth2: + return None + return targets[0] -def _is_aggregate_gateway_dcr_challenge_scope( +def _is_gateway_dcr_challenge_scope( route: str, mcp_servers: list[str] | None, mcp_auth_header: str | None, mcp_server_auth_headers: dict[str, dict[str, str]] | None, exc: Exception, + client_ip: str | None, ) -> bool: - """True when an unauthenticated request to the aggregate ``/mcp`` endpoint - should receive the RFC 9728 401 challenge that advertises the gateway as - the authorization server. + """True when an unauthenticated MCP request should receive the RFC 9728 401 + challenge that advertises the gateway as the authorization server. - Fires only for a genuine 401 on the aggregate scope: any named target - (path or ``x-mcp-servers``) belongs to the per-server challenge paths, and - client-supplied MCP auth headers mean the caller is not a cold-start DCR - client. Fails closed to the original admission error otherwise.""" + Fires only for a genuine 401 with no client-supplied MCP auth headers (those mean + the caller is not a cold-start DCR client), on the scopes the gateway's keyless + flow serves: the aggregate ``/mcp`` endpoint, an ``x-mcp-servers``-scoped request + (the resource the client configured is still ``/mcp``), or a per-server path whose + single target is a gateway-managed oauth2 server. Every other named target keeps + its existing behavior, failing closed to the original admission error.""" if not _is_litellm_auth_admission_error(exc): return False if _has_client_supplied_mcp_auth(mcp_auth_header, mcp_server_auth_headers): return False - return _is_aggregate_mcp_scope(route, mcp_servers) + if len(MCPRequestHandler._extract_target_server_names_from_path(route)) == 0: + return True + return _gateway_dcr_challenge_target(route, mcp_servers, client_ip) is not None -def _aggregate_gateway_dcr_challenge(request: Request, invalid_token: bool) -> HTTPException: - """The RFC 9728 challenge for the aggregate endpoint: points the client at - the gateway's own protected-resource metadata so a DCR client discovers - the gateway as its authorization server and starts the sign-in flow. +def _gateway_dcr_challenge( + request: Request, + route: str, + mcp_servers: list[str] | None, + invalid_token: bool, +) -> HTTPException: + """The RFC 9728 challenge pointing the client at the protected-resource metadata + matching the scope it requested: the per-server document (same URL spelling the + request arrived on) when the single target is a gateway-managed oauth2 server, + else the gateway's aggregate document. Either way the client discovers the gateway + as its authorization server and starts the same sign-in flow. ``invalid_token`` adds the RFC 6750 error code for a request that DID present a bearer that failed admission (expired or revoked), telling spec-compliant clients to re-authorize rather than retry; a request with no credentials at all gets the bare challenge per RFC 6750 section 3.1.""" - error_attr = 'error="invalid_token", ' if invalid_token else "" + target = _gateway_dcr_challenge_target(route, mcp_servers, IPAddressUtils.get_mcp_client_ip(request)) resource_metadata_url = ( - f"{get_request_base_url(request)}/.well-known/oauth-protected-resource{well_known_root_suffix()}/mcp" + get_passthrough_resource_metadata_url(request.scope, target) + if target is not None + else f"{get_request_base_url(request)}/.well-known/oauth-protected-resource{well_known_root_suffix()}/mcp" ) + error_attr = 'error="invalid_token", ' if invalid_token else "" return HTTPException( status_code=401, detail={ @@ -240,14 +272,15 @@ def _admission_failure_fallback( ): verbose_logger.debug("MCP pass-through cold start: deferring admission to route 401 emitter") return UserAPIKeyAuth() - if _is_aggregate_gateway_dcr_challenge_scope( + if _is_gateway_dcr_challenge_scope( route=request_route, mcp_servers=mcp_servers, mcp_auth_header=mcp_auth_header, mcp_server_auth_headers=mcp_server_auth_headers, exc=exc, + client_ip=IPAddressUtils.get_mcp_client_ip(request), ): - raise _aggregate_gateway_dcr_challenge(request, invalid_token=bearer_presented) from exc + raise _gateway_dcr_challenge(request, request_route, mcp_servers, invalid_token=bearer_presented) from exc raise exc @@ -399,18 +432,18 @@ class MCPRequestHandler: request=request, route=request_route, ) - elif ( - _is_aggregate_mcp_scope(request_route, mcp_servers) - and oauth2_headers - and is_session_bearer_shaped(oauth2_headers["Authorization"]) - ): - # A gateway DCR session bearer at the aggregate /mcp scope: open the identity-only session - # token and admit under the live litellm user. One that does not open fails closed with the - # aggregate invalid_token challenge; a non-session bearer falls through to the oauth2 arm. + elif oauth2_headers and is_session_bearer_shaped(oauth2_headers["Authorization"]): + # A gateway DCR session bearer at any MCP scope: open the identity-only session + # token and admit under the live litellm user; downstream grant resolution + # intersects the admitted subject's servers with any path or header target, so a + # per-server scope narrows and never broadens. One that does not open fails + # closed with the scope's invalid_token challenge; a non-session bearer falls + # through to the oauth2 arm. validated_user_api_key_auth = await MCPRequestHandler._admit_gateway_session( authorization_value=oauth2_headers["Authorization"], request=request, route=request_route, + mcp_servers=mcp_servers, ) elif oauth2_headers: # Authorization on a non-delegated server: the bearer must be a real @@ -746,6 +779,7 @@ class MCPRequestHandler: authorization_value: str, request: Request, route: str, + mcp_servers: list[str] | None, ) -> UserAPIKeyAuth: """Open a gateway DCR session bearer and admit the live litellm user it references. @@ -753,8 +787,8 @@ class MCPRequestHandler: upstream credential (those are vaulted per user, resolved at egress), so authorization is resolved fresh via :meth:`_reload_admitted_user` + the centralized policy gate rather than a mint-time snapshot. Pre-DB gates (size, IP, route allowlist) run first, mirroring the standard - pipeline. Fails closed with the aggregate ``invalid_token`` challenge on an expired, tampered, - foreign, or refresh token, or a missing/deactivated/policy-rejected user.""" + pipeline. Fails closed with the requested scope's ``invalid_token`` challenge on an expired, + tampered, foreign, or refresh token, or a missing/deactivated/policy-rejected user.""" from litellm.proxy._experimental.mcp_server.outbound_credentials.session_credentials import ( NotSessionBearer, SessionBearerAdmitted, @@ -780,20 +814,20 @@ class MCPRequestHandler: ) except HTTPException as exc: # A cryptographically valid bearer whose referenced user is now missing or - # SCIM-deactivated is an invalid_token at the aggregate scope: relay the RFC 9728 + # SCIM-deactivated is an invalid_token at the requested scope: relay the RFC 9728 # challenge so the DCR client re-authorizes, matching the SessionBearerInvalid # arm, instead of a bare 401 with no WWW-Authenticate. A 503 (DB outage) is a # transient availability failure, not an auth failure, so it passes through. if exc.status_code == 401: - raise _aggregate_gateway_dcr_challenge(request, invalid_token=True) from exc + raise _gateway_dcr_challenge(request, route, mcp_servers, invalid_token=True) from exc raise return admitted case SessionBearerInvalid(): - raise _aggregate_gateway_dcr_challenge(request, invalid_token=True) + raise _gateway_dcr_challenge(request, route, mcp_servers, invalid_token=True) case NotSessionBearer(): # Unreachable: the arm is entered only for an is_session_bearer_shaped # value. Kept for match exhaustiveness and fails closed regardless. - raise _aggregate_gateway_dcr_challenge(request, invalid_token=True) + raise _gateway_dcr_challenge(request, route, mcp_servers, invalid_token=True) case _: assert_never(result) diff --git a/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py b/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py index cdc3ac15b1a..865787d5a07 100644 --- a/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py +++ b/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py @@ -2097,6 +2097,15 @@ async def _build_oauth_protected_resource_response( it. Only the legacy ``is_oauth_passthrough`` opt-in rewrites ``resource`` to the gateway's own URL so clients present the bearer token back to the gateway. + An explicitly named gateway-managed oauth2 server (interactive with + gateway-vaulted per-user tokens, or M2M) advertises the gateway's own + authorization server (``{base}/mcp``): a keyless DCR client that configured the + per-server URL completes the same sign-in flow the aggregate ``/mcp`` endpoint + supports and is admitted with a gateway session bearer. The per-server relay + authorize/token endpoints stay registered for the keyed interactive flow (which + is challenged with an explicit ``authorization_uri``), and the root-resolved + (unnamed) legacy shape keeps the relay authorization server. + Args: request: FastAPI Request object mcp_server_name: Name of the MCP server @@ -2112,6 +2121,7 @@ async def _build_oauth_protected_resource_response( request_base_url = get_request_base_url(request) client_ip = IPAddressUtils.get_mcp_client_ip(request) + explicitly_named = mcp_server_name is not None # When no server name provided, try to resolve the single OAuth2 server if mcp_server_name is None: @@ -2186,6 +2196,13 @@ async def _build_oauth_protected_resource_response( if mcp_server is None or mcp_server.auth_type != MCPAuth.oauth2_token_exchange: _raise_unless_oauth2_discovery_server(mcp_server, mcp_server_name, "not an OAuth-protected resource") + if explicitly_named and mcp_server is not None and mcp_server.is_gateway_managed_oauth2: + return { + "authorization_servers": [f"{request_base_url}/mcp"], + "resource": resource_url, + "scopes_supported": (mcp_server.scopes if mcp_server.scopes else []), + } + return { "authorization_servers": [ (f"{request_base_url}/{mcp_server_name}" if mcp_server_name else f"{request_base_url}") diff --git a/litellm/proxy/_experimental/mcp_server/exceptions.py b/litellm/proxy/_experimental/mcp_server/exceptions.py index 74752809e86..ca2261139c9 100644 --- a/litellm/proxy/_experimental/mcp_server/exceptions.py +++ b/litellm/proxy/_experimental/mcp_server/exceptions.py @@ -57,7 +57,7 @@ class MCPUpstreamAuthError(Exception): ``/.well-known/oauth-protected-resource/mcp/{server_name}``. This keeps the ``resource_metadata`` URI aligned with the resource pattern the client originally targeted, matching the path-aware behaviour of - ``_get_passthrough_resource_metadata_url`` in ``server.py``. + ``get_passthrough_resource_metadata_url`` in ``oauth_utils.py``. """ challenge: Optional[str] = self.www_authenticate if challenge is None and self.status_code == 401 and base_url: diff --git a/litellm/proxy/_experimental/mcp_server/oauth_utils.py b/litellm/proxy/_experimental/mcp_server/oauth_utils.py index 5daec9f97be..8f47aa7344d 100644 --- a/litellm/proxy/_experimental/mcp_server/oauth_utils.py +++ b/litellm/proxy/_experimental/mcp_server/oauth_utils.py @@ -7,6 +7,7 @@ from typing import TYPE_CHECKING, Any, Dict, List, NoReturn, Optional from urllib.parse import ParseResult, urlparse, urlsplit, urlunparse, urlunsplit from fastapi import HTTPException, Request +from starlette.types import Scope from litellm._logging import verbose_logger from litellm.proxy._experimental.mcp_server.auth.token_endpoint_auth import ( @@ -179,6 +180,37 @@ def well_known_root_suffix() -> str: return "" if root == "/" else root +def get_passthrough_resource_metadata_url(scope: Scope, server_name: str) -> str: + """The per-server protected-resource metadata URL matching the spelling the request + arrived on, so a strict RFC 9728 client resolves the same route the proxy registered. + ``_original_path`` preserves the ``/{server}/mcp`` spelling through the + ``dynamic_mcp_route`` rewrite; the ``SERVER_ROOT_PATH`` segment is inserted exactly as + the route decorators insert it (see :func:`well_known_root_suffix`).""" + request = Request(scope) + base_url = get_request_base_url(request) + _path = scope.get("_original_path") or scope.get("path", "") or "" + + if _path.startswith(f"/{server_name}/mcp"): + return f"{base_url}/.well-known/oauth-protected-resource{well_known_root_suffix()}/{server_name}/mcp" + return f"{base_url}/.well-known/oauth-protected-resource{well_known_root_suffix()}/mcp/{server_name}" + + +def get_passthrough_www_authenticate( + scope: Scope, + server_name: str, + invalid_token: bool = False, +) -> str: + """The RFC 9728 ``WWW-Authenticate`` value advertising the per-server + protected-resource metadata, with the RFC 6750 ``invalid_token`` error code when the + caller presented a bearer that failed rather than no credential at all.""" + resource_metadata_url = get_passthrough_resource_metadata_url( + scope=scope, + server_name=server_name, + ) + error_attr = 'error="invalid_token", ' if invalid_token else "" + return f'Bearer {error_attr}resource_metadata="{resource_metadata_url}"' + + def validate_loopback_redirect_uri(redirect_uri: str) -> None: """Require a loopback ``redirect_uri`` (OAuth 2.1 §4.1.2.1 + RFC 8252 §7.3 native-app pattern). MCP clients are native apps that listen on diff --git a/litellm/proxy/_experimental/mcp_server/server.py b/litellm/proxy/_experimental/mcp_server/server.py index 06a3a5a61e4..ec07d33f24d 100644 --- a/litellm/proxy/_experimental/mcp_server/server.py +++ b/litellm/proxy/_experimental/mcp_server/server.py @@ -37,6 +37,7 @@ from litellm.llms.custom_httpx.http_handler import ( ) from litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp import ( MCPRequestHandler, + _is_mcp_admitted_user_subject, ) from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( get_request_base_url, @@ -53,6 +54,7 @@ from litellm.proxy._experimental.mcp_server.mcp_context import ( from litellm.proxy._experimental.mcp_server.mcp_debug import MCPDebug from litellm.proxy._experimental.mcp_server.oauth_utils import ( _redact_mcp_resource_url, + get_passthrough_www_authenticate, ) from litellm.proxy._experimental.mcp_server.utils import ( LITELLM_MCP_SERVER_DESCRIPTION, @@ -3650,30 +3652,6 @@ if MCP_AVAILABLE: ) return user_api_key_auth.model_copy(update={"object_permission": updated_op}) - def _get_passthrough_resource_metadata_url(scope: Scope, server_name: str) -> str: - request = StarletteRequest(scope) - base_url = get_request_base_url(request) - _path = scope.get("_original_path") or scope.get("path", "") or "" - - if _path.startswith(f"/{server_name}/mcp"): - return f"{base_url}/.well-known/oauth-protected-resource/{server_name}/mcp" - return f"{base_url}/.well-known/oauth-protected-resource/mcp/{server_name}" - - def _get_passthrough_www_authenticate( - scope: Scope, - server_name: str, - invalid_token: bool = False, - ) -> str: - resource_metadata_url = _get_passthrough_resource_metadata_url( - scope=scope, - server_name=server_name, - ) - params = [] - if invalid_token: - params.append('error="invalid_token"') - params.append(f'resource_metadata="{resource_metadata_url}"') - return "Bearer " + ", ".join(params) - async def _raise_preemptive_401_for_unauthenticated_servers( scope: Scope, mcp_servers: list[str] | None, @@ -3723,10 +3701,26 @@ if MCP_AVAILABLE: # challenge whenever one is absent, regardless of any bearer. # The v2 resolver owns the existence check, so every # authorization_code resolution (egress and this discovery - # challenge) runs through it. + # challenge) runs through it. A keyless admitted subject is + # challenged with the per-server resource_metadata (whose + # authorization server is the gateway itself, vaulting via the + # authorize interlude); the per-server relay advertised below + # cannot vault without a litellm key on its token request. if await global_mcp_server_manager.has_user_oauth_token(server, user_api_key_auth): continue + if _is_mcp_admitted_user_subject(user_api_key_auth): + raise HTTPException( + status_code=401, + detail="Unauthorized", + headers={ + "www-authenticate": get_passthrough_www_authenticate( + scope=scope, + server_name=server_name, + ) + }, + ) + request = StarletteRequest(scope) base_url = get_request_base_url(request) _path = scope.get("_original_path") or scope.get("path", "") or "" @@ -3751,7 +3745,7 @@ if MCP_AVAILABLE: # the proxied resource_metadata (RFC 9728), not the gateway # authorization_uri above which would authorize against the # gateway instead of the upstream IdP. - www_authenticate = _get_passthrough_www_authenticate( + www_authenticate = get_passthrough_www_authenticate( scope=scope, server_name=server_name, ) @@ -3807,7 +3801,7 @@ if MCP_AVAILABLE: and server.is_oauth_passthrough and not _client_has_passthrough_authorization(server, oauth2_headers, mcp_server_auth_headers) ): - www_authenticate = _get_passthrough_www_authenticate( + www_authenticate = get_passthrough_www_authenticate( scope=scope, server_name=server_name, ) @@ -3824,7 +3818,7 @@ if MCP_AVAILABLE: and _get_forwarded_auth_from_scope(scope) is None and not _client_has_per_server_auth_header(server, mcp_server_auth_headers) ): - www_authenticate = _get_passthrough_www_authenticate( + www_authenticate = get_passthrough_www_authenticate( scope=scope, server_name=server_name, ) @@ -3846,7 +3840,7 @@ if MCP_AVAILABLE: status_code=401, detail="Unauthorized", headers={ - "www-authenticate": _get_passthrough_www_authenticate( + "www-authenticate": get_passthrough_www_authenticate( scope=scope, server_name=server_name, ) @@ -4053,7 +4047,7 @@ if MCP_AVAILABLE: # Token is missing or expired: keep pass-through clients on the # protected-resource discovery flow so they re-authorize against # the upstream IdP metadata proxied by LiteLLM. - www_authenticate = _get_passthrough_www_authenticate( + www_authenticate = get_passthrough_www_authenticate( scope=scope, server_name=challenge_server_name, invalid_token=True, diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index f5a50d1697a..dc8bc961296 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -5,6 +5,7 @@ import math import time import traceback from datetime import datetime +from functools import lru_cache from typing import ( TYPE_CHECKING, Any, @@ -12,6 +13,7 @@ from typing import ( Callable, Dict, Literal, + Mapping, Optional, Tuple, Union, @@ -38,6 +40,9 @@ from litellm.constants import ( ) from litellm.integrations.custom_guardrail import CustomGuardrail from litellm.litellm_core_utils.dd_tracing import NullTracer, tracer +from litellm.litellm_core_utils.get_supported_openai_params import ( + get_supported_openai_params, +) from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.litellm_core_utils.llm_response_utils.get_headers import ( get_response_headers, @@ -244,6 +249,71 @@ async def _cancel_pending_gather_tasks(tasks: list["asyncio.Task[Any]"]) -> None pass +@lru_cache(maxsize=512) +def _litellm_model_supports_stream_options(litellm_model: str) -> bool: + try: + supported_params = get_supported_openai_params(model=litellm_model) + except Exception: # noqa: BLE001 # unmapped or malformed model strings must disable injection, not fail the request + return False + return supported_params is not None and "stream_options" in supported_params + + +def _deployment_litellm_model(deployment: Mapping[str, object]) -> str | None: + litellm_params = deployment.get("litellm_params") + if isinstance(litellm_params, Mapping): + litellm_model = litellm_params.get("model") + else: + litellm_model = getattr(litellm_params, "model", None) + return litellm_model if isinstance(litellm_model, str) else None + + +def _model_deployments_support_stream_options( + model: object, + llm_router: Router | None, + team_id: str | None, +) -> bool: + if not isinstance(model, str): + return False + deployments = llm_router.get_model_list(model_name=model, team_id=team_id) if llm_router is not None else None + deployment_models = tuple( + litellm_model + for deployment in deployments or () + if (litellm_model := _deployment_litellm_model(deployment)) is not None + ) + candidate_models = deployment_models if deployment_models else (model,) + return all(_litellm_model_supports_stream_options(m) for m in candidate_models) + + +def _stream_usage_tracking_updates( + data: Mapping[str, object], + general_settings: Mapping[str, object], + route_type: str, + supports_stream_options: Callable[[], bool], +) -> Mapping[str, object]: + scrub = {"_litellm_strip_stream_usage": False} if "_litellm_strip_stream_usage" in data else {} + if data.get("stream", False) is not True: + return scrub + always_include = general_settings.get("always_include_stream_usage") + stream_options = data.get("stream_options") + if always_include is True: + if "stream_options" not in data: + return {**scrub, "stream_options": {"include_usage": True}} + if isinstance(stream_options, dict) and "include_usage" not in stream_options: + return {**scrub, "stream_options": {**stream_options, "include_usage": True}} + return scrub + if always_include is False or route_type != "acompletion": + return scrub + if isinstance(stream_options, dict) and stream_options.get("include_usage") is True: + return scrub + if not supports_stream_options(): + return scrub + merged_stream_options = {**stream_options} if isinstance(stream_options, dict) else {} + return { + "stream_options": {**merged_stream_options, "include_usage": True}, + "_litellm_strip_stream_usage": True, + } + + def _serialize_http_exception_detail( detail: Any, ) -> Tuple[str, Optional[dict]]: @@ -1232,17 +1302,18 @@ class ProxyBaseLLMRequestProcessing: ) ### AUTO STREAM USAGE TRACKING ### - # If always_include_stream_usage is enabled and this is a streaming request - # automatically add stream_options={'include_usage': True} if not already set - if ( - general_settings.get("always_include_stream_usage", False) is True - and self.data.get("stream", False) is True - ): - # Only set if stream_options is not already provided by the client - if "stream_options" not in self.data: - self.data["stream_options"] = {"include_usage": True} - elif isinstance(self.data["stream_options"], dict) and "include_usage" not in self.data["stream_options"]: - self.data["stream_options"]["include_usage"] = True + self.data.update( + _stream_usage_tracking_updates( + data=self.data, + general_settings=general_settings, + route_type=route_type, + supports_stream_options=lambda: _model_deployments_support_stream_options( + model=self.data.get("model"), + llm_router=llm_router, + team_id=user_api_key_dict.team_id, + ), + ) + ) ### CALL HOOKS ### - modify/reject incoming data before calling the model ## LOGGING OBJECT ## - initialize logging object for logging success/failure events for call @@ -2730,9 +2801,7 @@ class ProxyBaseLLMRequestProcessing: and proxy_logging_obj is not None and user_api_key_dict is not None ): - await proxy_logging_obj._arelease_max_parallel_requests_on_disconnect( - user_api_key_dict, request_data - ) + await proxy_logging_obj._arelease_max_parallel_requests_on_disconnect(user_api_key_dict) if hasattr(response, "aclose"): try: diff --git a/litellm/proxy/guardrails/guardrail_hooks/compresr/compresr.py b/litellm/proxy/guardrails/guardrail_hooks/compresr/compresr.py index e512be23fc9..4627b298d09 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/compresr/compresr.py +++ b/litellm/proxy/guardrails/guardrail_hooks/compresr/compresr.py @@ -48,6 +48,7 @@ from litellm.llms.custom_httpx.http_handler import ( ) from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.guardrails.guardrail_hooks.content_text import ( + assistant_text_from_response, content_to_text, is_all_text_parts, merge_rewritten_text_parts, @@ -391,47 +392,6 @@ def _is_anthropic_messages_response(response: object) -> bool: return isinstance(get_attribute_or_key(response, "content", None), list) -def _assistant_text_from_response(response: object) -> str | None: - """The assistant's natural-language text from a model response, across chat, - Anthropic, and Responses shapes. Preserved when the turn is rebuilt for the - retrieval follow-up so the model's reasoning is not lost.""" - choices = get_attribute_or_key(response, "choices", None) - if isinstance(choices, list) and choices: - message = get_attribute_or_key(choices[0], "message", None) - if message is not None: - text = content_to_text(get_attribute_or_key(message, "content", None)) - if text: - return text - content = get_attribute_or_key(response, "content", None) - if isinstance(content, list): - parts = [ - text - for block in content - if get_attribute_or_key(block, "type", None) == "text" - for text in (get_attribute_or_key(block, "text", None),) - if isinstance(text, str) and text - ] - if parts: - return "".join(parts) - output = get_attribute_or_key(response, "output", None) - if isinstance(output, list): - parts = [] - for item in output: - if get_attribute_or_key(item, "type", None) != "message": - continue - item_content = get_attribute_or_key(item, "content", None) - if not isinstance(item_content, list): - continue - for chunk in item_content: - if get_attribute_or_key(chunk, "type", None) == "output_text": - text = get_attribute_or_key(chunk, "text", None) - if isinstance(text, str) and text: - parts.append(text) - if parts: - return "".join(parts) - return None - - def _build_assistant_message_from_response( response: object, retrieved: list[tuple[dict[str, object], str]], @@ -446,7 +406,7 @@ def _build_assistant_message_from_response( """ return { "role": "assistant", - "content": _assistant_text_from_response(response), + "content": assistant_text_from_response(response), "tool_calls": [ { "id": tool_call.get("id"), @@ -470,7 +430,7 @@ def _build_anthropic_followup_messages( assistant text is preserved; non-retrieve tool calls are re-planned by the follow-up (see _build_assistant_message_from_response).""" assistant_content: list[dict[str, object]] = [] - text = _assistant_text_from_response(response) + text = assistant_text_from_response(response) if text: assistant_content.append({"type": "text", "text": text}) assistant_content.extend( @@ -501,7 +461,7 @@ def _build_responses_followup_items( with a function_call_output keyed by the same call_id. The assistant text is preserved; non-retrieve tool calls are re-planned by the follow-up.""" items: list[dict[str, object]] = [] - text = _assistant_text_from_response(response) + text = assistant_text_from_response(response) if text: items.append({"role": "assistant", "content": text}) for tool_call, content in retrieved: diff --git a/litellm/proxy/guardrails/guardrail_hooks/content_text.py b/litellm/proxy/guardrails/guardrail_hooks/content_text.py index f4211e67512..4111c909d01 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/content_text.py +++ b/litellm/proxy/guardrails/guardrail_hooks/content_text.py @@ -14,6 +14,8 @@ non-text part, which is what ``is_all_text_parts`` gates. from collections.abc import Sequence +from litellm.litellm_core_utils.prompt_templates.factory import get_attribute_or_key + def content_to_text(content: object) -> str: """Collapse a message ``content`` (str or list-of-parts) to plain text. @@ -53,3 +55,41 @@ def merge_rewritten_text_parts(parts: Sequence[object], new_text: str) -> list[o breakpoints = tuple(part["cache_control"] for part in dict_parts if part.get("cache_control") is not None) base = {**dict_parts[0], "text": new_text} if dict_parts else {"type": "text", "text": new_text} return [{**base, "cache_control": breakpoints[-1]} if breakpoints else base] + + +def assistant_text_from_response(response: object) -> str | None: + """The assistant's natural-language text from a model response, across chat, + Anthropic, and Responses shapes. Preserved when the turn is rebuilt for the + retrieval follow-up so the model's reasoning is not lost.""" + choices = get_attribute_or_key(response, "choices", None) + if isinstance(choices, list) and choices: + message = get_attribute_or_key(choices[0], "message", None) + if message is not None: + text = content_to_text(get_attribute_or_key(message, "content", None)) + if text: + return text + content = get_attribute_or_key(response, "content", None) + if isinstance(content, list): + parts = [ + text + for block in content + if get_attribute_or_key(block, "type", None) == "text" + for text in (get_attribute_or_key(block, "text", None),) + if isinstance(text, str) and text + ] + if parts: + return "".join(parts) + output = get_attribute_or_key(response, "output", None) + if isinstance(output, list): + output_parts = [ + text + for item in output + if get_attribute_or_key(item, "type", None) == "message" + for chunk in (get_attribute_or_key(item, "content", None) or ()) + if get_attribute_or_key(chunk, "type", None) == "output_text" + for text in (get_attribute_or_key(chunk, "text", None),) + if isinstance(text, str) and text + ] + if output_parts: + return "".join(output_parts) + return None diff --git a/litellm/proxy/guardrails/guardrail_hooks/headroom/headroom.py b/litellm/proxy/guardrails/guardrail_hooks/headroom/headroom.py index 2735acd7787..1667bb604ba 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/headroom/headroom.py +++ b/litellm/proxy/guardrails/guardrail_hooks/headroom/headroom.py @@ -7,6 +7,7 @@ import uuid from typing import TYPE_CHECKING, Any, ClassVar, List, Literal, Optional import httpx +from collections.abc import Mapping, Sequence from fastapi import HTTPException import litellm @@ -15,6 +16,7 @@ from litellm.proxy.spend_tracking.compression_savings import HEADROOM_GUARDRAIL_ from typing_extensions import TypeGuard from litellm._logging import verbose_proxy_logger +from litellm.compression.compress import get_protected_indices from litellm.integrations.custom_guardrail import ( CustomGuardrail, log_guardrail_information, @@ -22,6 +24,7 @@ from litellm.integrations.custom_guardrail import ( from litellm.litellm_core_utils.prompt_templates.factory import ( get_attribute_or_key, get_tool_calls_from_response, + group_tool_exchanges, has_tool_with_name, ) from litellm.llms.custom_httpx.http_handler import ( @@ -29,6 +32,7 @@ from litellm.llms.custom_httpx.http_handler import ( httpxSpecialProvider, ) from litellm.proxy.guardrails.guardrail_hooks.content_text import ( + assistant_text_from_response, content_to_text, is_all_text_parts, merge_rewritten_text_parts, @@ -110,6 +114,42 @@ def _restore_content_shapes( return restored +def _protected_indices(messages: Sequence[Mapping[str, object]]) -> frozenset[int]: + """Indices headroom must not send to the compression service. + + ``get_protected_indices`` is litellm's own compression policy: the system + rows, the last user row, the last assistant row. It is expanded over whole + tool exchanges the way ``compress()`` expands it, so a protected assistant + tool call cannot end up answered by a marker standing in for the result the + model just asked for. + """ + protected = frozenset(get_protected_indices(messages)) + return protected | frozenset( + index + for group in group_tool_exchanges(messages) + if any(member in protected for member in group) + for index in group + ) + + +def _restore_protected_messages( + messages: Sequence[dict[str, object]], + compressed: Sequence[dict[str, object]], + protected_indices: frozenset[int], +) -> Sequence[dict[str, object]]: + """Put the rows that were held back from compression at their original positions. + + Requires one returned row per row actually sent, which ``_call_compress`` + enforces; a service that changed the row count is treated as a failure + there, because a reshaped conversation cannot be re-interleaved. + """ + sent_positions = tuple(index for index in range(len(messages)) if index not in protected_indices) + compressed_by_index = dict(zip(sent_positions, compressed)) + return [ + messages[index] if index in protected_indices else compressed_by_index[index] for index in range(len(messages)) + ] + + def extract_hashes_from_messages(messages: list[dict[str, object]]) -> list[str]: hashes: list[str] = [] for msg in messages: @@ -175,30 +215,33 @@ def _extract_headroom_tool_calls(response: object) -> list[dict[str, object]]: ] -def _build_assistant_message_from_response(response: object) -> dict[str, object]: - choices = getattr(response, "choices", None) - if not isinstance(choices, list) or not choices: - return {"role": "assistant", "content": None, "tool_calls": []} - message = getattr(choices[0], "message", None) - if message is None: - return {"role": "assistant", "content": None, "tool_calls": []} - content = getattr(message, "content", None) - tool_calls = getattr(message, "tool_calls", None) - raw_tool_calls: list[dict[str, object]] = [] - if isinstance(tool_calls, list): - for tc in tool_calls: - fn = getattr(tc, "function", None) - raw_tool_calls.append( - { - "id": getattr(tc, "id", None), - "type": "function", - "function": { - "name": getattr(fn, "name", None) if fn else None, - "arguments": getattr(fn, "arguments", "{}") if fn else "{}", - }, - } - ) - return {"role": "assistant", "content": content, "tool_calls": raw_tool_calls} +def _build_assistant_message_from_response( + response: object, + retrieved: Sequence[tuple[dict[str, object], str]], +) -> dict[str, object]: + """Rebuild the chat-completions assistant turn for the retrieval follow-up. + + Only the ``headroom_retrieve`` calls are echoed, each answered by a tool + result below. Other tool calls made in the same turn are omitted on purpose: + the follow-up re-runs the model with the recovered content so it re-plans + them. Echoing them would leave tool_calls with no matching tool result and + the provider would reject the request. + """ + return { + "role": "assistant", + "content": assistant_text_from_response(response), + "tool_calls": [ + { + "id": tool_call.get("id"), + "type": "function", + "function": { + "name": tool_call.get("name"), + "arguments": json.dumps(tool_call.get("arguments", {})), + }, + } + for tool_call, _ in retrieved + ], + } def _is_responses_api_response(response: object) -> bool: @@ -213,17 +256,22 @@ def _is_anthropic_messages_response(response: object) -> bool: def _build_anthropic_followup_messages( + response: object, retrieved: list[tuple[dict[str, object], str]], ) -> list[dict[str, object]]: """Build Anthropic Messages API follow-up messages for a tool round-trip. Anthropic requires the tool_use block to be echoed back in an assistant message, paired with a tool_result block in a user message keyed by the - same tool_use_id -- it does not accept chat-style tool-role messages. + same tool_use_id -- it does not accept chat-style tool-role messages. Any + text the model wrote alongside the tool call is preserved, so its reasoning + survives into the follow-up turn. """ + text = assistant_text_from_response(response) assistant_message: dict[str, object] = { "role": "assistant", - "content": [ + "content": ([{"type": "text", "text": text}] if text else []) + + [ { "type": "tool_use", "id": tool_call.get("id"), @@ -244,15 +292,18 @@ def _build_anthropic_followup_messages( def _build_responses_followup_items( + response: object, retrieved: list[tuple[dict[str, object], str]], ) -> list[dict[str, object]]: """Build Responses API input items for a tool round-trip. The Responses API does not accept chat-style assistant/tool messages as follow-up input; it requires the model's function_call to be echoed back - paired with a function_call_output keyed by the same call_id. + paired with a function_call_output keyed by the same call_id. Any text the + model wrote alongside the tool call is preserved. """ - items: list[dict[str, object]] = [] + text = assistant_text_from_response(response) + items: List[dict[str, object]] = [{"role": "assistant", "content": text}] if text else [] for tool_call, content in retrieved: call_id = tool_call.get("id") items.append( @@ -453,6 +504,19 @@ class HeadroomGuardrail(CustomGuardrail): {}, ) + if len(filtered) != len(messages): + # Rows are matched positionally when the never-compressed messages + # are put back, so a reshaped conversation cannot be applied at all. + return ( + self._handle_compress_failure( + messages, + "Headroom compression service changed the message count", + {"sent": len(messages), "returned": len(filtered)}, + ), + False, + {}, + ) + verbose_proxy_logger.debug( "Headroom: compressed %s tokens -> %s tokens (ratio %.2f)", body.get("tokens_before", "?"), @@ -547,14 +611,27 @@ class HeadroomGuardrail(CustomGuardrail): if not messages: return inputs + # The last user message is the instruction the model is being asked to + # act on, so replacing it with a marker means the model answers a + # retrieval result instead of the request. Protected rows are held back + # from the payload rather than pinned after the fact, so their tokens + # are not counted as savings we never apply; the Anthropic write-back + # discards a compressed system prompt outright. Keep it that way unless + # /v1/compress grows a field for sending the live turn as the retrieval + # query without compressing it: query-aware compression reads the newest + # user message, so it is withheld here at some cost to history ranking. + protected_indices = _protected_indices(messages) + compressible = [m for i, m in enumerate(messages) if i not in protected_indices] + if not compressible: + return inputs + model = self.headroom_model or request_data.get("model") start_time = time.time() - compressed, compression_succeeded, stats = await self._call_compress( - messages=_flatten_messages_for_compression(messages), + returned, compression_succeeded, stats = await self._call_compress( + messages=_flatten_messages_for_compression(compressible), model=model if isinstance(model, str) else None, ) end_time = time.time() - compressed = _restore_content_shapes(originals=messages, returned=compressed) from litellm.proxy.common_utils.callback_utils import ( add_guardrail_to_applied_guardrails_header, @@ -571,7 +648,17 @@ class HeadroomGuardrail(CustomGuardrail): duration=end_time - start_time, ) add_guardrail_to_applied_guardrails_header(request_data=request_data, guardrail_name=self.guardrail_name) - return {**inputs, "structured_messages": compressed} # pyright: ignore[reportReturnType] + # Hand back the caller's own inputs object. Translation handlers + # detect "the guardrail rewrote the messages" by identity, so + # returning a rebuilt copy sends an unchanged request through the + # write-back and restructures it for nothing. + return inputs + + compressed = _restore_protected_messages( + messages=messages, + compressed=_restore_content_shapes(originals=compressible, returned=returned), + protected_indices=protected_indices, + ) self.add_standard_logging_guardrail_information_to_request_data( guardrail_json_response=stats, @@ -668,11 +755,11 @@ class HeadroomGuardrail(CustomGuardrail): retrieved.append((tc, content)) if _is_responses_api_response(response): - follow_up_messages = list(messages) + _build_responses_followup_items(retrieved) + follow_up_messages = list(messages) + _build_responses_followup_items(response, retrieved) elif _is_anthropic_messages_response(response): - follow_up_messages = list(messages) + _build_anthropic_followup_messages(retrieved) + follow_up_messages = list(messages) + _build_anthropic_followup_messages(response, retrieved) else: - assistant_message = _build_assistant_message_from_response(response) + assistant_message = _build_assistant_message_from_response(response, retrieved) tool_results = [ {"role": "tool", "tool_call_id": tc.get("id"), "content": content} for tc, content in retrieved ] diff --git a/litellm/proxy/hooks/dynamic_rate_limiter_v3.py b/litellm/proxy/hooks/dynamic_rate_limiter_v3.py index 6e4a6fe1a51..932146800e2 100644 --- a/litellm/proxy/hooks/dynamic_rate_limiter_v3.py +++ b/litellm/proxy/hooks/dynamic_rate_limiter_v3.py @@ -21,7 +21,10 @@ from litellm.proxy.common_utils.proxy_rate_limit_error import ( from litellm.proxy.hooks.parallel_request_limiter_v3 import ( RateLimitDescriptor, RateLimitDescriptorRateLimitObject, + RateLimitResponse, _PROXY_MaxParallelRequestsHandler_v3, + claim_request_stash_for_data, + get_or_create_request_stash, ) from litellm.proxy.hooks.rate_limiter_utils import ( convert_priority_to_percent, @@ -373,7 +376,6 @@ class _PROXY_DynamicRateLimitHandlerV3(CustomLogger): user_api_key_dict: UserAPIKeyAuth, priority: Optional[str], saturation: float, - data: dict, ) -> None: """ Check rate limits using THREE-PHASE approach to prevent partial increments. @@ -400,7 +402,6 @@ class _PROXY_DynamicRateLimitHandlerV3(CustomLogger): user_api_key_dict: User authentication info priority: User's priority level saturation: Current saturation level - data: Request data dictionary Raises: HTTPException: If any limit is exceeded @@ -550,12 +551,12 @@ class _PROXY_DynamicRateLimitHandlerV3(CustomLogger): parent_otel_span=user_api_key_dict.parent_otel_span, read_only=False, ) - data["litellm_proxy_rate_limit_response"] = { - "overall_code": atomic_response["overall_code"], - "statuses": atomic_response["statuses"] + priority_tracking_response["statuses"], - } + get_or_create_request_stash().rate_limit_response = RateLimitResponse( + overall_code=atomic_response["overall_code"], + statuses=atomic_response["statuses"] + priority_tracking_response["statuses"], + ) else: - data["litellm_proxy_rate_limit_response"] = atomic_response + get_or_create_request_stash().rate_limit_response = atomic_response async def async_pre_call_hook( self, @@ -601,6 +602,7 @@ class _PROXY_DynamicRateLimitHandlerV3(CustomLogger): if "model" not in data: return None + claim_request_stash_for_data(data) model = data["model"] priority = self._get_priority_from_user_api_key_dict(user_api_key_dict=user_api_key_dict) @@ -632,7 +634,6 @@ class _PROXY_DynamicRateLimitHandlerV3(CustomLogger): user_api_key_dict=user_api_key_dict, priority=priority, saturation=saturation, - data=data, ) except HTTPException: diff --git a/litellm/proxy/hooks/parallel_request_limiter_v3.py b/litellm/proxy/hooks/parallel_request_limiter_v3.py index 719496785dd..b04ef5f7087 100644 --- a/litellm/proxy/hooks/parallel_request_limiter_v3.py +++ b/litellm/proxy/hooks/parallel_request_limiter_v3.py @@ -8,16 +8,18 @@ import asyncio import binascii import os import uuid +from contextvars import ContextVar +from dataclasses import dataclass, field from datetime import datetime from typing import ( TYPE_CHECKING, Any, Callable, Dict, + FrozenSet, List, Literal, Optional, - Set, Tuple, TypedDict, Union, @@ -28,7 +30,6 @@ from litellm import DualCache from litellm._logging import verbose_proxy_logger from litellm.constants import DYNAMIC_RATE_LIMIT_ERROR_THRESHOLD_PER_MINUTE from litellm.integrations.custom_logger import CustomLogger -from litellm.litellm_core_utils.core_helpers import get_or_create_metadata_bucket from litellm.litellm_core_utils.prompt_templates.common_utils import ( get_str_from_messages, ) @@ -291,53 +292,11 @@ DEFAULT_CHARS_PER_TOKEN = 4 # (baseline floor) and to the smallest configured TPM limit (capped floor for # small per-tenant TPM caps). _TPM_FLOOR_FRACTION = 4 -# Stash for the reserved-token count on the request data dict so success/ -# failure callbacks can reconcile against the upfront reservation. -TPM_RESERVED_TOKENS_KEY = "_litellm_tpm_reserved_tokens" -# Stash for the model identifier the reservation was charged against. -# Reconciliation must target the same key that was incremented at reservation -TPM_RESERVED_MODEL_KEY = "_litellm_tpm_reserved_model" -# Stash for the (scope_key, scope_value) pairs whose :tokens counter the -# upfront reservation incremented. Reconciliation applies the delta to these -# scopes only; scopes without a configured TPM limit were never charged at -# pre-call and must receive the full actual usage instead of the delta — -# otherwise their counters drift negative whenever actual < reserved. -TPM_RESERVED_SCOPES_KEY = "_litellm_tpm_reserved_scopes" -# Idempotency marker for the reservation refund path. Set when any failure -# callback releases the reservation so the next callback in the same flow -# (e.g. async_log_failure_event firing after async_post_call_failure_hook) -# does not double-refund. -TPM_RESERVATION_RELEASED_KEY = "_litellm_tpm_reservation_released" -RATE_LIMIT_DESCRIPTORS_KEY = "_litellm_rate_limit_descriptors" -# Pre-call RateLimitResponse stashed here so streaming success logging can -# mirror ``x-ratelimit-*`` headers into the SLP. Streaming exits -# common_request_processing before ``async_post_call_success_hook`` runs. -RATE_LIMIT_RESPONSE_KEY = "_litellm_proxy_rate_limit_response" -# Holds the acquisition the pre-call hook made for this request: the slot id -# plus the gauge counter keys it was registered under. The success/failure -# callbacks release only this exact acquisition: those callbacks also fire -# for requests rejected at pre-call (which never acquired a slot), and an -# id-less release would free a slot still owned by another in-flight request -# — every rejection would then raise effective concurrency above the -# configured limit. -MAX_PARALLEL_SLOT_ACQUIRED_KEY = "_litellm_max_parallel_slot_acquired" # How long an acquired slot counts toward the in-flight total before it is # considered leaked (worker crashed without any release callback firing) and # pruned. Also the longest request duration the gauge can track: a request # running longer than this stops occupying its slot. PARALLEL_REQUEST_SLOT_TTL_SECONDS = 3600 -# Stash keys live ONLY in metadata channels — never at the top level of the -# request body. Top-level keys are forwarded as body params to upstream -# providers, which reject unknown fields with 400/429 errors. -_LITELLM_STASH_KEYS: Tuple[str, ...] = ( - TPM_RESERVED_TOKENS_KEY, - TPM_RESERVED_MODEL_KEY, - TPM_RESERVED_SCOPES_KEY, - TPM_RESERVATION_RELEASED_KEY, - RATE_LIMIT_DESCRIPTORS_KEY, - RATE_LIMIT_RESPONSE_KEY, - MAX_PARALLEL_SLOT_ACQUIRED_KEY, -) class RateLimitDescriptorRateLimitObject(TypedDict, total=False): @@ -382,6 +341,79 @@ class RateLimitResponseWithDescriptors(TypedDict): response: RateLimitResponse +@dataclass(slots=True) +class RequestRateLimiterStash: + """ + Per-request bookkeeping the pre-call hook hands to the success/failure/ + disconnect callbacks. Lives on a ContextVar instead of the request body so + it never reaches provider-facing ``metadata`` channels. + + A single mutable instance is shared by every context forked from the + request task (the SDK call, streaming generators, and the logging worker's + captured context all see the same object), which is what makes the + ``reservation_released`` flag and ``parallel_slot`` clearing effective + across sibling callbacks: the first release wins, later callbacks observe + the cleared state. + + Because the stash is context-inherited, nested LiteLLM calls made inside + the request (LLM-judge guardrails, silent experiments) would also see it + from their own logging callbacks. ``owner_litellm_call_id`` pins the stash + to the proxy request's ``litellm_call_id`` so those callbacks can tell the + owning request's events apart from a nested call's: router retries and + fallbacks reuse the request's call id and keep access, while nested calls + mint fresh ids and are ignored. + """ + + owner_litellm_call_id: Optional[str] = None + rate_limit_response: Optional[RateLimitResponse] = None + parallel_slot: Optional[ParallelSlotAcquisition] = None + reserved_tokens: int = 0 + reserved_model: Optional[str] = None + reserved_scopes: FrozenSet[Tuple[str, str]] = field(default_factory=frozenset) + reservation_released: bool = False + + +_request_stash: ContextVar[Optional[RequestRateLimiterStash]] = ContextVar( + "litellm_v3_rate_limiter_request_stash", default=None +) + + +def get_request_stash() -> Optional[RequestRateLimiterStash]: + return _request_stash.get() + + +def get_or_create_request_stash() -> RequestRateLimiterStash: + stash = _request_stash.get() + if stash is None: + stash = RequestRateLimiterStash() + _request_stash.set(stash) + return stash + + +def claim_request_stash_for_data(data: dict) -> RequestRateLimiterStash: + stash = get_or_create_request_stash() + owner_call_id = data.get("litellm_call_id") + if isinstance(owner_call_id, str): + stash.owner_litellm_call_id = owner_call_id + return stash + + +def get_request_stash_for_call(litellm_call_id: Optional[str]) -> Optional[RequestRateLimiterStash]: + stash = _request_stash.get() + if stash is None: + return None + if stash.owner_litellm_call_id is None or litellm_call_id is None: + return stash + return stash if litellm_call_id == stash.owner_litellm_call_id else None + + +def _call_id_from_callback_kwargs(kwargs: object) -> Optional[str]: + if not isinstance(kwargs, dict): + return None + call_id = kwargs.get("litellm_call_id") + return call_id if isinstance(call_id, str) else None + + class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): def __init__( self, @@ -2343,12 +2375,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): """ verbose_proxy_logger.debug("Inside Rate Limit Pre-Call Hook") - # Reject caller-supplied stash values before any read/write. Otherwise - # a client can inject ``_litellm_rate_limit_descriptors`` / - # ``_litellm_tpm_reserved_tokens`` in body ``metadata`` and have - # ``async_post_call_failure_hook`` refund TPM counters against scopes - # they name (e.g. another tenant's api_key). - self._strip_stash_keys_from_all_channels(data) + stash = claim_request_stash_for_data(data) ######################################################### # Check if the call type has a specific rate limiter @@ -2444,23 +2471,11 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): requested_model=requested_model, ) else: - # add descriptors to request headers - data["litellm_proxy_rate_limit_response"] = response - # Mirror into metadata so streaming success logging can find - # it via ``kwargs["litellm_params"]["metadata"]``. - self._stash_value_in_internal_metadata( - data=data, - key=RATE_LIMIT_RESPONSE_KEY, - value=response, - ) + stash.rate_limit_response = response if parallel_slot_id is not None: - self._stash_value_in_internal_metadata( - data=data, - key=MAX_PARALLEL_SLOT_ACQUIRED_KEY, - value={ - "slot_id": parallel_slot_id, - "counter_keys": parallel_counter_keys, - }, + stash.parallel_slot = ParallelSlotAcquisition( + slot_id=parallel_slot_id, + counter_keys=parallel_counter_keys, ) # ---------------------------------------------------------------- @@ -2521,38 +2536,29 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): ) if tpm_response["overall_code"] == "OVER_LIMIT": - acquisition = self._get_parallel_slot_acquisition(kwargs=data) + acquisition = stash.parallel_slot if acquisition is not None: await self._release_parallel_request_slots( acquisition=acquisition, parent_otel_span=user_api_key_dict.parent_otel_span, ) - self._clear_parallel_slot_marker(data) + stash.parallel_slot = None self._handle_rate_limit_error( response=tpm_response, descriptors=descriptors, requested_model=requested_model, ) else: - self._stash_value_in_internal_metadata( - data=data, - key=RATE_LIMIT_DESCRIPTORS_KEY, - value=descriptors, - ) # Capture the exact (key, value) scopes the reservation # incremented so post-call reconciliation only applies # the (actual - reserved) delta to those — unreserved # scopes get charged the full actual usage instead. - reserved_scopes: List[Tuple[str, str]] = [ + stash.reserved_tokens = estimated_tokens + stash.reserved_model = requested_model + stash.reserved_scopes = frozenset( (d["key"], d["value"]) for d in descriptors if (d.get("rate_limit") or {}).get("tokens_per_unit") is not None - ] - self._stash_reservation_in_data( - data=data, - estimated_tokens=estimated_tokens, - reserved_model=requested_model, - reserved_scopes=reserved_scopes, ) # Merge TPM statuses into the stored rate-limit response @@ -2560,44 +2566,12 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): # headers reach the client. Without this, the RPM-only # response from should_rate_limit (skip_tpm_check=True) # silently drops all token headers. - stored_response = data.get("litellm_proxy_rate_limit_response") - if isinstance(stored_response, dict): - stored_response.setdefault("statuses", []).extend(tpm_response["statuses"]) - elif tpm_response["statuses"]: - data["litellm_proxy_rate_limit_response"] = tpm_response - # Keep the metadata stash in sync when this is the - # first snapshot written. - self._stash_value_in_internal_metadata( - data=data, - key=RATE_LIMIT_RESPONSE_KEY, - value=tpm_response, - ) + stored_response = stash.rate_limit_response + if stored_response is not None: + stored_response["statuses"].extend(tpm_response["statuses"]) verbose_proxy_logger.debug(f"TPM tokens reserved: {estimated_tokens} for model {requested_model}") - # Defense-in-depth: scrub any stash key that escaped onto data - # top-level (stale cache hit, router pass, test fixture) before the - # body is forwarded to the provider. - self._strip_stash_keys_from_top_level(data) - - @staticmethod - def _strip_stash_keys_from_top_level(data: Any) -> None: - if not isinstance(data, dict): - return - for stash_key in _LITELLM_STASH_KEYS: - data.pop(stash_key, None) - - @classmethod - def _strip_stash_keys_from_all_channels(cls, data: Any) -> None: - if not isinstance(data, dict): - return - cls._strip_stash_keys_from_top_level(data) - for channel in ("metadata", "litellm_metadata"): - channel_dict = data.get(channel) - if isinstance(channel_dict, dict): - for stash_key in _LITELLM_STASH_KEYS: - channel_dict.pop(stash_key, None) - def _create_pipeline_operations( self, key: str, @@ -2803,202 +2777,6 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): merged[f"{prefix}-limit-{status['rate_limit_type']}"] = status["current_limit"] return merged - @staticmethod - def _stash_value_in_internal_metadata( - data: Dict[str, Any], - key: str, - value: Any, - ) -> None: - # Writes only the proxy-internal bucket. Routes that own - # ``litellm_metadata`` (Responses, /v1/messages, batches, files) expose - # ``metadata`` as a provider request parameter, so creating or adding to - # it here would forward internal state upstream. - _, metadata_bucket = get_or_create_metadata_bucket(data) - metadata_bucket[key] = value - - @classmethod - def _stash_reservation_in_data( - cls, - data: Dict[str, Any], - estimated_tokens: int, - reserved_model: Optional[str], - reserved_scopes: Optional[List[Tuple[str, str]]] = None, - ) -> None: - """ - ``reserved_scopes`` is serialized as a list of [key, value] pairs so - it round-trips through JSON-based metadata transports. - """ - scopes_payload: Optional[List[List[str]]] = [[k, v] for k, v in reserved_scopes] if reserved_scopes else None - - cls._stash_value_in_internal_metadata(data=data, key=TPM_RESERVED_TOKENS_KEY, value=estimated_tokens) - if reserved_model: - cls._stash_value_in_internal_metadata(data=data, key=TPM_RESERVED_MODEL_KEY, value=reserved_model) - if scopes_payload is not None: - cls._stash_value_in_internal_metadata(data=data, key=TPM_RESERVED_SCOPES_KEY, value=scopes_payload) - - @staticmethod - def _lookup_stashed_value( - kwargs: Any, - standard_logging_metadata: Optional[Dict[str, Any]], - key: str, - ) -> Any: - """ - Resolve a stashed value from any metadata channel the request data - can flow through to a callback. Top-level ``kwargs`` is not checked - because stash keys must never live there. - """ - candidate: Any = None - if isinstance(kwargs, dict): - for channel in ("metadata", "litellm_metadata"): - channel_dict = kwargs.get(channel) - if isinstance(channel_dict, dict) and key in channel_dict: - candidate = channel_dict.get(key) - if candidate is not None: - return candidate - litellm_params = kwargs.get("litellm_params") - if isinstance(litellm_params, dict): - for channel in ("litellm_metadata", "metadata"): - lp_metadata = litellm_params.get(channel) - if isinstance(lp_metadata, dict) and lp_metadata.get(key) is not None: - return lp_metadata[key] - if candidate is None and isinstance(standard_logging_metadata, dict): - candidate = standard_logging_metadata.get(key) - return candidate - - @classmethod - def _get_reserved_tokens_from_kwargs( - cls, - kwargs: Any, - standard_logging_metadata: Optional[Dict[str, Any]] = None, - ) -> int: - candidate = cls._lookup_stashed_value(kwargs, standard_logging_metadata, TPM_RESERVED_TOKENS_KEY) - try: - return int(candidate or 0) - except (TypeError, ValueError): - return 0 - - @classmethod - def _get_reserved_model_from_kwargs( - cls, - kwargs: Any, - standard_logging_metadata: Optional[Dict[str, Any]] = None, - ) -> Optional[str]: - """ - Resolve the model the upfront reservation was charged against. Used to - target reconciliation at the same key that was incremented, regardless - of whether the router later set a different ``model_group`` in - ``litellm_params.metadata``. - """ - candidate = cls._lookup_stashed_value(kwargs, standard_logging_metadata, TPM_RESERVED_MODEL_KEY) - return candidate if isinstance(candidate, str) and candidate else None - - @classmethod - def _get_reserved_scopes_from_kwargs( - cls, - kwargs: Any, - standard_logging_metadata: Optional[Dict[str, Any]] = None, - ) -> Set[Tuple[str, str]]: - """ - Resolve the (scope_key, scope_value) pairs the upfront reservation - actually charged. Reconciliation distinguishes these from - unreserved scopes — applying the delta to reserved scopes (which - already carry +reserved on the counter) and the full actual to - unreserved ones (which were never charged). - """ - candidate = cls._lookup_stashed_value(kwargs, standard_logging_metadata, TPM_RESERVED_SCOPES_KEY) - if not isinstance(candidate, list): - return set() - scopes: Set[Tuple[str, str]] = set() - for entry in candidate: - if ( - isinstance(entry, (list, tuple)) - and len(entry) == 2 - and isinstance(entry[0], str) - and isinstance(entry[1], str) - ): - scopes.add((entry[0], entry[1])) - return scopes - - @classmethod - def _is_reservation_released( - cls, - kwargs: Any, - standard_logging_metadata: Optional[Dict[str, Any]] = None, - ) -> bool: - """True if a prior callback already refunded this request's reservation.""" - return bool(cls._lookup_stashed_value(kwargs, standard_logging_metadata, TPM_RESERVATION_RELEASED_KEY)) - - @classmethod - def _get_parallel_slot_acquisition( - cls, - kwargs: Any, - standard_logging_metadata: dict[str, Any] | None = None, - ) -> ParallelSlotAcquisition | None: - """The slot acquisition this request's pre-call hook made, if any.""" - candidate = cls._lookup_stashed_value(kwargs, standard_logging_metadata, MAX_PARALLEL_SLOT_ACQUIRED_KEY) - if not isinstance(candidate, dict): - return None - slot_id = candidate.get("slot_id") - counter_keys = candidate.get("counter_keys") - if not isinstance(slot_id, str) or not slot_id: - return None - if not isinstance(counter_keys, list) or not counter_keys: - return None - if not all(isinstance(key, str) and key for key in counter_keys): - return None - return ParallelSlotAcquisition(slot_id=slot_id, counter_keys=counter_keys) - - @staticmethod - def _clear_parallel_slot_marker(data: Any) -> None: - """ - Remove the acquired-slot marker from every metadata channel a sibling - callback might read, so one release per acquire is an invariant even - when multiple callbacks fire for the same request. - """ - if not isinstance(data, dict): - return - for channel in ("metadata", "litellm_metadata"): - channel_dict = data.get(channel) - if isinstance(channel_dict, dict): - channel_dict.pop(MAX_PARALLEL_SLOT_ACQUIRED_KEY, None) - litellm_params = data.get("litellm_params") - if isinstance(litellm_params, dict): - lp_metadata = litellm_params.get("metadata") - if isinstance(lp_metadata, dict): - lp_metadata.pop(MAX_PARALLEL_SLOT_ACQUIRED_KEY, None) - slo = data.get("standard_logging_object") - if isinstance(slo, dict): - slo_meta = slo.get("metadata") - if isinstance(slo_meta, dict): - slo_meta.pop(MAX_PARALLEL_SLOT_ACQUIRED_KEY, None) - - @staticmethod - def _mark_reservation_released(data: Any) -> None: - """ - Stamp the released flag into every metadata channel a sibling - callback might read from. async_post_call_failure_hook receives the - request data dict; async_log_failure_event reads kwargs + - standard_logging_object.metadata. Same dict identity across - ``request_data["metadata"]`` and ``kwargs["litellm_params"]["metadata"]`` - means writes here propagate to the other hook. - """ - if not isinstance(data, dict): - return - for channel in ("metadata", "litellm_metadata"): - existing = data.get(channel) - if isinstance(existing, dict): - existing[TPM_RESERVATION_RELEASED_KEY] = True - litellm_params = data.get("litellm_params") - if isinstance(litellm_params, dict): - lp_metadata = litellm_params.get("metadata") - if isinstance(lp_metadata, dict): - lp_metadata[TPM_RESERVATION_RELEASED_KEY] = True - slo = data.get("standard_logging_object") - if isinstance(slo, dict): - slo_meta = slo.get("metadata") - if isinstance(slo_meta, dict): - slo_meta[TPM_RESERVATION_RELEASED_KEY] = True - def _collect_tpm_scope_targets( self, standard_logging_metadata: Dict[str, Any], @@ -3064,7 +2842,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): def _build_reservation_aware_tpm_ops( self, targets: List[Tuple[str, str]], - reserved_scopes: Set[Tuple[str, str]], + reserved_scopes: FrozenSet[Tuple[str, str]], actual_tokens: int, reserved_tokens: int, ) -> List[RedisPipelineIncrementOperation]: @@ -3139,18 +2917,10 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): if total_tokens == 0: total_tokens = self._aggregate_only_total_tokens(usage=_usage) - reserved_tokens = self._get_reserved_tokens_from_kwargs( - kwargs=kwargs, - standard_logging_metadata=standard_logging_metadata, - ) - reserved_model = self._get_reserved_model_from_kwargs( - kwargs=kwargs, - standard_logging_metadata=standard_logging_metadata, - ) - reserved_scopes = self._get_reserved_scopes_from_kwargs( - kwargs=kwargs, - standard_logging_metadata=standard_logging_metadata, - ) + stash = get_request_stash_for_call(_call_id_from_callback_kwargs(kwargs)) + reserved_tokens = stash.reserved_tokens if stash is not None else 0 + reserved_model = stash.reserved_model if stash is not None else None + reserved_scopes: FrozenSet[Tuple[str, str]] = stash.reserved_scopes if stash is not None else frozenset() # Reconciliation must target the same model-scoped counter that the # pre-call reservation incremented. If a reservation was made, # ``reserved_model`` is authoritative; otherwise fall back to the @@ -3206,18 +2976,14 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): try: verbose_proxy_logger.debug("INSIDE parallel request limiter ASYNC SUCCESS LOGGING") - standard_logging_object = kwargs.get("standard_logging_object") or {} - standard_logging_metadata = standard_logging_object.get("metadata") or {} - acquisition = self._get_parallel_slot_acquisition( - kwargs=kwargs, - standard_logging_metadata=standard_logging_metadata, - ) - if acquisition is not None: + stash = get_request_stash_for_call(_call_id_from_callback_kwargs(kwargs)) + acquisition = stash.parallel_slot if stash is not None else None + if stash is not None and acquisition is not None: await self._release_parallel_request_slots( acquisition=acquisition, parent_otel_span=litellm_parent_otel_span, ) - self._clear_parallel_slot_marker(kwargs) + stash.parallel_slot = None pipeline_operations = self._build_success_event_pipeline_operations( kwargs=kwargs, @@ -3267,23 +3033,13 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): if not isinstance(kwargs, dict): return - standard_logging_object = kwargs.get("standard_logging_object") - standard_logging_metadata: Optional[Dict[str, Any]] = None - if isinstance(standard_logging_object, dict): - slp_metadata = standard_logging_object.get("metadata") - if isinstance(slp_metadata, dict): - standard_logging_metadata = slp_metadata - - statuses = self._narrow_ratelimit_statuses( - self._lookup_stashed_value( - kwargs=kwargs, - standard_logging_metadata=standard_logging_metadata, - key=RATE_LIMIT_RESPONSE_KEY, - ) - ) + stash = get_request_stash_for_call(_call_id_from_callback_kwargs(kwargs)) + rate_limit_response = stash.rate_limit_response if stash is not None else None + statuses = rate_limit_response["statuses"] if rate_limit_response is not None else [] if not statuses: return + standard_logging_object = kwargs.get("standard_logging_object") if isinstance(standard_logging_object, dict): hidden_params = standard_logging_object.get("hidden_params") if not isinstance(hidden_params, dict): @@ -3303,43 +3059,6 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): statuses=statuses, ) - @staticmethod - def _narrow_ratelimit_statuses(stashed: Any) -> List[RateLimitStatus]: - """ - Narrow a stashed ``RateLimitResponse``-shaped dict to a typed - ``statuses`` list. Entries missing any header-write field are dropped; - an empty list means "nothing to mirror". - """ - if not isinstance(stashed, dict): - return [] - raw_statuses = stashed.get("statuses") - if not isinstance(raw_statuses, list): - return [] - narrowed: List[RateLimitStatus] = [] - for entry in raw_statuses: - if not isinstance(entry, dict): - continue - descriptor_key = entry.get("descriptor_key") - rate_limit_type = entry.get("rate_limit_type") - current_limit = entry.get("current_limit") - limit_remaining = entry.get("limit_remaining") - if ( - isinstance(descriptor_key, str) - and rate_limit_type in ("requests", "tokens", "max_parallel_requests") - and isinstance(current_limit, int) - and isinstance(limit_remaining, int) - ): - narrowed.append( - RateLimitStatus( - code=entry.get("code", "OK") if isinstance(entry.get("code"), str) else "OK", - current_limit=current_limit, - limit_remaining=limit_remaining, - rate_limit_type=rate_limit_type, - descriptor_key=descriptor_key, - ) - ) - return narrowed - async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time): """ On failure: decrement max_parallel_requests and refund the upfront @@ -3353,55 +3072,36 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): try: litellm_parent_otel_span: Union[Span, None] = _get_parent_otel_span_from_kwargs(kwargs) - standard_logging_object = kwargs.get("standard_logging_object") or {} - standard_logging_metadata = standard_logging_object.get("metadata") or {} pipeline_operations: List[RedisPipelineIncrementOperation] = [] - acquisition = self._get_parallel_slot_acquisition( - kwargs=kwargs, - standard_logging_metadata=standard_logging_metadata, - ) - if acquisition is not None: + stash = get_request_stash_for_call(_call_id_from_callback_kwargs(kwargs)) + acquisition = stash.parallel_slot if stash is not None else None + if stash is not None and acquisition is not None: await self._release_parallel_request_slots( acquisition=acquisition, parent_otel_span=litellm_parent_otel_span, ) - self._clear_parallel_slot_marker(kwargs) + stash.parallel_slot = None # Skip the reservation refund if async_post_call_failure_hook # already released it (proxy-level rejection that also bubbles up # here as an LLM-error callback). max_parallel_requests is its # own counter and is always decremented per call. - already_released = self._is_reservation_released( - kwargs=kwargs, - standard_logging_metadata=standard_logging_metadata, - ) - reserved_tokens = ( - 0 - if already_released - else self._get_reserved_tokens_from_kwargs( - kwargs=kwargs, - standard_logging_metadata=standard_logging_metadata, - ) - ) - if reserved_tokens > 0: + reserved_tokens = 0 + if stash is not None and not stash.reservation_released: + reserved_tokens = stash.reserved_tokens + if stash is not None and reserved_tokens > 0: verbose_proxy_logger.debug(f"Releasing reserved TPM tokens on failure: {reserved_tokens}") # Refund only against the scopes the reservation actually # charged. _build_reservation_aware_tpm_ops with # actual_tokens=0 emits -reserved on reserved scopes and 0 # on unreserved (skipped), so unreserved scopes can't drift - # negative. Targets are derived purely from the reserved - # set so we don't even need to re-collect them from - # metadata. - reserved_scopes = self._get_reserved_scopes_from_kwargs( - kwargs=kwargs, - standard_logging_metadata=standard_logging_metadata, - ) + # negative. pipeline_operations.extend( self._build_reservation_aware_tpm_ops( - targets=list(reserved_scopes), - reserved_scopes=reserved_scopes, + targets=list(stash.reserved_scopes), + reserved_scopes=stash.reserved_scopes, actual_tokens=0, reserved_tokens=reserved_tokens, ) @@ -3412,15 +3112,14 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): increment_list=pipeline_operations, litellm_parent_otel_span=litellm_parent_otel_span, ) - if reserved_tokens > 0: - self._mark_reservation_released(kwargs) + if stash is not None and reserved_tokens > 0: + stash.reservation_released = True except Exception as e: verbose_proxy_logger.exception(f"Error in rate limit failure event: {str(e)}") async def async_release_max_parallel_requests_on_disconnect( self, user_api_key_dict: UserAPIKeyAuth, - request_data: dict | None = None, ) -> None: """ Release the api-key ``max_parallel_requests`` slot that @@ -3432,20 +3131,19 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): client cancels a stream mid-flight, the cancellation surfaces as ``asyncio.CancelledError`` / ``GeneratorExit`` and neither callback runs, so without this the slot leaks per cancelled stream until its - TTL prunes it. ``request_data`` carries the stashed acquisition; - its presence (not the key object's current max_parallel_requests - configuration, which can change mid-request) decides whether there - is anything to release. + TTL prunes it. The stashed acquisition's presence (not the key + object's current max_parallel_requests configuration, which can + change mid-request) decides whether there is anything to release. """ - acquisition = self._get_parallel_slot_acquisition(kwargs=request_data) - if acquisition is None: + stash = get_request_stash() + if stash is None or stash.parallel_slot is None: return await self._release_parallel_request_slots( - acquisition=acquisition, + acquisition=stash.parallel_slot, parent_otel_span=None, ) - self._clear_parallel_slot_marker(request_data) + stash.parallel_slot = None async def async_post_call_success_hook(self, data: dict, user_api_key_dict: UserAPIKeyAuth, response): """ @@ -3454,10 +3152,8 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): try: from pydantic import BaseModel - litellm_proxy_rate_limit_response = cast( - Optional[RateLimitResponse], - data.get("litellm_proxy_rate_limit_response", None), - ) + stash = get_request_stash() + litellm_proxy_rate_limit_response = stash.rate_limit_response if stash is not None else None if litellm_proxy_rate_limit_response is not None: # Update response headers @@ -3502,59 +3198,42 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): rejections, so a leaked slot would occupy the gauge for the full PARALLEL_REQUEST_SLOT_TTL_SECONDS. - Idempotent: the slot release clears the acquisition marker (and slot + Idempotent: the slot release clears the stashed acquisition (and slot removal is a no-op ZREM on a second run), and the TPM refund is - guarded by TPM_RESERVATION_RELEASED_KEY — if both this hook and - async_log_failure_event end up running in the same flow, only the - first release/refund applies. + guarded by the stash's ``reservation_released`` flag — if both this + hook and async_log_failure_event end up running in the same flow, only + the first release/refund applies. """ try: - acquisition = self._get_parallel_slot_acquisition(kwargs=request_data) - if acquisition is not None: + stash = get_request_stash() + if stash is None: + return + if stash.parallel_slot is not None: await self._release_parallel_request_slots( - acquisition=acquisition, + acquisition=stash.parallel_slot, parent_otel_span=user_api_key_dict.parent_otel_span, ) - self._clear_parallel_slot_marker(request_data) + stash.parallel_slot = None - if self._is_reservation_released(kwargs=request_data): + if stash.reservation_released: return - reserved_tokens = self._get_reserved_tokens_from_kwargs(kwargs=request_data) + reserved_tokens = stash.reserved_tokens if reserved_tokens <= 0: return - # Refund directly against the descriptors we reserved against — - # the pre-call hook stashes them in the request-data metadata - # channels before success/failure callbacks run. - stashed = self._lookup_stashed_value( - kwargs=request_data, - standard_logging_metadata=None, - key=RATE_LIMIT_DESCRIPTORS_KEY, + ops = self._build_reservation_aware_tpm_ops( + targets=list(stash.reserved_scopes), + reserved_scopes=stash.reserved_scopes, + actual_tokens=0, + reserved_tokens=reserved_tokens, ) - descriptors: List[RateLimitDescriptor] = stashed if isinstance(stashed, list) else [] - ops: List[RedisPipelineIncrementOperation] = [] - for descriptor in descriptors: - rate_limit = descriptor.get("rate_limit") or {} - if rate_limit.get("tokens_per_unit") is None: - continue - ops.append( - RedisPipelineIncrementOperation( - key=self.create_rate_limit_keys( - descriptor["key"], - descriptor["value"], - "tokens", - ), - increment_value=-reserved_tokens, - ttl=self.window_size, - ) - ) if ops: verbose_proxy_logger.debug(f"Releasing reserved TPM tokens on proxy-level rejection: {reserved_tokens}") await self.internal_usage_cache.dual_cache.async_increment_cache_pipeline( increment_list=ops, litellm_parent_otel_span=user_api_key_dict.parent_otel_span, ) - self._mark_reservation_released(request_data) + stash.reservation_released = True except Exception as e: verbose_proxy_logger.exception(f"Error releasing TPM reservation on post-call failure: {e}") return None diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index c72e3d4ee5b..a60ea2da019 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -119,6 +119,7 @@ from litellm.router_utils.add_retry_fallback_headers import ( from litellm.types.utils import ( ModelResponse, ModelResponseStream, + StreamingChoices, TextCompletionResponse, TokenCountResponse, ) @@ -7368,6 +7369,25 @@ def _serialize_streaming_chunk(chunk: BaseModel) -> Union[str, bytes]: return chunk.model_dump_json(exclude_none=True, exclude_unset=True) +def _is_injected_stream_usage_artifact(chunk: object) -> bool: + if not isinstance(chunk, ModelResponseStream): + return False + if chunk.provider_specific_fields is not None: + return False + return all(_is_empty_streaming_choice(choice) for choice in chunk.choices or []) + + +def _is_empty_streaming_choice(choice: StreamingChoices) -> bool: + if choice.finish_reason is not None: + return False + if getattr(choice, "logprobs", None) is not None: + return False + delta = getattr(choice, "delta", None) + if delta is None: + return True + return all(value is None for value in delta.model_dump().values()) + + async def _apply_streaming_chunk_hooks( *, chunk: Any, @@ -7447,6 +7467,7 @@ async def async_data_generator( needs_iterator_wrap = proxy_logging_obj.needs_iterator_wrap() needs_per_chunk_hook = proxy_logging_obj.needs_per_chunk_streaming_hook() is_raw_sse_stream = bool(request_data.get("_litellm_raw_sse_stream")) + strip_stream_usage = bool(request_data.get("_litellm_strip_stream_usage")) raw_sse_buffer = "" if needs_iterator_wrap: @@ -7498,6 +7519,15 @@ async def async_data_generator( fallback_model_from_metadata=fallback_model_from_metadata, ) + if strip_stream_usage and _is_injected_stream_usage_artifact(chunk): + if pending_fallback_event: + yield _format_fallback_metadata_sse_event( + fallback_model=fallback_model_from_metadata, + fallback_errors=fallback_errors, + ) + fallback_metadata_event_sent = True + continue + raw_passthrough = False if isinstance(chunk, BaseModel): chunk = _serialize_streaming_chunk(chunk) @@ -13470,6 +13500,7 @@ async def async_queue_request( data = {} try: data = await request.json() # type: ignore + data.pop("_litellm_strip_stream_usage", None) # Include original request and headers in the data data["proxy_server_request"] = { diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index d95d64c47f4..2ca251a3211 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -2814,7 +2814,6 @@ class ProxyLogging: async def _arelease_max_parallel_requests_on_disconnect( self, user_api_key_dict: UserAPIKeyAuth, - request_data: dict | None = None, ) -> None: """ Release the api-key max_parallel_requests slot when a streaming @@ -2834,7 +2833,7 @@ class ProxyLogging: limiter = self.get_proxy_hook("parallel_request_limiter") if not isinstance(limiter, _PROXY_MaxParallelRequestsHandler_v3): return - await limiter.async_release_max_parallel_requests_on_disconnect(user_api_key_dict, request_data) + await limiter.async_release_max_parallel_requests_on_disconnect(user_api_key_dict) def _init_response_taking_too_long_task(self, data: Optional[dict] = None): """ diff --git a/litellm/types/mcp_server/mcp_server_manager.py b/litellm/types/mcp_server/mcp_server_manager.py index a127e8dad11..d02778e9eac 100644 --- a/litellm/types/mcp_server/mcp_server_manager.py +++ b/litellm/types/mcp_server/mcp_server_manager.py @@ -194,6 +194,18 @@ class MCPServer(BaseModel): """True if this is an OAuth2 server that relies on per-user tokens (no client_credentials).""" return self.auth_type == MCPAuth.oauth2 and not self.has_client_credentials + @property + def is_gateway_managed_oauth2(self) -> bool: + """True when the gateway itself owns this server's OAuth custody: an ``oauth2`` server + (interactive authorization_code with gateway-vaulted per-user tokens, or M2M + client_credentials minted at egress) that has NOT opted into upstream-delegated auth. + These are the servers the keyless gateway-DCR flow can serve end to end, so the + per-server 401 challenge and protected-resource metadata advertise the gateway as the + authorization server for exactly this set. ``true_passthrough``, ``oauth_delegate``, + DCR-bridge, and token-exchange servers are their own auth types and client-forwarded, + so they are excluded by construction.""" + return self.auth_type == MCPAuth.oauth2 and not self.delegate_auth_to_upstream + @property def is_true_passthrough(self) -> bool: """True for the transparent-proxy mode: LiteLLM performs no admission auth and forwards the diff --git a/litellm/types/utils.py b/litellm/types/utils.py index ba9bba6a62a..668143d5950 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -3297,7 +3297,6 @@ all_litellm_params = ( "mock_response", "mock_timeout", "disable_add_transform_inline_image_block", - "litellm_proxy_rate_limit_response", "api_key", "api_version", "prompt_id", @@ -3313,6 +3312,7 @@ all_litellm_params = ( "model_file_id_mapping", "litellm_logging_obj", "litellm_call_id", + "_litellm_strip_stream_usage", "use_client", "id", "fallbacks", @@ -3391,11 +3391,6 @@ all_litellm_params = ( "enable_tag_filtering", "enable_json_schema_validation", "use_xai_oauth", - "_litellm_rate_limit_descriptors", - "_litellm_tpm_reserved_tokens", - "_litellm_tpm_reserved_model", - "_litellm_tpm_reserved_scopes", - "_litellm_tpm_reservation_released", "auto_router_config_path", "auto_router_config", "auto_router_default_model", diff --git a/tests/e2e/batches/test_batches_e2e.py b/tests/e2e/batches/test_batches_e2e.py index 5c25b7f2a93..1376bdbed38 100644 --- a/tests/e2e/batches/test_batches_e2e.py +++ b/tests/e2e/batches/test_batches_e2e.py @@ -397,6 +397,16 @@ def unattributed_rows(rows: list[SpendLogRow]) -> list[SpendLogRow]: return [row for row in rows if not row.api_key] +@pytest.mark.skip( + reason=( + "LIT-5027: the path under test hangs. The batch rate limiter reads the input file " + "to count tokens by awaiting litellm.afile_content with no timeout, so a slow Files " + "API holds POST /v1/batches open past any client deadline (63.6s observed on stage " + "against a 60s read timeout). The unattributed-spend-row contract below is never " + "reached, so the test reports a timeout rather than the behavior it guards. Unskip " + "once the fetch is bounded." + ) +) def test_rate_limited_batch_create_leaves_no_unattributed_spend_row( client: BatchClient, resources: ResourceManager, batch_deployments: None ) -> None: diff --git a/tests/test_litellm/compression/test_compress.py b/tests/test_litellm/compression/test_compress.py new file mode 100644 index 00000000000..6827c37dfd5 --- /dev/null +++ b/tests/test_litellm/compression/test_compress.py @@ -0,0 +1,55 @@ +""" +Unit tests for litellm.compression.compress helpers. + +get_protected_indices is the shared policy for which messages a compressor may +never rewrite. It is consumed by compress() and by the Headroom guardrail, so +the two agree on what "never compress this" means. +""" + +from litellm.compression.compress import get_protected_indices + + +def test_protects_system_last_user_and_last_assistant(): + messages = [ + {"role": "system", "content": "sys"}, + {"role": "user", "content": "old question"}, + {"role": "assistant", "content": "old answer"}, + {"role": "user", "content": "newer question"}, + {"role": "assistant", "content": "newer answer"}, + {"role": "user", "content": "live instruction"}, + ] + + assert sorted(get_protected_indices(messages)) == [0, 4, 5] + + +def test_history_is_not_protected(): + messages = [ + {"role": "user", "content": "old question"}, + {"role": "assistant", "content": "old answer"}, + {"role": "tool", "tool_call_id": "t1", "content": "old tool output"}, + {"role": "user", "content": "live instruction"}, + ] + + protected = sorted(get_protected_indices(messages)) + + assert protected == [1, 3] + # The tool row and the older user turn stay compressible; protection that + # covered everything would make compression a no-op. + assert 0 not in protected + assert 2 not in protected + + +def test_every_system_row_is_protected(): + messages = [ + {"role": "system", "content": "first"}, + {"role": "user", "content": "q"}, + {"role": "system", "content": "second, injected mid conversation"}, + {"role": "user", "content": "live"}, + ] + + assert sorted(get_protected_indices(messages)) == [0, 2, 3] + + +def test_no_user_or_assistant_rows(): + assert sorted(get_protected_indices([{"role": "system", "content": "sys"}])) == [0] + assert get_protected_indices([]) == () diff --git a/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py b/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py index 9565de1139c..dc745abb9e7 100644 --- a/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py +++ b/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py @@ -3197,3 +3197,75 @@ def test_get_tool_calls_from_response_include_all_choices_reads_every_choice(): names = [tc["name"] for tc in get_tool_calls_from_response(response, include_all_choices=True)] assert names == ["tool_alpha", "tool_beta"] + + +def test_group_tool_exchanges_pairs_assistant_with_its_tool_rows(): + from litellm.litellm_core_utils.prompt_templates.factory import group_tool_exchanges + + messages = [ + {"role": "user", "content": "first turn"}, + { + "role": "assistant", + "content": None, + "tool_calls": [ + {"id": "tu_1", "type": "function", "function": {"name": "Read", "arguments": "{}"}}, + {"id": "tu_2", "type": "function", "function": {"name": "Grep", "arguments": "{}"}}, + ], + }, + {"role": "tool", "tool_call_id": "tu_1", "content": "file body"}, + {"role": "tool", "tool_call_id": "tu_2", "content": "matches"}, + {"role": "user", "content": "live instruction"}, + ] + + assert group_tool_exchanges(messages) == ((0,), (1, 2, 3), (4,)) + + +def test_group_tool_exchanges_uses_ownership_not_adjacency(): + """A tool row answering some other call must not be swept into the exchange + it happens to sit next to.""" + from litellm.litellm_core_utils.prompt_templates.factory import group_tool_exchanges + + messages = [ + { + "role": "assistant", + "content": None, + "tool_calls": [{"id": "tu_1", "type": "function", "function": {"name": "Read", "arguments": "{}"}}], + }, + {"role": "tool", "tool_call_id": "unrelated", "content": "not an answer to tu_1"}, + {"role": "tool", "tool_call_id": "tu_1", "content": "file body"}, + ] + + assert group_tool_exchanges(messages) == ((0,), (1,), (2,)) + + +def test_group_tool_exchanges_assistant_without_tool_calls_stands_alone(): + from litellm.litellm_core_utils.prompt_templates.factory import group_tool_exchanges + + messages = [ + {"role": "assistant", "content": "no tools here"}, + {"role": "user", "content": "next"}, + ] + + assert group_tool_exchanges(messages) == ((0,), (1,)) + assert group_tool_exchanges([]) == () + + +def test_group_tool_exchanges_is_linear_in_message_count(): + """Grouping runs on every guardrail write-back, over a message array the + caller controls, so it has to stay linear. Accumulating groups by rebuilding + a tuple each iteration made this O(n^2): 20k standalone messages took 312ms + and 100k would take minutes. Linear finishes in single-digit ms, so this + ceiling has ~200x headroom while a quadratic rewrite blows straight past it. + """ + import time + + from litellm.litellm_core_utils.prompt_templates.factory import group_tool_exchanges + + messages = [{"role": "user", "content": "x"} for _ in range(100_000)] + + started = time.perf_counter() + groups = group_tool_exchanges(messages) + elapsed = time.perf_counter() - started + + assert len(groups) == 100_000 + assert elapsed < 3.0, f"grouping 100k messages took {elapsed:.2f}s; suspect superlinear accumulation" diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/auth/test_user_api_key_auth_mcp.py b/tests/test_litellm/proxy/_experimental/mcp_server/auth/test_user_api_key_auth_mcp.py index 4be2bb053ef..89b8f018e5c 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/auth/test_user_api_key_auth_mcp.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/auth/test_user_api_key_auth_mcp.py @@ -1154,21 +1154,27 @@ class TestMCPOAuth2AuthFlow: await MCPRequestHandler.process_mcp_request(scope) assert exc_info.value.status_code == 500 - async def test_proxy_exception_non_delegate_oauth2_propagates(self): + async def test_proxy_exception_non_delegate_oauth2_challenges_with_per_server_metadata(self): """ Production raises ProxyException (not HTTPException) on auth failure. For - a non-delegate oauth2 server the bearer is treated as a LiteLLM credential - and a 401 must propagate as a real auth error, not be exchanged for an - anonymous upstream-passthrough session. + a gateway-managed oauth2 server the bearer is treated as a LiteLLM + credential and its failure stays a 401, never an anonymous + upstream-passthrough session. The 401 now carries the RFC 9728 + invalid_token challenge with the per-server resource metadata (LIT-4864): + a keyless client holding a stale upstream token (the relayed gho_ shape) + re-discovers the gateway as this resource's authorization server instead + of dead-ending on a bare 401. """ from litellm.proxy._types import ProxyException from litellm.types.mcp import MCPAuth + from litellm.types.mcp_server.mcp_server_manager import MCPServer scope = { "type": "http", "method": "POST", "path": "/mcp/atlassian_mcp", "headers": [ + (b"host", b"testserver"), (b"authorization", b"Bearer atlassian-oauth2-access-token-xyz"), ], } @@ -1181,10 +1187,14 @@ class TestMCPOAuth2AuthFlow: code=401, ) - oauth2_server = MagicMock() - oauth2_server.auth_type = MCPAuth.oauth2 - oauth2_server.delegate_auth_to_upstream = False - oauth2_server.is_oauth_passthrough = False + oauth2_server = MCPServer( + server_id="atlassian-id", + name="atlassian_mcp", + server_name="atlassian_mcp", + url="https://upstream.example/mcp", + transport="http", + auth_type=MCPAuth.oauth2, + ) with ( patch( @@ -1194,9 +1204,14 @@ class TestMCPOAuth2AuthFlow: patch("litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, ): mock_mgr.get_mcp_server_by_name.return_value = oauth2_server - with pytest.raises(ProxyException) as exc_info: + with pytest.raises(HTTPException) as exc_info: await MCPRequestHandler.process_mcp_request(scope) - assert str(exc_info.value.code) == "401" + assert exc_info.value.status_code == 401 + www_authenticate = (exc_info.value.headers or {})["WWW-Authenticate"] + assert www_authenticate == ( + 'Bearer error="invalid_token", ' + 'resource_metadata="http://testserver/.well-known/oauth-protected-resource/mcp/atlassian_mcp"' + ) async def test_proxy_exception_non_auth_still_raises(self): """ @@ -6250,14 +6265,133 @@ class TestAggregateGatewayDcrChallenge: self._scope(extra_headers=((b"x-litellm-api-key", b"sk-typo"),)) ) - async def test_no_challenge_for_named_servers_header(self): - """x-mcp-servers names explicit targets; the per-server challenge paths - own those, so the aggregate challenge must not fire.""" + async def test_challenge_for_named_servers_header(self): + """x-mcp-servers scopes the fan-out but the resource the client configured is still + the aggregate /mcp URL, so an unauthenticated request gets the aggregate challenge + and completes the same keyless flow; the header names then narrow (never broaden) + the admitted subject's servers downstream (LIT-4864).""" with ( patch(self._AUTH_PATCH_TARGET, side_effect=self._auth_401()), ): - with pytest.raises(ProxyException): + with pytest.raises(HTTPException) as exc_info: await MCPRequestHandler.process_mcp_request(self._scope(extra_headers=((b"x-mcp-servers", b"github"),))) + assert exc_info.value.status_code == 401 + www_authenticate = (exc_info.value.headers or {})["WWW-Authenticate"] + assert www_authenticate == f"Bearer {self._EXPECTED_RESOURCE_METADATA}" + + async def test_per_server_challenge_for_gateway_managed_oauth2(self): + """Anonymous request to a per-server path whose single target is a gateway-managed + oauth2 server: 401 plus the RFC 9728 challenge advertising the PER-SERVER + protected-resource metadata in the same URL spelling the request used, so a keyless + DCR client configured with either per-server spelling discovers the gateway as the + authorization server (LIT-4864). Covers interactive and M2M, which the gateway can + both serve end to end.""" + from litellm.types.mcp import MCPAuth + from litellm.types.mcp_server.mcp_server_manager import MCPServer + + server = MCPServer( + server_id="gh-id", + name="github", + server_name="github", + url="https://upstream.example/mcp", + transport="http", + auth_type=MCPAuth.oauth2, + ) + for path, expected_metadata_path in ( + ("/mcp/github", "/.well-known/oauth-protected-resource/mcp/github"), + ("/github/mcp", "/.well-known/oauth-protected-resource/github/mcp"), + ): + with ( + patch(self._AUTH_PATCH_TARGET, side_effect=self._auth_401()), + patch( + "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager" + ) as mock_mgr, + ): + mock_mgr.get_mcp_server_by_name.return_value = server + with pytest.raises(HTTPException) as exc_info: + await MCPRequestHandler.process_mcp_request(self._scope(path=path)) + assert exc_info.value.status_code == 401 + www_authenticate = (exc_info.value.headers or {})["WWW-Authenticate"] + assert www_authenticate == f'Bearer resource_metadata="http://testserver{expected_metadata_path}"' + + async def test_no_per_server_challenge_for_non_gateway_managed_targets(self): + """The per-server challenge fires only for the server set the gateway's keyless flow + serves: an OBO server and a multi-server CSV path keep the original admission error + through the full pipeline, so no client-forwarded mode is redirected into the gateway + sign-in flow and no cell broadens (LIT-4864).""" + from litellm.types.mcp import MCPAuth + from litellm.types.mcp_server.mcp_server_manager import MCPServer + + obo_server = MCPServer( + server_id="o-id", + name="obo", + server_name="obo", + url="https://upstream.example/mcp", + transport="http", + auth_type=MCPAuth.oauth2_token_exchange, + ) + for path, resolved in ( + ("/mcp/obo", obo_server), + ("/mcp/github,linear", None), + ): + with ( + patch(self._AUTH_PATCH_TARGET, side_effect=self._auth_401()), + patch( + "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager" + ) as mock_mgr, + ): + mock_mgr.get_mcp_server_by_name.return_value = resolved + with pytest.raises(ProxyException): + await MCPRequestHandler.process_mcp_request( + self._scope(path=path, extra_headers=((b"authorization", b"Bearer not-a-key"),)) + ) + + def test_challenge_target_excludes_every_non_gateway_managed_mode(self): + """Unit pin of the challenge-target owner: only a resolved gateway-managed oauth2 + target (interactive or M2M) yields a per-server challenge; delegate-auth oauth2 + (whose keyless flow is upstream PKCE via the relay), every client-forwarded auth + type, OBO, api_key, unknown names, and CSV paths yield None (LIT-4864).""" + from litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp import ( + _gateway_dcr_challenge_target, + ) + from litellm.types.mcp import MCPAuth + from litellm.types.mcp_server.mcp_server_manager import MCPServer + + def _server(auth_type, **kw): + return MCPServer( + server_id="s-id", + name="srv", + server_name="srv", + url="https://upstream.example/mcp", + transport="http", + auth_type=auth_type, + **kw, + ) + + cases = [ + (_server(MCPAuth.oauth2), "srv"), + (_server(MCPAuth.oauth2, oauth2_flow="client_credentials"), "srv"), + (_server(MCPAuth.oauth2, delegate_auth_to_upstream=True), None), + (_server(MCPAuth.oauth2_token_exchange), None), + (_server(MCPAuth.true_passthrough), None), + (_server(MCPAuth.oauth_delegate), None), + (_server(MCPAuth.oauth_delegate, dcr_bridge=True), None), + (_server(MCPAuth.api_key), None), + (None, None), + ] + for resolved, expected in cases: + with patch( + "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager" + ) as mock_mgr: + mock_mgr.get_mcp_server_by_name.return_value = resolved + assert _gateway_dcr_challenge_target("/mcp/srv", None, None) == expected, resolved + assert _gateway_dcr_challenge_target("/mcp/a,b", None, None) is None + assert _gateway_dcr_challenge_target("/mcp", None, None) is None + with patch( + "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager" + ) as mock_mgr: + mock_mgr.get_mcp_server_by_name.return_value = _server(MCPAuth.oauth2) + assert _gateway_dcr_challenge_target("/mcp/srv", ["other"], None) is None async def test_no_challenge_for_path_named_server(self): """/mcp/{server} targets one server; the aggregate challenge must not @@ -6295,10 +6429,11 @@ class TestAggregateGatewayDcrChallenge: @pytest.mark.asyncio class TestGatewaySessionAdmission: - """The aggregate /mcp session-bearer admission arm (mcp_gateway_dcr). A valid session - token admits under the LIVE litellm user it references; an invalid/expired/refresh/foreign - token fails closed with the aggregate invalid_token challenge; the arm fires ONLY at the - aggregate scope, never for named servers or per-server flows.""" + """The session-bearer admission arm (mcp_gateway_dcr). A valid session token admits under + the LIVE litellm user it references at any MCP scope (aggregate, per-server path, or + x-mcp-servers scoped; LIT-4864) with downstream grant resolution narrowing to the + requested servers; an invalid/expired/refresh/foreign token fails closed with the + requested scope's invalid_token challenge.""" _MASTER_KEY = "sk-gateway-session-admission-master-key" @@ -6471,21 +6606,89 @@ class TestGatewaySessionAdmission: assert oauth2_headers is None assert not any(k.lower() == "authorization" for k in (raw_headers or {})) - async def test_arm_does_not_fire_for_named_server(self): - """A session-shaped bearer aimed at a named server (path scope) does not enter the - aggregate arm; it is treated as an ordinary bearer on that server.""" - token = self._access_token() + @pytest.mark.parametrize( + "path, original_path, extra_headers", + [ + ("/mcp/github", None, ()), + ("/mcp/github", "/github/mcp", ()), + ("/mcp", None, ((b"x-mcp-servers", b"github"),)), + ], + ) + async def test_arm_admits_session_bearer_on_per_server_scopes(self, path, original_path, extra_headers): + """A valid session bearer admits the live user on per-server paths (the standard + spelling and the legacy /{server}/mcp spelling as dynamic_mcp_route rewrites it) and + x-mcp-servers scoped requests, never touching user_api_key_auth; downstream grant + resolution then intersects the named servers against the admitted subject's grants, + so the narrower scope can never broaden access (LIT-4864).""" + token = self._access_token(user_id="sso-user-42") + scope = self._scope(token, path=path, extra_headers=extra_headers) + if original_path is not None: + scope["_original_path"] = original_path with ( patch("litellm.proxy.proxy_server.master_key", self._MASTER_KEY), patch( "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", new_callable=AsyncMock, - side_effect=ProxyException(message="bad key", type="auth_error", param="api_key", code=401), ) as mock_auth, + self._patch_user_reload(user_id="sso-user-42"), ): - with pytest.raises((HTTPException, ProxyException)): - await MCPRequestHandler.process_mcp_request(self._scope(token, path="/mcp/github")) - mock_auth.assert_called_once() + auth_result, *_rest = await MCPRequestHandler.process_mcp_request(scope) + assert auth_result.user_id == "sso-user-42" + assert auth_result.mcp_admitted_user_subject is True + mock_auth.assert_not_called() + + async def test_expired_session_bearer_on_per_server_path_gets_per_server_challenge(self): + """An expired session bearer on a per-server path targeting a gateway-managed oauth2 + server re-challenges with the PER-SERVER resource metadata (matching the resource the + client configured), so a spec client re-authorizes against the right document instead + of a bare 401 or the aggregate metadata (LIT-4864).""" + from datetime import datetime, timezone + + from litellm.types.mcp import MCPAuth + from litellm.types.mcp_server.mcp_server_manager import MCPServer + + mint, _refresh, principal, keys = self._session_bearer() + bearer = mint(principal, keys, datetime(2020, 1, 1, tzinfo=timezone.utc)).token.get_secret_value() + server = MCPServer( + server_id="gh-id", + name="github", + server_name="github", + url="https://upstream.example/mcp", + transport="http", + auth_type=MCPAuth.oauth2, + ) + with ( + patch("litellm.proxy.proxy_server.master_key", self._MASTER_KEY), + patch("litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, + ): + mock_mgr.get_mcp_server_by_name.return_value = server + with pytest.raises(HTTPException) as exc_info: + await MCPRequestHandler.process_mcp_request(self._scope(bearer, path="/mcp/github")) + assert exc_info.value.status_code == 401 + www_authenticate = (exc_info.value.headers or {})["WWW-Authenticate"] + assert www_authenticate == ( + 'Bearer error="invalid_token", ' + 'resource_metadata="http://testserver/.well-known/oauth-protected-resource/mcp/github"' + ) + + async def test_session_bearer_scrubbed_from_egress_on_per_server_path(self): + """After a per-server keyless admission the session bearer must be scrubbed from every + egress header context exactly as at the aggregate scope, so no per-server passthrough + egress can forward it upstream for replay (LIT-4864).""" + token = self._access_token(user_id="sso-user-42") + with ( + patch("litellm.proxy.proxy_server.master_key", self._MASTER_KEY), + patch( + "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", + new_callable=AsyncMock, + ), + self._patch_user_reload(user_id="sso-user-42"), + ): + _auth, _h, _servers, _msah, oauth2_headers, raw_headers = await MCPRequestHandler.process_mcp_request( + self._scope(token, path="/mcp/github") + ) + assert oauth2_headers is None + assert not any(k.lower() == "authorization" for k in (raw_headers or {})) def _make_team(team_id, mcp_servers, *, org_id=None, tool_perms=None, members=("sso-user",)): diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_discoverable_endpoints.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_discoverable_endpoints.py index 694583dde88..9bc84b43fc5 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_discoverable_endpoints.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_discoverable_endpoints.py @@ -2976,6 +2976,125 @@ async def test_oauth_protected_resource_returns_empty_scopes_when_none(): global_mcp_server_manager.registry.clear() +@pytest.mark.asyncio +async def test_oauth_protected_resource_gateway_managed_oauth2_advertises_gateway_as(): + """LIT-4864: an explicitly named gateway-managed oauth2 server (interactive or M2M) + advertises the gateway's own authorization server, so a keyless DCR client that + configured the per-server URL completes the same sign-in flow the aggregate /mcp + endpoint supports and returns with a gateway session bearer; the resource stays the + per-server URL in the requested spelling (RFC 9728 resource match). A delegate-auth + oauth2 server keeps the per-server relay authorization server (its keyless flow is + upstream PKCE via the relay), and the root-resolved unnamed legacy shape is unchanged.""" + try: + from fastapi import Request + + from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + _build_oauth_protected_resource_response, + ) + from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + global_mcp_server_manager, + ) + from litellm.proxy._types import MCPTransport + from litellm.types.mcp import MCPAuth + from litellm.types.mcp_server.mcp_server_manager import MCPServer + except ImportError: + pytest.skip("MCP discoverable endpoints not available") + + def _oauth2_server(name, **kw): + return MCPServer( + server_id=name, + name=name, + server_name=name, + alias=name, + transport=MCPTransport.http, + auth_type=MCPAuth.oauth2, + authorization_url="https://idp.example.com/authorize", + token_url="https://idp.example.com/oauth/token", + scopes=["read"], + **kw, + ) + + mock_request = MagicMock(spec=Request) + mock_request.base_url = "https://litellm.example.com/" + mock_request.headers = {} + + interactive = _oauth2_server("github_mcp") + m2m = _oauth2_server("m2m_mcp", oauth2_flow="client_credentials", client_id="cid", client_secret="cs") + delegated = _oauth2_server("delegated_mcp", delegate_auth_to_upstream=True) + + global_mcp_server_manager.registry.clear() + try: + for server in (interactive, m2m, delegated): + global_mcp_server_manager.registry[server.server_id] = server + + for name in ("github_mcp", "m2m_mcp"): + standard = await _build_oauth_protected_resource_response( + request=mock_request, mcp_server_name=name, use_standard_pattern=True + ) + assert standard["authorization_servers"] == ["https://litellm.example.com/mcp"], name + assert standard["resource"] == f"https://litellm.example.com/mcp/{name}" + assert standard["scopes_supported"] == ["read"] + legacy = await _build_oauth_protected_resource_response( + request=mock_request, mcp_server_name=name, use_standard_pattern=False + ) + assert legacy["authorization_servers"] == ["https://litellm.example.com/mcp"], name + assert legacy["resource"] == f"https://litellm.example.com/{name}/mcp" + + delegated_response = await _build_oauth_protected_resource_response( + request=mock_request, mcp_server_name="delegated_mcp", use_standard_pattern=True + ) + assert delegated_response["authorization_servers"] == ["https://litellm.example.com/delegated_mcp"] + finally: + global_mcp_server_manager.registry.clear() + + +@pytest.mark.asyncio +async def test_oauth_protected_resource_root_resolved_single_server_keeps_relay_as(): + """The unnamed (bare-root) legacy shape resolves the single configured oauth2 server and + must keep advertising the per-server relay authorization server: only an EXPLICITLY + named request opts into the gateway-as-AS flow (LIT-4864), so pre-existing single-server + deployments discovering through the root document are byte-identical.""" + try: + from fastapi import Request + + from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + _build_oauth_protected_resource_response, + ) + from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + global_mcp_server_manager, + ) + from litellm.proxy._types import MCPTransport + from litellm.types.mcp import MCPAuth + from litellm.types.mcp_server.mcp_server_manager import MCPServer + except ImportError: + pytest.skip("MCP discoverable endpoints not available") + + only_server = MCPServer( + server_id="solo_mcp", + name="solo_mcp", + server_name="solo_mcp", + alias="solo_mcp", + transport=MCPTransport.http, + auth_type=MCPAuth.oauth2, + authorization_url="https://idp.example.com/authorize", + token_url="https://idp.example.com/oauth/token", + ) + + mock_request = MagicMock(spec=Request) + mock_request.base_url = "https://litellm.example.com/" + mock_request.headers = {} + + global_mcp_server_manager.registry.clear() + try: + global_mcp_server_manager.registry[only_server.server_id] = only_server + response = await _build_oauth_protected_resource_response( + request=mock_request, mcp_server_name=None, use_standard_pattern=False + ) + assert response["authorization_servers"] == ["https://litellm.example.com/solo_mcp"] + finally: + global_mcp_server_manager.registry.clear() + + @pytest.mark.asyncio async def test_oauth_authorization_server_returns_empty_scopes_when_none(): """ diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_oauth_passthrough.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_oauth_passthrough.py index ec285f8eba0..fe583ace897 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_oauth_passthrough.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_oauth_passthrough.py @@ -442,7 +442,10 @@ async def test_fetch_upstream_metadata_returns_none_when_not_all_candidates_netw @pytest.mark.asyncio async def test_oauth_protected_resource_gateway_managed_unchanged(): - """Regression guard: OAuth2 servers still advertise the gateway as AS.""" + """Regression guard: gateway-managed OAuth2 servers advertise the gateway as AS and + never fetch upstream metadata. Since LIT-4864 the advertised document is the gateway's + own aggregate authorization server ({base}/mcp), which serves the keyless DCR flow for + per-server URLs; the per-server relay endpoints remain for the keyed flow.""" from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( global_mcp_server_manager, ) @@ -477,7 +480,7 @@ async def test_oauth_protected_resource_gateway_managed_unchanged(): ) mock_client.get.assert_not_awaited() - assert result["authorization_servers"] == ["https://gateway.example.com/keycloak_whoami"] + assert result["authorization_servers"] == ["https://gateway.example.com/mcp"] assert result["scopes_supported"] == ["read"] diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_stale_session.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_stale_session.py index e5173be45b9..7bdd3b36763 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_stale_session.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_stale_session.py @@ -664,6 +664,97 @@ async def test_per_user_oauth_missing_stored_token_returns_preemptive_401(): assert "Bearer authorization_uri=" in exc_info.value.headers["www-authenticate"] +@pytest.mark.asyncio +async def test_admitted_subject_missing_stored_token_challenged_with_resource_metadata(): + """ + LIT-4864: a keyless gateway-session subject (mcp_admitted_user_subject) with no stored + per-user token must be challenged with the per-server resource_metadata, whose + authorization server is the gateway itself, so the client re-runs the gateway sign-in + flow and vaults the upstream token through the authorize interlude. The keyed + authorization_uri challenge points at the per-server relay, which cannot vault a token + for a keyless client (its token request carries no litellm credential), so sending an + admitted subject there would dead-end the flow on a raw upstream token. + """ + from fastapi import HTTPException + + try: + from litellm.proxy._experimental.mcp_server.server import ( + handle_streamable_http_mcp, + session_manager_stateless, + ) + except ImportError: + pytest.skip("MCP server not available") + + scope = { + "type": "http", + "method": "POST", + "path": "/mcp/repro_oauth_server", + "scheme": "http", + "query_string": b"", + "root_path": "", + "server": ("localhost", 8000), + "headers": [ + (b"content-type", b"application/json"), + (b"host", b"localhost:8000"), + ], + } + receive = AsyncMock() + send = AsyncMock() + user_auth = MagicMock() + user_auth.user_id = "sso-user-42" + user_auth.mcp_admitted_user_subject = True + oauth_server = MagicMock() + oauth_server.auth_type = MCPAuth.oauth2 + oauth_server.needs_user_oauth_token = True + oauth_server.delegate_auth_to_upstream = False + + with ( + patch( + "litellm.proxy._experimental.mcp_server.server.extract_mcp_auth_context", + new_callable=AsyncMock, + return_value=(user_auth, None, ["repro_oauth_server"], None, None, None), + ), + patch( + "litellm.proxy._experimental.mcp_server.server.set_auth_context", + ), + patch( + "litellm.proxy._experimental.mcp_server.server._SESSION_MANAGERS_INITIALIZED", + True, + ), + patch( + "litellm.proxy._experimental.mcp_server.server._handle_stale_mcp_session", + new_callable=AsyncMock, + return_value=False, + ), + patch( + "litellm.proxy._experimental.mcp_server.server.global_mcp_server_manager.has_user_oauth_token", + new_callable=AsyncMock, + return_value=False, + ) as mock_has_token, + patch( + "litellm.proxy._experimental.mcp_server.server.global_mcp_server_manager.get_mcp_server_by_name", + return_value=oauth_server, + ), + patch.object( + session_manager_stateless, + "handle_request", + new_callable=AsyncMock, + ) as mock_handle_request, + ): + with pytest.raises(HTTPException) as exc_info: + await handle_streamable_http_mcp(scope, receive, send) + + assert mock_has_token.await_count == 1 + assert mock_handle_request.await_count == 0 + assert exc_info.value.status_code == 401 + challenge = exc_info.value.headers["www-authenticate"] + assert "authorization_uri=" not in challenge + assert challenge == ( + 'Bearer resource_metadata="http://localhost:8000' + '/.well-known/oauth-protected-resource/mcp/repro_oauth_server"' + ) + + @pytest.mark.asyncio @pytest.mark.parametrize( "m2m_fields", diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_headroom.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_headroom.py index 248893ed153..00ab39357b4 100644 --- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_headroom.py +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_headroom.py @@ -43,21 +43,35 @@ from litellm.types.utils import GenericGuardrailAPIInputs FAKE_API_BASE = "https://headroom.example.com" FAKE_API_KEY = "test-key" +# The system prompt, the last user turn and the last assistant turn are never +# sent to the compression service, so a fixture needs history for anything to +# be eligible: only index 1 is. ORIGINAL_MESSAGES = [ {"role": "system", "content": "You are a helpful assistant."}, {"role": "user", "content": "A" * 5000}, + {"role": "assistant", "content": "Understood."}, + {"role": "user", "content": "and what about B?"}, ] -COMPRESSED_MESSAGES = [ - {"role": "system", "content": "You are a helpful assistant."}, - {"role": "user", "content": "A" * 500}, -] +COMPRESSIBLE_MESSAGES = [ORIGINAL_MESSAGES[1]] +COMPRESSED_MESSAGES = [{"role": "user", "content": "A" * 500}] COMPRESSED_MESSAGES_WITH_HASH = [ - {"role": "system", "content": "You are a helpful assistant."}, { "role": "user", "content": "Summary. Retrieve more: hash=b573993006976af767214fac", }, ] +EXPECTED_MESSAGES = [ + ORIGINAL_MESSAGES[0], + COMPRESSED_MESSAGES[0], + ORIGINAL_MESSAGES[2], + ORIGINAL_MESSAGES[3], +] +EXPECTED_MESSAGES_WITH_HASH = [ + ORIGINAL_MESSAGES[0], + COMPRESSED_MESSAGES_WITH_HASH[0], + ORIGINAL_MESSAGES[2], + ORIGINAL_MESSAGES[3], +] def _make_guardrail(**kwargs) -> HeadroomGuardrail: @@ -161,7 +175,7 @@ async def test_apply_guardrail_compresses_and_returns_structured_messages( input_type="request", ) - assert result.get("structured_messages") == COMPRESSED_MESSAGES + assert result.get("structured_messages") == EXPECTED_MESSAGES entries = _recorded_guardrail_entries(request_data) assert len(entries) == 1 @@ -275,7 +289,7 @@ async def test_apply_guardrail_skips_derivation_for_non_numeric_token_counts( assert "tokens_saved" not in _recorded_guardrail_response(request_data) # Compression itself is unaffected by the skipped derivation. - assert result.get("structured_messages") == COMPRESSED_MESSAGES + assert result.get("structured_messages") == EXPECTED_MESSAGES @pytest.mark.asyncio @@ -1571,9 +1585,15 @@ PARTS_MESSAGES = [ "role": "system", "content": [ {"type": "text", "text": "You are Claude Code.", "cache_control": {"type": "ephemeral"}}, + ], + }, + { + "role": "user", + "content": [ + {"type": "text", "text": "Earlier turn.", "cache_control": {"type": "ephemeral"}}, { "type": "text", - "text": "Second system block. " + "B" * 5000, + "text": "Second block. " + "B" * 5000, "cache_control": {"type": "ephemeral", "ttl": "1h"}, }, ], @@ -1586,9 +1606,10 @@ PARTS_MESSAGES = [ ], }, {"role": "tool", "content": "tool output " + "C" * 500}, + {"role": "user", "content": "what does that file do?"}, ] -FLATTENED_SYSTEM_TEXT = "You are Claude Code.\n\nSecond system block. " + "B" * 5000 +FLATTENED_HISTORY_TEXT = "Earlier turn.\n\nSecond block. " + "B" * 5000 def _parts_copy() -> list: @@ -1596,10 +1617,13 @@ def _parts_copy() -> list: def _echo_wire_view() -> list: - """What the service receives (and echoes back when it changes nothing).""" + """What the service receives (and echoes back when it changes nothing). + + The system row and the trailing user row are never sent. + """ return [ - {"role": "system", "content": FLATTENED_SYSTEM_TEXT}, - json.loads(json.dumps(PARTS_MESSAGES[1])), + {"role": "user", "content": FLATTENED_HISTORY_TEXT}, + json.loads(json.dumps(PARTS_MESSAGES[2])), {"role": "tool", "content": "tool output " + "C" * 500}, ] @@ -1627,7 +1651,7 @@ async def test_apply_guardrail_flattens_all_text_rows_only( ) wire_messages = mock_post.call_args.kwargs["json"]["messages"] - assert wire_messages[0]["content"] == FLATTENED_SYSTEM_TEXT + assert wire_messages[0]["content"] == FLATTENED_HISTORY_TEXT # Mixed text+image row is never flattened: merging its text would move a # later cache_control breakpoint across the image part. assert isinstance(wire_messages[1]["content"], list) @@ -1643,7 +1667,7 @@ async def test_apply_guardrail_restores_rewritten_all_text_row( structured_messages=_parts_copy(), ) compressed = _echo_wire_view() - compressed[0]["content"] = "compressed system. Retrieve more: hash=b573993006976af767214fac" + compressed[0]["content"] = "compressed history. Retrieve more: hash=b573993006976af767214fac" mock_response = _make_compress_response(compressed) with patch.object( @@ -1659,17 +1683,17 @@ async def test_apply_guardrail_restores_rewritten_all_text_row( ) messages = result["structured_messages"] - system_content = messages[0]["content"] + history_content = messages[1]["content"] # Rewritten all-text row collapses to one part carrying the LAST declared # breakpoint: an Anthropic breakpoint caches the prefix ending at its # part, so after the merge the last one (and its TTL) still describes the # row. - assert isinstance(system_content, list) - assert len(system_content) == 1 - assert system_content[0]["text"] == "compressed system. Retrieve more: hash=b573993006976af767214fac" - assert system_content[0]["cache_control"] == {"type": "ephemeral", "ttl": "1h"} + assert isinstance(history_content, list) + assert len(history_content) == 1 + assert history_content[0]["text"] == "compressed history. Retrieve more: hash=b573993006976af767214fac" + assert history_content[0]["cache_control"] == {"type": "ephemeral", "ttl": "1h"} # Mixed row passes through byte-identical. - assert messages[1]["content"] == PARTS_MESSAGES[1]["content"] + assert messages[2]["content"] == PARTS_MESSAGES[2]["content"] # Hashes inside restored parts still drive retrieve-tool injection. assert has_headroom_retrieve_tool(result.get("tools") or []) @@ -1701,19 +1725,43 @@ async def test_apply_guardrail_keeps_originals_when_service_echoes_unchanged( @pytest.mark.asyncio -async def test_apply_guardrail_adopts_service_output_when_rows_dropped( +async def test_apply_guardrail_rejects_service_output_when_rows_dropped( guardrail: HeadroomGuardrail, ): + """A reshaped conversation cannot be applied at all: the rows held back from + compression are matched positionally, so a response with a different row + count goes through the fail policy instead of being adopted.""" inputs = GenericGuardrailAPIInputs( texts=["B" * 5000], structured_messages=_parts_copy(), ) - dropped = [ - {"role": "system", "content": FLATTENED_SYSTEM_TEXT}, - {"role": "user", "content": "B" * 50}, - ] + dropped = [{"role": "user", "content": "B" * 50}] mock_response = _make_compress_response(dropped) + with patch.object( + guardrail.async_handler, + "post", + new_callable=AsyncMock, + return_value=mock_response, + ): + with pytest.raises(HTTPException) as exc_info: + await guardrail.apply_guardrail( + inputs=inputs, + request_data={"model": "claude-fable-5"}, + input_type="request", + ) + + assert exc_info.value.status_code == 502 + assert "changed the message count" in str(exc_info.value.detail) + + +@pytest.mark.asyncio +async def test_apply_guardrail_forwards_original_when_rows_dropped_and_fail_open(): + guardrail = _make_guardrail(unreachable_fallback="fail_open") + original = _parts_copy() + inputs = GenericGuardrailAPIInputs(texts=["B" * 5000], structured_messages=original) + mock_response = _make_compress_response([{"role": "user", "content": "B" * 50}]) + with patch.object( guardrail.async_handler, "post", @@ -1726,7 +1774,10 @@ async def test_apply_guardrail_adopts_service_output_when_rows_dropped( input_type="request", ) - assert result["structured_messages"] == dropped + # Same object back, so translation handlers that detect a rewrite by + # identity leave the request alone instead of round-tripping it. + assert result is inputs + assert result["structured_messages"] is original @pytest.mark.asyncio @@ -1739,7 +1790,7 @@ async def test_apply_guardrail_sends_textless_parts_rows_unflattened( ] inputs = GenericGuardrailAPIInputs( texts=["D" * 5000], - structured_messages=json.loads(json.dumps(image_only)), + structured_messages=json.loads(json.dumps(image_only)) + [{"role": "user", "content": "and now?"}], ) mock_response = _make_compress_response(json.loads(json.dumps(image_only))) @@ -1782,3 +1833,243 @@ async def test_fail_open_returns_original_parts_shapes(): messages = result["structured_messages"] assert [m["content"] for m in messages] == [m["content"] for m in PARTS_MESSAGES] + + +# --------------------------------------------------------------------------- +# LIT-5018: the turn the model is being asked to act on is never compressed. +# +# A Claude Code request ends with the live instruction, preceded by the tool +# result answering the assistant's last tool call. Replacing either with a +# marker makes the model answer a retrieval result instead of the request. +# --------------------------------------------------------------------------- + +AGENTIC_MESSAGES = [ + {"role": "system", "content": "You are Claude Code. " + "S" * 5000}, + {"role": "user", "content": "H" * 5000}, + {"role": "assistant", "content": "Older answer. " + "O" * 5000}, + {"role": "tool", "tool_call_id": "old_1", "content": "older tool output " + "T" * 5000}, + { + "role": "assistant", + "content": "Reading the file now.", + "tool_calls": [{"id": "tu_1", "type": "function", "function": {"name": "Read", "arguments": "{}"}}], + }, + {"role": "tool", "tool_call_id": "tu_1", "content": "FILE BODY " + "F" * 5000}, + { + "role": "user", + "content": [ + {"type": "text", "text": " " + "E" * 5000}, + {"type": "text", "text": "can we run /team to fix this"}, + ], + }, +] + + +async def _wire_and_result(guardrail: HeadroomGuardrail, messages: list, returned: list | None = None): + inputs = GenericGuardrailAPIInputs(texts=["x"], structured_messages=json.loads(json.dumps(messages))) + sent: dict = {} + + def _echo(**kwargs): + sent["messages"] = kwargs["json"]["messages"] + return _make_compress_response( + returned if returned is not None else json.loads(json.dumps(kwargs["json"]["messages"])) + ) + + with patch.object(guardrail.async_handler, "post", new_callable=AsyncMock, side_effect=_echo): + result = await guardrail.apply_guardrail( + inputs=inputs, + request_data={"model": "claude-sonnet-4-5-20250929"}, + input_type="request", + ) + return sent["messages"], result + + +@pytest.mark.asyncio +async def test_live_user_turn_is_never_sent_for_compression(guardrail: HeadroomGuardrail): + wire, result = await _wire_and_result(guardrail, AGENTIC_MESSAGES) + + live_turn = AGENTIC_MESSAGES[-1] + assert live_turn not in wire + assert not any("can we run /team to fix this" in json.dumps(row) for row in wire) + # It reaches the model byte-identical, both text parts intact, so no + # marker and no retrieval round-trip stands in for the instruction. + assert result["structured_messages"][-1] == live_turn + + +@pytest.mark.asyncio +async def test_system_prompt_is_never_sent_for_compression(guardrail: HeadroomGuardrail): + wire, result = await _wire_and_result(guardrail, AGENTIC_MESSAGES) + + assert not any(row.get("role") == "system" for row in wire) + # The Anthropic write-back drops compressed system rows, so sending it + # only inflates the savings the service reports back. + assert result["structured_messages"][0] == AGENTIC_MESSAGES[0] + + +@pytest.mark.asyncio +async def test_trailing_tool_exchange_is_never_sent_for_compression(guardrail: HeadroomGuardrail): + """The tool result answering the last assistant's tool call is protected + with it: a marker there stands in for the result of the call the model just + made, forcing an immediate retrieval of data it already asked for.""" + wire, result = await _wire_and_result(guardrail, AGENTIC_MESSAGES) + + assert not any(row.get("tool_call_id") == "tu_1" for row in wire) + assert result["structured_messages"][5] == AGENTIC_MESSAGES[5] + + +@pytest.mark.asyncio +async def test_history_is_still_compressed(guardrail: HeadroomGuardrail): + """Negative control: protection must not turn compression into a no-op.""" + compressed_history = [ + {"role": "user", "content": "hist. hash=b573993006976af767214fac"}, + {"role": "assistant", "content": "older. hash=a73993006976af767214fac1"}, + {"role": "tool", "tool_call_id": "old_1", "content": "older tool. hash=c73993006976af767214fac2"}, + ] + wire, result = await _wire_and_result(guardrail, AGENTIC_MESSAGES, returned=compressed_history) + + # Exactly the three history rows go to the service, in order. + assert [row["role"] for row in wire] == ["user", "assistant", "tool"] + assert wire[0]["content"] == "H" * 5000 + assert wire[2]["tool_call_id"] == "old_1" + + messages = result["structured_messages"] + assert len(messages) == len(AGENTIC_MESSAGES) + assert messages[1] == compressed_history[0] + assert messages[2] == compressed_history[1] + assert messages[3] == compressed_history[2] + # Hashes in the compressed history still drive retrieve-tool injection. + assert has_headroom_retrieve_tool(result.get("tools") or []) + + +@pytest.mark.asyncio +async def test_nothing_compressible_returns_inputs_untouched(guardrail: HeadroomGuardrail): + """A single-turn request is all protected, so there is nothing to send and + the caller's own inputs object comes back.""" + inputs = GenericGuardrailAPIInputs( + texts=["A" * 5000], + structured_messages=[ + {"role": "system", "content": "sys"}, + {"role": "user", "content": "A" * 5000}, + ], + ) + + with patch.object(guardrail.async_handler, "post", new_callable=AsyncMock) as mock_post: + result = await guardrail.apply_guardrail( + inputs=inputs, + request_data={"model": "gpt-4o"}, + input_type="request", + ) + + mock_post.assert_not_called() + assert result is inputs + + +@pytest.mark.asyncio +async def test_fail_open_returns_the_caller_inputs_object(): + """Translation handlers detect a rewrite by object identity, so a request + that was not compressed must come back as the same object or it is + round-tripped through the write-back for nothing.""" + guardrail = _make_guardrail(unreachable_fallback="fail_open") + original = json.loads(json.dumps(AGENTIC_MESSAGES)) + inputs = GenericGuardrailAPIInputs(texts=["x"], structured_messages=original) + + with patch.object( + guardrail.async_handler, + "post", + new_callable=AsyncMock, + side_effect=httpx.ConnectError("boom"), + ): + result = await guardrail.apply_guardrail(inputs=inputs, request_data={}, input_type="request") + + assert result is inputs + assert result["structured_messages"] is original + + +# --------------------------------------------------------------------------- +# LIT-5018: the retrieval follow-up keeps the model's own text. +# --------------------------------------------------------------------------- + + +def _anthropic_response_with_text_and_tool_call() -> dict: + return { + "content": [ + {"type": "text", "text": "Let me pull the original back."}, + {"type": "tool_use", "id": "call_1", "name": HEADROOM_RETRIEVE_TOOL_NAME, "input": {"hash": "h" * 24}}, + ] + } + + +async def _plan_for(guardrail: HeadroomGuardrail, response, messages: list): + guardrail._issued_hashes_by_call_id["call-1"] = (frozenset({"h" * 24}), time.monotonic() + 60) + logging_obj = MagicMock() + logging_obj.litellm_call_id = "call-1" + logging_obj.model_call_details = {} + with patch.object( + guardrail.async_handler, + "get", + new_callable=AsyncMock, + return_value=_make_retrieve_response("ORIGINAL CONTENT"), + ): + return await guardrail.async_build_agentic_loop_plan( + tools={"tool_calls": [{"id": "call_1", "name": HEADROOM_RETRIEVE_TOOL_NAME, "arguments": {"hash": "h" * 24}}]}, + model="claude-sonnet-4-5-20250929", + messages=messages, + response=response, + anthropic_messages_provider_config=None, + anthropic_messages_optional_request_params={}, + logging_obj=logging_obj, + stream=False, + kwargs={}, + ) + + +@pytest.mark.asyncio +async def test_anthropic_followup_preserves_assistant_text(guardrail: HeadroomGuardrail): + plan = await _plan_for(guardrail, _anthropic_response_with_text_and_tool_call(), [{"role": "user", "content": "q"}]) + + assistant = plan.request_patch.messages[-2] # type: ignore[union-attr] + assert assistant["role"] == "assistant" + # Text first, then the tool_use it accompanied: dropping it loses the + # model's stated reason for the retrieval from its own transcript. + assert assistant["content"][0] == {"type": "text", "text": "Let me pull the original back."} + assert assistant["content"][1]["type"] == "tool_use" + + +@pytest.mark.asyncio +async def test_responses_followup_preserves_assistant_text(guardrail: HeadroomGuardrail): + response = { + "output": [ + {"type": "message", "content": [{"type": "output_text", "text": "Fetching the original."}]}, + {"type": "function_call", "call_id": "call_1", "name": HEADROOM_RETRIEVE_TOOL_NAME, "arguments": "{}"}, + ] + } + + plan = await _plan_for(guardrail, response, [{"role": "user", "content": "q"}]) + + items = plan.request_patch.messages # type: ignore[union-attr] + assert items[1] == {"role": "assistant", "content": "Fetching the original."} + assert items[2]["type"] == "function_call" + + +@pytest.mark.asyncio +async def test_chat_followup_echoes_only_the_retrieve_call(guardrail: HeadroomGuardrail): + """A turn that called another tool alongside headroom_retrieve must not + echo that call: only the retrieve call gets a tool result, and a tool_call + without one is rejected by the provider.""" + other = MagicMock() + other.id = "call_other" + other.type = "function" + other.function = MagicMock() + other.function.name = "Write" + other.function.arguments = "{}" + + response = _make_openai_response_with_tool_call(HEADROOM_RETRIEVE_TOOL_NAME, {"hash": "h" * 24}, "call_1") + response.choices[0].message.content = "Getting the original first." + response.choices[0].message.tool_calls = [response.choices[0].message.tool_calls[0], other] + + plan = await _plan_for(guardrail, response, [{"role": "user", "content": "q"}]) + + messages = plan.request_patch.messages # type: ignore[union-attr] + assistant = messages[1] + assert assistant["content"] == "Getting the original first." + assert [tc["id"] for tc in assistant["tool_calls"]] == ["call_1"] + assert [m["tool_call_id"] for m in messages[2:]] == ["call_1"] diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_structured_messages_writeback.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_structured_messages_writeback.py index 642dd51b37b..d2e5b407e30 100644 --- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_structured_messages_writeback.py +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_structured_messages_writeback.py @@ -7,6 +7,7 @@ For Anthropic: structured_messages (OpenAI format) converted back to Anthropic f via anthropic_messages_pt before writing to data["messages"]. """ +import json from unittest.mock import MagicMock, patch import pytest @@ -127,3 +128,75 @@ async def test_anthropic_handler_converts_structured_messages_to_anthropic_forma llm_provider="anthropic", ) assert result["messages"] == converted_back + + +# --------------------------------------------------------------------------- +# LIT-5018: the write-back must not restructure the conversation. +# +# anthropic_messages_pt merges every run of consecutive user/tool rows into one +# message, so a tool_result-only turn and the live user turn that follows it +# came back fused: the current instruction stopped being its own turn purely +# because a compression guardrail was enabled. +# --------------------------------------------------------------------------- + +AGENTIC_ANTHROPIC_MESSAGES = [ + {"role": "user", "content": [{"type": "text", "text": "first turn"}]}, + {"role": "assistant", "content": [{"type": "tool_use", "id": "tu_1", "name": "Read", "input": {}}]}, + {"role": "user", "content": [{"type": "tool_result", "tool_use_id": "tu_1", "content": "FILE BODY"}]}, + {"role": "user", "content": [{"type": "text", "text": "can we run /team to fix this"}]}, +] + + +async def _write_back_identity(messages: list) -> list: + """Run the request through a guardrail that changes nothing but returns a + new list, which is what puts a compression guardrail on the write-back + path, and return the resulting Anthropic messages.""" + from litellm.llms.anthropic.chat.guardrail_translation.handler import ( + AnthropicMessagesHandler, + ) + + guardrail = MagicMock() + guardrail.should_run_guardrail.return_value = True + guardrail.skip_system_message_in_guardrail = None + guardrail.skip_tool_message_in_guardrail = None + guardrail.experimental_use_latest_role_message_only = False + + async def apply_guardrail(inputs, request_data, input_type, logging_obj=None): + return {**inputs, "structured_messages": list(inputs["structured_messages"])} + + guardrail.apply_guardrail = apply_guardrail + + data = {"model": "claude-sonnet-4-5-20250929", "messages": messages, "max_tokens": 1024} + result = await AnthropicMessagesHandler().process_input_messages(data=data, guardrail_to_apply=guardrail) + return result["messages"] + + +@pytest.mark.asyncio +async def test_write_back_keeps_the_live_user_turn_separate_from_the_tool_result_turn(): + written = await _write_back_identity([dict(m) for m in AGENTIC_ANTHROPIC_MESSAGES]) + + assert [m["role"] for m in written] == ["user", "assistant", "user", "user"] + assert written[2]["content"] == [{"type": "tool_result", "tool_use_id": "tu_1", "content": "FILE BODY"}] + assert written[3]["content"] == [{"type": "text", "text": "can we run /team to fix this"}] + + +@pytest.mark.asyncio +async def test_write_back_keeps_real_tool_results_under_modify_params(): + """Converting one row at a time would keep the turns apart too, but an + assistant row whose results are converted separately reads as an orphaned + tool call: with modify_params on, the sanitizer answers it with a synthetic + "tool execution skipped" result and drops the real one.""" + import litellm + + original = litellm.modify_params + litellm.modify_params = True + try: + written = await _write_back_identity([dict(m) for m in AGENTIC_ANTHROPIC_MESSAGES]) + finally: + litellm.modify_params = original + + serialized = json.dumps(written) + assert "FILE BODY" in serialized + assert "skipped" not in serialized + assert "Please continue" not in serialized + assert [m["role"] for m in written] == ["user", "assistant", "user", "user"] diff --git a/tests/test_litellm/proxy/hooks/test_dynamic_rate_limiter_v3.py b/tests/test_litellm/proxy/hooks/test_dynamic_rate_limiter_v3.py index 00ed7e8cd6c..c8176ca6337 100644 --- a/tests/test_litellm/proxy/hooks/test_dynamic_rate_limiter_v3.py +++ b/tests/test_litellm/proxy/hooks/test_dynamic_rate_limiter_v3.py @@ -1754,7 +1754,6 @@ async def test_priority_429_includes_model_name_and_configured_limits(): user_api_key_dict=user, priority="prod", saturation=0.95, - data={"model": model}, ) assert exc_info.value.status_code == 429 diff --git a/tests/test_litellm/proxy/hooks/test_parallel_request_limiter_v3.py b/tests/test_litellm/proxy/hooks/test_parallel_request_limiter_v3.py index a4c42ff601e..56bfd1829b5 100644 --- a/tests/test_litellm/proxy/hooks/test_parallel_request_limiter_v3.py +++ b/tests/test_litellm/proxy/hooks/test_parallel_request_limiter_v3.py @@ -18,8 +18,12 @@ from litellm import Router from litellm.caching.caching import DualCache from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.hooks.parallel_request_limiter_v3 import ( - MAX_PARALLEL_SLOT_ACQUIRED_KEY, PARALLEL_REQUEST_SLOT_TTL_SECONDS, + ParallelSlotAcquisition, + RequestRateLimiterStash, + _request_stash, + get_or_create_request_stash, + get_request_stash, ) from litellm.proxy.hooks.parallel_request_limiter_v3 import ( _PROXY_MaxParallelRequestsHandler_v3 as _PROXY_MaxParallelRequestsHandler, @@ -52,6 +56,13 @@ def time_controller(monkeypatch): return controller +@pytest.fixture(autouse=True) +def _isolated_request_stash(): + token = _request_stash.set(None) + yield + _request_stash.reset(token) + + @pytest.mark.parametrize( "throttle_pct, expected_rpm, expected_tpm", [ @@ -673,35 +684,36 @@ async def test_async_log_failure_event_v3(): await _seed_max_parallel_requests_slots(local_cache, counter_key, ["slot-a", "slot-b"]) - def kwargs_with_slot(slot_id): - return { - "metadata": { - MAX_PARALLEL_SLOT_ACQUIRED_KEY: { - "slot_id": slot_id, - "counter_keys": [counter_key], - } - }, - "standard_logging_object": {"metadata": {"user_api_key_hash": _api_key}}, - } + def seed_slot(slot_id): + get_or_create_request_stash().parallel_slot = ParallelSlotAcquisition( + slot_id=slot_id, + counter_keys=[counter_key], + ) + + kwargs = {"standard_logging_object": {"metadata": {"user_api_key_hash": _api_key}}} async def in_flight(): return parallel_request_handler._gauge_in_flight_from_cache_value( await local_cache.async_get_cache(key=counter_key) ) + seed_slot("slot-a") await parallel_request_handler.async_log_failure_event( - kwargs=kwargs_with_slot("slot-a"), response_obj=None, start_time=None, end_time=None + kwargs=kwargs, response_obj=None, start_time=None, end_time=None ) + assert get_request_stash().parallel_slot is None assert await in_flight() == 1 for slot_id in ("slot-a", "slot-unknown", "slot-a"): + seed_slot(slot_id) await parallel_request_handler.async_log_failure_event( - kwargs=kwargs_with_slot(slot_id), response_obj=None, start_time=None, end_time=None + kwargs=kwargs, response_obj=None, start_time=None, end_time=None ) assert await in_flight() == 1 + seed_slot("slot-b") await parallel_request_handler.async_log_failure_event( - kwargs=kwargs_with_slot("slot-b"), response_obj=None, start_time=None, end_time=None + kwargs=kwargs, response_obj=None, start_time=None, end_time=None ) assert await in_flight() == 0 @@ -803,8 +815,9 @@ async def test_rejected_request_does_not_consume_parallel_slot_v3(): data=admitted_data, call_type="", ) - acquisition = admitted_data["metadata"][MAX_PARALLEL_SLOT_ACQUIRED_KEY] - assert isinstance(acquisition, dict) + assert "metadata" not in admitted_data + acquisition = get_request_stash().parallel_slot + assert acquisition is not None assert isinstance(acquisition["slot_id"], str) and acquisition["slot_id"] assert acquisition["counter_keys"] == [f"{{api_key:{_api_key}}}:max_parallel_requests"] @@ -816,10 +829,10 @@ async def test_rejected_request_does_not_consume_parallel_slot_v3(): data={"model": "gpt-3.5-turbo"}, call_type="", ) + assert get_request_stash().parallel_slot == acquisition await handler.async_log_failure_event( kwargs={ - "metadata": {MAX_PARALLEL_SLOT_ACQUIRED_KEY: acquisition}, "standard_logging_object": {"metadata": {"user_api_key_hash": _api_key}}, }, response_obj=None, @@ -866,8 +879,8 @@ async def test_parallel_gauge_uses_atomic_redis_script_v3(): data=data, call_type="", ) - stashed_acquisition = data["metadata"][MAX_PARALLEL_SLOT_ACQUIRED_KEY] - assert isinstance(stashed_acquisition, dict) + stashed_acquisition = get_request_stash().parallel_slot + assert stashed_acquisition is not None stashed_slot_id = stashed_acquisition["slot_id"] assert isinstance(stashed_slot_id, str) and stashed_slot_id assert stashed_acquisition["counter_keys"] == [counter_key] @@ -882,7 +895,7 @@ async def test_parallel_gauge_uses_atomic_redis_script_v3(): ) gauge_statuses = [ s - for s in data["litellm_proxy_rate_limit_response"]["statuses"] + for s in get_request_stash().rate_limit_response["statuses"] if s["rate_limit_type"] == "max_parallel_requests" ] assert gauge_statuses == [ @@ -3102,14 +3115,12 @@ async def test_project_model_rate_limits_not_triggered_for_other_model_v3(): @pytest.mark.asyncio -async def test_pre_call_hook_does_not_leak_internal_stash_to_request_body(): - """Regression for #27001: stash keys must stay in metadata, never on - the top level of ``data`` (which gets forwarded as the provider body).""" - from litellm.proxy.hooks.parallel_request_limiter_v3 import ( - _LITELLM_STASH_KEYS, - RATE_LIMIT_DESCRIPTORS_KEY, - TPM_RESERVED_TOKENS_KEY, - ) +async def test_pre_call_hook_keeps_internal_stash_out_of_request_body(): + """Regression for #27001 / #35197: the limiter's per-request bookkeeping + must never touch the outgoing request body — no top-level keys and no + created or mutated ``metadata`` / ``litellm_metadata`` buckets. The + reservation must land on the ContextVar stash instead.""" + import copy _api_key = hash_token("sk-leak-regression") user_api_key_dict = UserAPIKeyAuth( @@ -3149,6 +3160,7 @@ async def test_pre_call_hook_does_not_leak_internal_stash_to_request_body(): "messages": [{"role": "user", "content": "hello"}], "max_tokens": 10, } + body_before = copy.deepcopy(data) await parallel_request_handler.async_pre_call_hook( user_api_key_dict=user_api_key_dict, @@ -3157,31 +3169,27 @@ async def test_pre_call_hook_does_not_leak_internal_stash_to_request_body(): call_type="completion", ) - leaked = [k for k in _LITELLM_STASH_KEYS if k in data] - assert not leaked, f"stash keys leaked to top level: {leaked}" + assert data == body_before - metadata = data.get("metadata") or {} - assert metadata.get(TPM_RESERVED_TOKENS_KEY) - assert isinstance(metadata.get(RATE_LIMIT_DESCRIPTORS_KEY), list) + stash = get_request_stash() + assert stash is not None + assert stash.reserved_tokens > 0 + assert stash.reserved_model == "gpt-4o-mini" + assert stash.reserved_scopes == frozenset({("api_key", _api_key)}) @pytest.mark.asyncio -@pytest.mark.parametrize("caller_metadata", [None, {"user_tag": "abc"}]) -async def test_pre_call_hook_does_not_touch_provider_metadata_on_litellm_metadata_routes( - caller_metadata, -): - """Regression for #35197: routes that own ``litellm_metadata`` (Responses, - /v1/messages, batches, files) send ``metadata`` to the provider, so the - limiter must never create it or write stash keys into it.""" - from litellm.proxy.hooks.parallel_request_limiter_v3 import ( - _LITELLM_STASH_KEYS, - RATE_LIMIT_DESCRIPTORS_KEY, - RATE_LIMIT_RESPONSE_KEY, - TPM_RESERVED_TOKENS_KEY, - ) +@pytest.mark.parametrize("caller_metadata", [None, {"user_tag": "campaign-42"}]) +async def test_responses_route_body_untouched_by_pre_call_hook(caller_metadata): + """Regression for #35197: on routes where ``metadata`` is a provider + request parameter (Responses API), the pre-call hook must forward the + body byte-identical — creating or adding to ``metadata`` / + ``litellm_metadata`` produced upstream HTTP 400s.""" + import copy + _api_key = hash_token("sk-responses-regression") user_api_key_dict = UserAPIKeyAuth( - api_key=hash_token("sk-responses-metadata"), + api_key=_api_key, tpm_limit=1000, rpm_limit=5, ) @@ -3190,35 +3198,13 @@ async def test_pre_call_hook_does_not_touch_provider_metadata_on_litellm_metadat internal_usage_cache=InternalUsageCache(local_cache), ) - async def mock_should_rate_limit(descriptors, **kwargs): - return { - "overall_code": "OK", - "statuses": [ - { - "code": "OK", - "current_limit": 5, - "limit_remaining": 4, - "descriptor_key": d["key"], - "descriptor_value": d["value"], - "rate_limit_type": "requests", - } - for d in descriptors - ], - } - - async def mock_reserve_tpm_tokens(descriptors, estimated_tokens, **kwargs): - return {"overall_code": "OK", "statuses": []} - - handler.should_rate_limit = mock_should_rate_limit - handler.reserve_tpm_tokens = mock_reserve_tpm_tokens - data: Dict[str, Any] = { - "model": "responses-model", + "model": "gpt-4o-mini", "input": "hello", - "litellm_metadata": {}, } if caller_metadata is not None: data["metadata"] = dict(caller_metadata) + body_before = copy.deepcopy(data) await handler.async_pre_call_hook( user_api_key_dict=user_api_key_dict, @@ -3227,37 +3213,87 @@ async def test_pre_call_hook_does_not_touch_provider_metadata_on_litellm_metadat call_type="aresponses", ) + assert data == body_before if caller_metadata is None: - assert "metadata" not in data, f"limiter created provider metadata: {data.get('metadata')!r}" + assert "metadata" not in data else: assert data["metadata"] == caller_metadata + assert "litellm_metadata" not in data - litellm_metadata = data["litellm_metadata"] - assert litellm_metadata.get(TPM_RESERVED_TOKENS_KEY) - assert isinstance(litellm_metadata.get(RATE_LIMIT_DESCRIPTORS_KEY), list) - assert litellm_metadata.get(RATE_LIMIT_RESPONSE_KEY) - - leaked = [k for k in _LITELLM_STASH_KEYS if k in data] - assert not leaked, f"stash keys leaked to top level: {leaked}" - - for key in _LITELLM_STASH_KEYS: - assert handler._lookup_stashed_value( - kwargs={"litellm_params": {"litellm_metadata": litellm_metadata}}, - standard_logging_metadata=None, - key=key, - ) == litellm_metadata.get(key) + stash = get_request_stash() + assert stash is not None + assert stash.reserved_tokens > 0 + assert stash.rate_limit_response is not None @pytest.mark.asyncio -async def test_pre_call_hook_rejects_caller_supplied_stash_values(): - """Caller cannot pre-populate stash keys in body metadata to drive a - later TPM refund against an arbitrary scope.""" - from litellm.proxy.hooks.parallel_request_limiter_v3 import ( - _LITELLM_STASH_KEYS, - RATE_LIMIT_DESCRIPTORS_KEY, - TPM_RESERVED_TOKENS_KEY, +async def test_chat_tpm_refund_and_slot_release_via_context_stash(monkeypatch): + """ + Full chat lifecycle with no body stashing: pre-call reserves TPM tokens + and acquires a parallel slot on the ContextVar stash; the failure + callback refunds the reservation and frees the slot exactly once — a + second failure callback for the same request must not double-refund the + :tokens counter or double-release the gauge. + """ + monkeypatch.delenv("LITELLM_TPM_TOKEN_RESERVATION_ENABLED", raising=False) + _api_key = hash_token("sk-refund-lifecycle") + local_cache = DualCache() + handler = _PROXY_MaxParallelRequestsHandler( + internal_usage_cache=InternalUsageCache(local_cache) + ) + user_api_key_dict = UserAPIKeyAuth( + api_key=_api_key, + tpm_limit=10_000, + max_parallel_requests=2, + ) + tokens_key = handler.create_rate_limit_keys( + key="api_key", value=_api_key, rate_limit_type="tokens" + ) + parallel_key = f"{{api_key:{_api_key}}}:max_parallel_requests" + + await handler.async_pre_call_hook( + user_api_key_dict=user_api_key_dict, + cache=local_cache, + data={ + "model": "gpt-4o-mini", + "messages": [{"role": "user", "content": "hello"}], + "max_tokens": 50, + }, + call_type="completion", ) + reserved = get_request_stash().reserved_tokens + assert reserved > 0 + assert int(await local_cache.async_get_cache(key=tokens_key) or 0) == reserved + assert handler._gauge_in_flight_from_cache_value( + await local_cache.async_get_cache(key=parallel_key) + ) == 1 + + kwargs = {"standard_logging_object": {"metadata": {"user_api_key_hash": _api_key}}} + await handler.async_log_failure_event( + kwargs=kwargs, response_obj=None, start_time=None, end_time=None + ) + + assert int(await local_cache.async_get_cache(key=tokens_key) or 0) == 0 + assert handler._gauge_in_flight_from_cache_value( + await local_cache.async_get_cache(key=parallel_key) + ) == 0 + assert get_request_stash().reservation_released is True + + await handler.async_log_failure_event( + kwargs=kwargs, response_obj=None, start_time=None, end_time=None + ) + assert int(await local_cache.async_get_cache(key=tokens_key) or 0) == 0 + assert handler._gauge_in_flight_from_cache_value( + await local_cache.async_get_cache(key=parallel_key) + ) == 0 + + +@pytest.mark.asyncio +async def test_pre_call_hook_ignores_caller_supplied_stash_values(): + """Caller-supplied bookkeeping lookalikes in the body must not drive a + TPM refund against an arbitrary scope: the ContextVar stash is the only + source the refund path reads.""" user_api_key_dict = UserAPIKeyAuth(api_key=hash_token("sk-no-limits")) local_cache = DualCache() handler = _PROXY_MaxParallelRequestsHandler( @@ -3271,19 +3307,15 @@ async def test_pre_call_hook_rejects_caller_supplied_stash_values(): "rate_limit": {"tokens_per_unit": 10000, "window_size": 60}, } ] + injected = { + "_litellm_tpm_reserved_tokens": 9999, + "_litellm_rate_limit_descriptors": victim_descriptors, + } data: Dict[str, Any] = { "model": "gpt-4o-mini", "messages": [{"role": "user", "content": "hi"}], - TPM_RESERVED_TOKENS_KEY: 9999, - RATE_LIMIT_DESCRIPTORS_KEY: victim_descriptors, - "metadata": { - TPM_RESERVED_TOKENS_KEY: 9999, - RATE_LIMIT_DESCRIPTORS_KEY: victim_descriptors, - }, - "litellm_metadata": { - TPM_RESERVED_TOKENS_KEY: 9999, - RATE_LIMIT_DESCRIPTORS_KEY: victim_descriptors, - }, + "metadata": dict(injected), + "litellm_metadata": dict(injected), } await handler.async_pre_call_hook( @@ -3293,13 +3325,139 @@ async def test_pre_call_hook_rejects_caller_supplied_stash_values(): call_type="completion", ) - for channel in ( - data, - data.get("metadata") or {}, - data.get("litellm_metadata") or {}, - ): - leaked = [k for k in _LITELLM_STASH_KEYS if k in channel] - assert not leaked, f"caller-supplied stash survived in {channel!r}: {leaked}" + refund_calls = [] + + async def spy_increment_pipeline(increment_list, **kwargs): + refund_calls.append(increment_list) + + handler.internal_usage_cache.dual_cache.async_increment_cache_pipeline = ( + spy_increment_pipeline + ) + + await handler.async_post_call_failure_hook( + request_data=data, + original_exception=Exception("boom"), + user_api_key_dict=user_api_key_dict, + ) + + assert refund_calls == [] + stash = get_request_stash() + assert stash is not None + assert stash.reserved_tokens == 0 + + +@pytest.mark.asyncio +async def test_log_events_from_nested_calls_leave_owner_stash_alone(monkeypatch): + """ + A nested LiteLLM call made inside the request (LLM-judge guardrail, + silent experiment) inherits the request context and fires the same global + logging callbacks with a fresh ``litellm_call_id``. Those callbacks must + not release the owning request's parallel slot or refund its TPM + reservation; only events carrying the owner's call id may. + """ + monkeypatch.delenv("LITELLM_TPM_TOKEN_RESERVATION_ENABLED", raising=False) + _api_key = hash_token("sk-nested-guard") + local_cache = DualCache() + handler = _PROXY_MaxParallelRequestsHandler( + internal_usage_cache=InternalUsageCache(local_cache) + ) + user_api_key_dict = UserAPIKeyAuth( + api_key=_api_key, + tpm_limit=10_000, + max_parallel_requests=2, + ) + tokens_key = handler.create_rate_limit_keys( + key="api_key", value=_api_key, rate_limit_type="tokens" + ) + parallel_key = f"{{api_key:{_api_key}}}:max_parallel_requests" + + await handler.async_pre_call_hook( + user_api_key_dict=user_api_key_dict, + cache=local_cache, + data={ + "model": "gpt-4o-mini", + "messages": [{"role": "user", "content": "hello"}], + "max_tokens": 50, + "litellm_call_id": "owner-call-id", + }, + call_type="completion", + ) + + stash = get_request_stash() + assert stash is not None + assert stash.owner_litellm_call_id == "owner-call-id" + reserved = stash.reserved_tokens + assert reserved > 0 + + nested_kwargs = { + "litellm_call_id": "nested-guardrail-call", + "standard_logging_object": {"metadata": {"user_api_key_hash": _api_key}}, + } + await handler.async_log_success_event( + kwargs=nested_kwargs, response_obj=None, start_time=None, end_time=None + ) + await handler.async_log_failure_event( + kwargs=nested_kwargs, response_obj=None, start_time=None, end_time=None + ) + + assert stash.parallel_slot is not None + assert stash.reservation_released is False + assert handler._gauge_in_flight_from_cache_value( + await local_cache.async_get_cache(key=parallel_key) + ) == 1 + assert int(await local_cache.async_get_cache(key=tokens_key) or 0) == reserved + + owner_kwargs = { + "litellm_call_id": "owner-call-id", + "standard_logging_object": {"metadata": {"user_api_key_hash": _api_key}}, + } + await handler.async_log_failure_event( + kwargs=owner_kwargs, response_obj=None, start_time=None, end_time=None + ) + + assert stash.parallel_slot is None + assert stash.reservation_released is True + assert handler._gauge_in_flight_from_cache_value( + await local_cache.async_get_cache(key=parallel_key) + ) == 0 + assert int(await local_cache.async_get_cache(key=tokens_key) or 0) == 0 + + +@pytest.mark.asyncio +async def test_stash_applies_when_owner_or_callback_call_id_missing(): + """ + The owner guard only rejects a positive mismatch. A stash never claimed + by a pre-call hook (no owner id) must stay visible to any callback, and a + claimed stash must stay visible to callbacks whose kwargs carry no call + id — otherwise reservations and slots would strand on request paths that + do not thread ``litellm_call_id`` into their logging kwargs. + """ + local_cache = DualCache() + handler = _PROXY_MaxParallelRequestsHandler( + internal_usage_cache=InternalUsageCache(local_cache) + ) + + unclaimed = get_or_create_request_stash() + unclaimed.reserved_tokens = 42 + await handler.async_log_failure_event( + kwargs={"litellm_call_id": "any-id", "standard_logging_object": {}}, + response_obj=None, + start_time=None, + end_time=None, + ) + assert unclaimed.reservation_released is True + + claimed = RequestRateLimiterStash( + owner_litellm_call_id="owner-1", reserved_tokens=42 + ) + _request_stash.set(claimed) + await handler.async_log_failure_event( + kwargs={"standard_logging_object": {}}, + response_obj=None, + start_time=None, + end_time=None, + ) + assert claimed.reservation_released is True # ----------------------- Per-MCP-server rate limiting (v3) ----------------------- @@ -3594,18 +3752,13 @@ async def test_release_max_parallel_requests_on_disconnect_v3(): await local_cache.async_get_cache(key=counter_key) ) == 1 - await handler.async_release_max_parallel_requests_on_disconnect( - user_api_key_dict, - request_data={ - "metadata": { - MAX_PARALLEL_SLOT_ACQUIRED_KEY: { - "slot_id": _TEST_SLOT_ID, - "counter_keys": [counter_key], - } - } - }, + get_or_create_request_stash().parallel_slot = ParallelSlotAcquisition( + slot_id=_TEST_SLOT_ID, + counter_keys=[counter_key], ) + await handler.async_release_max_parallel_requests_on_disconnect(user_api_key_dict) + assert get_request_stash().parallel_slot is None assert handler._gauge_in_flight_from_cache_value( await local_cache.async_get_cache(key=counter_key) ) == 0 @@ -3627,16 +3780,12 @@ async def test_release_on_disconnect_works_when_key_config_changed_v3(): counter_key = f"{{api_key:{_api_key}}}:max_parallel_requests" await _seed_max_parallel_requests_slots(local_cache, counter_key, [_TEST_SLOT_ID]) + get_or_create_request_stash().parallel_slot = ParallelSlotAcquisition( + slot_id=_TEST_SLOT_ID, + counter_keys=[counter_key], + ) await handler.async_release_max_parallel_requests_on_disconnect( - UserAPIKeyAuth(api_key=_api_key, max_parallel_requests=None), - request_data={ - "metadata": { - MAX_PARALLEL_SLOT_ACQUIRED_KEY: { - "slot_id": _TEST_SLOT_ID, - "counter_keys": [counter_key], - } - } - }, + UserAPIKeyAuth(api_key=_api_key, max_parallel_requests=None) ) assert handler._gauge_in_flight_from_cache_value( await local_cache.async_get_cache(key=counter_key) @@ -3684,7 +3833,6 @@ async def test_post_call_failure_hook_releases_parallel_slot_v3(): await handler.async_log_failure_event( kwargs={ - "metadata": admitted_data["metadata"], "standard_logging_object": {"metadata": {"user_api_key_hash": _api_key}}, }, response_obj=None, @@ -3732,7 +3880,6 @@ async def test_success_event_releases_parallel_slot_v3(monkeypatch): await handler.async_log_success_event( kwargs={ - "metadata": admitted_data["metadata"], "standard_logging_object": {"metadata": {"user_api_key_hash": _api_key}}, }, response_obj=ModelResponse( @@ -3833,14 +3980,12 @@ async def test_redis_release_script_updates_local_mirror_v3(): handler.parallel_release_script = fake_release + get_or_create_request_stash().parallel_slot = ParallelSlotAcquisition( + slot_id="slot-redis-test", + counter_keys=[counter_key], + ) await handler.async_log_failure_event( kwargs={ - "metadata": { - MAX_PARALLEL_SLOT_ACQUIRED_KEY: { - "slot_id": "slot-redis-test", - "counter_keys": [counter_key], - } - }, "standard_logging_object": {"metadata": {"user_api_key_hash": _api_key}}, }, response_obj=None, @@ -3945,7 +4090,6 @@ async def test_in_memory_fallback_respects_mirrored_redis_count_v3(): await handler.async_log_failure_event( kwargs={ - "metadata": admitted_data["metadata"], "standard_logging_object": {"metadata": {"user_api_key_hash": _api_key}}, }, response_obj=None, @@ -4011,19 +4155,15 @@ async def test_async_streaming_data_generator_releases_counter_on_disconnect_v3( while True: yield ModelResponse() + get_or_create_request_stash().parallel_slot = ParallelSlotAcquisition( + slot_id=_TEST_SLOT_ID, + counter_keys=[counter_key], + ) with _override_litellm_callbacks([]): gen = ProxyBaseLLMRequestProcessing.async_sse_data_generator( response=upstream(), user_api_key_dict=user_api_key_dict, - request_data={ - "model": "claude-test", - "metadata": { - MAX_PARALLEL_SLOT_ACQUIRED_KEY: { - "slot_id": _TEST_SLOT_ID, - "counter_keys": [counter_key], - } - }, - }, + request_data={"model": "claude-test"}, proxy_logging_obj=proxy_logging_obj, ) await gen.__anext__() @@ -4064,21 +4204,17 @@ async def test_async_data_generator_releases_counter_on_disconnect_v3(disconnect while True: yield ModelResponse() + get_or_create_request_stash().parallel_slot = ParallelSlotAcquisition( + slot_id=_TEST_SLOT_ID, + counter_keys=[counter_key], + ) try: with _override_litellm_callbacks([]): assert proxy_logging_obj.needs_iterator_wrap() is False gen = proxy_server.async_data_generator( response=upstream(), user_api_key_dict=user_api_key_dict, - request_data={ - "model": "gpt-test", - "metadata": { - MAX_PARALLEL_SLOT_ACQUIRED_KEY: { - "slot_id": _TEST_SLOT_ID, - "counter_keys": [counter_key], - } - }, - }, + request_data={"model": "gpt-test"}, ) await gen.__anext__() if disconnect == "cancel": @@ -4127,21 +4263,17 @@ async def test_async_data_generator_releases_counter_when_wrapped_v3(): while True: yield ModelResponse() + get_or_create_request_stash().parallel_slot = ParallelSlotAcquisition( + slot_id=_TEST_SLOT_ID, + counter_keys=[counter_key], + ) try: with _override_litellm_callbacks([_PassthroughIteratorOverride()]): assert proxy_logging_obj.needs_iterator_wrap() is True gen = proxy_server.async_data_generator( response=upstream(), user_api_key_dict=user_api_key_dict, - request_data={ - "model": "gpt-test", - "metadata": { - MAX_PARALLEL_SLOT_ACQUIRED_KEY: { - "slot_id": _TEST_SLOT_ID, - "counter_keys": [counter_key], - } - }, - }, + request_data={"model": "gpt-test"}, ) await gen.__anext__() await gen.aclose() @@ -4258,12 +4390,7 @@ async def test_pre_call_hook_skips_reservation_when_disabled(monkeypatch): assert reserve_calls == [], "reservation must be skipped when disabled" assert should_rate_limit_calls[0]["skip_tpm_check"] is False - # No reservation stash leaks into the request metadata. - from litellm.proxy.hooks.parallel_request_limiter_v3 import ( - TPM_RESERVED_TOKENS_KEY, - ) - - assert TPM_RESERVED_TOKENS_KEY not in (data.get("metadata") or {}) + assert get_request_stash().reserved_tokens == 0 @pytest.mark.asyncio diff --git a/tests/test_litellm/proxy/hooks/test_proxy_rate_limit_provider_field.py b/tests/test_litellm/proxy/hooks/test_proxy_rate_limit_provider_field.py index 02b4e32db86..ec680317980 100644 --- a/tests/test_litellm/proxy/hooks/test_proxy_rate_limit_provider_field.py +++ b/tests/test_litellm/proxy/hooks/test_proxy_rate_limit_provider_field.py @@ -691,7 +691,6 @@ async def test_dynamic_rate_limiter_v3_model_capacity_path_populates_provider(): user_api_key_dict=user_api_key_dict, priority="default", saturation=1.0, - data={"model": "gpt-4o-mini"}, ) exc = exc_info.value @@ -741,7 +740,6 @@ async def test_dynamic_rate_limiter_v3_unknown_descriptor_path_populates_provide user_api_key_dict=user_api_key_dict, priority="default", saturation=1.0, - data={"model": "gpt-4o-mini"}, ) assert exc_info.value.llm_provider == "openai" diff --git a/tests/test_litellm/proxy/hooks/test_rate_limiter_toctou.py b/tests/test_litellm/proxy/hooks/test_rate_limiter_toctou.py index ceea5de7991..1c1e8eee145 100644 --- a/tests/test_litellm/proxy/hooks/test_rate_limiter_toctou.py +++ b/tests/test_litellm/proxy/hooks/test_rate_limiter_toctou.py @@ -253,7 +253,6 @@ async def test_dynamic_rate_limiter_v3_concurrent_bypasses_model_capacity(): user_api_key_dict=user, priority="high", saturation=0.0, - data={}, ) return "OK" except Exception as e: @@ -332,7 +331,6 @@ async def test_dynamic_rate_limiter_v3_uses_atomic_check_and_increment(): user_api_key_dict=user, priority="high", saturation=0.0, - data={}, ) assert atomic_descriptors_observed, ( @@ -482,7 +480,6 @@ async def test_dynamic_rate_limiter_v3_fails_closed_on_unknown_descriptor(): user_api_key_dict=user, priority="high", saturation=0.0, - data={}, ) assert ( exc.value.status_code == 429 diff --git a/tests/test_litellm/proxy/hooks/test_tpm_concurrent.py b/tests/test_litellm/proxy/hooks/test_tpm_concurrent.py index b02f6c15168..f7bd37b412a 100644 --- a/tests/test_litellm/proxy/hooks/test_tpm_concurrent.py +++ b/tests/test_litellm/proxy/hooks/test_tpm_concurrent.py @@ -23,13 +23,13 @@ import pytest from litellm.caching.caching import DualCache from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.hooks.parallel_request_limiter_v3 import ( - RATE_LIMIT_DESCRIPTORS_KEY, - TPM_RESERVATION_RELEASED_KEY, - TPM_RESERVED_MODEL_KEY, - TPM_RESERVED_SCOPES_KEY, - TPM_RESERVED_TOKENS_KEY, _PROXY_MaxParallelRequestsHandler_v3 as RateLimitHandler, ) +from litellm.proxy.hooks.parallel_request_limiter_v3 import ( + _request_stash, + get_or_create_request_stash, + get_request_stash, +) from litellm.proxy.utils import InternalUsageCache, hash_token from litellm.types.utils import ModelResponse, Usage @@ -41,6 +41,13 @@ def rate_limiter(): return handler, cache +@pytest.fixture(autouse=True) +def _isolated_request_stash(): + token = _request_stash.set(None) + yield + _request_stash.reset(token) + + @pytest.mark.asyncio async def test_token_reservation_prevents_concurrent_bypass(rate_limiter): """ @@ -79,7 +86,7 @@ async def test_token_reservation_prevents_concurrent_bypass(rate_limiter): return { "request_id": request_id, "success": True, - "reserved_tokens": data.get(TPM_RESERVED_TOKENS_KEY, 0), + "reserved_tokens": get_request_stash().reserved_tokens, } except Exception as e: return { @@ -167,12 +174,14 @@ async def test_token_adjustment_on_success(rate_limiter): api_key = hash_token("sk-test-adjust") + stash = get_or_create_request_stash() + stash.reserved_tokens = 100 + stash.reserved_scopes = frozenset({("api_key", api_key)}) + mock_kwargs = { "standard_logging_object": { "metadata": { "user_api_key_hash": api_key, - TPM_RESERVED_TOKENS_KEY: 100, - TPM_RESERVED_SCOPES_KEY: [["api_key", api_key]], } }, "model": "gpt-3.5-turbo", @@ -227,12 +236,14 @@ async def test_token_release_on_failure(rate_limiter): api_key = hash_token("sk-test-fail") + stash = get_or_create_request_stash() + stash.reserved_tokens = 100 + stash.reserved_scopes = frozenset({("api_key", api_key)}) + mock_kwargs = { "standard_logging_object": { "metadata": { "user_api_key_hash": api_key, - TPM_RESERVED_TOKENS_KEY: 100, - TPM_RESERVED_SCOPES_KEY: [["api_key", api_key]], } }, } @@ -285,6 +296,11 @@ async def test_model_scope_refund_targets_reserved_model(rate_limiter): team_id = "team-abc" reserved_model = "gpt-4o-mini" + stash = get_or_create_request_stash() + stash.reserved_tokens = 100 + stash.reserved_model = reserved_model + stash.reserved_scopes = frozenset({("model_per_team", f"{team_id}:{reserved_model}")}) + mock_kwargs = { # NOTE: no litellm_params.metadata.model_group — get_model_group_from_litellm_kwargs # returns None on this kwargs dict. @@ -292,11 +308,6 @@ async def test_model_scope_refund_targets_reserved_model(rate_limiter): "metadata": { "user_api_key_hash": api_key, "user_api_key_team_id": team_id, - TPM_RESERVED_TOKENS_KEY: 100, - TPM_RESERVED_MODEL_KEY: reserved_model, - TPM_RESERVED_SCOPES_KEY: [ - ["model_per_team", f"{team_id}:{reserved_model}"] - ], } }, } @@ -446,13 +457,15 @@ async def test_org_scope_refund_on_failure(rate_limiter): api_key = hash_token("sk-org-refund") org_id = "org-acme" + stash = get_or_create_request_stash() + stash.reserved_tokens = 100 + stash.reserved_scopes = frozenset({("organization", org_id)}) + mock_kwargs = { "standard_logging_object": { "metadata": { "user_api_key_hash": api_key, "user_api_key_org_id": org_id, - TPM_RESERVED_TOKENS_KEY: 100, - TPM_RESERVED_SCOPES_KEY: [["organization", org_id]], } }, } @@ -498,13 +511,15 @@ async def test_org_scope_reconciled_on_success(rate_limiter): api_key = hash_token("sk-org-success") org_id = "org-acme" + stash = get_or_create_request_stash() + stash.reserved_tokens = 100 + stash.reserved_scopes = frozenset({("organization", org_id)}) + mock_kwargs = { "standard_logging_object": { "metadata": { "user_api_key_hash": api_key, "user_api_key_org_id": org_id, - TPM_RESERVED_TOKENS_KEY: 100, - TPM_RESERVED_SCOPES_KEY: [["organization", org_id]], } }, "model": "gpt-3.5-turbo", @@ -607,9 +622,9 @@ async def test_contentless_request_reserves_minimum(rate_limiter): data=data, call_type="", ) - assert (data.get("metadata") or {}).get( - TPM_RESERVED_TOKENS_KEY - ) == 1, "Contentless request should reserve the floor of 1 token" + assert ( + get_request_stash().reserved_tokens == 1 + ), "Contentless request should reserve the floor of 1 token" counter_after_two = int( await cache.async_get_cache(key=counter_key, local_only=True) or 0 @@ -702,7 +717,7 @@ async def test_reservation_released_on_proxy_rejection(rate_limiter): data=data, call_type="", ) - reserved = (data.get("metadata") or {})[TPM_RESERVED_TOKENS_KEY] + reserved = get_request_stash().reserved_tokens assert reserved > 0 counter_key = handler.create_rate_limit_keys( @@ -727,8 +742,8 @@ async def test_reservation_released_on_proxy_rejection(rate_limiter): f"Reservation leaked: counter={counter_after_release} after " f"proxy-level rejection refund (expected 0)." ) - assert (data.get("metadata") or {}).get(TPM_RESERVATION_RELEASED_KEY) is True, ( - "Released marker must be stamped to prevent " + assert get_request_stash().reservation_released is True, ( + "Released flag must be set to prevent " "async_log_failure_event from double-refunding." ) @@ -754,28 +769,15 @@ async def test_reservation_release_idempotent(rate_limiter): mock_increment ) - # Shared metadata dict simulates the propagation between - # request_data["metadata"] and kwargs["litellm_params"]["metadata"] — - # the post-call-failure-hook stamps the released marker there, and the - # log-failure-event reads it. - shared_metadata = { - "user_api_key_hash": api_key, - TPM_RESERVED_TOKENS_KEY: 100, - RATE_LIMIT_DESCRIPTORS_KEY: [ - { - "key": "api_key", - "value": api_key, - "rate_limit": {"tokens_per_unit": 10000, "window_size": 60}, - } - ], - } - - request_data = { - "metadata": shared_metadata, - } + # Both hooks read the same per-request ContextVar stash: the + # post-call-failure-hook flips reservation_released on it, and the + # log-failure-event observes the flip. + stash = get_or_create_request_stash() + stash.reserved_tokens = 100 + stash.reserved_scopes = frozenset({("api_key", api_key)}) await handler.async_post_call_failure_hook( - request_data=request_data, + request_data={}, original_exception=Exception("rejected"), user_api_key_dict=UserAPIKeyAuth(api_key=api_key), ) @@ -784,11 +786,10 @@ async def test_reservation_release_idempotent(rate_limiter): assert first_refund_count > 0, "First refund should have applied" # Now simulate async_log_failure_event firing afterwards. It must see - # the released marker (via shared metadata) and not double-refund. + # the released flag on the stash and not double-refund. await handler.async_log_failure_event( kwargs={ - "litellm_params": {"metadata": shared_metadata}, - "standard_logging_object": {"metadata": shared_metadata}, + "standard_logging_object": {"metadata": {"user_api_key_hash": api_key}}, }, response_obj=None, start_time=datetime.now(), @@ -818,13 +819,15 @@ async def test_unreserved_scopes_charged_actual_not_delta_on_success(rate_limite team_id = "team-no-tpm-limit" # Reservation ONLY hit api_key — team had no TPM limit configured. + stash = get_or_create_request_stash() + stash.reserved_tokens = 100 + stash.reserved_scopes = frozenset({("api_key", api_key)}) + mock_kwargs = { "standard_logging_object": { "metadata": { "user_api_key_hash": api_key, "user_api_key_team_id": team_id, - TPM_RESERVED_TOKENS_KEY: 100, - TPM_RESERVED_SCOPES_KEY: [["api_key", api_key]], } }, "model": "gpt-3.5-turbo", @@ -888,13 +891,15 @@ async def test_unreserved_scopes_not_refunded_on_failure(rate_limiter): api_key = hash_token("sk-mixed-fail") team_id = "team-no-tpm" + stash = get_or_create_request_stash() + stash.reserved_tokens = 100 + stash.reserved_scopes = frozenset({("api_key", api_key)}) + mock_kwargs = { "standard_logging_object": { "metadata": { "user_api_key_hash": api_key, "user_api_key_team_id": team_id, - TPM_RESERVED_TOKENS_KEY: 100, - TPM_RESERVED_SCOPES_KEY: [["api_key", api_key]], } }, } @@ -939,10 +944,10 @@ async def test_unreserved_scopes_not_refunded_on_failure(rate_limiter): async def test_token_rate_limit_headers_present_in_stored_response(rate_limiter): """ With `skip_tpm_check=True` on the RPM sliding-window pass, token statuses - only come from `reserve_tpm_tokens`. They must be merged into - `data["litellm_proxy_rate_limit_response"]` so the post-call hook can - emit `x-ratelimit-{key}-remaining-tokens` / `-limit-tokens` headers to - the client. + only come from `reserve_tpm_tokens`. They must be merged into the stashed + rate-limit response so the post-call hook can emit + `x-ratelimit-{key}-remaining-tokens` / `-limit-tokens` headers to the + client. """ handler, cache = rate_limiter @@ -966,10 +971,10 @@ async def test_token_rate_limit_headers_present_in_stored_response(rate_limiter) call_type="", ) - response = data.get("litellm_proxy_rate_limit_response") + response = get_request_stash().rate_limit_response assert isinstance( response, dict - ), "Expected litellm_proxy_rate_limit_response to be set after pre-call" + ), "Expected the stashed rate-limit response to be set after pre-call" statuses = response.get("statuses") or [] token_statuses = [s for s in statuses if s.get("rate_limit_type") == "tokens"] @@ -1080,8 +1085,8 @@ async def test_small_tpm_cap_admits_no_max_tokens_request(rate_limiter): call_type="", ) - reserved = (data.get("metadata") or {}).get(TPM_RESERVED_TOKENS_KEY) - assert reserved is not None, "Reservation should have been stashed" + reserved = get_request_stash().reserved_tokens + assert reserved > 0, "Reservation should have been stashed" assert reserved <= 1000 // 2, ( f"Capped floor must keep the reservation well under the 1000 TPM " f"cap; got {reserved}" diff --git a/tests/test_litellm/proxy/test_common_request_processing.py b/tests/test_litellm/proxy/test_common_request_processing.py index 58f81cdad35..3bb84e095a0 100644 --- a/tests/test_litellm/proxy/test_common_request_processing.py +++ b/tests/test_litellm/proxy/test_common_request_processing.py @@ -1,7 +1,7 @@ import asyncio import copy import datetime -from typing import AsyncGenerator, Optional +from typing import AsyncGenerator, Callable, Optional from unittest.mock import AsyncMock, MagicMock, patch import httpx @@ -5111,3 +5111,246 @@ class TestStreamingClientDisconnectBilling: ) proxy_logging_obj._arelease_max_parallel_requests_on_disconnect.assert_awaited_once() + + +def _apply_stream_usage_tracking( + data: dict, + general_settings: dict, + route_type: str, + supports_stream_options: Callable[[], bool] = lambda: True, +) -> None: + from litellm.proxy.common_request_processing import _stream_usage_tracking_updates + + data.update( + _stream_usage_tracking_updates( + data=data, + general_settings=general_settings, + route_type=route_type, + supports_stream_options=supports_stream_options, + ) + ) + + +class TestApplyStreamUsageTracking: + def test_default_injects_usage_and_marks_strip_for_chat_completions(self): + data = {"stream": True, "model": "gpt-5.4-nano"} + + _apply_stream_usage_tracking(data=data, general_settings={}, route_type="acompletion") + + assert data["stream_options"] == {"include_usage": True} + assert data["_litellm_strip_stream_usage"] is True + + def test_default_preserves_other_client_stream_options_keys(self): + data = {"stream": True, "stream_options": {"include_obfuscation": True}} + + _apply_stream_usage_tracking(data=data, general_settings={}, route_type="acompletion") + + assert data["stream_options"] == {"include_obfuscation": True, "include_usage": True} + assert data["_litellm_strip_stream_usage"] is True + + def test_client_requested_usage_is_left_untouched_and_not_stripped(self): + data = {"stream": True, "stream_options": {"include_usage": True}} + + _apply_stream_usage_tracking(data=data, general_settings={}, route_type="acompletion") + + assert data["stream_options"] == {"include_usage": True} + assert "_litellm_strip_stream_usage" not in data + + def test_client_include_usage_false_is_overridden_and_stripped(self): + data = {"stream": True, "stream_options": {"include_usage": False}} + + _apply_stream_usage_tracking(data=data, general_settings={}, route_type="acompletion") + + assert data["stream_options"]["include_usage"] is True + assert data["_litellm_strip_stream_usage"] is True + + def test_explicit_false_flag_disables_injection_entirely(self): + data = {"stream": True} + + _apply_stream_usage_tracking( + data=data, + general_settings={"always_include_stream_usage": False}, + route_type="acompletion", + ) + + assert "stream_options" not in data + assert "_litellm_strip_stream_usage" not in data + + def test_flag_true_injects_without_strip_marker(self): + data = {"stream": True} + + _apply_stream_usage_tracking( + data=data, + general_settings={"always_include_stream_usage": True}, + route_type="acompletion", + ) + + assert data["stream_options"] == {"include_usage": True} + assert "_litellm_strip_stream_usage" not in data + + def test_flag_true_respects_client_explicit_include_usage_false(self): + data = {"stream": True, "stream_options": {"include_usage": False}} + + _apply_stream_usage_tracking( + data=data, + general_settings={"always_include_stream_usage": True}, + route_type="acompletion", + ) + + assert data["stream_options"] == {"include_usage": False} + assert "_litellm_strip_stream_usage" not in data + + def test_default_does_not_touch_non_chat_completion_routes(self): + data = {"stream": True} + + _apply_stream_usage_tracking(data=data, general_settings={}, route_type="anthropic_messages") + + assert "stream_options" not in data + assert "_litellm_strip_stream_usage" not in data + + def test_non_streaming_request_is_untouched(self): + data = {"model": "gpt-5.4-nano"} + + _apply_stream_usage_tracking(data=data, general_settings={}, route_type="acompletion") + + assert "stream_options" not in data + assert "_litellm_strip_stream_usage" not in data + + def test_default_skips_injection_when_provider_lacks_stream_options_support(self): + data = {"stream": True, "model": "bytez-model"} + + _apply_stream_usage_tracking( + data=data, + general_settings={}, + route_type="acompletion", + supports_stream_options=lambda: False, + ) + + assert "stream_options" not in data + assert "_litellm_strip_stream_usage" not in data + + def test_client_supplied_strip_marker_is_neutralized(self): + data = { + "stream": True, + "stream_options": {"include_usage": True}, + "_litellm_strip_stream_usage": True, + } + + _apply_stream_usage_tracking(data=data, general_settings={}, route_type="acompletion") + + assert data["_litellm_strip_stream_usage"] is False + assert data["stream_options"] == {"include_usage": True} + + def test_client_supplied_strip_marker_is_neutralized_with_flag_true(self): + data = { + "stream": True, + "stream_options": {"include_usage": True}, + "_litellm_strip_stream_usage": True, + } + + _apply_stream_usage_tracking( + data=data, + general_settings={"always_include_stream_usage": True}, + route_type="acompletion", + ) + + assert data["_litellm_strip_stream_usage"] is False + + def test_client_supplied_strip_marker_is_neutralized_on_non_streaming_request(self): + data = {"_litellm_strip_stream_usage": True} + + _apply_stream_usage_tracking(data=data, general_settings={}, route_type="acompletion") + + assert data["_litellm_strip_stream_usage"] is False + + +class TestModelDeploymentsSupportStreamOptions: + def _support(self, model, llm_router=None, team_id=None) -> bool: + from litellm.proxy.common_request_processing import ( + _model_deployments_support_stream_options, + ) + + return _model_deployments_support_stream_options(model=model, llm_router=llm_router, team_id=team_id) + + def test_openai_compatible_deployment_supports_stream_options(self): + router = litellm.Router( + model_list=[ + { + "model_name": "azure-nano", + "litellm_params": { + "model": "azure/gpt-5.4-nano", + "api_key": "fake", + "api_base": "https://example.openai.azure.com", + }, + } + ] + ) + + assert self._support("azure-nano", router) is True + + def test_deployment_on_provider_rejecting_stream_options_is_not_injected(self): + router = litellm.Router( + model_list=[ + { + "model_name": "tiny", + "litellm_params": {"model": "bytez/openai-community/gpt2", "api_key": "fake"}, + } + ] + ) + + assert self._support("tiny", router) is False + + def test_mixed_provider_model_group_is_not_injected(self): + router = litellm.Router( + model_list=[ + { + "model_name": "mixed", + "litellm_params": {"model": "openai/gpt-4o", "api_key": "fake"}, + }, + { + "model_name": "mixed", + "litellm_params": {"model": "oci/cohere.command-r-plus", "api_key": "fake"}, + }, + ] + ) + + assert self._support("mixed", router) is False + + def test_wildcard_route_resolves_provider_support(self): + router = litellm.Router( + model_list=[ + { + "model_name": "openai/*", + "litellm_params": {"model": "openai/*", "api_key": "fake"}, + } + ] + ) + + assert self._support("openai/gpt-4o", router) is True + + def test_provider_prefixed_model_without_router_is_resolved_directly(self): + assert self._support("openai/gpt-4o", None) is True + assert self._support("bytez/openai-community/gpt2", None) is False + + def test_unmapped_model_name_is_not_injected(self): + assert self._support("some-unmapped-public-alias", None) is False + + def test_team_alias_model_resolves_with_team_id(self): + router = litellm.Router( + model_list=[ + { + "model_name": "model_name_team-1_8b6a0b3f", + "litellm_params": {"model": "azure/gpt-5.4-nano", "api_key": "fake"}, + "model_info": { + "team_id": "team-1", + "team_public_model_name": "team-gpt", + }, + } + ] + ) + + assert self._support("team-gpt", router, team_id="team-1") is True + assert self._support("team-gpt", router, team_id=None) is False + + def test_non_string_model_is_not_injected(self): + assert self._support(None, None) is False diff --git a/tests/test_litellm/proxy/test_proxy_server.py b/tests/test_litellm/proxy/test_proxy_server.py index 5646d202e31..b9a33bd2cef 100644 --- a/tests/test_litellm/proxy/test_proxy_server.py +++ b/tests/test_litellm/proxy/test_proxy_server.py @@ -10572,3 +10572,109 @@ async def test_startup_survives_database_read_failure_for_coordination_redis(): ) assert result is None + + +def _stream_usage_test_chunks(): + from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices, Usage + + content_chunk = ModelResponseStream( + model="gpt-5.4-nano", + choices=[StreamingChoices(delta=Delta(content="pong"))], + ) + finish_chunk = ModelResponseStream( + model="gpt-5.4-nano", + choices=[StreamingChoices(finish_reason="stop")], + ) + usage_chunk = ModelResponseStream(model="gpt-5.4-nano", choices=[]) + usage_chunk.usage = Usage(prompt_tokens=50, completion_tokens=188, total_tokens=238) + return content_chunk, finish_chunk, usage_chunk + + +def _stream_usage_generator_chunks(): + from litellm.types.utils import ModelResponseStream + + content_chunk, finish_chunk, usage_chunk = _stream_usage_test_chunks() + prompt_filter_chunk = ModelResponseStream(model="gpt-5.4-nano", choices=[]) + return prompt_filter_chunk, content_chunk, finish_chunk, usage_chunk + + +def test_is_injected_stream_usage_artifact(): + from litellm.proxy.proxy_server import _is_injected_stream_usage_artifact + from litellm.types.utils import ModelResponseStream, Usage + + content_chunk, finish_chunk, empty_choices_usage_chunk = _stream_usage_test_chunks() + assert _is_injected_stream_usage_artifact(empty_choices_usage_chunk) is True + + synthetic_final_chunk = ModelResponseStream(model="gpt-5.4-nano") + synthetic_final_chunk.usage = Usage(prompt_tokens=50, completion_tokens=188, total_tokens=238) + assert _is_injected_stream_usage_artifact(synthetic_final_chunk) is True + + azure_prompt_filter_chunk = ModelResponseStream(model="gpt-5.4-nano", choices=[]) + assert _is_injected_stream_usage_artifact(azure_prompt_filter_chunk) is True + + assert _is_injected_stream_usage_artifact(content_chunk) is False + assert _is_injected_stream_usage_artifact(finish_chunk) is False + + content_chunk_with_usage, finish_chunk_with_usage, _ = _stream_usage_test_chunks() + content_chunk_with_usage.usage = Usage(prompt_tokens=50, completion_tokens=188, total_tokens=238) + finish_chunk_with_usage.usage = Usage(prompt_tokens=50, completion_tokens=188, total_tokens=238) + assert _is_injected_stream_usage_artifact(content_chunk_with_usage) is False + assert _is_injected_stream_usage_artifact(finish_chunk_with_usage) is False + + assert _is_injected_stream_usage_artifact({"usage": {"prompt_tokens": 1}}) is False + + +async def _collect_async_data_generator_frames(request_data: dict) -> list: + from litellm.proxy.proxy_server import async_data_generator + from litellm.proxy.utils import ProxyLogging + + chunks = _stream_usage_generator_chunks() + + class MockStream: + def __aiter__(self): + return self._stream() + + async def _stream(self): + for chunk in chunks: + yield chunk + + async def aclose(self): + pass + + mock_proxy_logging_obj = MagicMock(spec=ProxyLogging) + mock_proxy_logging_obj.needs_iterator_wrap.return_value = False + mock_proxy_logging_obj.needs_per_chunk_streaming_hook.return_value = False + mock_proxy_logging_obj.post_call_failure_hook = AsyncMock() + + with patch("litellm.proxy.proxy_server.proxy_logging_obj", mock_proxy_logging_obj): + with patch.object(proxy_server_module.ProxyLogging, "_fire_deferred_stream_logging"): + return [ + frame.decode("utf-8") if isinstance(frame, bytes) else frame + async for frame in async_data_generator( + MockStream(), MagicMock(spec=UserAPIKeyAuth), request_data + ) + ] + + +@pytest.mark.asyncio +async def test_async_data_generator_strips_injected_usage_chunk(): + frames = await _collect_async_data_generator_frames( + {"model": "gpt-5.4-nano", "_litellm_strip_stream_usage": True} + ) + + data_frames = [frame for frame in frames if frame.startswith("data: {")] + assert len(data_frames) == 2 + assert any("pong" in frame for frame in data_frames) + assert any("finish_reason" in frame for frame in data_frames) + assert not any('"usage"' in frame for frame in data_frames) + assert frames[-1] == "data: [DONE]\n\n" + + +@pytest.mark.asyncio +async def test_async_data_generator_forwards_usage_chunk_without_strip_marker(): + frames = await _collect_async_data_generator_frames({"model": "gpt-5.4-nano"}) + + data_frames = [frame for frame in frames if frame.startswith("data: {")] + assert len(data_frames) == 4 + assert any('"usage"' in frame and '"completion_tokens":188' in frame.replace(" ", "") for frame in data_frames) + assert frames[-1] == "data: [DONE]\n\n" diff --git a/tests/test_litellm/test_rate_limit_error_unification.py b/tests/test_litellm/test_rate_limit_error_unification.py index 8287e82ded0..99e9981857c 100644 --- a/tests/test_litellm/test_rate_limit_error_unification.py +++ b/tests/test_litellm/test_rate_limit_error_unification.py @@ -881,7 +881,6 @@ class TestProxyHooksActuallyRaiseProxyRateLimitError: user_api_key_dict=UserAPIKeyAuth(api_key="sk-test-v3"), priority="default", saturation=0.99, - data={}, ) e = exc_info.value assert e.status_code == 429 diff --git a/tests/test_litellm/types/test_types_utils.py b/tests/test_litellm/types/test_types_utils.py index 21f28f54b8b..320c46aed3b 100644 --- a/tests/test_litellm/types/test_types_utils.py +++ b/tests/test_litellm/types/test_types_utils.py @@ -321,29 +321,6 @@ class TestNativeFinishReason: assert choice.provider_specific_fields["native_finish_reason"] == "MAX_TOKENS" -def test_parallel_request_limiter_internal_fields_in_all_litellm_params(): - """ - Regression test: internal fields written by parallel_request_limiter_v3 must - be in all_litellm_params so they are stripped before forwarding to upstream - providers. If missing, they are sent as extra body parameters and providers - like OpenAI reject the request with a 400 invalid_request_error. - """ - from litellm.types.utils import all_litellm_params - - internal_fields = [ - "_litellm_rate_limit_descriptors", - "_litellm_tpm_reserved_tokens", - "_litellm_tpm_reserved_model", - "_litellm_tpm_reserved_scopes", - "_litellm_tpm_reservation_released", - ] - for field in internal_fields: - assert field in all_litellm_params, ( - f"{field!r} is not in all_litellm_params. " - "It will be forwarded to upstream providers and cause 400 errors." - ) - - def test_delta_maps_reasoning_to_reasoning_content(): """ Test that Delta maps 'reasoning' field to 'reasoning_content'. diff --git a/ui/litellm-dashboard/src/app/(dashboard)/users/_components/default-user-settings/DefaultUserSettingsForm.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/users/_components/default-user-settings/DefaultUserSettingsForm.test.tsx index bfdcc70fb0c..cc24931b2b3 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/users/_components/default-user-settings/DefaultUserSettingsForm.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/users/_components/default-user-settings/DefaultUserSettingsForm.test.tsx @@ -149,6 +149,34 @@ describe("DefaultUserSettingsForm", () => { expect(updateSettings).toHaveBeenCalledWith({ ...SAVED_BODY, max_budget: 250 }); }); + it("saves a sub-cent budget the browser would veto under a 0.01 step", async () => { + const user = userEvent.setup(); + const { updateSettings } = renderForm(); + + await enterEditMode(user); + const budget: HTMLInputElement = await screen.findByLabelText("Max Budget (USD)"); + await user.clear(budget); + await user.type(budget, "0.001"); + + const teamBudget: HTMLInputElement = screen.getByLabelText("Max Budget in Team (USD)"); + await user.clear(teamBudget); + await user.type(teamBudget, "0.002"); + + // jsdom never blocks the submit itself, so assert the constraint the real browser + // enforces before handleSubmit ever runs + expect(budget.checkValidity()).toBe(true); + expect(teamBudget.checkValidity()).toBe(true); + + await user.click(await saveButton()); + + await waitFor(() => expect(updateSettings).toHaveBeenCalledTimes(1)); + expect(updateSettings).toHaveBeenCalledWith({ + ...SAVED_BODY, + max_budget: 0.001, + teams: [{ team_id: "team-alpha", max_budget_in_team: 0.002, user_role: "user" }], + }); + }); + it("clears an emptied budget with null", async () => { const user = userEvent.setup(); const { updateSettings } = renderForm(); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/users/_components/default-user-settings/DefaultUserSettingsForm.tsx b/ui/litellm-dashboard/src/app/(dashboard)/users/_components/default-user-settings/DefaultUserSettingsForm.tsx index b1474e7cd0c..1e0ec6b6998 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/users/_components/default-user-settings/DefaultUserSettingsForm.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/users/_components/default-user-settings/DefaultUserSettingsForm.tsx @@ -133,7 +133,7 @@ const TeamsField = ({ control }: { control: SettingsControl }) => { {({ ref, ...budgetField }) => ( - + )} @@ -249,7 +249,7 @@ const SettingsForm = ({ initialValues, roleOptions, updateSettings, onCancel, on const onSubmit = form.handleSubmit((values) => mutation.mutate(values)); return ( -
+ - {({ ref, ...field }) => } + {({ ref, ...field }) => } { await waitFor(() => expect(screen.queryByLabelText("Organization Name")).not.toBeInTheDocument()); }); + it("creates with a sub-cent max budget the browser would veto under a 0.01 step", async () => { + const user = userEvent.setup(); + const { createOrganization } = renderDialog(); + + await user.type(screen.getByLabelText("Organization Name"), "new-org"); + const budget: HTMLInputElement = screen.getByLabelText("Max Budget (USD)"); + await user.type(budget, "0.001"); + + // jsdom never blocks the submit itself, so assert the constraint the real browser + // enforces before handleSubmit ever runs + expect(budget.checkValidity()).toBe(true); + + await user.click(screen.getByRole("button", { name: "Create Organization" })); + + await waitFor(() => expect(createOrganization).toHaveBeenCalledTimes(1)); + expect(createOrganization.mock.calls[0][0]).toStrictEqual({ + organization_alias: "new-org", + models: [], + max_budget: 0.001, + }); + }); + it("maps selectors and limits into the create body", async () => { const user = userEvent.setup(); const { createOrganization } = renderDialog(); diff --git a/ui/litellm-dashboard/src/components/organization/org-create/OrgCreateDialog.tsx b/ui/litellm-dashboard/src/components/organization/org-create/OrgCreateDialog.tsx index 998d9446365..4e1a00704e3 100644 --- a/ui/litellm-dashboard/src/components/organization/org-create/OrgCreateDialog.tsx +++ b/ui/litellm-dashboard/src/components/organization/org-create/OrgCreateDialog.tsx @@ -79,7 +79,7 @@ export const OrgCreateDialog = ({ Create Organization - + {({ ref, ...field }) => } @@ -97,7 +97,7 @@ export const OrgCreateDialog = ({ - {({ ref, ...field }) => } + {({ ref, ...field }) => } diff --git a/ui/litellm-dashboard/src/components/organization/org-settings/OrgSettingsForm.test.tsx b/ui/litellm-dashboard/src/components/organization/org-settings/OrgSettingsForm.test.tsx index 5bd809bcfd5..4dfd37e3466 100644 --- a/ui/litellm-dashboard/src/components/organization/org-settings/OrgSettingsForm.test.tsx +++ b/ui/litellm-dashboard/src/components/organization/org-settings/OrgSettingsForm.test.tsx @@ -113,6 +113,24 @@ describe("OrgSettingsForm", () => { expect(patchOrganization).toHaveBeenCalledWith("org-1", { organization_alias: "acme-2" }); }); + it("saves a sub-cent max budget the browser would veto under a 0.01 step", async () => { + const user = userEvent.setup(); + const { patchOrganization } = renderForm(); + + const budget: HTMLInputElement = screen.getByLabelText("Max Budget (USD)"); + await user.clear(budget); + await user.type(budget, "0.001"); + + // jsdom never blocks the submit itself, so assert the constraint the real browser + // enforces before handleSubmit ever runs + expect(budget.checkValidity()).toBe(true); + + await user.click(screen.getByRole("button", { name: "Save Changes" })); + + await waitFor(() => expect(patchOrganization).toHaveBeenCalledTimes(1)); + expect(patchOrganization).toHaveBeenCalledWith("org-1", { max_budget: 0.001 }); + }); + it("sends null when a limit is cleared", async () => { const user = userEvent.setup(); const { patchOrganization } = renderForm(); diff --git a/ui/litellm-dashboard/src/components/organization/org-settings/OrgSettingsForm.tsx b/ui/litellm-dashboard/src/components/organization/org-settings/OrgSettingsForm.tsx index fe4965adb3c..affe0ed2d4e 100644 --- a/ui/litellm-dashboard/src/components/organization/org-settings/OrgSettingsForm.tsx +++ b/ui/litellm-dashboard/src/components/organization/org-settings/OrgSettingsForm.tsx @@ -78,7 +78,7 @@ export const OrgSettingsForm = ({ }); return ( - + {({ ref, ...field }) => } @@ -96,7 +96,7 @@ export const OrgSettingsForm = ({ - {({ ref, ...field }) => } + {({ ref, ...field }) => }