From 631c02fe1294b95572f99eb9b053bb35a9a28494 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 30 Jul 2026 14:00:20 -0700 Subject: [PATCH 01/11] refactor(rate-limits): move the v3 limiter per-request stash off request metadata onto a ContextVar The v3 parallel-request limiter stashed its per-request bookkeeping (TPM reservation, descriptors, parallel slot, rate-limit response snapshot, released flag) in the request body's metadata channels. On routes where metadata is a provider request parameter (Responses API and the other LITELLM_METADATA_ROUTES) that leaked internal keys upstream and produced HTTP 400s, and it required denylist stripping plus dual-channel writes to contain. The stash now lives on an asyncio ContextVar holding a single typed RequestRateLimiterStash per request. The pre-call hook writes it, and the success/failure callbacks, disconnect release, and post-call hooks read and clear the same shared instance, which keeps the refund and slot release idempotent across sibling callbacks. The request body is never touched, so the stash-key stripping, the metadata mirror writes, and the all_litellm_params denylist entries are removed --- litellm/proxy/common_request_processing.py | 4 +- .../proxy/hooks/dynamic_rate_limiter_v3.py | 15 +- .../hooks/parallel_request_limiter_v3.py | 572 ++++-------------- litellm/proxy/utils.py | 3 +- litellm/types/utils.py | 6 - .../hooks/test_dynamic_rate_limiter_v3.py | 1 - .../hooks/test_parallel_request_limiter_v3.py | 331 ++++++---- .../test_proxy_rate_limit_provider_field.py | 2 - .../proxy/hooks/test_rate_limiter_toctou.py | 3 - .../proxy/hooks/test_tpm_concurrent.py | 125 ++-- tests/test_litellm/types/test_types_utils.py | 23 - 11 files changed, 397 insertions(+), 688 deletions(-) diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index f5a50d1697a..d265313fdbe 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -2730,9 +2730,7 @@ class ProxyBaseLLMRequestProcessing: and proxy_logging_obj is not None and user_api_key_dict is not None ): - await proxy_logging_obj._arelease_max_parallel_requests_on_disconnect( - user_api_key_dict, request_data - ) + await proxy_logging_obj._arelease_max_parallel_requests_on_disconnect(user_api_key_dict) if hasattr(response, "aclose"): try: diff --git a/litellm/proxy/hooks/dynamic_rate_limiter_v3.py b/litellm/proxy/hooks/dynamic_rate_limiter_v3.py index 6e4a6fe1a51..1aaeda6ba95 100644 --- a/litellm/proxy/hooks/dynamic_rate_limiter_v3.py +++ b/litellm/proxy/hooks/dynamic_rate_limiter_v3.py @@ -21,7 +21,9 @@ from litellm.proxy.common_utils.proxy_rate_limit_error import ( from litellm.proxy.hooks.parallel_request_limiter_v3 import ( RateLimitDescriptor, RateLimitDescriptorRateLimitObject, + RateLimitResponse, _PROXY_MaxParallelRequestsHandler_v3, + get_or_create_request_stash, ) from litellm.proxy.hooks.rate_limiter_utils import ( convert_priority_to_percent, @@ -373,7 +375,6 @@ class _PROXY_DynamicRateLimitHandlerV3(CustomLogger): user_api_key_dict: UserAPIKeyAuth, priority: Optional[str], saturation: float, - data: dict, ) -> None: """ Check rate limits using THREE-PHASE approach to prevent partial increments. @@ -400,7 +401,6 @@ class _PROXY_DynamicRateLimitHandlerV3(CustomLogger): user_api_key_dict: User authentication info priority: User's priority level saturation: Current saturation level - data: Request data dictionary Raises: HTTPException: If any limit is exceeded @@ -550,12 +550,12 @@ class _PROXY_DynamicRateLimitHandlerV3(CustomLogger): parent_otel_span=user_api_key_dict.parent_otel_span, read_only=False, ) - data["litellm_proxy_rate_limit_response"] = { - "overall_code": atomic_response["overall_code"], - "statuses": atomic_response["statuses"] + priority_tracking_response["statuses"], - } + get_or_create_request_stash().rate_limit_response = RateLimitResponse( + overall_code=atomic_response["overall_code"], + statuses=atomic_response["statuses"] + priority_tracking_response["statuses"], + ) else: - data["litellm_proxy_rate_limit_response"] = atomic_response + get_or_create_request_stash().rate_limit_response = atomic_response async def async_pre_call_hook( self, @@ -632,7 +632,6 @@ class _PROXY_DynamicRateLimitHandlerV3(CustomLogger): user_api_key_dict=user_api_key_dict, priority=priority, saturation=saturation, - data=data, ) except HTTPException: diff --git a/litellm/proxy/hooks/parallel_request_limiter_v3.py b/litellm/proxy/hooks/parallel_request_limiter_v3.py index f72492881b3..9d7423166ad 100644 --- a/litellm/proxy/hooks/parallel_request_limiter_v3.py +++ b/litellm/proxy/hooks/parallel_request_limiter_v3.py @@ -8,16 +8,18 @@ import asyncio import binascii import os import uuid +from contextvars import ContextVar +from dataclasses import dataclass, field from datetime import datetime from typing import ( TYPE_CHECKING, Any, Callable, Dict, + FrozenSet, List, Literal, Optional, - Set, Tuple, TypedDict, Union, @@ -290,53 +292,11 @@ DEFAULT_CHARS_PER_TOKEN = 4 # (baseline floor) and to the smallest configured TPM limit (capped floor for # small per-tenant TPM caps). _TPM_FLOOR_FRACTION = 4 -# Stash for the reserved-token count on the request data dict so success/ -# failure callbacks can reconcile against the upfront reservation. -TPM_RESERVED_TOKENS_KEY = "_litellm_tpm_reserved_tokens" -# Stash for the model identifier the reservation was charged against. -# Reconciliation must target the same key that was incremented at reservation -TPM_RESERVED_MODEL_KEY = "_litellm_tpm_reserved_model" -# Stash for the (scope_key, scope_value) pairs whose :tokens counter the -# upfront reservation incremented. Reconciliation applies the delta to these -# scopes only; scopes without a configured TPM limit were never charged at -# pre-call and must receive the full actual usage instead of the delta — -# otherwise their counters drift negative whenever actual < reserved. -TPM_RESERVED_SCOPES_KEY = "_litellm_tpm_reserved_scopes" -# Idempotency marker for the reservation refund path. Set when any failure -# callback releases the reservation so the next callback in the same flow -# (e.g. async_log_failure_event firing after async_post_call_failure_hook) -# does not double-refund. -TPM_RESERVATION_RELEASED_KEY = "_litellm_tpm_reservation_released" -RATE_LIMIT_DESCRIPTORS_KEY = "_litellm_rate_limit_descriptors" -# Pre-call RateLimitResponse stashed here so streaming success logging can -# mirror ``x-ratelimit-*`` headers into the SLP. Streaming exits -# common_request_processing before ``async_post_call_success_hook`` runs. -RATE_LIMIT_RESPONSE_KEY = "_litellm_proxy_rate_limit_response" -# Holds the acquisition the pre-call hook made for this request: the slot id -# plus the gauge counter keys it was registered under. The success/failure -# callbacks release only this exact acquisition: those callbacks also fire -# for requests rejected at pre-call (which never acquired a slot), and an -# id-less release would free a slot still owned by another in-flight request -# — every rejection would then raise effective concurrency above the -# configured limit. -MAX_PARALLEL_SLOT_ACQUIRED_KEY = "_litellm_max_parallel_slot_acquired" # How long an acquired slot counts toward the in-flight total before it is # considered leaked (worker crashed without any release callback firing) and # pruned. Also the longest request duration the gauge can track: a request # running longer than this stops occupying its slot. PARALLEL_REQUEST_SLOT_TTL_SECONDS = 3600 -# Stash keys live ONLY in metadata channels — never at the top level of the -# request body. Top-level keys are forwarded as body params to upstream -# providers, which reject unknown fields with 400/429 errors. -_LITELLM_STASH_KEYS: Tuple[str, ...] = ( - TPM_RESERVED_TOKENS_KEY, - TPM_RESERVED_MODEL_KEY, - TPM_RESERVED_SCOPES_KEY, - TPM_RESERVATION_RELEASED_KEY, - RATE_LIMIT_DESCRIPTORS_KEY, - RATE_LIMIT_RESPONSE_KEY, - MAX_PARALLEL_SLOT_ACQUIRED_KEY, -) class RateLimitDescriptorRateLimitObject(TypedDict, total=False): @@ -381,6 +341,46 @@ class RateLimitResponseWithDescriptors(TypedDict): response: RateLimitResponse +@dataclass(slots=True) +class RequestRateLimiterStash: + """ + Per-request bookkeeping the pre-call hook hands to the success/failure/ + disconnect callbacks. Lives on a ContextVar instead of the request body so + it never reaches provider-facing ``metadata`` channels. + + A single mutable instance is shared by every context forked from the + request task (the SDK call, streaming generators, and the logging worker's + captured context all see the same object), which is what makes the + ``reservation_released`` flag and ``parallel_slot`` clearing effective + across sibling callbacks: the first release wins, later callbacks observe + the cleared state. + """ + + rate_limit_response: Optional[RateLimitResponse] = None + parallel_slot: Optional[ParallelSlotAcquisition] = None + reserved_tokens: int = 0 + reserved_model: Optional[str] = None + reserved_scopes: FrozenSet[Tuple[str, str]] = field(default_factory=frozenset) + reservation_released: bool = False + + +_request_stash: ContextVar[Optional[RequestRateLimiterStash]] = ContextVar( + "litellm_v3_rate_limiter_request_stash", default=None +) + + +def get_request_stash() -> Optional[RequestRateLimiterStash]: + return _request_stash.get() + + +def get_or_create_request_stash() -> RequestRateLimiterStash: + stash = _request_stash.get() + if stash is None: + stash = RequestRateLimiterStash() + _request_stash.set(stash) + return stash + + class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): def __init__( self, @@ -2342,12 +2342,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): """ verbose_proxy_logger.debug("Inside Rate Limit Pre-Call Hook") - # Reject caller-supplied stash values before any read/write. Otherwise - # a client can inject ``_litellm_rate_limit_descriptors`` / - # ``_litellm_tpm_reserved_tokens`` in body ``metadata`` and have - # ``async_post_call_failure_hook`` refund TPM counters against scopes - # they name (e.g. another tenant's api_key). - self._strip_stash_keys_from_all_channels(data) + stash = get_or_create_request_stash() ######################################################### # Check if the call type has a specific rate limiter @@ -2443,23 +2438,11 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): requested_model=requested_model, ) else: - # add descriptors to request headers - data["litellm_proxy_rate_limit_response"] = response - # Mirror into metadata so streaming success logging can find - # it via ``kwargs["litellm_params"]["metadata"]``. - self._stash_value_in_metadata_channels( - data=data, - key=RATE_LIMIT_RESPONSE_KEY, - value=response, - ) + stash.rate_limit_response = response if parallel_slot_id is not None: - self._stash_value_in_metadata_channels( - data=data, - key=MAX_PARALLEL_SLOT_ACQUIRED_KEY, - value={ - "slot_id": parallel_slot_id, - "counter_keys": parallel_counter_keys, - }, + stash.parallel_slot = ParallelSlotAcquisition( + slot_id=parallel_slot_id, + counter_keys=parallel_counter_keys, ) # ---------------------------------------------------------------- @@ -2520,38 +2503,29 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): ) if tpm_response["overall_code"] == "OVER_LIMIT": - acquisition = self._get_parallel_slot_acquisition(kwargs=data) + acquisition = stash.parallel_slot if acquisition is not None: await self._release_parallel_request_slots( acquisition=acquisition, parent_otel_span=user_api_key_dict.parent_otel_span, ) - self._clear_parallel_slot_marker(data) + stash.parallel_slot = None self._handle_rate_limit_error( response=tpm_response, descriptors=descriptors, requested_model=requested_model, ) else: - self._stash_value_in_metadata_channels( - data=data, - key=RATE_LIMIT_DESCRIPTORS_KEY, - value=descriptors, - ) # Capture the exact (key, value) scopes the reservation # incremented so post-call reconciliation only applies # the (actual - reserved) delta to those — unreserved # scopes get charged the full actual usage instead. - reserved_scopes: List[Tuple[str, str]] = [ + stash.reserved_tokens = estimated_tokens + stash.reserved_model = requested_model + stash.reserved_scopes = frozenset( (d["key"], d["value"]) for d in descriptors if (d.get("rate_limit") or {}).get("tokens_per_unit") is not None - ] - self._stash_reservation_in_data( - data=data, - estimated_tokens=estimated_tokens, - reserved_model=requested_model, - reserved_scopes=reserved_scopes, ) # Merge TPM statuses into the stored rate-limit response @@ -2559,44 +2533,14 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): # headers reach the client. Without this, the RPM-only # response from should_rate_limit (skip_tpm_check=True) # silently drops all token headers. - stored_response = data.get("litellm_proxy_rate_limit_response") - if isinstance(stored_response, dict): - stored_response.setdefault("statuses", []).extend(tpm_response["statuses"]) + stored_response = stash.rate_limit_response + if stored_response is not None: + stored_response["statuses"].extend(tpm_response["statuses"]) elif tpm_response["statuses"]: - data["litellm_proxy_rate_limit_response"] = tpm_response - # Keep the metadata stash in sync when this is the - # first snapshot written. - self._stash_value_in_metadata_channels( - data=data, - key=RATE_LIMIT_RESPONSE_KEY, - value=tpm_response, - ) + stash.rate_limit_response = tpm_response verbose_proxy_logger.debug(f"TPM tokens reserved: {estimated_tokens} for model {requested_model}") - # Defense-in-depth: scrub any stash key that escaped onto data - # top-level (stale cache hit, router pass, test fixture) before the - # body is forwarded to the provider. - self._strip_stash_keys_from_top_level(data) - - @staticmethod - def _strip_stash_keys_from_top_level(data: Any) -> None: - if not isinstance(data, dict): - return - for stash_key in _LITELLM_STASH_KEYS: - data.pop(stash_key, None) - - @classmethod - def _strip_stash_keys_from_all_channels(cls, data: Any) -> None: - if not isinstance(data, dict): - return - cls._strip_stash_keys_from_top_level(data) - for channel in ("metadata", "litellm_metadata"): - channel_dict = data.get(channel) - if isinstance(channel_dict, dict): - for stash_key in _LITELLM_STASH_KEYS: - channel_dict.pop(stash_key, None) - def _create_pipeline_operations( self, key: str, @@ -2802,203 +2746,6 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): merged[f"{prefix}-limit-{status['rate_limit_type']}"] = status["current_limit"] return merged - @staticmethod - def _stash_value_in_metadata_channels( - data: Dict[str, Any], - key: str, - value: Any, - ) -> None: - for channel in ("metadata", "litellm_metadata"): - existing = data.get(channel) - if isinstance(existing, dict): - existing[key] = value - elif channel == "metadata": - # ``litellm_metadata`` is owned by the router; don't conjure - # it here. - data[channel] = {key: value} - - @classmethod - def _stash_reservation_in_data( - cls, - data: Dict[str, Any], - estimated_tokens: int, - reserved_model: Optional[str], - reserved_scopes: Optional[List[Tuple[str, str]]] = None, - ) -> None: - """ - ``reserved_scopes`` is serialized as a list of [key, value] pairs so - it round-trips through JSON-based metadata transports. - """ - scopes_payload: Optional[List[List[str]]] = [[k, v] for k, v in reserved_scopes] if reserved_scopes else None - - cls._stash_value_in_metadata_channels(data=data, key=TPM_RESERVED_TOKENS_KEY, value=estimated_tokens) - if reserved_model: - cls._stash_value_in_metadata_channels(data=data, key=TPM_RESERVED_MODEL_KEY, value=reserved_model) - if scopes_payload is not None: - cls._stash_value_in_metadata_channels(data=data, key=TPM_RESERVED_SCOPES_KEY, value=scopes_payload) - - @staticmethod - def _lookup_stashed_value( - kwargs: Any, - standard_logging_metadata: Optional[Dict[str, Any]], - key: str, - ) -> Any: - """ - Resolve a stashed value from any metadata channel the request data - can flow through to a callback. Top-level ``kwargs`` is not checked - because stash keys must never live there. - """ - candidate: Any = None - if isinstance(kwargs, dict): - for channel in ("metadata", "litellm_metadata"): - channel_dict = kwargs.get(channel) - if isinstance(channel_dict, dict) and key in channel_dict: - candidate = channel_dict.get(key) - if candidate is not None: - return candidate - litellm_params = kwargs.get("litellm_params") - if isinstance(litellm_params, dict): - lp_metadata = litellm_params.get("metadata") - if isinstance(lp_metadata, dict): - candidate = lp_metadata.get(key) - if candidate is None and isinstance(standard_logging_metadata, dict): - candidate = standard_logging_metadata.get(key) - return candidate - - @classmethod - def _get_reserved_tokens_from_kwargs( - cls, - kwargs: Any, - standard_logging_metadata: Optional[Dict[str, Any]] = None, - ) -> int: - candidate = cls._lookup_stashed_value(kwargs, standard_logging_metadata, TPM_RESERVED_TOKENS_KEY) - try: - return int(candidate or 0) - except (TypeError, ValueError): - return 0 - - @classmethod - def _get_reserved_model_from_kwargs( - cls, - kwargs: Any, - standard_logging_metadata: Optional[Dict[str, Any]] = None, - ) -> Optional[str]: - """ - Resolve the model the upfront reservation was charged against. Used to - target reconciliation at the same key that was incremented, regardless - of whether the router later set a different ``model_group`` in - ``litellm_params.metadata``. - """ - candidate = cls._lookup_stashed_value(kwargs, standard_logging_metadata, TPM_RESERVED_MODEL_KEY) - return candidate if isinstance(candidate, str) and candidate else None - - @classmethod - def _get_reserved_scopes_from_kwargs( - cls, - kwargs: Any, - standard_logging_metadata: Optional[Dict[str, Any]] = None, - ) -> Set[Tuple[str, str]]: - """ - Resolve the (scope_key, scope_value) pairs the upfront reservation - actually charged. Reconciliation distinguishes these from - unreserved scopes — applying the delta to reserved scopes (which - already carry +reserved on the counter) and the full actual to - unreserved ones (which were never charged). - """ - candidate = cls._lookup_stashed_value(kwargs, standard_logging_metadata, TPM_RESERVED_SCOPES_KEY) - if not isinstance(candidate, list): - return set() - scopes: Set[Tuple[str, str]] = set() - for entry in candidate: - if ( - isinstance(entry, (list, tuple)) - and len(entry) == 2 - and isinstance(entry[0], str) - and isinstance(entry[1], str) - ): - scopes.add((entry[0], entry[1])) - return scopes - - @classmethod - def _is_reservation_released( - cls, - kwargs: Any, - standard_logging_metadata: Optional[Dict[str, Any]] = None, - ) -> bool: - """True if a prior callback already refunded this request's reservation.""" - return bool(cls._lookup_stashed_value(kwargs, standard_logging_metadata, TPM_RESERVATION_RELEASED_KEY)) - - @classmethod - def _get_parallel_slot_acquisition( - cls, - kwargs: Any, - standard_logging_metadata: dict[str, Any] | None = None, - ) -> ParallelSlotAcquisition | None: - """The slot acquisition this request's pre-call hook made, if any.""" - candidate = cls._lookup_stashed_value(kwargs, standard_logging_metadata, MAX_PARALLEL_SLOT_ACQUIRED_KEY) - if not isinstance(candidate, dict): - return None - slot_id = candidate.get("slot_id") - counter_keys = candidate.get("counter_keys") - if not isinstance(slot_id, str) or not slot_id: - return None - if not isinstance(counter_keys, list) or not counter_keys: - return None - if not all(isinstance(key, str) and key for key in counter_keys): - return None - return ParallelSlotAcquisition(slot_id=slot_id, counter_keys=counter_keys) - - @staticmethod - def _clear_parallel_slot_marker(data: Any) -> None: - """ - Remove the acquired-slot marker from every metadata channel a sibling - callback might read, so one release per acquire is an invariant even - when multiple callbacks fire for the same request. - """ - if not isinstance(data, dict): - return - for channel in ("metadata", "litellm_metadata"): - channel_dict = data.get(channel) - if isinstance(channel_dict, dict): - channel_dict.pop(MAX_PARALLEL_SLOT_ACQUIRED_KEY, None) - litellm_params = data.get("litellm_params") - if isinstance(litellm_params, dict): - lp_metadata = litellm_params.get("metadata") - if isinstance(lp_metadata, dict): - lp_metadata.pop(MAX_PARALLEL_SLOT_ACQUIRED_KEY, None) - slo = data.get("standard_logging_object") - if isinstance(slo, dict): - slo_meta = slo.get("metadata") - if isinstance(slo_meta, dict): - slo_meta.pop(MAX_PARALLEL_SLOT_ACQUIRED_KEY, None) - - @staticmethod - def _mark_reservation_released(data: Any) -> None: - """ - Stamp the released flag into every metadata channel a sibling - callback might read from. async_post_call_failure_hook receives the - request data dict; async_log_failure_event reads kwargs + - standard_logging_object.metadata. Same dict identity across - ``request_data["metadata"]`` and ``kwargs["litellm_params"]["metadata"]`` - means writes here propagate to the other hook. - """ - if not isinstance(data, dict): - return - for channel in ("metadata", "litellm_metadata"): - existing = data.get(channel) - if isinstance(existing, dict): - existing[TPM_RESERVATION_RELEASED_KEY] = True - litellm_params = data.get("litellm_params") - if isinstance(litellm_params, dict): - lp_metadata = litellm_params.get("metadata") - if isinstance(lp_metadata, dict): - lp_metadata[TPM_RESERVATION_RELEASED_KEY] = True - slo = data.get("standard_logging_object") - if isinstance(slo, dict): - slo_meta = slo.get("metadata") - if isinstance(slo_meta, dict): - slo_meta[TPM_RESERVATION_RELEASED_KEY] = True - def _collect_tpm_scope_targets( self, standard_logging_metadata: Dict[str, Any], @@ -3064,7 +2811,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): def _build_reservation_aware_tpm_ops( self, targets: List[Tuple[str, str]], - reserved_scopes: Set[Tuple[str, str]], + reserved_scopes: FrozenSet[Tuple[str, str]], actual_tokens: int, reserved_tokens: int, ) -> List[RedisPipelineIncrementOperation]: @@ -3139,18 +2886,10 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): if total_tokens == 0: total_tokens = self._aggregate_only_total_tokens(usage=_usage) - reserved_tokens = self._get_reserved_tokens_from_kwargs( - kwargs=kwargs, - standard_logging_metadata=standard_logging_metadata, - ) - reserved_model = self._get_reserved_model_from_kwargs( - kwargs=kwargs, - standard_logging_metadata=standard_logging_metadata, - ) - reserved_scopes = self._get_reserved_scopes_from_kwargs( - kwargs=kwargs, - standard_logging_metadata=standard_logging_metadata, - ) + stash = get_request_stash() + reserved_tokens = stash.reserved_tokens if stash is not None else 0 + reserved_model = stash.reserved_model if stash is not None else None + reserved_scopes: FrozenSet[Tuple[str, str]] = stash.reserved_scopes if stash is not None else frozenset() # Reconciliation must target the same model-scoped counter that the # pre-call reservation incremented. If a reservation was made, # ``reserved_model`` is authoritative; otherwise fall back to the @@ -3206,18 +2945,14 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): try: verbose_proxy_logger.debug("INSIDE parallel request limiter ASYNC SUCCESS LOGGING") - standard_logging_object = kwargs.get("standard_logging_object") or {} - standard_logging_metadata = standard_logging_object.get("metadata") or {} - acquisition = self._get_parallel_slot_acquisition( - kwargs=kwargs, - standard_logging_metadata=standard_logging_metadata, - ) - if acquisition is not None: + stash = get_request_stash() + acquisition = stash.parallel_slot if stash is not None else None + if stash is not None and acquisition is not None: await self._release_parallel_request_slots( acquisition=acquisition, parent_otel_span=litellm_parent_otel_span, ) - self._clear_parallel_slot_marker(kwargs) + stash.parallel_slot = None pipeline_operations = self._build_success_event_pipeline_operations( kwargs=kwargs, @@ -3267,23 +3002,13 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): if not isinstance(kwargs, dict): return - standard_logging_object = kwargs.get("standard_logging_object") - standard_logging_metadata: Optional[Dict[str, Any]] = None - if isinstance(standard_logging_object, dict): - slp_metadata = standard_logging_object.get("metadata") - if isinstance(slp_metadata, dict): - standard_logging_metadata = slp_metadata - - statuses = self._narrow_ratelimit_statuses( - self._lookup_stashed_value( - kwargs=kwargs, - standard_logging_metadata=standard_logging_metadata, - key=RATE_LIMIT_RESPONSE_KEY, - ) - ) + stash = get_request_stash() + rate_limit_response = stash.rate_limit_response if stash is not None else None + statuses = rate_limit_response["statuses"] if rate_limit_response is not None else [] if not statuses: return + standard_logging_object = kwargs.get("standard_logging_object") if isinstance(standard_logging_object, dict): hidden_params = standard_logging_object.get("hidden_params") if not isinstance(hidden_params, dict): @@ -3303,43 +3028,6 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): statuses=statuses, ) - @staticmethod - def _narrow_ratelimit_statuses(stashed: Any) -> List[RateLimitStatus]: - """ - Narrow a stashed ``RateLimitResponse``-shaped dict to a typed - ``statuses`` list. Entries missing any header-write field are dropped; - an empty list means "nothing to mirror". - """ - if not isinstance(stashed, dict): - return [] - raw_statuses = stashed.get("statuses") - if not isinstance(raw_statuses, list): - return [] - narrowed: List[RateLimitStatus] = [] - for entry in raw_statuses: - if not isinstance(entry, dict): - continue - descriptor_key = entry.get("descriptor_key") - rate_limit_type = entry.get("rate_limit_type") - current_limit = entry.get("current_limit") - limit_remaining = entry.get("limit_remaining") - if ( - isinstance(descriptor_key, str) - and rate_limit_type in ("requests", "tokens", "max_parallel_requests") - and isinstance(current_limit, int) - and isinstance(limit_remaining, int) - ): - narrowed.append( - RateLimitStatus( - code=entry.get("code", "OK") if isinstance(entry.get("code"), str) else "OK", - current_limit=current_limit, - limit_remaining=limit_remaining, - rate_limit_type=rate_limit_type, - descriptor_key=descriptor_key, - ) - ) - return narrowed - async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time): """ On failure: decrement max_parallel_requests and refund the upfront @@ -3353,55 +3041,36 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): try: litellm_parent_otel_span: Union[Span, None] = _get_parent_otel_span_from_kwargs(kwargs) - standard_logging_object = kwargs.get("standard_logging_object") or {} - standard_logging_metadata = standard_logging_object.get("metadata") or {} pipeline_operations: List[RedisPipelineIncrementOperation] = [] - acquisition = self._get_parallel_slot_acquisition( - kwargs=kwargs, - standard_logging_metadata=standard_logging_metadata, - ) - if acquisition is not None: + stash = get_request_stash() + acquisition = stash.parallel_slot if stash is not None else None + if stash is not None and acquisition is not None: await self._release_parallel_request_slots( acquisition=acquisition, parent_otel_span=litellm_parent_otel_span, ) - self._clear_parallel_slot_marker(kwargs) + stash.parallel_slot = None # Skip the reservation refund if async_post_call_failure_hook # already released it (proxy-level rejection that also bubbles up # here as an LLM-error callback). max_parallel_requests is its # own counter and is always decremented per call. - already_released = self._is_reservation_released( - kwargs=kwargs, - standard_logging_metadata=standard_logging_metadata, - ) - reserved_tokens = ( - 0 - if already_released - else self._get_reserved_tokens_from_kwargs( - kwargs=kwargs, - standard_logging_metadata=standard_logging_metadata, - ) - ) - if reserved_tokens > 0: + reserved_tokens = 0 + if stash is not None and not stash.reservation_released: + reserved_tokens = stash.reserved_tokens + if stash is not None and reserved_tokens > 0: verbose_proxy_logger.debug(f"Releasing reserved TPM tokens on failure: {reserved_tokens}") # Refund only against the scopes the reservation actually # charged. _build_reservation_aware_tpm_ops with # actual_tokens=0 emits -reserved on reserved scopes and 0 # on unreserved (skipped), so unreserved scopes can't drift - # negative. Targets are derived purely from the reserved - # set so we don't even need to re-collect them from - # metadata. - reserved_scopes = self._get_reserved_scopes_from_kwargs( - kwargs=kwargs, - standard_logging_metadata=standard_logging_metadata, - ) + # negative. pipeline_operations.extend( self._build_reservation_aware_tpm_ops( - targets=list(reserved_scopes), - reserved_scopes=reserved_scopes, + targets=list(stash.reserved_scopes), + reserved_scopes=stash.reserved_scopes, actual_tokens=0, reserved_tokens=reserved_tokens, ) @@ -3412,15 +3081,14 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): increment_list=pipeline_operations, litellm_parent_otel_span=litellm_parent_otel_span, ) - if reserved_tokens > 0: - self._mark_reservation_released(kwargs) + if stash is not None and reserved_tokens > 0: + stash.reservation_released = True except Exception as e: verbose_proxy_logger.exception(f"Error in rate limit failure event: {str(e)}") async def async_release_max_parallel_requests_on_disconnect( self, user_api_key_dict: UserAPIKeyAuth, - request_data: dict | None = None, ) -> None: """ Release the api-key ``max_parallel_requests`` slot that @@ -3432,20 +3100,19 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): client cancels a stream mid-flight, the cancellation surfaces as ``asyncio.CancelledError`` / ``GeneratorExit`` and neither callback runs, so without this the slot leaks per cancelled stream until its - TTL prunes it. ``request_data`` carries the stashed acquisition; - its presence (not the key object's current max_parallel_requests - configuration, which can change mid-request) decides whether there - is anything to release. + TTL prunes it. The stashed acquisition's presence (not the key + object's current max_parallel_requests configuration, which can + change mid-request) decides whether there is anything to release. """ - acquisition = self._get_parallel_slot_acquisition(kwargs=request_data) - if acquisition is None: + stash = get_request_stash() + if stash is None or stash.parallel_slot is None: return await self._release_parallel_request_slots( - acquisition=acquisition, + acquisition=stash.parallel_slot, parent_otel_span=None, ) - self._clear_parallel_slot_marker(request_data) + stash.parallel_slot = None async def async_post_call_success_hook(self, data: dict, user_api_key_dict: UserAPIKeyAuth, response): """ @@ -3454,10 +3121,8 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): try: from pydantic import BaseModel - litellm_proxy_rate_limit_response = cast( - Optional[RateLimitResponse], - data.get("litellm_proxy_rate_limit_response", None), - ) + stash = get_request_stash() + litellm_proxy_rate_limit_response = stash.rate_limit_response if stash is not None else None if litellm_proxy_rate_limit_response is not None: # Update response headers @@ -3502,59 +3167,42 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): rejections, so a leaked slot would occupy the gauge for the full PARALLEL_REQUEST_SLOT_TTL_SECONDS. - Idempotent: the slot release clears the acquisition marker (and slot + Idempotent: the slot release clears the stashed acquisition (and slot removal is a no-op ZREM on a second run), and the TPM refund is - guarded by TPM_RESERVATION_RELEASED_KEY — if both this hook and - async_log_failure_event end up running in the same flow, only the - first release/refund applies. + guarded by the stash's ``reservation_released`` flag — if both this + hook and async_log_failure_event end up running in the same flow, only + the first release/refund applies. """ try: - acquisition = self._get_parallel_slot_acquisition(kwargs=request_data) - if acquisition is not None: + stash = get_request_stash() + if stash is None: + return + if stash.parallel_slot is not None: await self._release_parallel_request_slots( - acquisition=acquisition, + acquisition=stash.parallel_slot, parent_otel_span=user_api_key_dict.parent_otel_span, ) - self._clear_parallel_slot_marker(request_data) + stash.parallel_slot = None - if self._is_reservation_released(kwargs=request_data): + if stash.reservation_released: return - reserved_tokens = self._get_reserved_tokens_from_kwargs(kwargs=request_data) + reserved_tokens = stash.reserved_tokens if reserved_tokens <= 0: return - # Refund directly against the descriptors we reserved against — - # the pre-call hook stashes them in the request-data metadata - # channels before success/failure callbacks run. - stashed = self._lookup_stashed_value( - kwargs=request_data, - standard_logging_metadata=None, - key=RATE_LIMIT_DESCRIPTORS_KEY, + ops = self._build_reservation_aware_tpm_ops( + targets=list(stash.reserved_scopes), + reserved_scopes=stash.reserved_scopes, + actual_tokens=0, + reserved_tokens=reserved_tokens, ) - descriptors: List[RateLimitDescriptor] = stashed if isinstance(stashed, list) else [] - ops: List[RedisPipelineIncrementOperation] = [] - for descriptor in descriptors: - rate_limit = descriptor.get("rate_limit") or {} - if rate_limit.get("tokens_per_unit") is None: - continue - ops.append( - RedisPipelineIncrementOperation( - key=self.create_rate_limit_keys( - descriptor["key"], - descriptor["value"], - "tokens", - ), - increment_value=-reserved_tokens, - ttl=self.window_size, - ) - ) if ops: verbose_proxy_logger.debug(f"Releasing reserved TPM tokens on proxy-level rejection: {reserved_tokens}") await self.internal_usage_cache.dual_cache.async_increment_cache_pipeline( increment_list=ops, litellm_parent_otel_span=user_api_key_dict.parent_otel_span, ) - self._mark_reservation_released(request_data) + stash.reservation_released = True except Exception as e: verbose_proxy_logger.exception(f"Error releasing TPM reservation on post-call failure: {e}") return None diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 924189fed4b..b1196ecfe1d 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -2730,7 +2730,6 @@ class ProxyLogging: async def _arelease_max_parallel_requests_on_disconnect( self, user_api_key_dict: UserAPIKeyAuth, - request_data: dict | None = None, ) -> None: """ Release the api-key max_parallel_requests slot when a streaming @@ -2750,7 +2749,7 @@ class ProxyLogging: limiter = self.get_proxy_hook("parallel_request_limiter") if not isinstance(limiter, _PROXY_MaxParallelRequestsHandler_v3): return - await limiter.async_release_max_parallel_requests_on_disconnect(user_api_key_dict, request_data) + await limiter.async_release_max_parallel_requests_on_disconnect(user_api_key_dict) def _init_response_taking_too_long_task(self, data: Optional[dict] = None): """ diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 9df44c6202c..ee70e615e88 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -3280,7 +3280,6 @@ all_litellm_params = ( "mock_response", "mock_timeout", "disable_add_transform_inline_image_block", - "litellm_proxy_rate_limit_response", "api_key", "api_version", "prompt_id", @@ -3374,11 +3373,6 @@ all_litellm_params = ( "enable_tag_filtering", "enable_json_schema_validation", "use_xai_oauth", - "_litellm_rate_limit_descriptors", - "_litellm_tpm_reserved_tokens", - "_litellm_tpm_reserved_model", - "_litellm_tpm_reserved_scopes", - "_litellm_tpm_reservation_released", "auto_router_config_path", "auto_router_config", "auto_router_default_model", diff --git a/tests/test_litellm/proxy/hooks/test_dynamic_rate_limiter_v3.py b/tests/test_litellm/proxy/hooks/test_dynamic_rate_limiter_v3.py index 00ed7e8cd6c..c8176ca6337 100644 --- a/tests/test_litellm/proxy/hooks/test_dynamic_rate_limiter_v3.py +++ b/tests/test_litellm/proxy/hooks/test_dynamic_rate_limiter_v3.py @@ -1754,7 +1754,6 @@ async def test_priority_429_includes_model_name_and_configured_limits(): user_api_key_dict=user, priority="prod", saturation=0.95, - data={"model": model}, ) assert exc_info.value.status_code == 429 diff --git a/tests/test_litellm/proxy/hooks/test_parallel_request_limiter_v3.py b/tests/test_litellm/proxy/hooks/test_parallel_request_limiter_v3.py index 9337050b61c..d546b629f0f 100644 --- a/tests/test_litellm/proxy/hooks/test_parallel_request_limiter_v3.py +++ b/tests/test_litellm/proxy/hooks/test_parallel_request_limiter_v3.py @@ -18,8 +18,11 @@ from litellm import Router from litellm.caching.caching import DualCache from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.hooks.parallel_request_limiter_v3 import ( - MAX_PARALLEL_SLOT_ACQUIRED_KEY, PARALLEL_REQUEST_SLOT_TTL_SECONDS, + ParallelSlotAcquisition, + _request_stash, + get_or_create_request_stash, + get_request_stash, ) from litellm.proxy.hooks.parallel_request_limiter_v3 import ( _PROXY_MaxParallelRequestsHandler_v3 as _PROXY_MaxParallelRequestsHandler, @@ -52,6 +55,13 @@ def time_controller(monkeypatch): return controller +@pytest.fixture(autouse=True) +def _isolated_request_stash(): + token = _request_stash.set(None) + yield + _request_stash.reset(token) + + @pytest.mark.parametrize( "throttle_pct, expected_rpm, expected_tpm", [ @@ -673,35 +683,36 @@ async def test_async_log_failure_event_v3(): await _seed_max_parallel_requests_slots(local_cache, counter_key, ["slot-a", "slot-b"]) - def kwargs_with_slot(slot_id): - return { - "metadata": { - MAX_PARALLEL_SLOT_ACQUIRED_KEY: { - "slot_id": slot_id, - "counter_keys": [counter_key], - } - }, - "standard_logging_object": {"metadata": {"user_api_key_hash": _api_key}}, - } + def seed_slot(slot_id): + get_or_create_request_stash().parallel_slot = ParallelSlotAcquisition( + slot_id=slot_id, + counter_keys=[counter_key], + ) + + kwargs = {"standard_logging_object": {"metadata": {"user_api_key_hash": _api_key}}} async def in_flight(): return parallel_request_handler._gauge_in_flight_from_cache_value( await local_cache.async_get_cache(key=counter_key) ) + seed_slot("slot-a") await parallel_request_handler.async_log_failure_event( - kwargs=kwargs_with_slot("slot-a"), response_obj=None, start_time=None, end_time=None + kwargs=kwargs, response_obj=None, start_time=None, end_time=None ) + assert get_request_stash().parallel_slot is None assert await in_flight() == 1 for slot_id in ("slot-a", "slot-unknown", "slot-a"): + seed_slot(slot_id) await parallel_request_handler.async_log_failure_event( - kwargs=kwargs_with_slot(slot_id), response_obj=None, start_time=None, end_time=None + kwargs=kwargs, response_obj=None, start_time=None, end_time=None ) assert await in_flight() == 1 + seed_slot("slot-b") await parallel_request_handler.async_log_failure_event( - kwargs=kwargs_with_slot("slot-b"), response_obj=None, start_time=None, end_time=None + kwargs=kwargs, response_obj=None, start_time=None, end_time=None ) assert await in_flight() == 0 @@ -803,8 +814,9 @@ async def test_rejected_request_does_not_consume_parallel_slot_v3(): data=admitted_data, call_type="", ) - acquisition = admitted_data["metadata"][MAX_PARALLEL_SLOT_ACQUIRED_KEY] - assert isinstance(acquisition, dict) + assert "metadata" not in admitted_data + acquisition = get_request_stash().parallel_slot + assert acquisition is not None assert isinstance(acquisition["slot_id"], str) and acquisition["slot_id"] assert acquisition["counter_keys"] == [f"{{api_key:{_api_key}}}:max_parallel_requests"] @@ -816,10 +828,10 @@ async def test_rejected_request_does_not_consume_parallel_slot_v3(): data={"model": "gpt-3.5-turbo"}, call_type="", ) + assert get_request_stash().parallel_slot == acquisition await handler.async_log_failure_event( kwargs={ - "metadata": {MAX_PARALLEL_SLOT_ACQUIRED_KEY: acquisition}, "standard_logging_object": {"metadata": {"user_api_key_hash": _api_key}}, }, response_obj=None, @@ -866,8 +878,8 @@ async def test_parallel_gauge_uses_atomic_redis_script_v3(): data=data, call_type="", ) - stashed_acquisition = data["metadata"][MAX_PARALLEL_SLOT_ACQUIRED_KEY] - assert isinstance(stashed_acquisition, dict) + stashed_acquisition = get_request_stash().parallel_slot + assert stashed_acquisition is not None stashed_slot_id = stashed_acquisition["slot_id"] assert isinstance(stashed_slot_id, str) and stashed_slot_id assert stashed_acquisition["counter_keys"] == [counter_key] @@ -882,7 +894,7 @@ async def test_parallel_gauge_uses_atomic_redis_script_v3(): ) gauge_statuses = [ s - for s in data["litellm_proxy_rate_limit_response"]["statuses"] + for s in get_request_stash().rate_limit_response["statuses"] if s["rate_limit_type"] == "max_parallel_requests" ] assert gauge_statuses == [ @@ -3102,14 +3114,12 @@ async def test_project_model_rate_limits_not_triggered_for_other_model_v3(): @pytest.mark.asyncio -async def test_pre_call_hook_does_not_leak_internal_stash_to_request_body(): - """Regression for #27001: stash keys must stay in metadata, never on - the top level of ``data`` (which gets forwarded as the provider body).""" - from litellm.proxy.hooks.parallel_request_limiter_v3 import ( - _LITELLM_STASH_KEYS, - RATE_LIMIT_DESCRIPTORS_KEY, - TPM_RESERVED_TOKENS_KEY, - ) +async def test_pre_call_hook_keeps_internal_stash_out_of_request_body(): + """Regression for #27001 / #35197: the limiter's per-request bookkeeping + must never touch the outgoing request body — no top-level keys and no + created or mutated ``metadata`` / ``litellm_metadata`` buckets. The + reservation must land on the ContextVar stash instead.""" + import copy _api_key = hash_token("sk-leak-regression") user_api_key_dict = UserAPIKeyAuth( @@ -3149,6 +3159,7 @@ async def test_pre_call_hook_does_not_leak_internal_stash_to_request_body(): "messages": [{"role": "user", "content": "hello"}], "max_tokens": 10, } + body_before = copy.deepcopy(data) await parallel_request_handler.async_pre_call_hook( user_api_key_dict=user_api_key_dict, @@ -3157,24 +3168,131 @@ async def test_pre_call_hook_does_not_leak_internal_stash_to_request_body(): call_type="completion", ) - leaked = [k for k in _LITELLM_STASH_KEYS if k in data] - assert not leaked, f"stash keys leaked to top level: {leaked}" + assert data == body_before - metadata = data.get("metadata") or {} - assert metadata.get(TPM_RESERVED_TOKENS_KEY) - assert isinstance(metadata.get(RATE_LIMIT_DESCRIPTORS_KEY), list) + stash = get_request_stash() + assert stash is not None + assert stash.reserved_tokens > 0 + assert stash.reserved_model == "gpt-4o-mini" + assert stash.reserved_scopes == frozenset({("api_key", _api_key)}) @pytest.mark.asyncio -async def test_pre_call_hook_rejects_caller_supplied_stash_values(): - """Caller cannot pre-populate stash keys in body metadata to drive a - later TPM refund against an arbitrary scope.""" - from litellm.proxy.hooks.parallel_request_limiter_v3 import ( - _LITELLM_STASH_KEYS, - RATE_LIMIT_DESCRIPTORS_KEY, - TPM_RESERVED_TOKENS_KEY, +@pytest.mark.parametrize("caller_metadata", [None, {"user_tag": "campaign-42"}]) +async def test_responses_route_body_untouched_by_pre_call_hook(caller_metadata): + """Regression for #35197: on routes where ``metadata`` is a provider + request parameter (Responses API), the pre-call hook must forward the + body byte-identical — creating or adding to ``metadata`` / + ``litellm_metadata`` produced upstream HTTP 400s.""" + import copy + + _api_key = hash_token("sk-responses-regression") + user_api_key_dict = UserAPIKeyAuth( + api_key=_api_key, + tpm_limit=1000, + rpm_limit=5, + ) + local_cache = DualCache() + handler = _PROXY_MaxParallelRequestsHandler( + internal_usage_cache=InternalUsageCache(local_cache), ) + data: Dict[str, Any] = { + "model": "gpt-4o-mini", + "input": "hello", + } + if caller_metadata is not None: + data["metadata"] = dict(caller_metadata) + body_before = copy.deepcopy(data) + + await handler.async_pre_call_hook( + user_api_key_dict=user_api_key_dict, + cache=local_cache, + data=data, + call_type="aresponses", + ) + + assert data == body_before + if caller_metadata is None: + assert "metadata" not in data + else: + assert data["metadata"] == caller_metadata + assert "litellm_metadata" not in data + + stash = get_request_stash() + assert stash is not None + assert stash.reserved_tokens > 0 + assert stash.rate_limit_response is not None + + +@pytest.mark.asyncio +async def test_chat_tpm_refund_and_slot_release_via_context_stash(monkeypatch): + """ + Full chat lifecycle with no body stashing: pre-call reserves TPM tokens + and acquires a parallel slot on the ContextVar stash; the failure + callback refunds the reservation and frees the slot exactly once — a + second failure callback for the same request must not double-refund the + :tokens counter or double-release the gauge. + """ + monkeypatch.delenv("LITELLM_TPM_TOKEN_RESERVATION_ENABLED", raising=False) + _api_key = hash_token("sk-refund-lifecycle") + local_cache = DualCache() + handler = _PROXY_MaxParallelRequestsHandler( + internal_usage_cache=InternalUsageCache(local_cache) + ) + user_api_key_dict = UserAPIKeyAuth( + api_key=_api_key, + tpm_limit=10_000, + max_parallel_requests=2, + ) + tokens_key = handler.create_rate_limit_keys( + key="api_key", value=_api_key, rate_limit_type="tokens" + ) + parallel_key = f"{{api_key:{_api_key}}}:max_parallel_requests" + + await handler.async_pre_call_hook( + user_api_key_dict=user_api_key_dict, + cache=local_cache, + data={ + "model": "gpt-4o-mini", + "messages": [{"role": "user", "content": "hello"}], + "max_tokens": 50, + }, + call_type="completion", + ) + + reserved = get_request_stash().reserved_tokens + assert reserved > 0 + assert int(await local_cache.async_get_cache(key=tokens_key) or 0) == reserved + assert handler._gauge_in_flight_from_cache_value( + await local_cache.async_get_cache(key=parallel_key) + ) == 1 + + kwargs = {"standard_logging_object": {"metadata": {"user_api_key_hash": _api_key}}} + await handler.async_log_failure_event( + kwargs=kwargs, response_obj=None, start_time=None, end_time=None + ) + + assert int(await local_cache.async_get_cache(key=tokens_key) or 0) == 0 + assert handler._gauge_in_flight_from_cache_value( + await local_cache.async_get_cache(key=parallel_key) + ) == 0 + assert get_request_stash().reservation_released is True + + await handler.async_log_failure_event( + kwargs=kwargs, response_obj=None, start_time=None, end_time=None + ) + assert int(await local_cache.async_get_cache(key=tokens_key) or 0) == 0 + assert handler._gauge_in_flight_from_cache_value( + await local_cache.async_get_cache(key=parallel_key) + ) == 0 + + +@pytest.mark.asyncio +async def test_pre_call_hook_ignores_caller_supplied_stash_values(): + """Caller-supplied bookkeeping lookalikes in the body must not drive a + TPM refund against an arbitrary scope: the ContextVar stash is the only + source the refund path reads.""" user_api_key_dict = UserAPIKeyAuth(api_key=hash_token("sk-no-limits")) local_cache = DualCache() handler = _PROXY_MaxParallelRequestsHandler( @@ -3188,19 +3306,15 @@ async def test_pre_call_hook_rejects_caller_supplied_stash_values(): "rate_limit": {"tokens_per_unit": 10000, "window_size": 60}, } ] + injected = { + "_litellm_tpm_reserved_tokens": 9999, + "_litellm_rate_limit_descriptors": victim_descriptors, + } data: Dict[str, Any] = { "model": "gpt-4o-mini", "messages": [{"role": "user", "content": "hi"}], - TPM_RESERVED_TOKENS_KEY: 9999, - RATE_LIMIT_DESCRIPTORS_KEY: victim_descriptors, - "metadata": { - TPM_RESERVED_TOKENS_KEY: 9999, - RATE_LIMIT_DESCRIPTORS_KEY: victim_descriptors, - }, - "litellm_metadata": { - TPM_RESERVED_TOKENS_KEY: 9999, - RATE_LIMIT_DESCRIPTORS_KEY: victim_descriptors, - }, + "metadata": dict(injected), + "litellm_metadata": dict(injected), } await handler.async_pre_call_hook( @@ -3210,13 +3324,25 @@ async def test_pre_call_hook_rejects_caller_supplied_stash_values(): call_type="completion", ) - for channel in ( - data, - data.get("metadata") or {}, - data.get("litellm_metadata") or {}, - ): - leaked = [k for k in _LITELLM_STASH_KEYS if k in channel] - assert not leaked, f"caller-supplied stash survived in {channel!r}: {leaked}" + refund_calls = [] + + async def spy_increment_pipeline(increment_list, **kwargs): + refund_calls.append(increment_list) + + handler.internal_usage_cache.dual_cache.async_increment_cache_pipeline = ( + spy_increment_pipeline + ) + + await handler.async_post_call_failure_hook( + request_data=data, + original_exception=Exception("boom"), + user_api_key_dict=user_api_key_dict, + ) + + assert refund_calls == [] + stash = get_request_stash() + assert stash is not None + assert stash.reserved_tokens == 0 # ----------------------- Per-MCP-server rate limiting (v3) ----------------------- @@ -3511,18 +3637,13 @@ async def test_release_max_parallel_requests_on_disconnect_v3(): await local_cache.async_get_cache(key=counter_key) ) == 1 - await handler.async_release_max_parallel_requests_on_disconnect( - user_api_key_dict, - request_data={ - "metadata": { - MAX_PARALLEL_SLOT_ACQUIRED_KEY: { - "slot_id": _TEST_SLOT_ID, - "counter_keys": [counter_key], - } - } - }, + get_or_create_request_stash().parallel_slot = ParallelSlotAcquisition( + slot_id=_TEST_SLOT_ID, + counter_keys=[counter_key], ) + await handler.async_release_max_parallel_requests_on_disconnect(user_api_key_dict) + assert get_request_stash().parallel_slot is None assert handler._gauge_in_flight_from_cache_value( await local_cache.async_get_cache(key=counter_key) ) == 0 @@ -3544,16 +3665,12 @@ async def test_release_on_disconnect_works_when_key_config_changed_v3(): counter_key = f"{{api_key:{_api_key}}}:max_parallel_requests" await _seed_max_parallel_requests_slots(local_cache, counter_key, [_TEST_SLOT_ID]) + get_or_create_request_stash().parallel_slot = ParallelSlotAcquisition( + slot_id=_TEST_SLOT_ID, + counter_keys=[counter_key], + ) await handler.async_release_max_parallel_requests_on_disconnect( - UserAPIKeyAuth(api_key=_api_key, max_parallel_requests=None), - request_data={ - "metadata": { - MAX_PARALLEL_SLOT_ACQUIRED_KEY: { - "slot_id": _TEST_SLOT_ID, - "counter_keys": [counter_key], - } - } - }, + UserAPIKeyAuth(api_key=_api_key, max_parallel_requests=None) ) assert handler._gauge_in_flight_from_cache_value( await local_cache.async_get_cache(key=counter_key) @@ -3601,7 +3718,6 @@ async def test_post_call_failure_hook_releases_parallel_slot_v3(): await handler.async_log_failure_event( kwargs={ - "metadata": admitted_data["metadata"], "standard_logging_object": {"metadata": {"user_api_key_hash": _api_key}}, }, response_obj=None, @@ -3649,7 +3765,6 @@ async def test_success_event_releases_parallel_slot_v3(monkeypatch): await handler.async_log_success_event( kwargs={ - "metadata": admitted_data["metadata"], "standard_logging_object": {"metadata": {"user_api_key_hash": _api_key}}, }, response_obj=ModelResponse( @@ -3750,14 +3865,12 @@ async def test_redis_release_script_updates_local_mirror_v3(): handler.parallel_release_script = fake_release + get_or_create_request_stash().parallel_slot = ParallelSlotAcquisition( + slot_id="slot-redis-test", + counter_keys=[counter_key], + ) await handler.async_log_failure_event( kwargs={ - "metadata": { - MAX_PARALLEL_SLOT_ACQUIRED_KEY: { - "slot_id": "slot-redis-test", - "counter_keys": [counter_key], - } - }, "standard_logging_object": {"metadata": {"user_api_key_hash": _api_key}}, }, response_obj=None, @@ -3862,7 +3975,6 @@ async def test_in_memory_fallback_respects_mirrored_redis_count_v3(): await handler.async_log_failure_event( kwargs={ - "metadata": admitted_data["metadata"], "standard_logging_object": {"metadata": {"user_api_key_hash": _api_key}}, }, response_obj=None, @@ -3928,19 +4040,15 @@ async def test_async_streaming_data_generator_releases_counter_on_disconnect_v3( while True: yield ModelResponse() + get_or_create_request_stash().parallel_slot = ParallelSlotAcquisition( + slot_id=_TEST_SLOT_ID, + counter_keys=[counter_key], + ) with _override_litellm_callbacks([]): gen = ProxyBaseLLMRequestProcessing.async_sse_data_generator( response=upstream(), user_api_key_dict=user_api_key_dict, - request_data={ - "model": "claude-test", - "metadata": { - MAX_PARALLEL_SLOT_ACQUIRED_KEY: { - "slot_id": _TEST_SLOT_ID, - "counter_keys": [counter_key], - } - }, - }, + request_data={"model": "claude-test"}, proxy_logging_obj=proxy_logging_obj, ) await gen.__anext__() @@ -3981,21 +4089,17 @@ async def test_async_data_generator_releases_counter_on_disconnect_v3(disconnect while True: yield ModelResponse() + get_or_create_request_stash().parallel_slot = ParallelSlotAcquisition( + slot_id=_TEST_SLOT_ID, + counter_keys=[counter_key], + ) try: with _override_litellm_callbacks([]): assert proxy_logging_obj.needs_iterator_wrap() is False gen = proxy_server.async_data_generator( response=upstream(), user_api_key_dict=user_api_key_dict, - request_data={ - "model": "gpt-test", - "metadata": { - MAX_PARALLEL_SLOT_ACQUIRED_KEY: { - "slot_id": _TEST_SLOT_ID, - "counter_keys": [counter_key], - } - }, - }, + request_data={"model": "gpt-test"}, ) await gen.__anext__() if disconnect == "cancel": @@ -4044,21 +4148,17 @@ async def test_async_data_generator_releases_counter_when_wrapped_v3(): while True: yield ModelResponse() + get_or_create_request_stash().parallel_slot = ParallelSlotAcquisition( + slot_id=_TEST_SLOT_ID, + counter_keys=[counter_key], + ) try: with _override_litellm_callbacks([_PassthroughIteratorOverride()]): assert proxy_logging_obj.needs_iterator_wrap() is True gen = proxy_server.async_data_generator( response=upstream(), user_api_key_dict=user_api_key_dict, - request_data={ - "model": "gpt-test", - "metadata": { - MAX_PARALLEL_SLOT_ACQUIRED_KEY: { - "slot_id": _TEST_SLOT_ID, - "counter_keys": [counter_key], - } - }, - }, + request_data={"model": "gpt-test"}, ) await gen.__anext__() await gen.aclose() @@ -4175,12 +4275,7 @@ async def test_pre_call_hook_skips_reservation_when_disabled(monkeypatch): assert reserve_calls == [], "reservation must be skipped when disabled" assert should_rate_limit_calls[0]["skip_tpm_check"] is False - # No reservation stash leaks into the request metadata. - from litellm.proxy.hooks.parallel_request_limiter_v3 import ( - TPM_RESERVED_TOKENS_KEY, - ) - - assert TPM_RESERVED_TOKENS_KEY not in (data.get("metadata") or {}) + assert get_request_stash().reserved_tokens == 0 @pytest.mark.asyncio diff --git a/tests/test_litellm/proxy/hooks/test_proxy_rate_limit_provider_field.py b/tests/test_litellm/proxy/hooks/test_proxy_rate_limit_provider_field.py index 02b4e32db86..ec680317980 100644 --- a/tests/test_litellm/proxy/hooks/test_proxy_rate_limit_provider_field.py +++ b/tests/test_litellm/proxy/hooks/test_proxy_rate_limit_provider_field.py @@ -691,7 +691,6 @@ async def test_dynamic_rate_limiter_v3_model_capacity_path_populates_provider(): user_api_key_dict=user_api_key_dict, priority="default", saturation=1.0, - data={"model": "gpt-4o-mini"}, ) exc = exc_info.value @@ -741,7 +740,6 @@ async def test_dynamic_rate_limiter_v3_unknown_descriptor_path_populates_provide user_api_key_dict=user_api_key_dict, priority="default", saturation=1.0, - data={"model": "gpt-4o-mini"}, ) assert exc_info.value.llm_provider == "openai" diff --git a/tests/test_litellm/proxy/hooks/test_rate_limiter_toctou.py b/tests/test_litellm/proxy/hooks/test_rate_limiter_toctou.py index ceea5de7991..1c1e8eee145 100644 --- a/tests/test_litellm/proxy/hooks/test_rate_limiter_toctou.py +++ b/tests/test_litellm/proxy/hooks/test_rate_limiter_toctou.py @@ -253,7 +253,6 @@ async def test_dynamic_rate_limiter_v3_concurrent_bypasses_model_capacity(): user_api_key_dict=user, priority="high", saturation=0.0, - data={}, ) return "OK" except Exception as e: @@ -332,7 +331,6 @@ async def test_dynamic_rate_limiter_v3_uses_atomic_check_and_increment(): user_api_key_dict=user, priority="high", saturation=0.0, - data={}, ) assert atomic_descriptors_observed, ( @@ -482,7 +480,6 @@ async def test_dynamic_rate_limiter_v3_fails_closed_on_unknown_descriptor(): user_api_key_dict=user, priority="high", saturation=0.0, - data={}, ) assert ( exc.value.status_code == 429 diff --git a/tests/test_litellm/proxy/hooks/test_tpm_concurrent.py b/tests/test_litellm/proxy/hooks/test_tpm_concurrent.py index b02f6c15168..f7bd37b412a 100644 --- a/tests/test_litellm/proxy/hooks/test_tpm_concurrent.py +++ b/tests/test_litellm/proxy/hooks/test_tpm_concurrent.py @@ -23,13 +23,13 @@ import pytest from litellm.caching.caching import DualCache from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.hooks.parallel_request_limiter_v3 import ( - RATE_LIMIT_DESCRIPTORS_KEY, - TPM_RESERVATION_RELEASED_KEY, - TPM_RESERVED_MODEL_KEY, - TPM_RESERVED_SCOPES_KEY, - TPM_RESERVED_TOKENS_KEY, _PROXY_MaxParallelRequestsHandler_v3 as RateLimitHandler, ) +from litellm.proxy.hooks.parallel_request_limiter_v3 import ( + _request_stash, + get_or_create_request_stash, + get_request_stash, +) from litellm.proxy.utils import InternalUsageCache, hash_token from litellm.types.utils import ModelResponse, Usage @@ -41,6 +41,13 @@ def rate_limiter(): return handler, cache +@pytest.fixture(autouse=True) +def _isolated_request_stash(): + token = _request_stash.set(None) + yield + _request_stash.reset(token) + + @pytest.mark.asyncio async def test_token_reservation_prevents_concurrent_bypass(rate_limiter): """ @@ -79,7 +86,7 @@ async def test_token_reservation_prevents_concurrent_bypass(rate_limiter): return { "request_id": request_id, "success": True, - "reserved_tokens": data.get(TPM_RESERVED_TOKENS_KEY, 0), + "reserved_tokens": get_request_stash().reserved_tokens, } except Exception as e: return { @@ -167,12 +174,14 @@ async def test_token_adjustment_on_success(rate_limiter): api_key = hash_token("sk-test-adjust") + stash = get_or_create_request_stash() + stash.reserved_tokens = 100 + stash.reserved_scopes = frozenset({("api_key", api_key)}) + mock_kwargs = { "standard_logging_object": { "metadata": { "user_api_key_hash": api_key, - TPM_RESERVED_TOKENS_KEY: 100, - TPM_RESERVED_SCOPES_KEY: [["api_key", api_key]], } }, "model": "gpt-3.5-turbo", @@ -227,12 +236,14 @@ async def test_token_release_on_failure(rate_limiter): api_key = hash_token("sk-test-fail") + stash = get_or_create_request_stash() + stash.reserved_tokens = 100 + stash.reserved_scopes = frozenset({("api_key", api_key)}) + mock_kwargs = { "standard_logging_object": { "metadata": { "user_api_key_hash": api_key, - TPM_RESERVED_TOKENS_KEY: 100, - TPM_RESERVED_SCOPES_KEY: [["api_key", api_key]], } }, } @@ -285,6 +296,11 @@ async def test_model_scope_refund_targets_reserved_model(rate_limiter): team_id = "team-abc" reserved_model = "gpt-4o-mini" + stash = get_or_create_request_stash() + stash.reserved_tokens = 100 + stash.reserved_model = reserved_model + stash.reserved_scopes = frozenset({("model_per_team", f"{team_id}:{reserved_model}")}) + mock_kwargs = { # NOTE: no litellm_params.metadata.model_group — get_model_group_from_litellm_kwargs # returns None on this kwargs dict. @@ -292,11 +308,6 @@ async def test_model_scope_refund_targets_reserved_model(rate_limiter): "metadata": { "user_api_key_hash": api_key, "user_api_key_team_id": team_id, - TPM_RESERVED_TOKENS_KEY: 100, - TPM_RESERVED_MODEL_KEY: reserved_model, - TPM_RESERVED_SCOPES_KEY: [ - ["model_per_team", f"{team_id}:{reserved_model}"] - ], } }, } @@ -446,13 +457,15 @@ async def test_org_scope_refund_on_failure(rate_limiter): api_key = hash_token("sk-org-refund") org_id = "org-acme" + stash = get_or_create_request_stash() + stash.reserved_tokens = 100 + stash.reserved_scopes = frozenset({("organization", org_id)}) + mock_kwargs = { "standard_logging_object": { "metadata": { "user_api_key_hash": api_key, "user_api_key_org_id": org_id, - TPM_RESERVED_TOKENS_KEY: 100, - TPM_RESERVED_SCOPES_KEY: [["organization", org_id]], } }, } @@ -498,13 +511,15 @@ async def test_org_scope_reconciled_on_success(rate_limiter): api_key = hash_token("sk-org-success") org_id = "org-acme" + stash = get_or_create_request_stash() + stash.reserved_tokens = 100 + stash.reserved_scopes = frozenset({("organization", org_id)}) + mock_kwargs = { "standard_logging_object": { "metadata": { "user_api_key_hash": api_key, "user_api_key_org_id": org_id, - TPM_RESERVED_TOKENS_KEY: 100, - TPM_RESERVED_SCOPES_KEY: [["organization", org_id]], } }, "model": "gpt-3.5-turbo", @@ -607,9 +622,9 @@ async def test_contentless_request_reserves_minimum(rate_limiter): data=data, call_type="", ) - assert (data.get("metadata") or {}).get( - TPM_RESERVED_TOKENS_KEY - ) == 1, "Contentless request should reserve the floor of 1 token" + assert ( + get_request_stash().reserved_tokens == 1 + ), "Contentless request should reserve the floor of 1 token" counter_after_two = int( await cache.async_get_cache(key=counter_key, local_only=True) or 0 @@ -702,7 +717,7 @@ async def test_reservation_released_on_proxy_rejection(rate_limiter): data=data, call_type="", ) - reserved = (data.get("metadata") or {})[TPM_RESERVED_TOKENS_KEY] + reserved = get_request_stash().reserved_tokens assert reserved > 0 counter_key = handler.create_rate_limit_keys( @@ -727,8 +742,8 @@ async def test_reservation_released_on_proxy_rejection(rate_limiter): f"Reservation leaked: counter={counter_after_release} after " f"proxy-level rejection refund (expected 0)." ) - assert (data.get("metadata") or {}).get(TPM_RESERVATION_RELEASED_KEY) is True, ( - "Released marker must be stamped to prevent " + assert get_request_stash().reservation_released is True, ( + "Released flag must be set to prevent " "async_log_failure_event from double-refunding." ) @@ -754,28 +769,15 @@ async def test_reservation_release_idempotent(rate_limiter): mock_increment ) - # Shared metadata dict simulates the propagation between - # request_data["metadata"] and kwargs["litellm_params"]["metadata"] — - # the post-call-failure-hook stamps the released marker there, and the - # log-failure-event reads it. - shared_metadata = { - "user_api_key_hash": api_key, - TPM_RESERVED_TOKENS_KEY: 100, - RATE_LIMIT_DESCRIPTORS_KEY: [ - { - "key": "api_key", - "value": api_key, - "rate_limit": {"tokens_per_unit": 10000, "window_size": 60}, - } - ], - } - - request_data = { - "metadata": shared_metadata, - } + # Both hooks read the same per-request ContextVar stash: the + # post-call-failure-hook flips reservation_released on it, and the + # log-failure-event observes the flip. + stash = get_or_create_request_stash() + stash.reserved_tokens = 100 + stash.reserved_scopes = frozenset({("api_key", api_key)}) await handler.async_post_call_failure_hook( - request_data=request_data, + request_data={}, original_exception=Exception("rejected"), user_api_key_dict=UserAPIKeyAuth(api_key=api_key), ) @@ -784,11 +786,10 @@ async def test_reservation_release_idempotent(rate_limiter): assert first_refund_count > 0, "First refund should have applied" # Now simulate async_log_failure_event firing afterwards. It must see - # the released marker (via shared metadata) and not double-refund. + # the released flag on the stash and not double-refund. await handler.async_log_failure_event( kwargs={ - "litellm_params": {"metadata": shared_metadata}, - "standard_logging_object": {"metadata": shared_metadata}, + "standard_logging_object": {"metadata": {"user_api_key_hash": api_key}}, }, response_obj=None, start_time=datetime.now(), @@ -818,13 +819,15 @@ async def test_unreserved_scopes_charged_actual_not_delta_on_success(rate_limite team_id = "team-no-tpm-limit" # Reservation ONLY hit api_key — team had no TPM limit configured. + stash = get_or_create_request_stash() + stash.reserved_tokens = 100 + stash.reserved_scopes = frozenset({("api_key", api_key)}) + mock_kwargs = { "standard_logging_object": { "metadata": { "user_api_key_hash": api_key, "user_api_key_team_id": team_id, - TPM_RESERVED_TOKENS_KEY: 100, - TPM_RESERVED_SCOPES_KEY: [["api_key", api_key]], } }, "model": "gpt-3.5-turbo", @@ -888,13 +891,15 @@ async def test_unreserved_scopes_not_refunded_on_failure(rate_limiter): api_key = hash_token("sk-mixed-fail") team_id = "team-no-tpm" + stash = get_or_create_request_stash() + stash.reserved_tokens = 100 + stash.reserved_scopes = frozenset({("api_key", api_key)}) + mock_kwargs = { "standard_logging_object": { "metadata": { "user_api_key_hash": api_key, "user_api_key_team_id": team_id, - TPM_RESERVED_TOKENS_KEY: 100, - TPM_RESERVED_SCOPES_KEY: [["api_key", api_key]], } }, } @@ -939,10 +944,10 @@ async def test_unreserved_scopes_not_refunded_on_failure(rate_limiter): async def test_token_rate_limit_headers_present_in_stored_response(rate_limiter): """ With `skip_tpm_check=True` on the RPM sliding-window pass, token statuses - only come from `reserve_tpm_tokens`. They must be merged into - `data["litellm_proxy_rate_limit_response"]` so the post-call hook can - emit `x-ratelimit-{key}-remaining-tokens` / `-limit-tokens` headers to - the client. + only come from `reserve_tpm_tokens`. They must be merged into the stashed + rate-limit response so the post-call hook can emit + `x-ratelimit-{key}-remaining-tokens` / `-limit-tokens` headers to the + client. """ handler, cache = rate_limiter @@ -966,10 +971,10 @@ async def test_token_rate_limit_headers_present_in_stored_response(rate_limiter) call_type="", ) - response = data.get("litellm_proxy_rate_limit_response") + response = get_request_stash().rate_limit_response assert isinstance( response, dict - ), "Expected litellm_proxy_rate_limit_response to be set after pre-call" + ), "Expected the stashed rate-limit response to be set after pre-call" statuses = response.get("statuses") or [] token_statuses = [s for s in statuses if s.get("rate_limit_type") == "tokens"] @@ -1080,8 +1085,8 @@ async def test_small_tpm_cap_admits_no_max_tokens_request(rate_limiter): call_type="", ) - reserved = (data.get("metadata") or {}).get(TPM_RESERVED_TOKENS_KEY) - assert reserved is not None, "Reservation should have been stashed" + reserved = get_request_stash().reserved_tokens + assert reserved > 0, "Reservation should have been stashed" assert reserved <= 1000 // 2, ( f"Capped floor must keep the reservation well under the 1000 TPM " f"cap; got {reserved}" diff --git a/tests/test_litellm/types/test_types_utils.py b/tests/test_litellm/types/test_types_utils.py index 21f28f54b8b..320c46aed3b 100644 --- a/tests/test_litellm/types/test_types_utils.py +++ b/tests/test_litellm/types/test_types_utils.py @@ -321,29 +321,6 @@ class TestNativeFinishReason: assert choice.provider_specific_fields["native_finish_reason"] == "MAX_TOKENS" -def test_parallel_request_limiter_internal_fields_in_all_litellm_params(): - """ - Regression test: internal fields written by parallel_request_limiter_v3 must - be in all_litellm_params so they are stripped before forwarding to upstream - providers. If missing, they are sent as extra body parameters and providers - like OpenAI reject the request with a 400 invalid_request_error. - """ - from litellm.types.utils import all_litellm_params - - internal_fields = [ - "_litellm_rate_limit_descriptors", - "_litellm_tpm_reserved_tokens", - "_litellm_tpm_reserved_model", - "_litellm_tpm_reserved_scopes", - "_litellm_tpm_reservation_released", - ] - for field in internal_fields: - assert field in all_litellm_params, ( - f"{field!r} is not in all_litellm_params. " - "It will be forwarded to upstream providers and cause 400 errors." - ) - - def test_delta_maps_reasoning_to_reasoning_content(): """ Test that Delta maps 'reasoning' field to 'reasoning_content'. From f507a118af66eda884026b7d4cfeaec9fcb805cd Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 30 Jul 2026 14:54:28 -0700 Subject: [PATCH 02/11] fix(rate-limits): pin the request stash to its owning litellm_call_id so nested calls cannot release it --- .../proxy/hooks/dynamic_rate_limiter_v3.py | 2 + .../hooks/parallel_request_limiter_v3.py | 45 +++++-- .../hooks/test_parallel_request_limiter_v3.py | 115 ++++++++++++++++++ 3 files changed, 155 insertions(+), 7 deletions(-) diff --git a/litellm/proxy/hooks/dynamic_rate_limiter_v3.py b/litellm/proxy/hooks/dynamic_rate_limiter_v3.py index 1aaeda6ba95..932146800e2 100644 --- a/litellm/proxy/hooks/dynamic_rate_limiter_v3.py +++ b/litellm/proxy/hooks/dynamic_rate_limiter_v3.py @@ -23,6 +23,7 @@ from litellm.proxy.hooks.parallel_request_limiter_v3 import ( RateLimitDescriptorRateLimitObject, RateLimitResponse, _PROXY_MaxParallelRequestsHandler_v3, + claim_request_stash_for_data, get_or_create_request_stash, ) from litellm.proxy.hooks.rate_limiter_utils import ( @@ -601,6 +602,7 @@ class _PROXY_DynamicRateLimitHandlerV3(CustomLogger): if "model" not in data: return None + claim_request_stash_for_data(data) model = data["model"] priority = self._get_priority_from_user_api_key_dict(user_api_key_dict=user_api_key_dict) diff --git a/litellm/proxy/hooks/parallel_request_limiter_v3.py b/litellm/proxy/hooks/parallel_request_limiter_v3.py index 9d7423166ad..b04ef5f7087 100644 --- a/litellm/proxy/hooks/parallel_request_limiter_v3.py +++ b/litellm/proxy/hooks/parallel_request_limiter_v3.py @@ -354,8 +354,17 @@ class RequestRateLimiterStash: ``reservation_released`` flag and ``parallel_slot`` clearing effective across sibling callbacks: the first release wins, later callbacks observe the cleared state. + + Because the stash is context-inherited, nested LiteLLM calls made inside + the request (LLM-judge guardrails, silent experiments) would also see it + from their own logging callbacks. ``owner_litellm_call_id`` pins the stash + to the proxy request's ``litellm_call_id`` so those callbacks can tell the + owning request's events apart from a nested call's: router retries and + fallbacks reuse the request's call id and keep access, while nested calls + mint fresh ids and are ignored. """ + owner_litellm_call_id: Optional[str] = None rate_limit_response: Optional[RateLimitResponse] = None parallel_slot: Optional[ParallelSlotAcquisition] = None reserved_tokens: int = 0 @@ -381,6 +390,30 @@ def get_or_create_request_stash() -> RequestRateLimiterStash: return stash +def claim_request_stash_for_data(data: dict) -> RequestRateLimiterStash: + stash = get_or_create_request_stash() + owner_call_id = data.get("litellm_call_id") + if isinstance(owner_call_id, str): + stash.owner_litellm_call_id = owner_call_id + return stash + + +def get_request_stash_for_call(litellm_call_id: Optional[str]) -> Optional[RequestRateLimiterStash]: + stash = _request_stash.get() + if stash is None: + return None + if stash.owner_litellm_call_id is None or litellm_call_id is None: + return stash + return stash if litellm_call_id == stash.owner_litellm_call_id else None + + +def _call_id_from_callback_kwargs(kwargs: object) -> Optional[str]: + if not isinstance(kwargs, dict): + return None + call_id = kwargs.get("litellm_call_id") + return call_id if isinstance(call_id, str) else None + + class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): def __init__( self, @@ -2342,7 +2375,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): """ verbose_proxy_logger.debug("Inside Rate Limit Pre-Call Hook") - stash = get_or_create_request_stash() + stash = claim_request_stash_for_data(data) ######################################################### # Check if the call type has a specific rate limiter @@ -2536,8 +2569,6 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): stored_response = stash.rate_limit_response if stored_response is not None: stored_response["statuses"].extend(tpm_response["statuses"]) - elif tpm_response["statuses"]: - stash.rate_limit_response = tpm_response verbose_proxy_logger.debug(f"TPM tokens reserved: {estimated_tokens} for model {requested_model}") @@ -2886,7 +2917,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): if total_tokens == 0: total_tokens = self._aggregate_only_total_tokens(usage=_usage) - stash = get_request_stash() + stash = get_request_stash_for_call(_call_id_from_callback_kwargs(kwargs)) reserved_tokens = stash.reserved_tokens if stash is not None else 0 reserved_model = stash.reserved_model if stash is not None else None reserved_scopes: FrozenSet[Tuple[str, str]] = stash.reserved_scopes if stash is not None else frozenset() @@ -2945,7 +2976,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): try: verbose_proxy_logger.debug("INSIDE parallel request limiter ASYNC SUCCESS LOGGING") - stash = get_request_stash() + stash = get_request_stash_for_call(_call_id_from_callback_kwargs(kwargs)) acquisition = stash.parallel_slot if stash is not None else None if stash is not None and acquisition is not None: await self._release_parallel_request_slots( @@ -3002,7 +3033,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): if not isinstance(kwargs, dict): return - stash = get_request_stash() + stash = get_request_stash_for_call(_call_id_from_callback_kwargs(kwargs)) rate_limit_response = stash.rate_limit_response if stash is not None else None statuses = rate_limit_response["statuses"] if rate_limit_response is not None else [] if not statuses: @@ -3044,7 +3075,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): pipeline_operations: List[RedisPipelineIncrementOperation] = [] - stash = get_request_stash() + stash = get_request_stash_for_call(_call_id_from_callback_kwargs(kwargs)) acquisition = stash.parallel_slot if stash is not None else None if stash is not None and acquisition is not None: await self._release_parallel_request_slots( diff --git a/tests/test_litellm/proxy/hooks/test_parallel_request_limiter_v3.py b/tests/test_litellm/proxy/hooks/test_parallel_request_limiter_v3.py index d546b629f0f..56bfd1829b5 100644 --- a/tests/test_litellm/proxy/hooks/test_parallel_request_limiter_v3.py +++ b/tests/test_litellm/proxy/hooks/test_parallel_request_limiter_v3.py @@ -20,6 +20,7 @@ from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.hooks.parallel_request_limiter_v3 import ( PARALLEL_REQUEST_SLOT_TTL_SECONDS, ParallelSlotAcquisition, + RequestRateLimiterStash, _request_stash, get_or_create_request_stash, get_request_stash, @@ -3345,6 +3346,120 @@ async def test_pre_call_hook_ignores_caller_supplied_stash_values(): assert stash.reserved_tokens == 0 +@pytest.mark.asyncio +async def test_log_events_from_nested_calls_leave_owner_stash_alone(monkeypatch): + """ + A nested LiteLLM call made inside the request (LLM-judge guardrail, + silent experiment) inherits the request context and fires the same global + logging callbacks with a fresh ``litellm_call_id``. Those callbacks must + not release the owning request's parallel slot or refund its TPM + reservation; only events carrying the owner's call id may. + """ + monkeypatch.delenv("LITELLM_TPM_TOKEN_RESERVATION_ENABLED", raising=False) + _api_key = hash_token("sk-nested-guard") + local_cache = DualCache() + handler = _PROXY_MaxParallelRequestsHandler( + internal_usage_cache=InternalUsageCache(local_cache) + ) + user_api_key_dict = UserAPIKeyAuth( + api_key=_api_key, + tpm_limit=10_000, + max_parallel_requests=2, + ) + tokens_key = handler.create_rate_limit_keys( + key="api_key", value=_api_key, rate_limit_type="tokens" + ) + parallel_key = f"{{api_key:{_api_key}}}:max_parallel_requests" + + await handler.async_pre_call_hook( + user_api_key_dict=user_api_key_dict, + cache=local_cache, + data={ + "model": "gpt-4o-mini", + "messages": [{"role": "user", "content": "hello"}], + "max_tokens": 50, + "litellm_call_id": "owner-call-id", + }, + call_type="completion", + ) + + stash = get_request_stash() + assert stash is not None + assert stash.owner_litellm_call_id == "owner-call-id" + reserved = stash.reserved_tokens + assert reserved > 0 + + nested_kwargs = { + "litellm_call_id": "nested-guardrail-call", + "standard_logging_object": {"metadata": {"user_api_key_hash": _api_key}}, + } + await handler.async_log_success_event( + kwargs=nested_kwargs, response_obj=None, start_time=None, end_time=None + ) + await handler.async_log_failure_event( + kwargs=nested_kwargs, response_obj=None, start_time=None, end_time=None + ) + + assert stash.parallel_slot is not None + assert stash.reservation_released is False + assert handler._gauge_in_flight_from_cache_value( + await local_cache.async_get_cache(key=parallel_key) + ) == 1 + assert int(await local_cache.async_get_cache(key=tokens_key) or 0) == reserved + + owner_kwargs = { + "litellm_call_id": "owner-call-id", + "standard_logging_object": {"metadata": {"user_api_key_hash": _api_key}}, + } + await handler.async_log_failure_event( + kwargs=owner_kwargs, response_obj=None, start_time=None, end_time=None + ) + + assert stash.parallel_slot is None + assert stash.reservation_released is True + assert handler._gauge_in_flight_from_cache_value( + await local_cache.async_get_cache(key=parallel_key) + ) == 0 + assert int(await local_cache.async_get_cache(key=tokens_key) or 0) == 0 + + +@pytest.mark.asyncio +async def test_stash_applies_when_owner_or_callback_call_id_missing(): + """ + The owner guard only rejects a positive mismatch. A stash never claimed + by a pre-call hook (no owner id) must stay visible to any callback, and a + claimed stash must stay visible to callbacks whose kwargs carry no call + id — otherwise reservations and slots would strand on request paths that + do not thread ``litellm_call_id`` into their logging kwargs. + """ + local_cache = DualCache() + handler = _PROXY_MaxParallelRequestsHandler( + internal_usage_cache=InternalUsageCache(local_cache) + ) + + unclaimed = get_or_create_request_stash() + unclaimed.reserved_tokens = 42 + await handler.async_log_failure_event( + kwargs={"litellm_call_id": "any-id", "standard_logging_object": {}}, + response_obj=None, + start_time=None, + end_time=None, + ) + assert unclaimed.reservation_released is True + + claimed = RequestRateLimiterStash( + owner_litellm_call_id="owner-1", reserved_tokens=42 + ) + _request_stash.set(claimed) + await handler.async_log_failure_event( + kwargs={"standard_logging_object": {}}, + response_obj=None, + start_time=None, + end_time=None, + ) + assert claimed.reservation_released is True + + # ----------------------- Per-MCP-server rate limiting (v3) ----------------------- From 3b62b90b55f2d66e86066045e3228f0b5a4ab132 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 30 Jul 2026 15:20:44 -0700 Subject: [PATCH 03/11] test(rate-limits): drop the removed data kwarg from the v3 dynamic limiter raise-branch test --- tests/test_litellm/test_rate_limit_error_unification.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_litellm/test_rate_limit_error_unification.py b/tests/test_litellm/test_rate_limit_error_unification.py index 8287e82ded0..99e9981857c 100644 --- a/tests/test_litellm/test_rate_limit_error_unification.py +++ b/tests/test_litellm/test_rate_limit_error_unification.py @@ -881,7 +881,6 @@ class TestProxyHooksActuallyRaiseProxyRateLimitError: user_api_key_dict=UserAPIKeyAuth(api_key="sk-test-v3"), priority="default", saturation=0.99, - data={}, ) e = exc_info.value assert e.status_code == 429 From 43efd02d35ec475db1e96d89c49a4dbe88f4af0b Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 30 Jul 2026 16:35:23 -0700 Subject: [PATCH 04/11] fix(proxy): request stream usage upstream by default and strip it from client streams Streamed chat completions that did not opt into stream_options.include_usage were logged with tiktoken estimates over the visible text, so hidden reasoning tokens (billed as output by OpenAI-compatible providers) were never counted and SpendLogs could undercount output tokens by 90%+ on reasoning models. The proxy now injects include_usage upstream for /v1/chat/completions streams by default and strips the injection artifacts (the final usage chunk and the empty prompt-filter chunk) from the client-facing SSE stream, so accounting uses provider-billed usage while the client-visible stream stays byte-identical to today. always_include_stream_usage keeps its existing semantics: true forwards the usage chunk to clients as before, and an explicit false now acts as a kill switch that disables the upstream injection for OpenAI-compatible backends that reject stream_options. --- litellm/proxy/common_request_processing.py | 45 ++++++-- litellm/proxy/proxy_server.py | 30 +++++ litellm/types/utils.py | 1 + .../proxy/test_common_request_processing.py | 92 +++++++++++++++ tests/test_litellm/proxy/test_proxy_server.py | 106 ++++++++++++++++++ 5 files changed, 263 insertions(+), 11 deletions(-) diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index f5a50d1697a..750954b69f7 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -12,6 +12,7 @@ from typing import ( Callable, Dict, Literal, + Mapping, Optional, Tuple, Union, @@ -244,6 +245,32 @@ async def _cancel_pending_gather_tasks(tasks: list["asyncio.Task[Any]"]) -> None pass +def _stream_usage_tracking_updates( + data: Mapping[str, object], + general_settings: Mapping[str, object], + route_type: str, +) -> Mapping[str, object]: + if data.get("stream", False) is not True: + return {} + always_include = general_settings.get("always_include_stream_usage") + stream_options = data.get("stream_options") + if always_include is True: + if "stream_options" not in data: + return {"stream_options": {"include_usage": True}} + if isinstance(stream_options, dict) and "include_usage" not in stream_options: + return {"stream_options": {**stream_options, "include_usage": True}} + return {} + if always_include is False or route_type != "acompletion": + return {} + if isinstance(stream_options, dict) and stream_options.get("include_usage") is True: + return {} + merged_stream_options = {**stream_options} if isinstance(stream_options, dict) else {} + return { + "stream_options": {**merged_stream_options, "include_usage": True}, + "_litellm_strip_stream_usage": True, + } + + def _serialize_http_exception_detail( detail: Any, ) -> Tuple[str, Optional[dict]]: @@ -1232,17 +1259,13 @@ class ProxyBaseLLMRequestProcessing: ) ### AUTO STREAM USAGE TRACKING ### - # If always_include_stream_usage is enabled and this is a streaming request - # automatically add stream_options={'include_usage': True} if not already set - if ( - general_settings.get("always_include_stream_usage", False) is True - and self.data.get("stream", False) is True - ): - # Only set if stream_options is not already provided by the client - if "stream_options" not in self.data: - self.data["stream_options"] = {"include_usage": True} - elif isinstance(self.data["stream_options"], dict) and "include_usage" not in self.data["stream_options"]: - self.data["stream_options"]["include_usage"] = True + self.data.update( + _stream_usage_tracking_updates( + data=self.data, + general_settings=general_settings, + route_type=route_type, + ) + ) ### CALL HOOKS ### - modify/reject incoming data before calling the model ## LOGGING OBJECT ## - initialize logging object for logging success/failure events for call diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index c72e3d4ee5b..ab55e885cda 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -119,6 +119,7 @@ from litellm.router_utils.add_retry_fallback_headers import ( from litellm.types.utils import ( ModelResponse, ModelResponseStream, + StreamingChoices, TextCompletionResponse, TokenCountResponse, ) @@ -7368,6 +7369,25 @@ def _serialize_streaming_chunk(chunk: BaseModel) -> Union[str, bytes]: return chunk.model_dump_json(exclude_none=True, exclude_unset=True) +def _is_injected_stream_usage_artifact(chunk: object) -> bool: + if not isinstance(chunk, ModelResponseStream): + return False + if chunk.provider_specific_fields is not None: + return False + return all(_is_empty_streaming_choice(choice) for choice in chunk.choices or []) + + +def _is_empty_streaming_choice(choice: StreamingChoices) -> bool: + if choice.finish_reason is not None: + return False + if getattr(choice, "logprobs", None) is not None: + return False + delta = getattr(choice, "delta", None) + if delta is None: + return True + return all(value is None for value in delta.model_dump().values()) + + async def _apply_streaming_chunk_hooks( *, chunk: Any, @@ -7447,6 +7467,7 @@ async def async_data_generator( needs_iterator_wrap = proxy_logging_obj.needs_iterator_wrap() needs_per_chunk_hook = proxy_logging_obj.needs_per_chunk_streaming_hook() is_raw_sse_stream = bool(request_data.get("_litellm_raw_sse_stream")) + strip_stream_usage = bool(request_data.get("_litellm_strip_stream_usage")) raw_sse_buffer = "" if needs_iterator_wrap: @@ -7498,6 +7519,15 @@ async def async_data_generator( fallback_model_from_metadata=fallback_model_from_metadata, ) + if strip_stream_usage and _is_injected_stream_usage_artifact(chunk): + if pending_fallback_event: + yield _format_fallback_metadata_sse_event( + fallback_model=fallback_model_from_metadata, + fallback_errors=fallback_errors, + ) + fallback_metadata_event_sent = True + continue + raw_passthrough = False if isinstance(chunk, BaseModel): chunk = _serialize_streaming_chunk(chunk) diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 9df44c6202c..5ba994b921c 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -3296,6 +3296,7 @@ all_litellm_params = ( "model_file_id_mapping", "litellm_logging_obj", "litellm_call_id", + "_litellm_strip_stream_usage", "use_client", "id", "fallbacks", diff --git a/tests/test_litellm/proxy/test_common_request_processing.py b/tests/test_litellm/proxy/test_common_request_processing.py index 58f81cdad35..24b2007217e 100644 --- a/tests/test_litellm/proxy/test_common_request_processing.py +++ b/tests/test_litellm/proxy/test_common_request_processing.py @@ -5111,3 +5111,95 @@ class TestStreamingClientDisconnectBilling: ) proxy_logging_obj._arelease_max_parallel_requests_on_disconnect.assert_awaited_once() + + +def _apply_stream_usage_tracking(data: dict, general_settings: dict, route_type: str) -> None: + from litellm.proxy.common_request_processing import _stream_usage_tracking_updates + + data.update(_stream_usage_tracking_updates(data=data, general_settings=general_settings, route_type=route_type)) + + +class TestApplyStreamUsageTracking: + def test_default_injects_usage_and_marks_strip_for_chat_completions(self): + data = {"stream": True, "model": "gpt-5.4-nano"} + + _apply_stream_usage_tracking(data=data, general_settings={}, route_type="acompletion") + + assert data["stream_options"] == {"include_usage": True} + assert data["_litellm_strip_stream_usage"] is True + + def test_default_preserves_other_client_stream_options_keys(self): + data = {"stream": True, "stream_options": {"include_obfuscation": True}} + + _apply_stream_usage_tracking(data=data, general_settings={}, route_type="acompletion") + + assert data["stream_options"] == {"include_obfuscation": True, "include_usage": True} + assert data["_litellm_strip_stream_usage"] is True + + def test_client_requested_usage_is_left_untouched_and_not_stripped(self): + data = {"stream": True, "stream_options": {"include_usage": True}} + + _apply_stream_usage_tracking(data=data, general_settings={}, route_type="acompletion") + + assert data["stream_options"] == {"include_usage": True} + assert "_litellm_strip_stream_usage" not in data + + def test_client_include_usage_false_is_overridden_and_stripped(self): + data = {"stream": True, "stream_options": {"include_usage": False}} + + _apply_stream_usage_tracking(data=data, general_settings={}, route_type="acompletion") + + assert data["stream_options"]["include_usage"] is True + assert data["_litellm_strip_stream_usage"] is True + + def test_explicit_false_flag_disables_injection_entirely(self): + data = {"stream": True} + + _apply_stream_usage_tracking( + data=data, + general_settings={"always_include_stream_usage": False}, + route_type="acompletion", + ) + + assert "stream_options" not in data + assert "_litellm_strip_stream_usage" not in data + + def test_flag_true_injects_without_strip_marker(self): + data = {"stream": True} + + _apply_stream_usage_tracking( + data=data, + general_settings={"always_include_stream_usage": True}, + route_type="acompletion", + ) + + assert data["stream_options"] == {"include_usage": True} + assert "_litellm_strip_stream_usage" not in data + + def test_flag_true_respects_client_explicit_include_usage_false(self): + data = {"stream": True, "stream_options": {"include_usage": False}} + + _apply_stream_usage_tracking( + data=data, + general_settings={"always_include_stream_usage": True}, + route_type="acompletion", + ) + + assert data["stream_options"] == {"include_usage": False} + assert "_litellm_strip_stream_usage" not in data + + def test_default_does_not_touch_non_chat_completion_routes(self): + data = {"stream": True} + + _apply_stream_usage_tracking(data=data, general_settings={}, route_type="anthropic_messages") + + assert "stream_options" not in data + assert "_litellm_strip_stream_usage" not in data + + def test_non_streaming_request_is_untouched(self): + data = {"model": "gpt-5.4-nano"} + + _apply_stream_usage_tracking(data=data, general_settings={}, route_type="acompletion") + + assert "stream_options" not in data + assert "_litellm_strip_stream_usage" not in data diff --git a/tests/test_litellm/proxy/test_proxy_server.py b/tests/test_litellm/proxy/test_proxy_server.py index 5646d202e31..b9a33bd2cef 100644 --- a/tests/test_litellm/proxy/test_proxy_server.py +++ b/tests/test_litellm/proxy/test_proxy_server.py @@ -10572,3 +10572,109 @@ async def test_startup_survives_database_read_failure_for_coordination_redis(): ) assert result is None + + +def _stream_usage_test_chunks(): + from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices, Usage + + content_chunk = ModelResponseStream( + model="gpt-5.4-nano", + choices=[StreamingChoices(delta=Delta(content="pong"))], + ) + finish_chunk = ModelResponseStream( + model="gpt-5.4-nano", + choices=[StreamingChoices(finish_reason="stop")], + ) + usage_chunk = ModelResponseStream(model="gpt-5.4-nano", choices=[]) + usage_chunk.usage = Usage(prompt_tokens=50, completion_tokens=188, total_tokens=238) + return content_chunk, finish_chunk, usage_chunk + + +def _stream_usage_generator_chunks(): + from litellm.types.utils import ModelResponseStream + + content_chunk, finish_chunk, usage_chunk = _stream_usage_test_chunks() + prompt_filter_chunk = ModelResponseStream(model="gpt-5.4-nano", choices=[]) + return prompt_filter_chunk, content_chunk, finish_chunk, usage_chunk + + +def test_is_injected_stream_usage_artifact(): + from litellm.proxy.proxy_server import _is_injected_stream_usage_artifact + from litellm.types.utils import ModelResponseStream, Usage + + content_chunk, finish_chunk, empty_choices_usage_chunk = _stream_usage_test_chunks() + assert _is_injected_stream_usage_artifact(empty_choices_usage_chunk) is True + + synthetic_final_chunk = ModelResponseStream(model="gpt-5.4-nano") + synthetic_final_chunk.usage = Usage(prompt_tokens=50, completion_tokens=188, total_tokens=238) + assert _is_injected_stream_usage_artifact(synthetic_final_chunk) is True + + azure_prompt_filter_chunk = ModelResponseStream(model="gpt-5.4-nano", choices=[]) + assert _is_injected_stream_usage_artifact(azure_prompt_filter_chunk) is True + + assert _is_injected_stream_usage_artifact(content_chunk) is False + assert _is_injected_stream_usage_artifact(finish_chunk) is False + + content_chunk_with_usage, finish_chunk_with_usage, _ = _stream_usage_test_chunks() + content_chunk_with_usage.usage = Usage(prompt_tokens=50, completion_tokens=188, total_tokens=238) + finish_chunk_with_usage.usage = Usage(prompt_tokens=50, completion_tokens=188, total_tokens=238) + assert _is_injected_stream_usage_artifact(content_chunk_with_usage) is False + assert _is_injected_stream_usage_artifact(finish_chunk_with_usage) is False + + assert _is_injected_stream_usage_artifact({"usage": {"prompt_tokens": 1}}) is False + + +async def _collect_async_data_generator_frames(request_data: dict) -> list: + from litellm.proxy.proxy_server import async_data_generator + from litellm.proxy.utils import ProxyLogging + + chunks = _stream_usage_generator_chunks() + + class MockStream: + def __aiter__(self): + return self._stream() + + async def _stream(self): + for chunk in chunks: + yield chunk + + async def aclose(self): + pass + + mock_proxy_logging_obj = MagicMock(spec=ProxyLogging) + mock_proxy_logging_obj.needs_iterator_wrap.return_value = False + mock_proxy_logging_obj.needs_per_chunk_streaming_hook.return_value = False + mock_proxy_logging_obj.post_call_failure_hook = AsyncMock() + + with patch("litellm.proxy.proxy_server.proxy_logging_obj", mock_proxy_logging_obj): + with patch.object(proxy_server_module.ProxyLogging, "_fire_deferred_stream_logging"): + return [ + frame.decode("utf-8") if isinstance(frame, bytes) else frame + async for frame in async_data_generator( + MockStream(), MagicMock(spec=UserAPIKeyAuth), request_data + ) + ] + + +@pytest.mark.asyncio +async def test_async_data_generator_strips_injected_usage_chunk(): + frames = await _collect_async_data_generator_frames( + {"model": "gpt-5.4-nano", "_litellm_strip_stream_usage": True} + ) + + data_frames = [frame for frame in frames if frame.startswith("data: {")] + assert len(data_frames) == 2 + assert any("pong" in frame for frame in data_frames) + assert any("finish_reason" in frame for frame in data_frames) + assert not any('"usage"' in frame for frame in data_frames) + assert frames[-1] == "data: [DONE]\n\n" + + +@pytest.mark.asyncio +async def test_async_data_generator_forwards_usage_chunk_without_strip_marker(): + frames = await _collect_async_data_generator_frames({"model": "gpt-5.4-nano"}) + + data_frames = [frame for frame in frames if frame.startswith("data: {")] + assert len(data_frames) == 4 + assert any('"usage"' in frame and '"completion_tokens":188' in frame.replace(" ", "") for frame in data_frames) + assert frames[-1] == "data: [DONE]\n\n" From a770b437d53f4ddd03a4bf24c113268ac9d36dc4 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 30 Jul 2026 17:01:16 -0700 Subject: [PATCH 05/11] fix(proxy): gate default stream usage injection on provider support and neutralize client-sent strip marker Bytez and OCI param maps raise on stream_options when drop_params is unset, so the default injection would have broken every streamed chat completion routed to them. Injection now only happens when every router deployment behind the requested model (wildcards and aliases included) declares stream_options in its supported OpenAI params; providers that do not declare it either reject the param or already stream usage natively, so skipping them keeps old behavior instead of erroring. _litellm_strip_stream_usage arriving in the client request body is now overwritten at ingress (and popped in the experimental queue endpoint), so a client can no longer suppress the usage chunk it explicitly requested by planting the internal marker. --- litellm/proxy/common_request_processing.py | 59 +++++++- litellm/proxy/proxy_server.py | 1 + .../proxy/test_common_request_processing.py | 140 +++++++++++++++++- 3 files changed, 191 insertions(+), 9 deletions(-) diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index 750954b69f7..d25ef8a3039 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -5,6 +5,7 @@ import math import time import traceback from datetime import datetime +from functools import lru_cache from typing import ( TYPE_CHECKING, Any, @@ -39,6 +40,9 @@ from litellm.constants import ( ) from litellm.integrations.custom_guardrail import CustomGuardrail from litellm.litellm_core_utils.dd_tracing import NullTracer, tracer +from litellm.litellm_core_utils.get_supported_openai_params import ( + get_supported_openai_params, +) from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.litellm_core_utils.llm_response_utils.get_headers import ( get_response_headers, @@ -245,25 +249,64 @@ async def _cancel_pending_gather_tasks(tasks: list["asyncio.Task[Any]"]) -> None pass +@lru_cache(maxsize=512) +def _litellm_model_supports_stream_options(litellm_model: str) -> bool: + try: + supported_params = get_supported_openai_params(model=litellm_model) + except Exception: # noqa: BLE001 # unmapped or malformed model strings must disable injection, not fail the request + return False + return supported_params is not None and "stream_options" in supported_params + + +def _deployment_litellm_model(deployment: Mapping[str, object]) -> Optional[str]: + litellm_params = deployment.get("litellm_params") + if isinstance(litellm_params, Mapping): + litellm_model = litellm_params.get("model") + else: + litellm_model = getattr(litellm_params, "model", None) + return litellm_model if isinstance(litellm_model, str) else None + + +def _model_deployments_support_stream_options( + model: object, + llm_router: Optional[Router], +) -> bool: + if not isinstance(model, str): + return False + deployments = llm_router.get_model_list(model_name=model) if llm_router is not None else None + deployment_models = tuple( + litellm_model + for deployment in deployments or () + for litellm_model in (_deployment_litellm_model(deployment),) + if litellm_model is not None + ) + candidate_models = deployment_models if deployment_models else (model,) + return all(_litellm_model_supports_stream_options(m) for m in candidate_models) + + def _stream_usage_tracking_updates( data: Mapping[str, object], general_settings: Mapping[str, object], route_type: str, + supports_stream_options: Callable[[], bool], ) -> Mapping[str, object]: + scrub = {"_litellm_strip_stream_usage": False} if "_litellm_strip_stream_usage" in data else {} if data.get("stream", False) is not True: - return {} + return scrub always_include = general_settings.get("always_include_stream_usage") stream_options = data.get("stream_options") if always_include is True: if "stream_options" not in data: - return {"stream_options": {"include_usage": True}} + return {**scrub, "stream_options": {"include_usage": True}} if isinstance(stream_options, dict) and "include_usage" not in stream_options: - return {"stream_options": {**stream_options, "include_usage": True}} - return {} + return {**scrub, "stream_options": {**stream_options, "include_usage": True}} + return scrub if always_include is False or route_type != "acompletion": - return {} + return scrub if isinstance(stream_options, dict) and stream_options.get("include_usage") is True: - return {} + return scrub + if not supports_stream_options(): + return scrub merged_stream_options = {**stream_options} if isinstance(stream_options, dict) else {} return { "stream_options": {**merged_stream_options, "include_usage": True}, @@ -1264,6 +1307,10 @@ class ProxyBaseLLMRequestProcessing: data=self.data, general_settings=general_settings, route_type=route_type, + supports_stream_options=lambda: _model_deployments_support_stream_options( + model=self.data.get("model"), + llm_router=llm_router, + ), ) ) ### CALL HOOKS ### - modify/reject incoming data before calling the model diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index ab55e885cda..a60ea2da019 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -13500,6 +13500,7 @@ async def async_queue_request( data = {} try: data = await request.json() # type: ignore + data.pop("_litellm_strip_stream_usage", None) # Include original request and headers in the data data["proxy_server_request"] = { diff --git a/tests/test_litellm/proxy/test_common_request_processing.py b/tests/test_litellm/proxy/test_common_request_processing.py index 24b2007217e..28fb97006d8 100644 --- a/tests/test_litellm/proxy/test_common_request_processing.py +++ b/tests/test_litellm/proxy/test_common_request_processing.py @@ -1,7 +1,7 @@ import asyncio import copy import datetime -from typing import AsyncGenerator, Optional +from typing import AsyncGenerator, Callable, Optional from unittest.mock import AsyncMock, MagicMock, patch import httpx @@ -5113,10 +5113,22 @@ class TestStreamingClientDisconnectBilling: proxy_logging_obj._arelease_max_parallel_requests_on_disconnect.assert_awaited_once() -def _apply_stream_usage_tracking(data: dict, general_settings: dict, route_type: str) -> None: +def _apply_stream_usage_tracking( + data: dict, + general_settings: dict, + route_type: str, + supports_stream_options: Callable[[], bool] = lambda: True, +) -> None: from litellm.proxy.common_request_processing import _stream_usage_tracking_updates - data.update(_stream_usage_tracking_updates(data=data, general_settings=general_settings, route_type=route_type)) + data.update( + _stream_usage_tracking_updates( + data=data, + general_settings=general_settings, + route_type=route_type, + supports_stream_options=supports_stream_options, + ) + ) class TestApplyStreamUsageTracking: @@ -5203,3 +5215,125 @@ class TestApplyStreamUsageTracking: assert "stream_options" not in data assert "_litellm_strip_stream_usage" not in data + + def test_default_skips_injection_when_provider_lacks_stream_options_support(self): + data = {"stream": True, "model": "bytez-model"} + + _apply_stream_usage_tracking( + data=data, + general_settings={}, + route_type="acompletion", + supports_stream_options=lambda: False, + ) + + assert "stream_options" not in data + assert "_litellm_strip_stream_usage" not in data + + def test_client_supplied_strip_marker_is_neutralized(self): + data = { + "stream": True, + "stream_options": {"include_usage": True}, + "_litellm_strip_stream_usage": True, + } + + _apply_stream_usage_tracking(data=data, general_settings={}, route_type="acompletion") + + assert data["_litellm_strip_stream_usage"] is False + assert data["stream_options"] == {"include_usage": True} + + def test_client_supplied_strip_marker_is_neutralized_with_flag_true(self): + data = { + "stream": True, + "stream_options": {"include_usage": True}, + "_litellm_strip_stream_usage": True, + } + + _apply_stream_usage_tracking( + data=data, + general_settings={"always_include_stream_usage": True}, + route_type="acompletion", + ) + + assert data["_litellm_strip_stream_usage"] is False + + def test_client_supplied_strip_marker_is_neutralized_on_non_streaming_request(self): + data = {"_litellm_strip_stream_usage": True} + + _apply_stream_usage_tracking(data=data, general_settings={}, route_type="acompletion") + + assert data["_litellm_strip_stream_usage"] is False + + +class TestModelDeploymentsSupportStreamOptions: + def _support(self, model, llm_router=None) -> bool: + from litellm.proxy.common_request_processing import ( + _model_deployments_support_stream_options, + ) + + return _model_deployments_support_stream_options(model=model, llm_router=llm_router) + + def test_openai_compatible_deployment_supports_stream_options(self): + router = litellm.Router( + model_list=[ + { + "model_name": "azure-nano", + "litellm_params": { + "model": "azure/gpt-5.4-nano", + "api_key": "fake", + "api_base": "https://example.openai.azure.com", + }, + } + ] + ) + + assert self._support("azure-nano", router) is True + + def test_deployment_on_provider_rejecting_stream_options_is_not_injected(self): + router = litellm.Router( + model_list=[ + { + "model_name": "tiny", + "litellm_params": {"model": "bytez/openai-community/gpt2", "api_key": "fake"}, + } + ] + ) + + assert self._support("tiny", router) is False + + def test_mixed_provider_model_group_is_not_injected(self): + router = litellm.Router( + model_list=[ + { + "model_name": "mixed", + "litellm_params": {"model": "openai/gpt-4o", "api_key": "fake"}, + }, + { + "model_name": "mixed", + "litellm_params": {"model": "oci/cohere.command-r-plus", "api_key": "fake"}, + }, + ] + ) + + assert self._support("mixed", router) is False + + def test_wildcard_route_resolves_provider_support(self): + router = litellm.Router( + model_list=[ + { + "model_name": "openai/*", + "litellm_params": {"model": "openai/*", "api_key": "fake"}, + } + ] + ) + + assert self._support("openai/gpt-4o", router) is True + + def test_provider_prefixed_model_without_router_is_resolved_directly(self): + assert self._support("openai/gpt-4o", None) is True + assert self._support("bytez/openai-community/gpt2", None) is False + + def test_unmapped_model_name_is_not_injected(self): + assert self._support("some-unmapped-public-alias", None) is False + + def test_non_string_model_is_not_injected(self): + assert self._support(None, None) is False From 07321225360cc77bd1910366a567311c66492454 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 30 Jul 2026 17:33:33 -0700 Subject: [PATCH 06/11] refactor(proxy): use a walrus assignment in the deployment model gate The single-element tuple loop that bound the extracted deployment model inside the comprehension read poorly; an assignment expression in the filter clause does the same call-once-and-filter in one line. --- litellm/proxy/common_request_processing.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index d25ef8a3039..07dea5268d8 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -277,8 +277,7 @@ def _model_deployments_support_stream_options( deployment_models = tuple( litellm_model for deployment in deployments or () - for litellm_model in (_deployment_litellm_model(deployment),) - if litellm_model is not None + if (litellm_model := _deployment_litellm_model(deployment)) is not None ) candidate_models = deployment_models if deployment_models else (model,) return all(_litellm_model_supports_stream_options(m) for m in candidate_models) From 79d49620e72163619cdaadf271cc9a82165afb65 Mon Sep 17 00:00:00 2001 From: tin-berri Date: Thu, 30 Jul 2026 17:40:32 -0700 Subject: [PATCH 07/11] feat(mcp): extend keyless gateway OAuth flow to per-server MCP URL paths (#34856) The keyless flow (gateway as authorization server, no virtual key) worked only at the aggregate /mcp scope: the session-bearer admission arm was gated on _is_aggregate_mcp_scope, the 401 fallback only challenged at aggregate scope, and per-server protected-resource metadata for plain oauth2 servers pointed clients at the per-server relay, whose flow returns the raw upstream token that ingress can never accept keylessly (401 "LiteLLM Virtual Key expected. Received=gho_****"). Per-server spellings now join the same gateway flow for gateway-managed oauth2 servers (auth_type oauth2 without delegate_auth_to_upstream, new MCPServer.is_gateway_managed_oauth2 owner): - the session-bearer arm admits at any MCP scope; downstream grant resolution already intersects the admitted subject's servers with the path or header targets fail-closed, so a narrower scope never broadens - the 401 challenge is scope-aware: a single gateway-managed oauth2 path target gets the per-server resource_metadata in the spelling the request used, everything else gets the aggregate document; unknown names, CSV multi-target paths, and every client-forwarded or delegated mode keep their existing behavior - per-server PRM for explicitly named gateway-managed oauth2 servers advertises the gateway AS ({base}/mcp); delegate, passthrough, bridge, OBO, and the root-resolved unnamed shape are byte-identical - the preemptive 401 for an admitted keyless subject with no vaulted token challenges with resource_metadata (re-entering the gateway flow, whose authorize interlude vaults the upstream token) instead of the relay authorization_uri, which cannot vault without a litellm key The per-server challenge URL builder moved from server.py to oauth_utils.py (shared with the auth module) and now inserts the SERVER_ROOT_PATH segment exactly as the discovery routes do. Resolves LIT-4864 --- .../mcp_server/auth/user_api_key_auth_mcp.py | 114 +++++--- .../mcp_server/discoverable_endpoints.py | 17 ++ .../_experimental/mcp_server/exceptions.py | 2 +- .../_experimental/mcp_server/oauth_utils.py | 32 +++ .../proxy/_experimental/mcp_server/server.py | 54 ++-- .../types/mcp_server/mcp_server_manager.py | 12 + .../auth/test_user_api_key_auth_mcp.py | 255 ++++++++++++++++-- .../mcp_server/test_discoverable_endpoints.py | 119 ++++++++ .../mcp_server/test_mcp_oauth_passthrough.py | 7 +- .../mcp_server/test_mcp_stale_session.py | 91 +++++++ 10 files changed, 604 insertions(+), 99 deletions(-) diff --git a/litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py b/litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py index 423cda5eea2..5d8ac8d678f 100644 --- a/litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py +++ b/litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py @@ -11,6 +11,7 @@ from typing_extensions import assert_never import litellm from litellm._logging import verbose_logger from litellm.proxy._experimental.mcp_server.oauth_utils import ( + get_passthrough_resource_metadata_url, get_request_base_url, well_known_root_suffix, ) @@ -152,52 +153,83 @@ def _is_mcp_admitted_user_subject(user_api_key_auth: UserAPIKeyAuth | None) -> b return user_api_key_auth is not None and user_api_key_auth.mcp_admitted_user_subject is True -def _is_aggregate_mcp_scope(route: str, mcp_servers: list[str] | None) -> bool: - """True when a request targets the aggregate ``/mcp`` endpoint rather than any named - server. Named targets arrive either through ``x-mcp-servers`` (``mcp_servers``) or a - path segment (``/mcp/{server}`` / ``/{server}/mcp``); the aggregate scope has neither. - The gateway-DCR session arm and challenge fire only here, so a per-server flow is never - affected.""" - if mcp_servers: - return False - return len(MCPRequestHandler._extract_target_server_names_from_path(route)) == 0 +def _gateway_dcr_challenge_target( + route: str, + mcp_servers: list[str] | None, + client_ip: str | None, +) -> str | None: + """The single path-named server this request targets, iff it resolves to a + gateway-managed oauth2 server — the one per-server shape the gateway's own keyless + DCR flow serves end to end, so the 401 challenge may advertise the per-server + protected-resource metadata (whose ``authorization_servers`` names the gateway). + + Multi-server CSV paths, header/path mismatches, unknown names, and every + client-forwarded or delegated mode return ``None``: those cells keep their existing + challenge (or absence of one), and a challenge is never emitted for a name the + public discovery routes would 404, so this reveals exactly the server set the + per-server protected-resource metadata already reveals.""" + from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + global_mcp_server_manager, + ) + + targets = _parse_mcp_server_names_from_path(route, mcp_servers) + if targets is None: + return None + server = global_mcp_server_manager.get_mcp_server_by_name(targets[0], client_ip=client_ip) + if server is None or not server.is_gateway_managed_oauth2: + return None + return targets[0] -def _is_aggregate_gateway_dcr_challenge_scope( +def _is_gateway_dcr_challenge_scope( route: str, mcp_servers: list[str] | None, mcp_auth_header: str | None, mcp_server_auth_headers: dict[str, dict[str, str]] | None, exc: Exception, + client_ip: str | None, ) -> bool: - """True when an unauthenticated request to the aggregate ``/mcp`` endpoint - should receive the RFC 9728 401 challenge that advertises the gateway as - the authorization server. + """True when an unauthenticated MCP request should receive the RFC 9728 401 + challenge that advertises the gateway as the authorization server. - Fires only for a genuine 401 on the aggregate scope: any named target - (path or ``x-mcp-servers``) belongs to the per-server challenge paths, and - client-supplied MCP auth headers mean the caller is not a cold-start DCR - client. Fails closed to the original admission error otherwise.""" + Fires only for a genuine 401 with no client-supplied MCP auth headers (those mean + the caller is not a cold-start DCR client), on the scopes the gateway's keyless + flow serves: the aggregate ``/mcp`` endpoint, an ``x-mcp-servers``-scoped request + (the resource the client configured is still ``/mcp``), or a per-server path whose + single target is a gateway-managed oauth2 server. Every other named target keeps + its existing behavior, failing closed to the original admission error.""" if not _is_litellm_auth_admission_error(exc): return False if _has_client_supplied_mcp_auth(mcp_auth_header, mcp_server_auth_headers): return False - return _is_aggregate_mcp_scope(route, mcp_servers) + if len(MCPRequestHandler._extract_target_server_names_from_path(route)) == 0: + return True + return _gateway_dcr_challenge_target(route, mcp_servers, client_ip) is not None -def _aggregate_gateway_dcr_challenge(request: Request, invalid_token: bool) -> HTTPException: - """The RFC 9728 challenge for the aggregate endpoint: points the client at - the gateway's own protected-resource metadata so a DCR client discovers - the gateway as its authorization server and starts the sign-in flow. +def _gateway_dcr_challenge( + request: Request, + route: str, + mcp_servers: list[str] | None, + invalid_token: bool, +) -> HTTPException: + """The RFC 9728 challenge pointing the client at the protected-resource metadata + matching the scope it requested: the per-server document (same URL spelling the + request arrived on) when the single target is a gateway-managed oauth2 server, + else the gateway's aggregate document. Either way the client discovers the gateway + as its authorization server and starts the same sign-in flow. ``invalid_token`` adds the RFC 6750 error code for a request that DID present a bearer that failed admission (expired or revoked), telling spec-compliant clients to re-authorize rather than retry; a request with no credentials at all gets the bare challenge per RFC 6750 section 3.1.""" - error_attr = 'error="invalid_token", ' if invalid_token else "" + target = _gateway_dcr_challenge_target(route, mcp_servers, IPAddressUtils.get_mcp_client_ip(request)) resource_metadata_url = ( - f"{get_request_base_url(request)}/.well-known/oauth-protected-resource{well_known_root_suffix()}/mcp" + get_passthrough_resource_metadata_url(request.scope, target) + if target is not None + else f"{get_request_base_url(request)}/.well-known/oauth-protected-resource{well_known_root_suffix()}/mcp" ) + error_attr = 'error="invalid_token", ' if invalid_token else "" return HTTPException( status_code=401, detail={ @@ -240,14 +272,15 @@ def _admission_failure_fallback( ): verbose_logger.debug("MCP pass-through cold start: deferring admission to route 401 emitter") return UserAPIKeyAuth() - if _is_aggregate_gateway_dcr_challenge_scope( + if _is_gateway_dcr_challenge_scope( route=request_route, mcp_servers=mcp_servers, mcp_auth_header=mcp_auth_header, mcp_server_auth_headers=mcp_server_auth_headers, exc=exc, + client_ip=IPAddressUtils.get_mcp_client_ip(request), ): - raise _aggregate_gateway_dcr_challenge(request, invalid_token=bearer_presented) from exc + raise _gateway_dcr_challenge(request, request_route, mcp_servers, invalid_token=bearer_presented) from exc raise exc @@ -399,18 +432,18 @@ class MCPRequestHandler: request=request, route=request_route, ) - elif ( - _is_aggregate_mcp_scope(request_route, mcp_servers) - and oauth2_headers - and is_session_bearer_shaped(oauth2_headers["Authorization"]) - ): - # A gateway DCR session bearer at the aggregate /mcp scope: open the identity-only session - # token and admit under the live litellm user. One that does not open fails closed with the - # aggregate invalid_token challenge; a non-session bearer falls through to the oauth2 arm. + elif oauth2_headers and is_session_bearer_shaped(oauth2_headers["Authorization"]): + # A gateway DCR session bearer at any MCP scope: open the identity-only session + # token and admit under the live litellm user; downstream grant resolution + # intersects the admitted subject's servers with any path or header target, so a + # per-server scope narrows and never broadens. One that does not open fails + # closed with the scope's invalid_token challenge; a non-session bearer falls + # through to the oauth2 arm. validated_user_api_key_auth = await MCPRequestHandler._admit_gateway_session( authorization_value=oauth2_headers["Authorization"], request=request, route=request_route, + mcp_servers=mcp_servers, ) elif oauth2_headers: # Authorization on a non-delegated server: the bearer must be a real @@ -746,6 +779,7 @@ class MCPRequestHandler: authorization_value: str, request: Request, route: str, + mcp_servers: list[str] | None, ) -> UserAPIKeyAuth: """Open a gateway DCR session bearer and admit the live litellm user it references. @@ -753,8 +787,8 @@ class MCPRequestHandler: upstream credential (those are vaulted per user, resolved at egress), so authorization is resolved fresh via :meth:`_reload_admitted_user` + the centralized policy gate rather than a mint-time snapshot. Pre-DB gates (size, IP, route allowlist) run first, mirroring the standard - pipeline. Fails closed with the aggregate ``invalid_token`` challenge on an expired, tampered, - foreign, or refresh token, or a missing/deactivated/policy-rejected user.""" + pipeline. Fails closed with the requested scope's ``invalid_token`` challenge on an expired, + tampered, foreign, or refresh token, or a missing/deactivated/policy-rejected user.""" from litellm.proxy._experimental.mcp_server.outbound_credentials.session_credentials import ( NotSessionBearer, SessionBearerAdmitted, @@ -780,20 +814,20 @@ class MCPRequestHandler: ) except HTTPException as exc: # A cryptographically valid bearer whose referenced user is now missing or - # SCIM-deactivated is an invalid_token at the aggregate scope: relay the RFC 9728 + # SCIM-deactivated is an invalid_token at the requested scope: relay the RFC 9728 # challenge so the DCR client re-authorizes, matching the SessionBearerInvalid # arm, instead of a bare 401 with no WWW-Authenticate. A 503 (DB outage) is a # transient availability failure, not an auth failure, so it passes through. if exc.status_code == 401: - raise _aggregate_gateway_dcr_challenge(request, invalid_token=True) from exc + raise _gateway_dcr_challenge(request, route, mcp_servers, invalid_token=True) from exc raise return admitted case SessionBearerInvalid(): - raise _aggregate_gateway_dcr_challenge(request, invalid_token=True) + raise _gateway_dcr_challenge(request, route, mcp_servers, invalid_token=True) case NotSessionBearer(): # Unreachable: the arm is entered only for an is_session_bearer_shaped # value. Kept for match exhaustiveness and fails closed regardless. - raise _aggregate_gateway_dcr_challenge(request, invalid_token=True) + raise _gateway_dcr_challenge(request, route, mcp_servers, invalid_token=True) case _: assert_never(result) diff --git a/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py b/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py index cdc3ac15b1a..865787d5a07 100644 --- a/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py +++ b/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py @@ -2097,6 +2097,15 @@ async def _build_oauth_protected_resource_response( it. Only the legacy ``is_oauth_passthrough`` opt-in rewrites ``resource`` to the gateway's own URL so clients present the bearer token back to the gateway. + An explicitly named gateway-managed oauth2 server (interactive with + gateway-vaulted per-user tokens, or M2M) advertises the gateway's own + authorization server (``{base}/mcp``): a keyless DCR client that configured the + per-server URL completes the same sign-in flow the aggregate ``/mcp`` endpoint + supports and is admitted with a gateway session bearer. The per-server relay + authorize/token endpoints stay registered for the keyed interactive flow (which + is challenged with an explicit ``authorization_uri``), and the root-resolved + (unnamed) legacy shape keeps the relay authorization server. + Args: request: FastAPI Request object mcp_server_name: Name of the MCP server @@ -2112,6 +2121,7 @@ async def _build_oauth_protected_resource_response( request_base_url = get_request_base_url(request) client_ip = IPAddressUtils.get_mcp_client_ip(request) + explicitly_named = mcp_server_name is not None # When no server name provided, try to resolve the single OAuth2 server if mcp_server_name is None: @@ -2186,6 +2196,13 @@ async def _build_oauth_protected_resource_response( if mcp_server is None or mcp_server.auth_type != MCPAuth.oauth2_token_exchange: _raise_unless_oauth2_discovery_server(mcp_server, mcp_server_name, "not an OAuth-protected resource") + if explicitly_named and mcp_server is not None and mcp_server.is_gateway_managed_oauth2: + return { + "authorization_servers": [f"{request_base_url}/mcp"], + "resource": resource_url, + "scopes_supported": (mcp_server.scopes if mcp_server.scopes else []), + } + return { "authorization_servers": [ (f"{request_base_url}/{mcp_server_name}" if mcp_server_name else f"{request_base_url}") diff --git a/litellm/proxy/_experimental/mcp_server/exceptions.py b/litellm/proxy/_experimental/mcp_server/exceptions.py index 74752809e86..ca2261139c9 100644 --- a/litellm/proxy/_experimental/mcp_server/exceptions.py +++ b/litellm/proxy/_experimental/mcp_server/exceptions.py @@ -57,7 +57,7 @@ class MCPUpstreamAuthError(Exception): ``/.well-known/oauth-protected-resource/mcp/{server_name}``. This keeps the ``resource_metadata`` URI aligned with the resource pattern the client originally targeted, matching the path-aware behaviour of - ``_get_passthrough_resource_metadata_url`` in ``server.py``. + ``get_passthrough_resource_metadata_url`` in ``oauth_utils.py``. """ challenge: Optional[str] = self.www_authenticate if challenge is None and self.status_code == 401 and base_url: diff --git a/litellm/proxy/_experimental/mcp_server/oauth_utils.py b/litellm/proxy/_experimental/mcp_server/oauth_utils.py index 5daec9f97be..8f47aa7344d 100644 --- a/litellm/proxy/_experimental/mcp_server/oauth_utils.py +++ b/litellm/proxy/_experimental/mcp_server/oauth_utils.py @@ -7,6 +7,7 @@ from typing import TYPE_CHECKING, Any, Dict, List, NoReturn, Optional from urllib.parse import ParseResult, urlparse, urlsplit, urlunparse, urlunsplit from fastapi import HTTPException, Request +from starlette.types import Scope from litellm._logging import verbose_logger from litellm.proxy._experimental.mcp_server.auth.token_endpoint_auth import ( @@ -179,6 +180,37 @@ def well_known_root_suffix() -> str: return "" if root == "/" else root +def get_passthrough_resource_metadata_url(scope: Scope, server_name: str) -> str: + """The per-server protected-resource metadata URL matching the spelling the request + arrived on, so a strict RFC 9728 client resolves the same route the proxy registered. + ``_original_path`` preserves the ``/{server}/mcp`` spelling through the + ``dynamic_mcp_route`` rewrite; the ``SERVER_ROOT_PATH`` segment is inserted exactly as + the route decorators insert it (see :func:`well_known_root_suffix`).""" + request = Request(scope) + base_url = get_request_base_url(request) + _path = scope.get("_original_path") or scope.get("path", "") or "" + + if _path.startswith(f"/{server_name}/mcp"): + return f"{base_url}/.well-known/oauth-protected-resource{well_known_root_suffix()}/{server_name}/mcp" + return f"{base_url}/.well-known/oauth-protected-resource{well_known_root_suffix()}/mcp/{server_name}" + + +def get_passthrough_www_authenticate( + scope: Scope, + server_name: str, + invalid_token: bool = False, +) -> str: + """The RFC 9728 ``WWW-Authenticate`` value advertising the per-server + protected-resource metadata, with the RFC 6750 ``invalid_token`` error code when the + caller presented a bearer that failed rather than no credential at all.""" + resource_metadata_url = get_passthrough_resource_metadata_url( + scope=scope, + server_name=server_name, + ) + error_attr = 'error="invalid_token", ' if invalid_token else "" + return f'Bearer {error_attr}resource_metadata="{resource_metadata_url}"' + + def validate_loopback_redirect_uri(redirect_uri: str) -> None: """Require a loopback ``redirect_uri`` (OAuth 2.1 §4.1.2.1 + RFC 8252 §7.3 native-app pattern). MCP clients are native apps that listen on diff --git a/litellm/proxy/_experimental/mcp_server/server.py b/litellm/proxy/_experimental/mcp_server/server.py index 06a3a5a61e4..ec07d33f24d 100644 --- a/litellm/proxy/_experimental/mcp_server/server.py +++ b/litellm/proxy/_experimental/mcp_server/server.py @@ -37,6 +37,7 @@ from litellm.llms.custom_httpx.http_handler import ( ) from litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp import ( MCPRequestHandler, + _is_mcp_admitted_user_subject, ) from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( get_request_base_url, @@ -53,6 +54,7 @@ from litellm.proxy._experimental.mcp_server.mcp_context import ( from litellm.proxy._experimental.mcp_server.mcp_debug import MCPDebug from litellm.proxy._experimental.mcp_server.oauth_utils import ( _redact_mcp_resource_url, + get_passthrough_www_authenticate, ) from litellm.proxy._experimental.mcp_server.utils import ( LITELLM_MCP_SERVER_DESCRIPTION, @@ -3650,30 +3652,6 @@ if MCP_AVAILABLE: ) return user_api_key_auth.model_copy(update={"object_permission": updated_op}) - def _get_passthrough_resource_metadata_url(scope: Scope, server_name: str) -> str: - request = StarletteRequest(scope) - base_url = get_request_base_url(request) - _path = scope.get("_original_path") or scope.get("path", "") or "" - - if _path.startswith(f"/{server_name}/mcp"): - return f"{base_url}/.well-known/oauth-protected-resource/{server_name}/mcp" - return f"{base_url}/.well-known/oauth-protected-resource/mcp/{server_name}" - - def _get_passthrough_www_authenticate( - scope: Scope, - server_name: str, - invalid_token: bool = False, - ) -> str: - resource_metadata_url = _get_passthrough_resource_metadata_url( - scope=scope, - server_name=server_name, - ) - params = [] - if invalid_token: - params.append('error="invalid_token"') - params.append(f'resource_metadata="{resource_metadata_url}"') - return "Bearer " + ", ".join(params) - async def _raise_preemptive_401_for_unauthenticated_servers( scope: Scope, mcp_servers: list[str] | None, @@ -3723,10 +3701,26 @@ if MCP_AVAILABLE: # challenge whenever one is absent, regardless of any bearer. # The v2 resolver owns the existence check, so every # authorization_code resolution (egress and this discovery - # challenge) runs through it. + # challenge) runs through it. A keyless admitted subject is + # challenged with the per-server resource_metadata (whose + # authorization server is the gateway itself, vaulting via the + # authorize interlude); the per-server relay advertised below + # cannot vault without a litellm key on its token request. if await global_mcp_server_manager.has_user_oauth_token(server, user_api_key_auth): continue + if _is_mcp_admitted_user_subject(user_api_key_auth): + raise HTTPException( + status_code=401, + detail="Unauthorized", + headers={ + "www-authenticate": get_passthrough_www_authenticate( + scope=scope, + server_name=server_name, + ) + }, + ) + request = StarletteRequest(scope) base_url = get_request_base_url(request) _path = scope.get("_original_path") or scope.get("path", "") or "" @@ -3751,7 +3745,7 @@ if MCP_AVAILABLE: # the proxied resource_metadata (RFC 9728), not the gateway # authorization_uri above which would authorize against the # gateway instead of the upstream IdP. - www_authenticate = _get_passthrough_www_authenticate( + www_authenticate = get_passthrough_www_authenticate( scope=scope, server_name=server_name, ) @@ -3807,7 +3801,7 @@ if MCP_AVAILABLE: and server.is_oauth_passthrough and not _client_has_passthrough_authorization(server, oauth2_headers, mcp_server_auth_headers) ): - www_authenticate = _get_passthrough_www_authenticate( + www_authenticate = get_passthrough_www_authenticate( scope=scope, server_name=server_name, ) @@ -3824,7 +3818,7 @@ if MCP_AVAILABLE: and _get_forwarded_auth_from_scope(scope) is None and not _client_has_per_server_auth_header(server, mcp_server_auth_headers) ): - www_authenticate = _get_passthrough_www_authenticate( + www_authenticate = get_passthrough_www_authenticate( scope=scope, server_name=server_name, ) @@ -3846,7 +3840,7 @@ if MCP_AVAILABLE: status_code=401, detail="Unauthorized", headers={ - "www-authenticate": _get_passthrough_www_authenticate( + "www-authenticate": get_passthrough_www_authenticate( scope=scope, server_name=server_name, ) @@ -4053,7 +4047,7 @@ if MCP_AVAILABLE: # Token is missing or expired: keep pass-through clients on the # protected-resource discovery flow so they re-authorize against # the upstream IdP metadata proxied by LiteLLM. - www_authenticate = _get_passthrough_www_authenticate( + www_authenticate = get_passthrough_www_authenticate( scope=scope, server_name=challenge_server_name, invalid_token=True, diff --git a/litellm/types/mcp_server/mcp_server_manager.py b/litellm/types/mcp_server/mcp_server_manager.py index a127e8dad11..d02778e9eac 100644 --- a/litellm/types/mcp_server/mcp_server_manager.py +++ b/litellm/types/mcp_server/mcp_server_manager.py @@ -194,6 +194,18 @@ class MCPServer(BaseModel): """True if this is an OAuth2 server that relies on per-user tokens (no client_credentials).""" return self.auth_type == MCPAuth.oauth2 and not self.has_client_credentials + @property + def is_gateway_managed_oauth2(self) -> bool: + """True when the gateway itself owns this server's OAuth custody: an ``oauth2`` server + (interactive authorization_code with gateway-vaulted per-user tokens, or M2M + client_credentials minted at egress) that has NOT opted into upstream-delegated auth. + These are the servers the keyless gateway-DCR flow can serve end to end, so the + per-server 401 challenge and protected-resource metadata advertise the gateway as the + authorization server for exactly this set. ``true_passthrough``, ``oauth_delegate``, + DCR-bridge, and token-exchange servers are their own auth types and client-forwarded, + so they are excluded by construction.""" + return self.auth_type == MCPAuth.oauth2 and not self.delegate_auth_to_upstream + @property def is_true_passthrough(self) -> bool: """True for the transparent-proxy mode: LiteLLM performs no admission auth and forwards the diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/auth/test_user_api_key_auth_mcp.py b/tests/test_litellm/proxy/_experimental/mcp_server/auth/test_user_api_key_auth_mcp.py index 4be2bb053ef..89b8f018e5c 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/auth/test_user_api_key_auth_mcp.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/auth/test_user_api_key_auth_mcp.py @@ -1154,21 +1154,27 @@ class TestMCPOAuth2AuthFlow: await MCPRequestHandler.process_mcp_request(scope) assert exc_info.value.status_code == 500 - async def test_proxy_exception_non_delegate_oauth2_propagates(self): + async def test_proxy_exception_non_delegate_oauth2_challenges_with_per_server_metadata(self): """ Production raises ProxyException (not HTTPException) on auth failure. For - a non-delegate oauth2 server the bearer is treated as a LiteLLM credential - and a 401 must propagate as a real auth error, not be exchanged for an - anonymous upstream-passthrough session. + a gateway-managed oauth2 server the bearer is treated as a LiteLLM + credential and its failure stays a 401, never an anonymous + upstream-passthrough session. The 401 now carries the RFC 9728 + invalid_token challenge with the per-server resource metadata (LIT-4864): + a keyless client holding a stale upstream token (the relayed gho_ shape) + re-discovers the gateway as this resource's authorization server instead + of dead-ending on a bare 401. """ from litellm.proxy._types import ProxyException from litellm.types.mcp import MCPAuth + from litellm.types.mcp_server.mcp_server_manager import MCPServer scope = { "type": "http", "method": "POST", "path": "/mcp/atlassian_mcp", "headers": [ + (b"host", b"testserver"), (b"authorization", b"Bearer atlassian-oauth2-access-token-xyz"), ], } @@ -1181,10 +1187,14 @@ class TestMCPOAuth2AuthFlow: code=401, ) - oauth2_server = MagicMock() - oauth2_server.auth_type = MCPAuth.oauth2 - oauth2_server.delegate_auth_to_upstream = False - oauth2_server.is_oauth_passthrough = False + oauth2_server = MCPServer( + server_id="atlassian-id", + name="atlassian_mcp", + server_name="atlassian_mcp", + url="https://upstream.example/mcp", + transport="http", + auth_type=MCPAuth.oauth2, + ) with ( patch( @@ -1194,9 +1204,14 @@ class TestMCPOAuth2AuthFlow: patch("litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, ): mock_mgr.get_mcp_server_by_name.return_value = oauth2_server - with pytest.raises(ProxyException) as exc_info: + with pytest.raises(HTTPException) as exc_info: await MCPRequestHandler.process_mcp_request(scope) - assert str(exc_info.value.code) == "401" + assert exc_info.value.status_code == 401 + www_authenticate = (exc_info.value.headers or {})["WWW-Authenticate"] + assert www_authenticate == ( + 'Bearer error="invalid_token", ' + 'resource_metadata="http://testserver/.well-known/oauth-protected-resource/mcp/atlassian_mcp"' + ) async def test_proxy_exception_non_auth_still_raises(self): """ @@ -6250,14 +6265,133 @@ class TestAggregateGatewayDcrChallenge: self._scope(extra_headers=((b"x-litellm-api-key", b"sk-typo"),)) ) - async def test_no_challenge_for_named_servers_header(self): - """x-mcp-servers names explicit targets; the per-server challenge paths - own those, so the aggregate challenge must not fire.""" + async def test_challenge_for_named_servers_header(self): + """x-mcp-servers scopes the fan-out but the resource the client configured is still + the aggregate /mcp URL, so an unauthenticated request gets the aggregate challenge + and completes the same keyless flow; the header names then narrow (never broaden) + the admitted subject's servers downstream (LIT-4864).""" with ( patch(self._AUTH_PATCH_TARGET, side_effect=self._auth_401()), ): - with pytest.raises(ProxyException): + with pytest.raises(HTTPException) as exc_info: await MCPRequestHandler.process_mcp_request(self._scope(extra_headers=((b"x-mcp-servers", b"github"),))) + assert exc_info.value.status_code == 401 + www_authenticate = (exc_info.value.headers or {})["WWW-Authenticate"] + assert www_authenticate == f"Bearer {self._EXPECTED_RESOURCE_METADATA}" + + async def test_per_server_challenge_for_gateway_managed_oauth2(self): + """Anonymous request to a per-server path whose single target is a gateway-managed + oauth2 server: 401 plus the RFC 9728 challenge advertising the PER-SERVER + protected-resource metadata in the same URL spelling the request used, so a keyless + DCR client configured with either per-server spelling discovers the gateway as the + authorization server (LIT-4864). Covers interactive and M2M, which the gateway can + both serve end to end.""" + from litellm.types.mcp import MCPAuth + from litellm.types.mcp_server.mcp_server_manager import MCPServer + + server = MCPServer( + server_id="gh-id", + name="github", + server_name="github", + url="https://upstream.example/mcp", + transport="http", + auth_type=MCPAuth.oauth2, + ) + for path, expected_metadata_path in ( + ("/mcp/github", "/.well-known/oauth-protected-resource/mcp/github"), + ("/github/mcp", "/.well-known/oauth-protected-resource/github/mcp"), + ): + with ( + patch(self._AUTH_PATCH_TARGET, side_effect=self._auth_401()), + patch( + "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager" + ) as mock_mgr, + ): + mock_mgr.get_mcp_server_by_name.return_value = server + with pytest.raises(HTTPException) as exc_info: + await MCPRequestHandler.process_mcp_request(self._scope(path=path)) + assert exc_info.value.status_code == 401 + www_authenticate = (exc_info.value.headers or {})["WWW-Authenticate"] + assert www_authenticate == f'Bearer resource_metadata="http://testserver{expected_metadata_path}"' + + async def test_no_per_server_challenge_for_non_gateway_managed_targets(self): + """The per-server challenge fires only for the server set the gateway's keyless flow + serves: an OBO server and a multi-server CSV path keep the original admission error + through the full pipeline, so no client-forwarded mode is redirected into the gateway + sign-in flow and no cell broadens (LIT-4864).""" + from litellm.types.mcp import MCPAuth + from litellm.types.mcp_server.mcp_server_manager import MCPServer + + obo_server = MCPServer( + server_id="o-id", + name="obo", + server_name="obo", + url="https://upstream.example/mcp", + transport="http", + auth_type=MCPAuth.oauth2_token_exchange, + ) + for path, resolved in ( + ("/mcp/obo", obo_server), + ("/mcp/github,linear", None), + ): + with ( + patch(self._AUTH_PATCH_TARGET, side_effect=self._auth_401()), + patch( + "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager" + ) as mock_mgr, + ): + mock_mgr.get_mcp_server_by_name.return_value = resolved + with pytest.raises(ProxyException): + await MCPRequestHandler.process_mcp_request( + self._scope(path=path, extra_headers=((b"authorization", b"Bearer not-a-key"),)) + ) + + def test_challenge_target_excludes_every_non_gateway_managed_mode(self): + """Unit pin of the challenge-target owner: only a resolved gateway-managed oauth2 + target (interactive or M2M) yields a per-server challenge; delegate-auth oauth2 + (whose keyless flow is upstream PKCE via the relay), every client-forwarded auth + type, OBO, api_key, unknown names, and CSV paths yield None (LIT-4864).""" + from litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp import ( + _gateway_dcr_challenge_target, + ) + from litellm.types.mcp import MCPAuth + from litellm.types.mcp_server.mcp_server_manager import MCPServer + + def _server(auth_type, **kw): + return MCPServer( + server_id="s-id", + name="srv", + server_name="srv", + url="https://upstream.example/mcp", + transport="http", + auth_type=auth_type, + **kw, + ) + + cases = [ + (_server(MCPAuth.oauth2), "srv"), + (_server(MCPAuth.oauth2, oauth2_flow="client_credentials"), "srv"), + (_server(MCPAuth.oauth2, delegate_auth_to_upstream=True), None), + (_server(MCPAuth.oauth2_token_exchange), None), + (_server(MCPAuth.true_passthrough), None), + (_server(MCPAuth.oauth_delegate), None), + (_server(MCPAuth.oauth_delegate, dcr_bridge=True), None), + (_server(MCPAuth.api_key), None), + (None, None), + ] + for resolved, expected in cases: + with patch( + "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager" + ) as mock_mgr: + mock_mgr.get_mcp_server_by_name.return_value = resolved + assert _gateway_dcr_challenge_target("/mcp/srv", None, None) == expected, resolved + assert _gateway_dcr_challenge_target("/mcp/a,b", None, None) is None + assert _gateway_dcr_challenge_target("/mcp", None, None) is None + with patch( + "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager" + ) as mock_mgr: + mock_mgr.get_mcp_server_by_name.return_value = _server(MCPAuth.oauth2) + assert _gateway_dcr_challenge_target("/mcp/srv", ["other"], None) is None async def test_no_challenge_for_path_named_server(self): """/mcp/{server} targets one server; the aggregate challenge must not @@ -6295,10 +6429,11 @@ class TestAggregateGatewayDcrChallenge: @pytest.mark.asyncio class TestGatewaySessionAdmission: - """The aggregate /mcp session-bearer admission arm (mcp_gateway_dcr). A valid session - token admits under the LIVE litellm user it references; an invalid/expired/refresh/foreign - token fails closed with the aggregate invalid_token challenge; the arm fires ONLY at the - aggregate scope, never for named servers or per-server flows.""" + """The session-bearer admission arm (mcp_gateway_dcr). A valid session token admits under + the LIVE litellm user it references at any MCP scope (aggregate, per-server path, or + x-mcp-servers scoped; LIT-4864) with downstream grant resolution narrowing to the + requested servers; an invalid/expired/refresh/foreign token fails closed with the + requested scope's invalid_token challenge.""" _MASTER_KEY = "sk-gateway-session-admission-master-key" @@ -6471,21 +6606,89 @@ class TestGatewaySessionAdmission: assert oauth2_headers is None assert not any(k.lower() == "authorization" for k in (raw_headers or {})) - async def test_arm_does_not_fire_for_named_server(self): - """A session-shaped bearer aimed at a named server (path scope) does not enter the - aggregate arm; it is treated as an ordinary bearer on that server.""" - token = self._access_token() + @pytest.mark.parametrize( + "path, original_path, extra_headers", + [ + ("/mcp/github", None, ()), + ("/mcp/github", "/github/mcp", ()), + ("/mcp", None, ((b"x-mcp-servers", b"github"),)), + ], + ) + async def test_arm_admits_session_bearer_on_per_server_scopes(self, path, original_path, extra_headers): + """A valid session bearer admits the live user on per-server paths (the standard + spelling and the legacy /{server}/mcp spelling as dynamic_mcp_route rewrites it) and + x-mcp-servers scoped requests, never touching user_api_key_auth; downstream grant + resolution then intersects the named servers against the admitted subject's grants, + so the narrower scope can never broaden access (LIT-4864).""" + token = self._access_token(user_id="sso-user-42") + scope = self._scope(token, path=path, extra_headers=extra_headers) + if original_path is not None: + scope["_original_path"] = original_path with ( patch("litellm.proxy.proxy_server.master_key", self._MASTER_KEY), patch( "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", new_callable=AsyncMock, - side_effect=ProxyException(message="bad key", type="auth_error", param="api_key", code=401), ) as mock_auth, + self._patch_user_reload(user_id="sso-user-42"), ): - with pytest.raises((HTTPException, ProxyException)): - await MCPRequestHandler.process_mcp_request(self._scope(token, path="/mcp/github")) - mock_auth.assert_called_once() + auth_result, *_rest = await MCPRequestHandler.process_mcp_request(scope) + assert auth_result.user_id == "sso-user-42" + assert auth_result.mcp_admitted_user_subject is True + mock_auth.assert_not_called() + + async def test_expired_session_bearer_on_per_server_path_gets_per_server_challenge(self): + """An expired session bearer on a per-server path targeting a gateway-managed oauth2 + server re-challenges with the PER-SERVER resource metadata (matching the resource the + client configured), so a spec client re-authorizes against the right document instead + of a bare 401 or the aggregate metadata (LIT-4864).""" + from datetime import datetime, timezone + + from litellm.types.mcp import MCPAuth + from litellm.types.mcp_server.mcp_server_manager import MCPServer + + mint, _refresh, principal, keys = self._session_bearer() + bearer = mint(principal, keys, datetime(2020, 1, 1, tzinfo=timezone.utc)).token.get_secret_value() + server = MCPServer( + server_id="gh-id", + name="github", + server_name="github", + url="https://upstream.example/mcp", + transport="http", + auth_type=MCPAuth.oauth2, + ) + with ( + patch("litellm.proxy.proxy_server.master_key", self._MASTER_KEY), + patch("litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, + ): + mock_mgr.get_mcp_server_by_name.return_value = server + with pytest.raises(HTTPException) as exc_info: + await MCPRequestHandler.process_mcp_request(self._scope(bearer, path="/mcp/github")) + assert exc_info.value.status_code == 401 + www_authenticate = (exc_info.value.headers or {})["WWW-Authenticate"] + assert www_authenticate == ( + 'Bearer error="invalid_token", ' + 'resource_metadata="http://testserver/.well-known/oauth-protected-resource/mcp/github"' + ) + + async def test_session_bearer_scrubbed_from_egress_on_per_server_path(self): + """After a per-server keyless admission the session bearer must be scrubbed from every + egress header context exactly as at the aggregate scope, so no per-server passthrough + egress can forward it upstream for replay (LIT-4864).""" + token = self._access_token(user_id="sso-user-42") + with ( + patch("litellm.proxy.proxy_server.master_key", self._MASTER_KEY), + patch( + "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", + new_callable=AsyncMock, + ), + self._patch_user_reload(user_id="sso-user-42"), + ): + _auth, _h, _servers, _msah, oauth2_headers, raw_headers = await MCPRequestHandler.process_mcp_request( + self._scope(token, path="/mcp/github") + ) + assert oauth2_headers is None + assert not any(k.lower() == "authorization" for k in (raw_headers or {})) def _make_team(team_id, mcp_servers, *, org_id=None, tool_perms=None, members=("sso-user",)): diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_discoverable_endpoints.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_discoverable_endpoints.py index 694583dde88..9bc84b43fc5 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_discoverable_endpoints.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_discoverable_endpoints.py @@ -2976,6 +2976,125 @@ async def test_oauth_protected_resource_returns_empty_scopes_when_none(): global_mcp_server_manager.registry.clear() +@pytest.mark.asyncio +async def test_oauth_protected_resource_gateway_managed_oauth2_advertises_gateway_as(): + """LIT-4864: an explicitly named gateway-managed oauth2 server (interactive or M2M) + advertises the gateway's own authorization server, so a keyless DCR client that + configured the per-server URL completes the same sign-in flow the aggregate /mcp + endpoint supports and returns with a gateway session bearer; the resource stays the + per-server URL in the requested spelling (RFC 9728 resource match). A delegate-auth + oauth2 server keeps the per-server relay authorization server (its keyless flow is + upstream PKCE via the relay), and the root-resolved unnamed legacy shape is unchanged.""" + try: + from fastapi import Request + + from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + _build_oauth_protected_resource_response, + ) + from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + global_mcp_server_manager, + ) + from litellm.proxy._types import MCPTransport + from litellm.types.mcp import MCPAuth + from litellm.types.mcp_server.mcp_server_manager import MCPServer + except ImportError: + pytest.skip("MCP discoverable endpoints not available") + + def _oauth2_server(name, **kw): + return MCPServer( + server_id=name, + name=name, + server_name=name, + alias=name, + transport=MCPTransport.http, + auth_type=MCPAuth.oauth2, + authorization_url="https://idp.example.com/authorize", + token_url="https://idp.example.com/oauth/token", + scopes=["read"], + **kw, + ) + + mock_request = MagicMock(spec=Request) + mock_request.base_url = "https://litellm.example.com/" + mock_request.headers = {} + + interactive = _oauth2_server("github_mcp") + m2m = _oauth2_server("m2m_mcp", oauth2_flow="client_credentials", client_id="cid", client_secret="cs") + delegated = _oauth2_server("delegated_mcp", delegate_auth_to_upstream=True) + + global_mcp_server_manager.registry.clear() + try: + for server in (interactive, m2m, delegated): + global_mcp_server_manager.registry[server.server_id] = server + + for name in ("github_mcp", "m2m_mcp"): + standard = await _build_oauth_protected_resource_response( + request=mock_request, mcp_server_name=name, use_standard_pattern=True + ) + assert standard["authorization_servers"] == ["https://litellm.example.com/mcp"], name + assert standard["resource"] == f"https://litellm.example.com/mcp/{name}" + assert standard["scopes_supported"] == ["read"] + legacy = await _build_oauth_protected_resource_response( + request=mock_request, mcp_server_name=name, use_standard_pattern=False + ) + assert legacy["authorization_servers"] == ["https://litellm.example.com/mcp"], name + assert legacy["resource"] == f"https://litellm.example.com/{name}/mcp" + + delegated_response = await _build_oauth_protected_resource_response( + request=mock_request, mcp_server_name="delegated_mcp", use_standard_pattern=True + ) + assert delegated_response["authorization_servers"] == ["https://litellm.example.com/delegated_mcp"] + finally: + global_mcp_server_manager.registry.clear() + + +@pytest.mark.asyncio +async def test_oauth_protected_resource_root_resolved_single_server_keeps_relay_as(): + """The unnamed (bare-root) legacy shape resolves the single configured oauth2 server and + must keep advertising the per-server relay authorization server: only an EXPLICITLY + named request opts into the gateway-as-AS flow (LIT-4864), so pre-existing single-server + deployments discovering through the root document are byte-identical.""" + try: + from fastapi import Request + + from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + _build_oauth_protected_resource_response, + ) + from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + global_mcp_server_manager, + ) + from litellm.proxy._types import MCPTransport + from litellm.types.mcp import MCPAuth + from litellm.types.mcp_server.mcp_server_manager import MCPServer + except ImportError: + pytest.skip("MCP discoverable endpoints not available") + + only_server = MCPServer( + server_id="solo_mcp", + name="solo_mcp", + server_name="solo_mcp", + alias="solo_mcp", + transport=MCPTransport.http, + auth_type=MCPAuth.oauth2, + authorization_url="https://idp.example.com/authorize", + token_url="https://idp.example.com/oauth/token", + ) + + mock_request = MagicMock(spec=Request) + mock_request.base_url = "https://litellm.example.com/" + mock_request.headers = {} + + global_mcp_server_manager.registry.clear() + try: + global_mcp_server_manager.registry[only_server.server_id] = only_server + response = await _build_oauth_protected_resource_response( + request=mock_request, mcp_server_name=None, use_standard_pattern=False + ) + assert response["authorization_servers"] == ["https://litellm.example.com/solo_mcp"] + finally: + global_mcp_server_manager.registry.clear() + + @pytest.mark.asyncio async def test_oauth_authorization_server_returns_empty_scopes_when_none(): """ diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_oauth_passthrough.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_oauth_passthrough.py index ec285f8eba0..fe583ace897 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_oauth_passthrough.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_oauth_passthrough.py @@ -442,7 +442,10 @@ async def test_fetch_upstream_metadata_returns_none_when_not_all_candidates_netw @pytest.mark.asyncio async def test_oauth_protected_resource_gateway_managed_unchanged(): - """Regression guard: OAuth2 servers still advertise the gateway as AS.""" + """Regression guard: gateway-managed OAuth2 servers advertise the gateway as AS and + never fetch upstream metadata. Since LIT-4864 the advertised document is the gateway's + own aggregate authorization server ({base}/mcp), which serves the keyless DCR flow for + per-server URLs; the per-server relay endpoints remain for the keyed flow.""" from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( global_mcp_server_manager, ) @@ -477,7 +480,7 @@ async def test_oauth_protected_resource_gateway_managed_unchanged(): ) mock_client.get.assert_not_awaited() - assert result["authorization_servers"] == ["https://gateway.example.com/keycloak_whoami"] + assert result["authorization_servers"] == ["https://gateway.example.com/mcp"] assert result["scopes_supported"] == ["read"] diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_stale_session.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_stale_session.py index e5173be45b9..7bdd3b36763 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_stale_session.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_stale_session.py @@ -664,6 +664,97 @@ async def test_per_user_oauth_missing_stored_token_returns_preemptive_401(): assert "Bearer authorization_uri=" in exc_info.value.headers["www-authenticate"] +@pytest.mark.asyncio +async def test_admitted_subject_missing_stored_token_challenged_with_resource_metadata(): + """ + LIT-4864: a keyless gateway-session subject (mcp_admitted_user_subject) with no stored + per-user token must be challenged with the per-server resource_metadata, whose + authorization server is the gateway itself, so the client re-runs the gateway sign-in + flow and vaults the upstream token through the authorize interlude. The keyed + authorization_uri challenge points at the per-server relay, which cannot vault a token + for a keyless client (its token request carries no litellm credential), so sending an + admitted subject there would dead-end the flow on a raw upstream token. + """ + from fastapi import HTTPException + + try: + from litellm.proxy._experimental.mcp_server.server import ( + handle_streamable_http_mcp, + session_manager_stateless, + ) + except ImportError: + pytest.skip("MCP server not available") + + scope = { + "type": "http", + "method": "POST", + "path": "/mcp/repro_oauth_server", + "scheme": "http", + "query_string": b"", + "root_path": "", + "server": ("localhost", 8000), + "headers": [ + (b"content-type", b"application/json"), + (b"host", b"localhost:8000"), + ], + } + receive = AsyncMock() + send = AsyncMock() + user_auth = MagicMock() + user_auth.user_id = "sso-user-42" + user_auth.mcp_admitted_user_subject = True + oauth_server = MagicMock() + oauth_server.auth_type = MCPAuth.oauth2 + oauth_server.needs_user_oauth_token = True + oauth_server.delegate_auth_to_upstream = False + + with ( + patch( + "litellm.proxy._experimental.mcp_server.server.extract_mcp_auth_context", + new_callable=AsyncMock, + return_value=(user_auth, None, ["repro_oauth_server"], None, None, None), + ), + patch( + "litellm.proxy._experimental.mcp_server.server.set_auth_context", + ), + patch( + "litellm.proxy._experimental.mcp_server.server._SESSION_MANAGERS_INITIALIZED", + True, + ), + patch( + "litellm.proxy._experimental.mcp_server.server._handle_stale_mcp_session", + new_callable=AsyncMock, + return_value=False, + ), + patch( + "litellm.proxy._experimental.mcp_server.server.global_mcp_server_manager.has_user_oauth_token", + new_callable=AsyncMock, + return_value=False, + ) as mock_has_token, + patch( + "litellm.proxy._experimental.mcp_server.server.global_mcp_server_manager.get_mcp_server_by_name", + return_value=oauth_server, + ), + patch.object( + session_manager_stateless, + "handle_request", + new_callable=AsyncMock, + ) as mock_handle_request, + ): + with pytest.raises(HTTPException) as exc_info: + await handle_streamable_http_mcp(scope, receive, send) + + assert mock_has_token.await_count == 1 + assert mock_handle_request.await_count == 0 + assert exc_info.value.status_code == 401 + challenge = exc_info.value.headers["www-authenticate"] + assert "authorization_uri=" not in challenge + assert challenge == ( + 'Bearer resource_metadata="http://localhost:8000' + '/.well-known/oauth-protected-resource/mcp/repro_oauth_server"' + ) + + @pytest.mark.asyncio @pytest.mark.parametrize( "m2m_fields", From c0de87d08d4bee0069280071a0fb1854bdd0a646 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 30 Jul 2026 17:46:37 -0700 Subject: [PATCH 08/11] fix(proxy): resolve team-alias models in the stream usage support gate Team-scoped models store an internal model_name_{team_id}_{uuid} name with the public alias only in team_public_model_name, so resolving them through get_model_list without team_id returned no deployments and the gate skipped injection, leaving those streams on tiktoken estimates. Thread user_api_key_dict.team_id through the gate. --- litellm/proxy/common_request_processing.py | 8 ++++--- .../proxy/test_common_request_processing.py | 21 +++++++++++++++++-- 2 files changed, 24 insertions(+), 5 deletions(-) diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index 07dea5268d8..1ca14ea3684 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -258,7 +258,7 @@ def _litellm_model_supports_stream_options(litellm_model: str) -> bool: return supported_params is not None and "stream_options" in supported_params -def _deployment_litellm_model(deployment: Mapping[str, object]) -> Optional[str]: +def _deployment_litellm_model(deployment: Mapping[str, object]) -> str | None: litellm_params = deployment.get("litellm_params") if isinstance(litellm_params, Mapping): litellm_model = litellm_params.get("model") @@ -269,11 +269,12 @@ def _deployment_litellm_model(deployment: Mapping[str, object]) -> Optional[str] def _model_deployments_support_stream_options( model: object, - llm_router: Optional[Router], + llm_router: Router | None, + team_id: str | None, ) -> bool: if not isinstance(model, str): return False - deployments = llm_router.get_model_list(model_name=model) if llm_router is not None else None + deployments = llm_router.get_model_list(model_name=model, team_id=team_id) if llm_router is not None else None deployment_models = tuple( litellm_model for deployment in deployments or () @@ -1309,6 +1310,7 @@ class ProxyBaseLLMRequestProcessing: supports_stream_options=lambda: _model_deployments_support_stream_options( model=self.data.get("model"), llm_router=llm_router, + team_id=user_api_key_dict.team_id, ), ) ) diff --git a/tests/test_litellm/proxy/test_common_request_processing.py b/tests/test_litellm/proxy/test_common_request_processing.py index 28fb97006d8..3bb84e095a0 100644 --- a/tests/test_litellm/proxy/test_common_request_processing.py +++ b/tests/test_litellm/proxy/test_common_request_processing.py @@ -5265,12 +5265,12 @@ class TestApplyStreamUsageTracking: class TestModelDeploymentsSupportStreamOptions: - def _support(self, model, llm_router=None) -> bool: + def _support(self, model, llm_router=None, team_id=None) -> bool: from litellm.proxy.common_request_processing import ( _model_deployments_support_stream_options, ) - return _model_deployments_support_stream_options(model=model, llm_router=llm_router) + return _model_deployments_support_stream_options(model=model, llm_router=llm_router, team_id=team_id) def test_openai_compatible_deployment_supports_stream_options(self): router = litellm.Router( @@ -5335,5 +5335,22 @@ class TestModelDeploymentsSupportStreamOptions: def test_unmapped_model_name_is_not_injected(self): assert self._support("some-unmapped-public-alias", None) is False + def test_team_alias_model_resolves_with_team_id(self): + router = litellm.Router( + model_list=[ + { + "model_name": "model_name_team-1_8b6a0b3f", + "litellm_params": {"model": "azure/gpt-5.4-nano", "api_key": "fake"}, + "model_info": { + "team_id": "team-1", + "team_public_model_name": "team-gpt", + }, + } + ] + ) + + assert self._support("team-gpt", router, team_id="team-1") is True + assert self._support("team-gpt", router, team_id=None) is False + def test_non_string_model_is_not_injected(self): assert self._support(None, None) is False From 8ccbc3e735788bf1db686d04a08102b4add8b0de Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Thu, 30 Jul 2026 18:20:04 -0700 Subject: [PATCH 09/11] test(e2e): skip the batch rate-limiter spend-row test pending LIT-5027 (#35301) The batch rate limiter counts input tokens by awaiting litellm.afile_content with no timeout, so a slow Files API holds POST /v1/batches open past any client deadline; stage saw 63.6s against the harness's 60s read timeout. The test times out before reaching the unattributed-spend-row assertion it exists to guard, so it reports an infrastructure hang rather than the contract. Skipping keeps the signal honest until the fetch is bounded. --- tests/e2e/batches/test_batches_e2e.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/e2e/batches/test_batches_e2e.py b/tests/e2e/batches/test_batches_e2e.py index 5c25b7f2a93..1376bdbed38 100644 --- a/tests/e2e/batches/test_batches_e2e.py +++ b/tests/e2e/batches/test_batches_e2e.py @@ -397,6 +397,16 @@ def unattributed_rows(rows: list[SpendLogRow]) -> list[SpendLogRow]: return [row for row in rows if not row.api_key] +@pytest.mark.skip( + reason=( + "LIT-5027: the path under test hangs. The batch rate limiter reads the input file " + "to count tokens by awaiting litellm.afile_content with no timeout, so a slow Files " + "API holds POST /v1/batches open past any client deadline (63.6s observed on stage " + "against a 60s read timeout). The unattributed-spend-row contract below is never " + "reached, so the test reports a timeout rather than the behavior it guards. Unskip " + "once the fetch is bounded." + ) +) def test_rate_limited_batch_create_leaves_no_unattributed_spend_row( client: BatchClient, resources: ResourceManager, batch_deployments: None ) -> None: From b408b1d6dcddf7008bbd6737c3a921b34afb3b8b Mon Sep 17 00:00:00 2001 From: tin-berri Date: Thu, 30 Jul 2026 18:53:31 -0700 Subject: [PATCH 10/11] fix(guardrails/headroom): stop compressing the turn the model must act on (#35294) The Headroom guardrail sent every message to /v1/compress, including the system prompt and the user's current instruction. On an agentic /v1/messages request the live turn is the largest compressible blob, so it came back as a hash marker; the model then called headroom_retrieve and got its own instruction returned in a tool_result block, which reads as data it fetched rather than a request to act on, so it described the content instead of doing the work. litellm already owns the policy for what a compressor may never rewrite: get_protected_indices covers the system rows, the last user row and the last assistant row, and compress() expands it over whole tool exchanges. Headroom now consults it (promoted from a private name and given tests) and expands it the same way, so the trailing tool result cannot come back as a marker standing in for the result of the call the model just made. Protected rows are withheld from the payload rather than pinned afterwards, so their tokens are not reported as savings that are never applied; the write-back discards a compressed system prompt outright, so that saving never existed. The cost is that a query-aware service no longer sees the newest user message. A response whose row count differs from what was sent can no longer be interleaved with the withheld rows, so it goes through the configured fail policy instead of being adopted. Fail-open now returns the caller's own inputs object: translation handlers detect a rewrite by identity, so a rebuilt copy sent an unchanged request through the Anthropic write-back for nothing. That write-back rebuilt the request with one anthropic_messages_pt call, which merges every run of consecutive user/tool rows, so a tool_result turn and the user turn after it arrived fused. Converting a row at a time would separate them but breaks tool pairing: with modify_params on, an assistant row whose results are converted separately reads as an orphaned tool call and the sanitizer answers it with a synthetic "tool execution skipped" result while dropping the real one. Conversion is now grouped by tool_call_id ownership, which satisfies both, and the same grouping decides which rows headroom protects, so the two agree by construction. The CCR follow-up also dropped any text the model wrote alongside its tool call, and echoed tool calls it had no results for. Both are fixed by reusing compresr's extraction helper, now shared instead of duplicated. Resolves LIT-5018 --- .github/workflows/test-unit-misc.yml | 1 + litellm/compression/compress.py | 33 +- .../prompt_templates/factory.py | 45 ++- .../chat/guardrail_translation/handler.py | 24 +- .../guardrail_hooks/compresr/compresr.py | 48 +-- .../guardrail_hooks/content_text.py | 40 ++ .../guardrail_hooks/headroom/headroom.py | 157 ++++++-- .../test_litellm/compression/test_compress.py | 55 +++ ...llm_core_utils_prompt_templates_factory.py | 72 ++++ .../guardrail_hooks/test_headroom.py | 345 ++++++++++++++++-- .../test_structured_messages_writeback.py | 73 ++++ 11 files changed, 762 insertions(+), 131 deletions(-) create mode 100644 tests/test_litellm/compression/test_compress.py diff --git a/.github/workflows/test-unit-misc.yml b/.github/workflows/test-unit-misc.yml index 7c3b195f0ad..9afaaaead93 100644 --- a/.github/workflows/test-unit-misc.yml +++ b/.github/workflows/test-unit-misc.yml @@ -27,6 +27,7 @@ jobs: tests/test_litellm/a2a_protocol tests/test_litellm/anthropic_interface tests/test_litellm/completion_extras + tests/test_litellm/compression tests/test_litellm/containers tests/test_litellm/experimental_mcp_client tests/test_litellm/models diff --git a/litellm/compression/compress.py b/litellm/compression/compress.py index 004dd82cbaa..9c5f57bc98f 100644 --- a/litellm/compression/compress.py +++ b/litellm/compression/compress.py @@ -3,6 +3,7 @@ Main compress() function — normalizes input messages, orchestrates BM25/embedd scoring, message stubbing, and retrieval tool injection. """ +from collections.abc import Mapping, Sequence from typing import Any, Dict, List, Optional, Set, Tuple, Union, cast from litellm.caching.dual_cache import DualCache @@ -204,33 +205,21 @@ def _extract_anthropic_tool_exchange_spans( return spans, None -def _get_protected_indices(messages: List[dict]) -> List[int]: +def get_protected_indices(messages: Sequence[Mapping[str, object]]) -> tuple[int, ...]: """ Return indices of messages that must never be compressed: - All system messages - The last user message - The last assistant message + + The last user message is what the model is being asked to act on right now, + so compressing it replaces the live instruction with a marker. Compression + guardrails share this policy; see the Headroom guardrail. """ - protected: List[int] = [] - - last_user_idx = None - last_assistant_idx = None - - for i, msg in enumerate(messages): - role = msg.get("role", "") - if role == "system": - protected.append(i) - elif role == "user": - last_user_idx = i - elif role == "assistant": - last_assistant_idx = i - - if last_user_idx is not None: - protected.append(last_user_idx) - if last_assistant_idx is not None: - protected.append(last_assistant_idx) - - return protected + system_indices = tuple(index for index, msg in enumerate(messages) if msg.get("role", "") == "system") + last_user = tuple(index for index, msg in enumerate(messages) if msg.get("role", "") == "user")[-1:] + last_assistant = tuple(index for index, msg in enumerate(messages) if msg.get("role", "") == "assistant")[-1:] + return system_indices + last_user + last_assistant def _combine_scores( @@ -432,7 +421,7 @@ def compress( combined_scores = bm25_scores # Protected messages are never compressed - protected_indices = _get_protected_indices(normalized_messages) + protected_indices = get_protected_indices(normalized_messages) kept_indices: Set[int] = set(protected_indices) tool_exchange_spans: List[Set[int]] = [] diff --git a/litellm/litellm_core_utils/prompt_templates/factory.py b/litellm/litellm_core_utils/prompt_templates/factory.py index d8ce48f05de..0752bf2d771 100644 --- a/litellm/litellm_core_utils/prompt_templates/factory.py +++ b/litellm/litellm_core_utils/prompt_templates/factory.py @@ -6,7 +6,7 @@ import mimetypes import re import xml.etree.ElementTree as ET from enum import Enum -from collections.abc import Mapping +from collections.abc import Iterator, Mapping, Sequence from typing import Any, Dict, List, Optional, Set, Tuple, TypedDict, Union, cast, overload from jinja2.sandbox import ImmutableSandboxedEnvironment @@ -2210,6 +2210,49 @@ def _is_orphaned_tool_result( return False +def _declared_tool_call_ids(message: Mapping[str, Any]) -> frozenset[str]: + tool_calls = message.get("tool_calls") + if not isinstance(tool_calls, list): + return frozenset() + return frozenset( + str(tool_call["id"]) for tool_call in tool_calls if isinstance(tool_call, Mapping) and tool_call.get("id") + ) + + +def group_tool_exchanges(messages: Sequence[Mapping[str, Any]]) -> tuple[tuple[int, ...], ...]: + """Group message indices into tool exchanges: an assistant row that made + tool calls, together with the tool rows answering the ids it declared. + + Membership is by ``tool_call_id`` ownership rather than adjacency, so a tool + row belonging to some other call opens its own group instead of being swept + into the exchange it happens to sit next to. Every other row is its own + group. Groups stay contiguous and in order, so a caller can convert or + protect them without reordering the conversation. + + Callers need this because an assistant row and the tool rows answering it + are only well-formed together: ``sanitize_messages_for_tool_calling`` reads + an assistant row whose results are missing as an orphaned tool call, and + a tool row whose call is missing as an orphaned result. + """ + return tuple(_iter_tool_exchange_groups(messages)) + + +def _iter_tool_exchange_groups(messages: Sequence[Mapping[str, Any]]) -> Iterator[tuple[int, ...]]: + index = 0 + while index < len(messages): + declared = _declared_tool_call_ids(messages[index]) + end = index + 1 + while ( + declared + and end < len(messages) + and messages[end].get("role") in ("tool", "function") + and str(messages[end].get("tool_call_id")) in declared + ): + end += 1 + yield tuple(range(index, end)) + index = end + + def sanitize_messages_for_tool_calling( messages: List[AllMessageValues], ) -> List[AllMessageValues]: diff --git a/litellm/llms/anthropic/chat/guardrail_translation/handler.py b/litellm/llms/anthropic/chat/guardrail_translation/handler.py index 90f735707bf..a549db94224 100644 --- a/litellm/llms/anthropic/chat/guardrail_translation/handler.py +++ b/litellm/llms/anthropic/chat/guardrail_translation/handler.py @@ -361,14 +361,34 @@ class AnthropicMessagesHandler(BaseTranslation): @staticmethod def _write_back_structured_messages(data: dict, structured_messages: list) -> None: - """Convert compressed structured_messages back to Anthropic format and write to data.""" + """Convert compressed structured_messages back to Anthropic format and write to data. + + ``anthropic_messages_pt`` merges every run of consecutive user/tool rows + into a single message, so a turn carrying only tool results and the user + turn that follows it come back fused, and the request the model sees no + longer has the boundaries the client sent. Converting a row at a time + would keep them apart but breaks tool pairing: an assistant row whose + tool results sit outside its own call reads as an orphaned tool call, + and under ``modify_params`` the sanitizer answers it with a synthetic + "tool execution skipped" result and drops the real one. Converting each + assistant row together with the tool rows that answer it, and every + other row on its own, satisfies both. + """ from litellm.litellm_core_utils.prompt_templates.factory import ( anthropic_messages_pt, + group_tool_exchanges, ) model = str(data.get("model") or "") non_system = [m for m in structured_messages if m.get("role") != "system"] - converted = anthropic_messages_pt(messages=non_system, model=model, llm_provider="anthropic") + groups = tuple([non_system[index] for index in group] for group in group_tool_exchanges(non_system)) or ( + non_system, + ) + converted = [ + message + for group in groups + for message in anthropic_messages_pt(messages=group, model=model, llm_provider="anthropic") + ] for msg in converted: content = msg.get("content") if isinstance(content, list): diff --git a/litellm/proxy/guardrails/guardrail_hooks/compresr/compresr.py b/litellm/proxy/guardrails/guardrail_hooks/compresr/compresr.py index e512be23fc9..4627b298d09 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/compresr/compresr.py +++ b/litellm/proxy/guardrails/guardrail_hooks/compresr/compresr.py @@ -48,6 +48,7 @@ from litellm.llms.custom_httpx.http_handler import ( ) from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.guardrails.guardrail_hooks.content_text import ( + assistant_text_from_response, content_to_text, is_all_text_parts, merge_rewritten_text_parts, @@ -391,47 +392,6 @@ def _is_anthropic_messages_response(response: object) -> bool: return isinstance(get_attribute_or_key(response, "content", None), list) -def _assistant_text_from_response(response: object) -> str | None: - """The assistant's natural-language text from a model response, across chat, - Anthropic, and Responses shapes. Preserved when the turn is rebuilt for the - retrieval follow-up so the model's reasoning is not lost.""" - choices = get_attribute_or_key(response, "choices", None) - if isinstance(choices, list) and choices: - message = get_attribute_or_key(choices[0], "message", None) - if message is not None: - text = content_to_text(get_attribute_or_key(message, "content", None)) - if text: - return text - content = get_attribute_or_key(response, "content", None) - if isinstance(content, list): - parts = [ - text - for block in content - if get_attribute_or_key(block, "type", None) == "text" - for text in (get_attribute_or_key(block, "text", None),) - if isinstance(text, str) and text - ] - if parts: - return "".join(parts) - output = get_attribute_or_key(response, "output", None) - if isinstance(output, list): - parts = [] - for item in output: - if get_attribute_or_key(item, "type", None) != "message": - continue - item_content = get_attribute_or_key(item, "content", None) - if not isinstance(item_content, list): - continue - for chunk in item_content: - if get_attribute_or_key(chunk, "type", None) == "output_text": - text = get_attribute_or_key(chunk, "text", None) - if isinstance(text, str) and text: - parts.append(text) - if parts: - return "".join(parts) - return None - - def _build_assistant_message_from_response( response: object, retrieved: list[tuple[dict[str, object], str]], @@ -446,7 +406,7 @@ def _build_assistant_message_from_response( """ return { "role": "assistant", - "content": _assistant_text_from_response(response), + "content": assistant_text_from_response(response), "tool_calls": [ { "id": tool_call.get("id"), @@ -470,7 +430,7 @@ def _build_anthropic_followup_messages( assistant text is preserved; non-retrieve tool calls are re-planned by the follow-up (see _build_assistant_message_from_response).""" assistant_content: list[dict[str, object]] = [] - text = _assistant_text_from_response(response) + text = assistant_text_from_response(response) if text: assistant_content.append({"type": "text", "text": text}) assistant_content.extend( @@ -501,7 +461,7 @@ def _build_responses_followup_items( with a function_call_output keyed by the same call_id. The assistant text is preserved; non-retrieve tool calls are re-planned by the follow-up.""" items: list[dict[str, object]] = [] - text = _assistant_text_from_response(response) + text = assistant_text_from_response(response) if text: items.append({"role": "assistant", "content": text}) for tool_call, content in retrieved: diff --git a/litellm/proxy/guardrails/guardrail_hooks/content_text.py b/litellm/proxy/guardrails/guardrail_hooks/content_text.py index f4211e67512..4111c909d01 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/content_text.py +++ b/litellm/proxy/guardrails/guardrail_hooks/content_text.py @@ -14,6 +14,8 @@ non-text part, which is what ``is_all_text_parts`` gates. from collections.abc import Sequence +from litellm.litellm_core_utils.prompt_templates.factory import get_attribute_or_key + def content_to_text(content: object) -> str: """Collapse a message ``content`` (str or list-of-parts) to plain text. @@ -53,3 +55,41 @@ def merge_rewritten_text_parts(parts: Sequence[object], new_text: str) -> list[o breakpoints = tuple(part["cache_control"] for part in dict_parts if part.get("cache_control") is not None) base = {**dict_parts[0], "text": new_text} if dict_parts else {"type": "text", "text": new_text} return [{**base, "cache_control": breakpoints[-1]} if breakpoints else base] + + +def assistant_text_from_response(response: object) -> str | None: + """The assistant's natural-language text from a model response, across chat, + Anthropic, and Responses shapes. Preserved when the turn is rebuilt for the + retrieval follow-up so the model's reasoning is not lost.""" + choices = get_attribute_or_key(response, "choices", None) + if isinstance(choices, list) and choices: + message = get_attribute_or_key(choices[0], "message", None) + if message is not None: + text = content_to_text(get_attribute_or_key(message, "content", None)) + if text: + return text + content = get_attribute_or_key(response, "content", None) + if isinstance(content, list): + parts = [ + text + for block in content + if get_attribute_or_key(block, "type", None) == "text" + for text in (get_attribute_or_key(block, "text", None),) + if isinstance(text, str) and text + ] + if parts: + return "".join(parts) + output = get_attribute_or_key(response, "output", None) + if isinstance(output, list): + output_parts = [ + text + for item in output + if get_attribute_or_key(item, "type", None) == "message" + for chunk in (get_attribute_or_key(item, "content", None) or ()) + if get_attribute_or_key(chunk, "type", None) == "output_text" + for text in (get_attribute_or_key(chunk, "text", None),) + if isinstance(text, str) and text + ] + if output_parts: + return "".join(output_parts) + return None diff --git a/litellm/proxy/guardrails/guardrail_hooks/headroom/headroom.py b/litellm/proxy/guardrails/guardrail_hooks/headroom/headroom.py index 2735acd7787..1667bb604ba 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/headroom/headroom.py +++ b/litellm/proxy/guardrails/guardrail_hooks/headroom/headroom.py @@ -7,6 +7,7 @@ import uuid from typing import TYPE_CHECKING, Any, ClassVar, List, Literal, Optional import httpx +from collections.abc import Mapping, Sequence from fastapi import HTTPException import litellm @@ -15,6 +16,7 @@ from litellm.proxy.spend_tracking.compression_savings import HEADROOM_GUARDRAIL_ from typing_extensions import TypeGuard from litellm._logging import verbose_proxy_logger +from litellm.compression.compress import get_protected_indices from litellm.integrations.custom_guardrail import ( CustomGuardrail, log_guardrail_information, @@ -22,6 +24,7 @@ from litellm.integrations.custom_guardrail import ( from litellm.litellm_core_utils.prompt_templates.factory import ( get_attribute_or_key, get_tool_calls_from_response, + group_tool_exchanges, has_tool_with_name, ) from litellm.llms.custom_httpx.http_handler import ( @@ -29,6 +32,7 @@ from litellm.llms.custom_httpx.http_handler import ( httpxSpecialProvider, ) from litellm.proxy.guardrails.guardrail_hooks.content_text import ( + assistant_text_from_response, content_to_text, is_all_text_parts, merge_rewritten_text_parts, @@ -110,6 +114,42 @@ def _restore_content_shapes( return restored +def _protected_indices(messages: Sequence[Mapping[str, object]]) -> frozenset[int]: + """Indices headroom must not send to the compression service. + + ``get_protected_indices`` is litellm's own compression policy: the system + rows, the last user row, the last assistant row. It is expanded over whole + tool exchanges the way ``compress()`` expands it, so a protected assistant + tool call cannot end up answered by a marker standing in for the result the + model just asked for. + """ + protected = frozenset(get_protected_indices(messages)) + return protected | frozenset( + index + for group in group_tool_exchanges(messages) + if any(member in protected for member in group) + for index in group + ) + + +def _restore_protected_messages( + messages: Sequence[dict[str, object]], + compressed: Sequence[dict[str, object]], + protected_indices: frozenset[int], +) -> Sequence[dict[str, object]]: + """Put the rows that were held back from compression at their original positions. + + Requires one returned row per row actually sent, which ``_call_compress`` + enforces; a service that changed the row count is treated as a failure + there, because a reshaped conversation cannot be re-interleaved. + """ + sent_positions = tuple(index for index in range(len(messages)) if index not in protected_indices) + compressed_by_index = dict(zip(sent_positions, compressed)) + return [ + messages[index] if index in protected_indices else compressed_by_index[index] for index in range(len(messages)) + ] + + def extract_hashes_from_messages(messages: list[dict[str, object]]) -> list[str]: hashes: list[str] = [] for msg in messages: @@ -175,30 +215,33 @@ def _extract_headroom_tool_calls(response: object) -> list[dict[str, object]]: ] -def _build_assistant_message_from_response(response: object) -> dict[str, object]: - choices = getattr(response, "choices", None) - if not isinstance(choices, list) or not choices: - return {"role": "assistant", "content": None, "tool_calls": []} - message = getattr(choices[0], "message", None) - if message is None: - return {"role": "assistant", "content": None, "tool_calls": []} - content = getattr(message, "content", None) - tool_calls = getattr(message, "tool_calls", None) - raw_tool_calls: list[dict[str, object]] = [] - if isinstance(tool_calls, list): - for tc in tool_calls: - fn = getattr(tc, "function", None) - raw_tool_calls.append( - { - "id": getattr(tc, "id", None), - "type": "function", - "function": { - "name": getattr(fn, "name", None) if fn else None, - "arguments": getattr(fn, "arguments", "{}") if fn else "{}", - }, - } - ) - return {"role": "assistant", "content": content, "tool_calls": raw_tool_calls} +def _build_assistant_message_from_response( + response: object, + retrieved: Sequence[tuple[dict[str, object], str]], +) -> dict[str, object]: + """Rebuild the chat-completions assistant turn for the retrieval follow-up. + + Only the ``headroom_retrieve`` calls are echoed, each answered by a tool + result below. Other tool calls made in the same turn are omitted on purpose: + the follow-up re-runs the model with the recovered content so it re-plans + them. Echoing them would leave tool_calls with no matching tool result and + the provider would reject the request. + """ + return { + "role": "assistant", + "content": assistant_text_from_response(response), + "tool_calls": [ + { + "id": tool_call.get("id"), + "type": "function", + "function": { + "name": tool_call.get("name"), + "arguments": json.dumps(tool_call.get("arguments", {})), + }, + } + for tool_call, _ in retrieved + ], + } def _is_responses_api_response(response: object) -> bool: @@ -213,17 +256,22 @@ def _is_anthropic_messages_response(response: object) -> bool: def _build_anthropic_followup_messages( + response: object, retrieved: list[tuple[dict[str, object], str]], ) -> list[dict[str, object]]: """Build Anthropic Messages API follow-up messages for a tool round-trip. Anthropic requires the tool_use block to be echoed back in an assistant message, paired with a tool_result block in a user message keyed by the - same tool_use_id -- it does not accept chat-style tool-role messages. + same tool_use_id -- it does not accept chat-style tool-role messages. Any + text the model wrote alongside the tool call is preserved, so its reasoning + survives into the follow-up turn. """ + text = assistant_text_from_response(response) assistant_message: dict[str, object] = { "role": "assistant", - "content": [ + "content": ([{"type": "text", "text": text}] if text else []) + + [ { "type": "tool_use", "id": tool_call.get("id"), @@ -244,15 +292,18 @@ def _build_anthropic_followup_messages( def _build_responses_followup_items( + response: object, retrieved: list[tuple[dict[str, object], str]], ) -> list[dict[str, object]]: """Build Responses API input items for a tool round-trip. The Responses API does not accept chat-style assistant/tool messages as follow-up input; it requires the model's function_call to be echoed back - paired with a function_call_output keyed by the same call_id. + paired with a function_call_output keyed by the same call_id. Any text the + model wrote alongside the tool call is preserved. """ - items: list[dict[str, object]] = [] + text = assistant_text_from_response(response) + items: List[dict[str, object]] = [{"role": "assistant", "content": text}] if text else [] for tool_call, content in retrieved: call_id = tool_call.get("id") items.append( @@ -453,6 +504,19 @@ class HeadroomGuardrail(CustomGuardrail): {}, ) + if len(filtered) != len(messages): + # Rows are matched positionally when the never-compressed messages + # are put back, so a reshaped conversation cannot be applied at all. + return ( + self._handle_compress_failure( + messages, + "Headroom compression service changed the message count", + {"sent": len(messages), "returned": len(filtered)}, + ), + False, + {}, + ) + verbose_proxy_logger.debug( "Headroom: compressed %s tokens -> %s tokens (ratio %.2f)", body.get("tokens_before", "?"), @@ -547,14 +611,27 @@ class HeadroomGuardrail(CustomGuardrail): if not messages: return inputs + # The last user message is the instruction the model is being asked to + # act on, so replacing it with a marker means the model answers a + # retrieval result instead of the request. Protected rows are held back + # from the payload rather than pinned after the fact, so their tokens + # are not counted as savings we never apply; the Anthropic write-back + # discards a compressed system prompt outright. Keep it that way unless + # /v1/compress grows a field for sending the live turn as the retrieval + # query without compressing it: query-aware compression reads the newest + # user message, so it is withheld here at some cost to history ranking. + protected_indices = _protected_indices(messages) + compressible = [m for i, m in enumerate(messages) if i not in protected_indices] + if not compressible: + return inputs + model = self.headroom_model or request_data.get("model") start_time = time.time() - compressed, compression_succeeded, stats = await self._call_compress( - messages=_flatten_messages_for_compression(messages), + returned, compression_succeeded, stats = await self._call_compress( + messages=_flatten_messages_for_compression(compressible), model=model if isinstance(model, str) else None, ) end_time = time.time() - compressed = _restore_content_shapes(originals=messages, returned=compressed) from litellm.proxy.common_utils.callback_utils import ( add_guardrail_to_applied_guardrails_header, @@ -571,7 +648,17 @@ class HeadroomGuardrail(CustomGuardrail): duration=end_time - start_time, ) add_guardrail_to_applied_guardrails_header(request_data=request_data, guardrail_name=self.guardrail_name) - return {**inputs, "structured_messages": compressed} # pyright: ignore[reportReturnType] + # Hand back the caller's own inputs object. Translation handlers + # detect "the guardrail rewrote the messages" by identity, so + # returning a rebuilt copy sends an unchanged request through the + # write-back and restructures it for nothing. + return inputs + + compressed = _restore_protected_messages( + messages=messages, + compressed=_restore_content_shapes(originals=compressible, returned=returned), + protected_indices=protected_indices, + ) self.add_standard_logging_guardrail_information_to_request_data( guardrail_json_response=stats, @@ -668,11 +755,11 @@ class HeadroomGuardrail(CustomGuardrail): retrieved.append((tc, content)) if _is_responses_api_response(response): - follow_up_messages = list(messages) + _build_responses_followup_items(retrieved) + follow_up_messages = list(messages) + _build_responses_followup_items(response, retrieved) elif _is_anthropic_messages_response(response): - follow_up_messages = list(messages) + _build_anthropic_followup_messages(retrieved) + follow_up_messages = list(messages) + _build_anthropic_followup_messages(response, retrieved) else: - assistant_message = _build_assistant_message_from_response(response) + assistant_message = _build_assistant_message_from_response(response, retrieved) tool_results = [ {"role": "tool", "tool_call_id": tc.get("id"), "content": content} for tc, content in retrieved ] diff --git a/tests/test_litellm/compression/test_compress.py b/tests/test_litellm/compression/test_compress.py new file mode 100644 index 00000000000..6827c37dfd5 --- /dev/null +++ b/tests/test_litellm/compression/test_compress.py @@ -0,0 +1,55 @@ +""" +Unit tests for litellm.compression.compress helpers. + +get_protected_indices is the shared policy for which messages a compressor may +never rewrite. It is consumed by compress() and by the Headroom guardrail, so +the two agree on what "never compress this" means. +""" + +from litellm.compression.compress import get_protected_indices + + +def test_protects_system_last_user_and_last_assistant(): + messages = [ + {"role": "system", "content": "sys"}, + {"role": "user", "content": "old question"}, + {"role": "assistant", "content": "old answer"}, + {"role": "user", "content": "newer question"}, + {"role": "assistant", "content": "newer answer"}, + {"role": "user", "content": "live instruction"}, + ] + + assert sorted(get_protected_indices(messages)) == [0, 4, 5] + + +def test_history_is_not_protected(): + messages = [ + {"role": "user", "content": "old question"}, + {"role": "assistant", "content": "old answer"}, + {"role": "tool", "tool_call_id": "t1", "content": "old tool output"}, + {"role": "user", "content": "live instruction"}, + ] + + protected = sorted(get_protected_indices(messages)) + + assert protected == [1, 3] + # The tool row and the older user turn stay compressible; protection that + # covered everything would make compression a no-op. + assert 0 not in protected + assert 2 not in protected + + +def test_every_system_row_is_protected(): + messages = [ + {"role": "system", "content": "first"}, + {"role": "user", "content": "q"}, + {"role": "system", "content": "second, injected mid conversation"}, + {"role": "user", "content": "live"}, + ] + + assert sorted(get_protected_indices(messages)) == [0, 2, 3] + + +def test_no_user_or_assistant_rows(): + assert sorted(get_protected_indices([{"role": "system", "content": "sys"}])) == [0] + assert get_protected_indices([]) == () diff --git a/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py b/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py index 9565de1139c..dc745abb9e7 100644 --- a/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py +++ b/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py @@ -3197,3 +3197,75 @@ def test_get_tool_calls_from_response_include_all_choices_reads_every_choice(): names = [tc["name"] for tc in get_tool_calls_from_response(response, include_all_choices=True)] assert names == ["tool_alpha", "tool_beta"] + + +def test_group_tool_exchanges_pairs_assistant_with_its_tool_rows(): + from litellm.litellm_core_utils.prompt_templates.factory import group_tool_exchanges + + messages = [ + {"role": "user", "content": "first turn"}, + { + "role": "assistant", + "content": None, + "tool_calls": [ + {"id": "tu_1", "type": "function", "function": {"name": "Read", "arguments": "{}"}}, + {"id": "tu_2", "type": "function", "function": {"name": "Grep", "arguments": "{}"}}, + ], + }, + {"role": "tool", "tool_call_id": "tu_1", "content": "file body"}, + {"role": "tool", "tool_call_id": "tu_2", "content": "matches"}, + {"role": "user", "content": "live instruction"}, + ] + + assert group_tool_exchanges(messages) == ((0,), (1, 2, 3), (4,)) + + +def test_group_tool_exchanges_uses_ownership_not_adjacency(): + """A tool row answering some other call must not be swept into the exchange + it happens to sit next to.""" + from litellm.litellm_core_utils.prompt_templates.factory import group_tool_exchanges + + messages = [ + { + "role": "assistant", + "content": None, + "tool_calls": [{"id": "tu_1", "type": "function", "function": {"name": "Read", "arguments": "{}"}}], + }, + {"role": "tool", "tool_call_id": "unrelated", "content": "not an answer to tu_1"}, + {"role": "tool", "tool_call_id": "tu_1", "content": "file body"}, + ] + + assert group_tool_exchanges(messages) == ((0,), (1,), (2,)) + + +def test_group_tool_exchanges_assistant_without_tool_calls_stands_alone(): + from litellm.litellm_core_utils.prompt_templates.factory import group_tool_exchanges + + messages = [ + {"role": "assistant", "content": "no tools here"}, + {"role": "user", "content": "next"}, + ] + + assert group_tool_exchanges(messages) == ((0,), (1,)) + assert group_tool_exchanges([]) == () + + +def test_group_tool_exchanges_is_linear_in_message_count(): + """Grouping runs on every guardrail write-back, over a message array the + caller controls, so it has to stay linear. Accumulating groups by rebuilding + a tuple each iteration made this O(n^2): 20k standalone messages took 312ms + and 100k would take minutes. Linear finishes in single-digit ms, so this + ceiling has ~200x headroom while a quadratic rewrite blows straight past it. + """ + import time + + from litellm.litellm_core_utils.prompt_templates.factory import group_tool_exchanges + + messages = [{"role": "user", "content": "x"} for _ in range(100_000)] + + started = time.perf_counter() + groups = group_tool_exchanges(messages) + elapsed = time.perf_counter() - started + + assert len(groups) == 100_000 + assert elapsed < 3.0, f"grouping 100k messages took {elapsed:.2f}s; suspect superlinear accumulation" diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_headroom.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_headroom.py index 248893ed153..00ab39357b4 100644 --- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_headroom.py +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_headroom.py @@ -43,21 +43,35 @@ from litellm.types.utils import GenericGuardrailAPIInputs FAKE_API_BASE = "https://headroom.example.com" FAKE_API_KEY = "test-key" +# The system prompt, the last user turn and the last assistant turn are never +# sent to the compression service, so a fixture needs history for anything to +# be eligible: only index 1 is. ORIGINAL_MESSAGES = [ {"role": "system", "content": "You are a helpful assistant."}, {"role": "user", "content": "A" * 5000}, + {"role": "assistant", "content": "Understood."}, + {"role": "user", "content": "and what about B?"}, ] -COMPRESSED_MESSAGES = [ - {"role": "system", "content": "You are a helpful assistant."}, - {"role": "user", "content": "A" * 500}, -] +COMPRESSIBLE_MESSAGES = [ORIGINAL_MESSAGES[1]] +COMPRESSED_MESSAGES = [{"role": "user", "content": "A" * 500}] COMPRESSED_MESSAGES_WITH_HASH = [ - {"role": "system", "content": "You are a helpful assistant."}, { "role": "user", "content": "Summary. Retrieve more: hash=b573993006976af767214fac", }, ] +EXPECTED_MESSAGES = [ + ORIGINAL_MESSAGES[0], + COMPRESSED_MESSAGES[0], + ORIGINAL_MESSAGES[2], + ORIGINAL_MESSAGES[3], +] +EXPECTED_MESSAGES_WITH_HASH = [ + ORIGINAL_MESSAGES[0], + COMPRESSED_MESSAGES_WITH_HASH[0], + ORIGINAL_MESSAGES[2], + ORIGINAL_MESSAGES[3], +] def _make_guardrail(**kwargs) -> HeadroomGuardrail: @@ -161,7 +175,7 @@ async def test_apply_guardrail_compresses_and_returns_structured_messages( input_type="request", ) - assert result.get("structured_messages") == COMPRESSED_MESSAGES + assert result.get("structured_messages") == EXPECTED_MESSAGES entries = _recorded_guardrail_entries(request_data) assert len(entries) == 1 @@ -275,7 +289,7 @@ async def test_apply_guardrail_skips_derivation_for_non_numeric_token_counts( assert "tokens_saved" not in _recorded_guardrail_response(request_data) # Compression itself is unaffected by the skipped derivation. - assert result.get("structured_messages") == COMPRESSED_MESSAGES + assert result.get("structured_messages") == EXPECTED_MESSAGES @pytest.mark.asyncio @@ -1571,9 +1585,15 @@ PARTS_MESSAGES = [ "role": "system", "content": [ {"type": "text", "text": "You are Claude Code.", "cache_control": {"type": "ephemeral"}}, + ], + }, + { + "role": "user", + "content": [ + {"type": "text", "text": "Earlier turn.", "cache_control": {"type": "ephemeral"}}, { "type": "text", - "text": "Second system block. " + "B" * 5000, + "text": "Second block. " + "B" * 5000, "cache_control": {"type": "ephemeral", "ttl": "1h"}, }, ], @@ -1586,9 +1606,10 @@ PARTS_MESSAGES = [ ], }, {"role": "tool", "content": "tool output " + "C" * 500}, + {"role": "user", "content": "what does that file do?"}, ] -FLATTENED_SYSTEM_TEXT = "You are Claude Code.\n\nSecond system block. " + "B" * 5000 +FLATTENED_HISTORY_TEXT = "Earlier turn.\n\nSecond block. " + "B" * 5000 def _parts_copy() -> list: @@ -1596,10 +1617,13 @@ def _parts_copy() -> list: def _echo_wire_view() -> list: - """What the service receives (and echoes back when it changes nothing).""" + """What the service receives (and echoes back when it changes nothing). + + The system row and the trailing user row are never sent. + """ return [ - {"role": "system", "content": FLATTENED_SYSTEM_TEXT}, - json.loads(json.dumps(PARTS_MESSAGES[1])), + {"role": "user", "content": FLATTENED_HISTORY_TEXT}, + json.loads(json.dumps(PARTS_MESSAGES[2])), {"role": "tool", "content": "tool output " + "C" * 500}, ] @@ -1627,7 +1651,7 @@ async def test_apply_guardrail_flattens_all_text_rows_only( ) wire_messages = mock_post.call_args.kwargs["json"]["messages"] - assert wire_messages[0]["content"] == FLATTENED_SYSTEM_TEXT + assert wire_messages[0]["content"] == FLATTENED_HISTORY_TEXT # Mixed text+image row is never flattened: merging its text would move a # later cache_control breakpoint across the image part. assert isinstance(wire_messages[1]["content"], list) @@ -1643,7 +1667,7 @@ async def test_apply_guardrail_restores_rewritten_all_text_row( structured_messages=_parts_copy(), ) compressed = _echo_wire_view() - compressed[0]["content"] = "compressed system. Retrieve more: hash=b573993006976af767214fac" + compressed[0]["content"] = "compressed history. Retrieve more: hash=b573993006976af767214fac" mock_response = _make_compress_response(compressed) with patch.object( @@ -1659,17 +1683,17 @@ async def test_apply_guardrail_restores_rewritten_all_text_row( ) messages = result["structured_messages"] - system_content = messages[0]["content"] + history_content = messages[1]["content"] # Rewritten all-text row collapses to one part carrying the LAST declared # breakpoint: an Anthropic breakpoint caches the prefix ending at its # part, so after the merge the last one (and its TTL) still describes the # row. - assert isinstance(system_content, list) - assert len(system_content) == 1 - assert system_content[0]["text"] == "compressed system. Retrieve more: hash=b573993006976af767214fac" - assert system_content[0]["cache_control"] == {"type": "ephemeral", "ttl": "1h"} + assert isinstance(history_content, list) + assert len(history_content) == 1 + assert history_content[0]["text"] == "compressed history. Retrieve more: hash=b573993006976af767214fac" + assert history_content[0]["cache_control"] == {"type": "ephemeral", "ttl": "1h"} # Mixed row passes through byte-identical. - assert messages[1]["content"] == PARTS_MESSAGES[1]["content"] + assert messages[2]["content"] == PARTS_MESSAGES[2]["content"] # Hashes inside restored parts still drive retrieve-tool injection. assert has_headroom_retrieve_tool(result.get("tools") or []) @@ -1701,19 +1725,43 @@ async def test_apply_guardrail_keeps_originals_when_service_echoes_unchanged( @pytest.mark.asyncio -async def test_apply_guardrail_adopts_service_output_when_rows_dropped( +async def test_apply_guardrail_rejects_service_output_when_rows_dropped( guardrail: HeadroomGuardrail, ): + """A reshaped conversation cannot be applied at all: the rows held back from + compression are matched positionally, so a response with a different row + count goes through the fail policy instead of being adopted.""" inputs = GenericGuardrailAPIInputs( texts=["B" * 5000], structured_messages=_parts_copy(), ) - dropped = [ - {"role": "system", "content": FLATTENED_SYSTEM_TEXT}, - {"role": "user", "content": "B" * 50}, - ] + dropped = [{"role": "user", "content": "B" * 50}] mock_response = _make_compress_response(dropped) + with patch.object( + guardrail.async_handler, + "post", + new_callable=AsyncMock, + return_value=mock_response, + ): + with pytest.raises(HTTPException) as exc_info: + await guardrail.apply_guardrail( + inputs=inputs, + request_data={"model": "claude-fable-5"}, + input_type="request", + ) + + assert exc_info.value.status_code == 502 + assert "changed the message count" in str(exc_info.value.detail) + + +@pytest.mark.asyncio +async def test_apply_guardrail_forwards_original_when_rows_dropped_and_fail_open(): + guardrail = _make_guardrail(unreachable_fallback="fail_open") + original = _parts_copy() + inputs = GenericGuardrailAPIInputs(texts=["B" * 5000], structured_messages=original) + mock_response = _make_compress_response([{"role": "user", "content": "B" * 50}]) + with patch.object( guardrail.async_handler, "post", @@ -1726,7 +1774,10 @@ async def test_apply_guardrail_adopts_service_output_when_rows_dropped( input_type="request", ) - assert result["structured_messages"] == dropped + # Same object back, so translation handlers that detect a rewrite by + # identity leave the request alone instead of round-tripping it. + assert result is inputs + assert result["structured_messages"] is original @pytest.mark.asyncio @@ -1739,7 +1790,7 @@ async def test_apply_guardrail_sends_textless_parts_rows_unflattened( ] inputs = GenericGuardrailAPIInputs( texts=["D" * 5000], - structured_messages=json.loads(json.dumps(image_only)), + structured_messages=json.loads(json.dumps(image_only)) + [{"role": "user", "content": "and now?"}], ) mock_response = _make_compress_response(json.loads(json.dumps(image_only))) @@ -1782,3 +1833,243 @@ async def test_fail_open_returns_original_parts_shapes(): messages = result["structured_messages"] assert [m["content"] for m in messages] == [m["content"] for m in PARTS_MESSAGES] + + +# --------------------------------------------------------------------------- +# LIT-5018: the turn the model is being asked to act on is never compressed. +# +# A Claude Code request ends with the live instruction, preceded by the tool +# result answering the assistant's last tool call. Replacing either with a +# marker makes the model answer a retrieval result instead of the request. +# --------------------------------------------------------------------------- + +AGENTIC_MESSAGES = [ + {"role": "system", "content": "You are Claude Code. " + "S" * 5000}, + {"role": "user", "content": "H" * 5000}, + {"role": "assistant", "content": "Older answer. " + "O" * 5000}, + {"role": "tool", "tool_call_id": "old_1", "content": "older tool output " + "T" * 5000}, + { + "role": "assistant", + "content": "Reading the file now.", + "tool_calls": [{"id": "tu_1", "type": "function", "function": {"name": "Read", "arguments": "{}"}}], + }, + {"role": "tool", "tool_call_id": "tu_1", "content": "FILE BODY " + "F" * 5000}, + { + "role": "user", + "content": [ + {"type": "text", "text": " " + "E" * 5000}, + {"type": "text", "text": "can we run /team to fix this"}, + ], + }, +] + + +async def _wire_and_result(guardrail: HeadroomGuardrail, messages: list, returned: list | None = None): + inputs = GenericGuardrailAPIInputs(texts=["x"], structured_messages=json.loads(json.dumps(messages))) + sent: dict = {} + + def _echo(**kwargs): + sent["messages"] = kwargs["json"]["messages"] + return _make_compress_response( + returned if returned is not None else json.loads(json.dumps(kwargs["json"]["messages"])) + ) + + with patch.object(guardrail.async_handler, "post", new_callable=AsyncMock, side_effect=_echo): + result = await guardrail.apply_guardrail( + inputs=inputs, + request_data={"model": "claude-sonnet-4-5-20250929"}, + input_type="request", + ) + return sent["messages"], result + + +@pytest.mark.asyncio +async def test_live_user_turn_is_never_sent_for_compression(guardrail: HeadroomGuardrail): + wire, result = await _wire_and_result(guardrail, AGENTIC_MESSAGES) + + live_turn = AGENTIC_MESSAGES[-1] + assert live_turn not in wire + assert not any("can we run /team to fix this" in json.dumps(row) for row in wire) + # It reaches the model byte-identical, both text parts intact, so no + # marker and no retrieval round-trip stands in for the instruction. + assert result["structured_messages"][-1] == live_turn + + +@pytest.mark.asyncio +async def test_system_prompt_is_never_sent_for_compression(guardrail: HeadroomGuardrail): + wire, result = await _wire_and_result(guardrail, AGENTIC_MESSAGES) + + assert not any(row.get("role") == "system" for row in wire) + # The Anthropic write-back drops compressed system rows, so sending it + # only inflates the savings the service reports back. + assert result["structured_messages"][0] == AGENTIC_MESSAGES[0] + + +@pytest.mark.asyncio +async def test_trailing_tool_exchange_is_never_sent_for_compression(guardrail: HeadroomGuardrail): + """The tool result answering the last assistant's tool call is protected + with it: a marker there stands in for the result of the call the model just + made, forcing an immediate retrieval of data it already asked for.""" + wire, result = await _wire_and_result(guardrail, AGENTIC_MESSAGES) + + assert not any(row.get("tool_call_id") == "tu_1" for row in wire) + assert result["structured_messages"][5] == AGENTIC_MESSAGES[5] + + +@pytest.mark.asyncio +async def test_history_is_still_compressed(guardrail: HeadroomGuardrail): + """Negative control: protection must not turn compression into a no-op.""" + compressed_history = [ + {"role": "user", "content": "hist. hash=b573993006976af767214fac"}, + {"role": "assistant", "content": "older. hash=a73993006976af767214fac1"}, + {"role": "tool", "tool_call_id": "old_1", "content": "older tool. hash=c73993006976af767214fac2"}, + ] + wire, result = await _wire_and_result(guardrail, AGENTIC_MESSAGES, returned=compressed_history) + + # Exactly the three history rows go to the service, in order. + assert [row["role"] for row in wire] == ["user", "assistant", "tool"] + assert wire[0]["content"] == "H" * 5000 + assert wire[2]["tool_call_id"] == "old_1" + + messages = result["structured_messages"] + assert len(messages) == len(AGENTIC_MESSAGES) + assert messages[1] == compressed_history[0] + assert messages[2] == compressed_history[1] + assert messages[3] == compressed_history[2] + # Hashes in the compressed history still drive retrieve-tool injection. + assert has_headroom_retrieve_tool(result.get("tools") or []) + + +@pytest.mark.asyncio +async def test_nothing_compressible_returns_inputs_untouched(guardrail: HeadroomGuardrail): + """A single-turn request is all protected, so there is nothing to send and + the caller's own inputs object comes back.""" + inputs = GenericGuardrailAPIInputs( + texts=["A" * 5000], + structured_messages=[ + {"role": "system", "content": "sys"}, + {"role": "user", "content": "A" * 5000}, + ], + ) + + with patch.object(guardrail.async_handler, "post", new_callable=AsyncMock) as mock_post: + result = await guardrail.apply_guardrail( + inputs=inputs, + request_data={"model": "gpt-4o"}, + input_type="request", + ) + + mock_post.assert_not_called() + assert result is inputs + + +@pytest.mark.asyncio +async def test_fail_open_returns_the_caller_inputs_object(): + """Translation handlers detect a rewrite by object identity, so a request + that was not compressed must come back as the same object or it is + round-tripped through the write-back for nothing.""" + guardrail = _make_guardrail(unreachable_fallback="fail_open") + original = json.loads(json.dumps(AGENTIC_MESSAGES)) + inputs = GenericGuardrailAPIInputs(texts=["x"], structured_messages=original) + + with patch.object( + guardrail.async_handler, + "post", + new_callable=AsyncMock, + side_effect=httpx.ConnectError("boom"), + ): + result = await guardrail.apply_guardrail(inputs=inputs, request_data={}, input_type="request") + + assert result is inputs + assert result["structured_messages"] is original + + +# --------------------------------------------------------------------------- +# LIT-5018: the retrieval follow-up keeps the model's own text. +# --------------------------------------------------------------------------- + + +def _anthropic_response_with_text_and_tool_call() -> dict: + return { + "content": [ + {"type": "text", "text": "Let me pull the original back."}, + {"type": "tool_use", "id": "call_1", "name": HEADROOM_RETRIEVE_TOOL_NAME, "input": {"hash": "h" * 24}}, + ] + } + + +async def _plan_for(guardrail: HeadroomGuardrail, response, messages: list): + guardrail._issued_hashes_by_call_id["call-1"] = (frozenset({"h" * 24}), time.monotonic() + 60) + logging_obj = MagicMock() + logging_obj.litellm_call_id = "call-1" + logging_obj.model_call_details = {} + with patch.object( + guardrail.async_handler, + "get", + new_callable=AsyncMock, + return_value=_make_retrieve_response("ORIGINAL CONTENT"), + ): + return await guardrail.async_build_agentic_loop_plan( + tools={"tool_calls": [{"id": "call_1", "name": HEADROOM_RETRIEVE_TOOL_NAME, "arguments": {"hash": "h" * 24}}]}, + model="claude-sonnet-4-5-20250929", + messages=messages, + response=response, + anthropic_messages_provider_config=None, + anthropic_messages_optional_request_params={}, + logging_obj=logging_obj, + stream=False, + kwargs={}, + ) + + +@pytest.mark.asyncio +async def test_anthropic_followup_preserves_assistant_text(guardrail: HeadroomGuardrail): + plan = await _plan_for(guardrail, _anthropic_response_with_text_and_tool_call(), [{"role": "user", "content": "q"}]) + + assistant = plan.request_patch.messages[-2] # type: ignore[union-attr] + assert assistant["role"] == "assistant" + # Text first, then the tool_use it accompanied: dropping it loses the + # model's stated reason for the retrieval from its own transcript. + assert assistant["content"][0] == {"type": "text", "text": "Let me pull the original back."} + assert assistant["content"][1]["type"] == "tool_use" + + +@pytest.mark.asyncio +async def test_responses_followup_preserves_assistant_text(guardrail: HeadroomGuardrail): + response = { + "output": [ + {"type": "message", "content": [{"type": "output_text", "text": "Fetching the original."}]}, + {"type": "function_call", "call_id": "call_1", "name": HEADROOM_RETRIEVE_TOOL_NAME, "arguments": "{}"}, + ] + } + + plan = await _plan_for(guardrail, response, [{"role": "user", "content": "q"}]) + + items = plan.request_patch.messages # type: ignore[union-attr] + assert items[1] == {"role": "assistant", "content": "Fetching the original."} + assert items[2]["type"] == "function_call" + + +@pytest.mark.asyncio +async def test_chat_followup_echoes_only_the_retrieve_call(guardrail: HeadroomGuardrail): + """A turn that called another tool alongside headroom_retrieve must not + echo that call: only the retrieve call gets a tool result, and a tool_call + without one is rejected by the provider.""" + other = MagicMock() + other.id = "call_other" + other.type = "function" + other.function = MagicMock() + other.function.name = "Write" + other.function.arguments = "{}" + + response = _make_openai_response_with_tool_call(HEADROOM_RETRIEVE_TOOL_NAME, {"hash": "h" * 24}, "call_1") + response.choices[0].message.content = "Getting the original first." + response.choices[0].message.tool_calls = [response.choices[0].message.tool_calls[0], other] + + plan = await _plan_for(guardrail, response, [{"role": "user", "content": "q"}]) + + messages = plan.request_patch.messages # type: ignore[union-attr] + assistant = messages[1] + assert assistant["content"] == "Getting the original first." + assert [tc["id"] for tc in assistant["tool_calls"]] == ["call_1"] + assert [m["tool_call_id"] for m in messages[2:]] == ["call_1"] diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_structured_messages_writeback.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_structured_messages_writeback.py index 642dd51b37b..d2e5b407e30 100644 --- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_structured_messages_writeback.py +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_structured_messages_writeback.py @@ -7,6 +7,7 @@ For Anthropic: structured_messages (OpenAI format) converted back to Anthropic f via anthropic_messages_pt before writing to data["messages"]. """ +import json from unittest.mock import MagicMock, patch import pytest @@ -127,3 +128,75 @@ async def test_anthropic_handler_converts_structured_messages_to_anthropic_forma llm_provider="anthropic", ) assert result["messages"] == converted_back + + +# --------------------------------------------------------------------------- +# LIT-5018: the write-back must not restructure the conversation. +# +# anthropic_messages_pt merges every run of consecutive user/tool rows into one +# message, so a tool_result-only turn and the live user turn that follows it +# came back fused: the current instruction stopped being its own turn purely +# because a compression guardrail was enabled. +# --------------------------------------------------------------------------- + +AGENTIC_ANTHROPIC_MESSAGES = [ + {"role": "user", "content": [{"type": "text", "text": "first turn"}]}, + {"role": "assistant", "content": [{"type": "tool_use", "id": "tu_1", "name": "Read", "input": {}}]}, + {"role": "user", "content": [{"type": "tool_result", "tool_use_id": "tu_1", "content": "FILE BODY"}]}, + {"role": "user", "content": [{"type": "text", "text": "can we run /team to fix this"}]}, +] + + +async def _write_back_identity(messages: list) -> list: + """Run the request through a guardrail that changes nothing but returns a + new list, which is what puts a compression guardrail on the write-back + path, and return the resulting Anthropic messages.""" + from litellm.llms.anthropic.chat.guardrail_translation.handler import ( + AnthropicMessagesHandler, + ) + + guardrail = MagicMock() + guardrail.should_run_guardrail.return_value = True + guardrail.skip_system_message_in_guardrail = None + guardrail.skip_tool_message_in_guardrail = None + guardrail.experimental_use_latest_role_message_only = False + + async def apply_guardrail(inputs, request_data, input_type, logging_obj=None): + return {**inputs, "structured_messages": list(inputs["structured_messages"])} + + guardrail.apply_guardrail = apply_guardrail + + data = {"model": "claude-sonnet-4-5-20250929", "messages": messages, "max_tokens": 1024} + result = await AnthropicMessagesHandler().process_input_messages(data=data, guardrail_to_apply=guardrail) + return result["messages"] + + +@pytest.mark.asyncio +async def test_write_back_keeps_the_live_user_turn_separate_from_the_tool_result_turn(): + written = await _write_back_identity([dict(m) for m in AGENTIC_ANTHROPIC_MESSAGES]) + + assert [m["role"] for m in written] == ["user", "assistant", "user", "user"] + assert written[2]["content"] == [{"type": "tool_result", "tool_use_id": "tu_1", "content": "FILE BODY"}] + assert written[3]["content"] == [{"type": "text", "text": "can we run /team to fix this"}] + + +@pytest.mark.asyncio +async def test_write_back_keeps_real_tool_results_under_modify_params(): + """Converting one row at a time would keep the turns apart too, but an + assistant row whose results are converted separately reads as an orphaned + tool call: with modify_params on, the sanitizer answers it with a synthetic + "tool execution skipped" result and drops the real one.""" + import litellm + + original = litellm.modify_params + litellm.modify_params = True + try: + written = await _write_back_identity([dict(m) for m in AGENTIC_ANTHROPIC_MESSAGES]) + finally: + litellm.modify_params = original + + serialized = json.dumps(written) + assert "FILE BODY" in serialized + assert "skipped" not in serialized + assert "Please continue" not in serialized + assert [m["role"] for m in written] == ["user", "assistant", "user", "user"] From 7d97bbc3bb592f33313a9e0f0b7138b30a5e96c2 Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Thu, 30 Jul 2026 18:56:28 -0700 Subject: [PATCH 11/11] fix(ui): let the internal user and org forms save sub-cent budgets (#35302) The Default User Settings form on Internal Users, the org settings form and the org create dialog all rendered their money fields as `` inside a form that never opted out of native constraint validation. Any value with more than two decimals, such as a 0.001 max budget, failed the browser's step check, so Chrome vetoed the submit before react-hook-form ran. No request went out, no field error was shown, and the read view kept displaying the old value; it looked like the budget silently refused to stick. Money fields now use `step="any"`, and the three react-hook-form forms carry `noValidate` so zod stays the only validator and a DOM-level constraint can never swallow a submit again. --- .../DefaultUserSettingsForm.test.tsx | 28 +++++++++++++++++++ .../DefaultUserSettingsForm.tsx | 6 ++-- .../org-create/OrgCreateDialog.test.tsx | 22 +++++++++++++++ .../org-create/OrgCreateDialog.tsx | 4 +-- .../org-settings/OrgSettingsForm.test.tsx | 18 ++++++++++++ .../org-settings/OrgSettingsForm.tsx | 4 +-- 6 files changed, 75 insertions(+), 7 deletions(-) diff --git a/ui/litellm-dashboard/src/app/(dashboard)/users/_components/default-user-settings/DefaultUserSettingsForm.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/users/_components/default-user-settings/DefaultUserSettingsForm.test.tsx index bfdcc70fb0c..cc24931b2b3 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/users/_components/default-user-settings/DefaultUserSettingsForm.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/users/_components/default-user-settings/DefaultUserSettingsForm.test.tsx @@ -149,6 +149,34 @@ describe("DefaultUserSettingsForm", () => { expect(updateSettings).toHaveBeenCalledWith({ ...SAVED_BODY, max_budget: 250 }); }); + it("saves a sub-cent budget the browser would veto under a 0.01 step", async () => { + const user = userEvent.setup(); + const { updateSettings } = renderForm(); + + await enterEditMode(user); + const budget: HTMLInputElement = await screen.findByLabelText("Max Budget (USD)"); + await user.clear(budget); + await user.type(budget, "0.001"); + + const teamBudget: HTMLInputElement = screen.getByLabelText("Max Budget in Team (USD)"); + await user.clear(teamBudget); + await user.type(teamBudget, "0.002"); + + // jsdom never blocks the submit itself, so assert the constraint the real browser + // enforces before handleSubmit ever runs + expect(budget.checkValidity()).toBe(true); + expect(teamBudget.checkValidity()).toBe(true); + + await user.click(await saveButton()); + + await waitFor(() => expect(updateSettings).toHaveBeenCalledTimes(1)); + expect(updateSettings).toHaveBeenCalledWith({ + ...SAVED_BODY, + max_budget: 0.001, + teams: [{ team_id: "team-alpha", max_budget_in_team: 0.002, user_role: "user" }], + }); + }); + it("clears an emptied budget with null", async () => { const user = userEvent.setup(); const { updateSettings } = renderForm(); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/users/_components/default-user-settings/DefaultUserSettingsForm.tsx b/ui/litellm-dashboard/src/app/(dashboard)/users/_components/default-user-settings/DefaultUserSettingsForm.tsx index b1474e7cd0c..1e0ec6b6998 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/users/_components/default-user-settings/DefaultUserSettingsForm.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/users/_components/default-user-settings/DefaultUserSettingsForm.tsx @@ -133,7 +133,7 @@ const TeamsField = ({ control }: { control: SettingsControl }) => { {({ ref, ...budgetField }) => ( - + )} @@ -249,7 +249,7 @@ const SettingsForm = ({ initialValues, roleOptions, updateSettings, onCancel, on const onSubmit = form.handleSubmit((values) => mutation.mutate(values)); return ( -
+ - {({ ref, ...field }) => } + {({ ref, ...field }) => } { await waitFor(() => expect(screen.queryByLabelText("Organization Name")).not.toBeInTheDocument()); }); + it("creates with a sub-cent max budget the browser would veto under a 0.01 step", async () => { + const user = userEvent.setup(); + const { createOrganization } = renderDialog(); + + await user.type(screen.getByLabelText("Organization Name"), "new-org"); + const budget: HTMLInputElement = screen.getByLabelText("Max Budget (USD)"); + await user.type(budget, "0.001"); + + // jsdom never blocks the submit itself, so assert the constraint the real browser + // enforces before handleSubmit ever runs + expect(budget.checkValidity()).toBe(true); + + await user.click(screen.getByRole("button", { name: "Create Organization" })); + + await waitFor(() => expect(createOrganization).toHaveBeenCalledTimes(1)); + expect(createOrganization.mock.calls[0][0]).toStrictEqual({ + organization_alias: "new-org", + models: [], + max_budget: 0.001, + }); + }); + it("maps selectors and limits into the create body", async () => { const user = userEvent.setup(); const { createOrganization } = renderDialog(); diff --git a/ui/litellm-dashboard/src/components/organization/org-create/OrgCreateDialog.tsx b/ui/litellm-dashboard/src/components/organization/org-create/OrgCreateDialog.tsx index 998d9446365..4e1a00704e3 100644 --- a/ui/litellm-dashboard/src/components/organization/org-create/OrgCreateDialog.tsx +++ b/ui/litellm-dashboard/src/components/organization/org-create/OrgCreateDialog.tsx @@ -79,7 +79,7 @@ export const OrgCreateDialog = ({ Create Organization - + {({ ref, ...field }) => } @@ -97,7 +97,7 @@ export const OrgCreateDialog = ({ - {({ ref, ...field }) => } + {({ ref, ...field }) => } diff --git a/ui/litellm-dashboard/src/components/organization/org-settings/OrgSettingsForm.test.tsx b/ui/litellm-dashboard/src/components/organization/org-settings/OrgSettingsForm.test.tsx index 5bd809bcfd5..4dfd37e3466 100644 --- a/ui/litellm-dashboard/src/components/organization/org-settings/OrgSettingsForm.test.tsx +++ b/ui/litellm-dashboard/src/components/organization/org-settings/OrgSettingsForm.test.tsx @@ -113,6 +113,24 @@ describe("OrgSettingsForm", () => { expect(patchOrganization).toHaveBeenCalledWith("org-1", { organization_alias: "acme-2" }); }); + it("saves a sub-cent max budget the browser would veto under a 0.01 step", async () => { + const user = userEvent.setup(); + const { patchOrganization } = renderForm(); + + const budget: HTMLInputElement = screen.getByLabelText("Max Budget (USD)"); + await user.clear(budget); + await user.type(budget, "0.001"); + + // jsdom never blocks the submit itself, so assert the constraint the real browser + // enforces before handleSubmit ever runs + expect(budget.checkValidity()).toBe(true); + + await user.click(screen.getByRole("button", { name: "Save Changes" })); + + await waitFor(() => expect(patchOrganization).toHaveBeenCalledTimes(1)); + expect(patchOrganization).toHaveBeenCalledWith("org-1", { max_budget: 0.001 }); + }); + it("sends null when a limit is cleared", async () => { const user = userEvent.setup(); const { patchOrganization } = renderForm(); diff --git a/ui/litellm-dashboard/src/components/organization/org-settings/OrgSettingsForm.tsx b/ui/litellm-dashboard/src/components/organization/org-settings/OrgSettingsForm.tsx index fe4965adb3c..affe0ed2d4e 100644 --- a/ui/litellm-dashboard/src/components/organization/org-settings/OrgSettingsForm.tsx +++ b/ui/litellm-dashboard/src/components/organization/org-settings/OrgSettingsForm.tsx @@ -78,7 +78,7 @@ export const OrgSettingsForm = ({ }); return ( - + {({ ref, ...field }) => } @@ -96,7 +96,7 @@ export const OrgSettingsForm = ({ - {({ ref, ...field }) => } + {({ ref, ...field }) => }