From 4cd8f9174dda26c9af10832b8c9663dac4062ebf Mon Sep 17 00:00:00 2001 From: kenneth-berri Date: Wed, 11 Nov 2026 22:15:14 -0800 Subject: [PATCH 001/168] fix(anthropic): close strict JSON schemas on the Messages to Responses bridge (#51337) * fix(anthropic): close strict JSON schemas on the Messages to Responses bridge Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * perf(anthropic): copy each strict schema subtree once Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(anthropic): close nullable objects in strict JSON schemas Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../messages/adapters/transformation.py | 54 ++----------- .../responses_adapters/transformation.py | 26 +++--- litellm/llms/anthropic/messages/utils.py | 66 +++++++++++++-- .../test_responses_adapters_transformation.py | 80 ++++++++++++++++++- .../count_tokens/test_token_counter.py | 9 ++- 5 files changed, 167 insertions(+), 68 deletions(-) diff --git a/litellm/llms/anthropic/messages/adapters/transformation.py b/litellm/llms/anthropic/messages/adapters/transformation.py index e68360e69b9..660c7c1dbc9 100644 --- a/litellm/llms/anthropic/messages/adapters/transformation.py +++ b/litellm/llms/anthropic/messages/adapters/transformation.py @@ -1,4 +1,3 @@ -import copy import hashlib import json from collections.abc import AsyncIterator, Iterator, Mapping, Sequence @@ -20,6 +19,8 @@ from litellm.llms.anthropic.messages.convertible_blocks import tool_result_text_ from litellm.llms.anthropic.messages.utils import ( is_reasoning_auto_summary_enabled, json_copy_or_original, + object_mapping, + openai_strict_json_schema, prompt_cache_key_from_user_id, ) from litellm.responses.utils import auto_reasoning_summary @@ -1017,66 +1018,23 @@ class LiteLLMAnthropicMessagesAdapter: Returns: OpenAI-compatible response_format dict, or None if invalid """ - if not isinstance(output_format, dict): + output_format_object: Final = object_mapping(output_format) + if output_format_object is None or output_format_object.get("type") != "json_schema": return None - output_type: Final = output_format.get("type") - if output_type != "json_schema": - return None - - schema = output_format.get("schema") + schema: Final = output_format_object.get("schema") if not schema: return None - # Deep copy to avoid mutating the original schema - schema = copy.deepcopy(schema) - # OpenAI strict mode requires additionalProperties: false on every object - self._add_additional_properties_false(schema) - - # Convert to OpenAI response_format structure return { "type": "json_schema", "json_schema": { "name": "structured_output", - "schema": schema, + "schema": openai_strict_json_schema(schema), "strict": True, }, } - @staticmethod - def _add_additional_properties_false(schema: dict) -> None: - """ - Recursively ensure object schemas comply with OpenAI strict mode. - - OpenAI's strict mode requires: - 1. 'additionalProperties': false at every object nesting level - 2. All property keys listed in 'required' - """ - if not isinstance(schema, dict): - return - - if schema.get("type") == "object" and "properties" in schema: - schema["additionalProperties"] = False - schema["required"] = list(schema["properties"].keys()) - for prop in schema["properties"].values(): - LiteLLMAnthropicMessagesAdapter._add_additional_properties_false(prop) - - # Handle array items - if "items" in schema: - LiteLLMAnthropicMessagesAdapter._add_additional_properties_false(schema["items"]) - - # Handle anyOf/oneOf/allOf - for key in ("anyOf", "oneOf", "allOf"): - if key in schema: - for sub_schema in schema[key]: - LiteLLMAnthropicMessagesAdapter._add_additional_properties_false(sub_schema) - - # Handle $defs / definitions - for key in ("$defs", "definitions"): - if key in schema: - for def_schema in schema[key].values(): - LiteLLMAnthropicMessagesAdapter._add_additional_properties_false(def_schema) - def _translate_midturn_system_message_to_openai( self, message: AnthropicMessagesSystemMessageParam, diff --git a/litellm/llms/anthropic/messages/responses_adapters/transformation.py b/litellm/llms/anthropic/messages/responses_adapters/transformation.py index d7b8592f45c..1d186c8aee4 100644 --- a/litellm/llms/anthropic/messages/responses_adapters/transformation.py +++ b/litellm/llms/anthropic/messages/responses_adapters/transformation.py @@ -35,6 +35,8 @@ from litellm.llms.anthropic.messages.native.utils import ( from litellm.llms.anthropic.messages.utils import ( is_reasoning_auto_summary_enabled, json_copy_or_original, + object_mapping, + openai_strict_json_schema, prompt_cache_key_from_user_id, ) from litellm.types.llms.anthropic import ( @@ -588,10 +590,8 @@ class LiteLLMAnthropicToResponsesAPIAdapter: ) @staticmethod - def _resolve_output_format_strict(strict: bool | None) -> bool: - if strict is None: - return True - return strict + def _resolve_output_format_strict(strict: object) -> bool: + return True if strict is None else bool(strict) def translate_request( self, @@ -690,19 +690,23 @@ class LiteLLMAnthropicToResponsesAPIAdapter: # output_format / output_config.format -> text format # output_format: {"type": "json_schema", "schema": {...}} # output_config: {"format": {"type": "json_schema", "schema": {...}}} - output_format: object = anthropic_request.get("output_format") - output_config = anthropic_request.get("output_config") - if not isinstance(output_format, dict) and isinstance(output_config, dict): - output_format = output_config.get("format") - if isinstance(output_format, dict) and output_format.get("type") == "json_schema": + output_config_object: Final = object_mapping(anthropic_request.get("output_config")) + top_level_output_format: Final = object_mapping(anthropic_request.get("output_format")) + output_format: Final = ( + top_level_output_format + if top_level_output_format is not None or output_config_object is None + else object_mapping(output_config_object.get("format")) + ) + if output_format is not None and output_format.get("type") == "json_schema": schema: Final = output_format.get("schema") if schema: + output_strict: Final = self._resolve_output_format_strict(output_format.get("strict")) responses_kwargs["text"] = { "format": { "type": "json_schema", "name": "structured_output", - "schema": schema, - "strict": self._resolve_output_format_strict(output_format.get("strict")), + "schema": openai_strict_json_schema(schema) if output_strict else schema, + "strict": output_strict, } } diff --git a/litellm/llms/anthropic/messages/utils.py b/litellm/llms/anthropic/messages/utils.py index 95c7914befa..94f81eadb3b 100644 --- a/litellm/llms/anthropic/messages/utils.py +++ b/litellm/llms/anthropic/messages/utils.py @@ -1,6 +1,7 @@ +import copy import json import os -from collections.abc import Mapping +from collections.abc import Mapping, Sequence from types import MappingProxyType from typing import TYPE_CHECKING, Final, TypeVar @@ -37,6 +38,61 @@ def json_copy_or_original(value: _JsonValue) -> _JsonValue: return value +_SUBSCHEMA_LIST_KEYWORDS: Final = ("anyOf", "oneOf", "allOf") +_SUBSCHEMA_MAP_KEYWORDS: Final = ("$defs", "definitions") + + +_JSON_ARRAY: Final = TypeAdapter(list[object]) + + +def _json_array(value: object) -> Sequence[object] | None: + try: + return _JSON_ARRAY.validate_python(value, strict=True) + except ValidationError: + return None + + +def openai_strict_json_schema(schema: object) -> object: + """New copy of a JSON schema where every object with properties forbids extra keys and requires all of them. + + OpenAI strict structured outputs rejects any schema that leaves an object open or a property optional. + """ + source: Final = object_mapping(schema) + if source is None: + return copy.deepcopy(schema) + properties: Final = object_mapping(source.get("properties")) + closing: Final[Mapping[str, object]] = ( + {"required": list(properties), "additionalProperties": False} + if properties is not None and _is_object_type(source.get("type")) + else {} + ) + return {**{key: _strict_subschema(key, value) for key, value in source.items()}, **closing} + + +def _is_object_type(schema_type: object) -> bool: + """Whether a ``type`` keyword names an object, alone or in a union such as ``["object", "null"]``.""" + types: Final = _json_array(schema_type) + return schema_type == "object" if types is None else "object" in types + + +def _strict_subschema(key: str, value: object) -> object: + if key == "items": + return openai_strict_json_schema(value) + if key == "properties" or key in _SUBSCHEMA_MAP_KEYWORDS: + named: Final = object_mapping(value) + return ( + copy.deepcopy(value) + if named is None + else {name: openai_strict_json_schema(subschema) for name, subschema in named.items()} + ) + if key in _SUBSCHEMA_LIST_KEYWORDS: + variants: Final = _json_array(value) + return ( + copy.deepcopy(value) if variants is None else [openai_strict_json_schema(variant) for variant in variants] + ) + return copy.deepcopy(value) + + class _ClaudeCodeUserId(LiteLLMBaseModel): """The JSON Claude Code packs into ``metadata.user_id``; only ``session_id`` is per conversation.""" @@ -136,7 +192,7 @@ def proxy_sub_call_logging_kwargs(parent_kwargs: Mapping[str, object]) -> ProxyS if disable_header is None else {**parent_params, "proxy_server_request": {"headers": {X_LITELLM_DISABLE_CALLBACKS: disable_header}}} ) - trusted_vars: Final = _object_mapping(parent_kwargs.get(TRUSTED_CALLBACK_VARS_FIELD)) + trusted_vars: Final = object_mapping(parent_kwargs.get(TRUSTED_CALLBACK_VARS_FIELD)) if trusted_vars is None: return dynamic_params return {**dynamic_params, "litellm_trusted_callback_vars": trusted_vars} @@ -154,13 +210,13 @@ def proxy_sub_call_trace_id(parent_kwargs: Mapping[str, object]) -> str | None: def _disable_callbacks_header(parent_kwargs: Mapping[str, object]) -> str | None: - parent_request: Final = _object_mapping(parent_kwargs.get("proxy_server_request")) - headers: Final = _object_mapping(parent_request.get("headers")) if parent_request is not None else None + parent_request: Final = object_mapping(parent_kwargs.get("proxy_server_request")) + headers: Final = object_mapping(parent_request.get("headers")) if parent_request is not None else None header: Final = headers.get(X_LITELLM_DISABLE_CALLBACKS) if headers is not None else None return header if isinstance(header, str) else None -def _object_mapping(value: object) -> Mapping[str, object] | None: +def object_mapping(value: object) -> Mapping[str, object] | None: try: return _OBJECT_MAPPING.validate_python(value) except ValidationError: diff --git a/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_transformation.py b/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_transformation.py index 660f00cd722..7ef69d4c791 100644 --- a/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_transformation.py +++ b/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_transformation.py @@ -3,6 +3,7 @@ Tests for LiteLLMAnthropicToResponsesAPIAdapter (litellm/llms/anthropic/messages/responses_adapters/transformation.py) """ +import copy import json import os from collections.abc import Mapping @@ -217,8 +218,8 @@ class TestOutputConfigStructuredOutput: assert kwargs["text"]["format"]["schema"] == self._SCHEMA assert kwargs["text"]["format"]["strict"] is False - def test_optional_property_stays_out_of_required_list(self): - """A property absent from required must stay absent from required in the translated schema.""" + def test_optional_property_stays_out_of_required_list_when_not_strict(self): + """Without strict, a property absent from required stays absent from required in the translated schema.""" schema = { "type": "object", "properties": { @@ -228,7 +229,7 @@ class TestOutputConfigStructuredOutput: "required": ["name"], "additionalProperties": False, } - req = _make_request(output_format={"type": "json_schema", "schema": schema}) + req = _make_request(output_format={"type": "json_schema", "schema": schema, "strict": False}) kwargs = _ADAPTER.translate_request(req) fmt_schema = kwargs["text"]["format"]["schema"] assert fmt_schema["required"] == ["name"] @@ -247,6 +248,79 @@ class TestOutputConfigStructuredOutput: assert output_format == snapshot assert req["output_format"] == snapshot + @pytest.mark.parametrize("format_field", ("output_format", "output_config")) + @pytest.mark.parametrize("strict_options", ({}, {"strict": True})) + def test_strict_schema_closes_every_object_like_messages_to_chat_adapter( + self, + format_field: Literal["output_format", "output_config"], + strict_options: Mapping[str, bool], + ) -> None: + """Regression for #51200: strict output schemas must be closed at every nesting level.""" + + def open_object(properties: Mapping[str, object], required: list[str]) -> dict[str, object]: + return {"type": "object", "properties": properties, "required": required, "additionalProperties": True} + + def closed_object(properties: Mapping[str, object]) -> dict[str, object]: + return { + "type": "object", + "properties": properties, + "required": list(properties), + "additionalProperties": False, + } + + schema: Final = { + **open_object( + { + "entry": open_object({"value": {"type": "string"}, "note": {"type": "string"}}, ["value"]), + "rows": {"type": "array", "items": open_object({"id": {"type": "integer"}}, [])}, + "choice": {"anyOf": [open_object({"a": {"type": "string"}}, []), {"type": "null"}]}, + "ref": {"$ref": "#/$defs/Item"}, + "maybe": {**open_object({"flag": {"type": "boolean"}}, []), "type": ["object", "null"]}, + }, + ["entry"], + ), + "$defs": {"Item": open_object({"label": {"type": "string"}}, [])}, + } + output_format: Final = {"type": "json_schema", "schema": schema, **strict_options} + snapshot: Final = copy.deepcopy(output_format) + request: Final = _make_request( + **{format_field: output_format if format_field == "output_format" else {"format": output_format}} + ) + expected_schema: Final = { + **closed_object( + { + "entry": closed_object({"value": {"type": "string"}, "note": {"type": "string"}}), + "rows": {"type": "array", "items": closed_object({"id": {"type": "integer"}})}, + "choice": {"anyOf": [closed_object({"a": {"type": "string"}}), {"type": "null"}]}, + "ref": {"$ref": "#/$defs/Item"}, + "maybe": {**closed_object({"flag": {"type": "boolean"}}), "type": ["object", "null"]}, + } + ), + "$defs": {"Item": closed_object({"label": {"type": "string"}})}, + } + + translated: Final = _ADAPTER.translate_request(request) + chat, _ = LiteLLMAnthropicMessagesAdapter().translate_anthropic_to_openai(request) + + assert translated["text"] == { + "format": {"type": "json_schema", "name": "structured_output", "schema": expected_schema, "strict": True} + } + assert chat.get("response_format") == { + "type": "json_schema", + "json_schema": {"name": "structured_output", "schema": expected_schema, "strict": True}, + } + assert output_format == snapshot + + def test_non_strict_schema_is_forwarded_unchanged(self) -> None: + schema: Final = { + "type": "object", + "properties": {"entry": {"type": "object", "properties": {"value": {"type": "string"}}}}, + "additionalProperties": True, + } + request: Final = _make_request(output_format={"type": "json_schema", "schema": schema, "strict": False}) + + assert _ADAPTER.translate_request(request)["text"]["format"]["schema"] == schema + # --------------------------------------------------------------------------- # translate_messages_to_responses_input diff --git a/tests/unit/llms/openai/responses/count_tokens/test_token_counter.py b/tests/unit/llms/openai/responses/count_tokens/test_token_counter.py index 35b544be31f..d3722cff1d0 100644 --- a/tests/unit/llms/openai/responses/count_tokens/test_token_counter.py +++ b/tests/unit/llms/openai/responses/count_tokens/test_token_counter.py @@ -301,7 +301,14 @@ async def test_anthropic_format_counts_the_structured_output_schema(monkeypatch: assert sent == { "model": "gpt-4o", "input": [{"type": "message", "role": "user", "content": [{"type": "input_text", "text": "Answer in JSON."}]}], - "text": {"format": {"type": "json_schema", "name": "structured_output", "schema": schema, "strict": True}}, + "text": { + "format": { + "type": "json_schema", + "name": "structured_output", + "schema": {**schema, "required": ["answer"], "additionalProperties": False}, + "strict": True, + } + }, } From 3d389262a4fc5c07e487c50a8bacdf73e077b83b Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 12 Nov 2026 06:24:23 +0000 Subject: [PATCH 002/168] test(providers): expect Rust routing for idle Messages streams (#51364) Co-authored-by: mateo <277851410+mateo-berri@users.noreply.github.com> Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../providers/test_idle_upstream_reconnect_routes_wire.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/providers/test_idle_upstream_reconnect_routes_wire.py b/tests/integration/providers/test_idle_upstream_reconnect_routes_wire.py index 2a8b640f654..43fd256e08c 100644 --- a/tests/integration/providers/test_idle_upstream_reconnect_routes_wire.py +++ b/tests/integration/providers/test_idle_upstream_reconnect_routes_wire.py @@ -33,7 +33,7 @@ pytestmark: Final = pytest.mark.timeout(int(2 * graceful_stop_seconds() + 120 + ROWS: Final = ( Row("messages-anthropic-sdk-sync", anthropic_deployment, messages_sdk), - Row("messages-anthropic-sdk-async-stream", anthropic_deployment, messages_sdk_stream), + Row("messages-anthropic-sdk-async-stream", anthropic_deployment, messages_sdk_stream, rust=True), Row("messages-upstream-resets-stale-connection", anthropic_deployment, messages_sdk, drop=Drop.RESET), Row("custom-pass-through-raw-httpx", pass_through_deployment, pass_through), Row("generic-guardrail-api-pre-call", guardrail_deployment, chat_with_guardrail), From ece1484bc7428da8f66683d4897c9c1d7d504e19 Mon Sep 17 00:00:00 2001 From: Kerry Lu Date: Wed, 11 Nov 2026 22:55:59 -0800 Subject: [PATCH 003/168] feat(rate-limit): team_model_limit_max_borrow_multiple caps how far a team borrows idle model capacity (#51359) * feat(rate-limit): team_model_limit_max_borrow_multiple caps how far a team borrows idle model capacity With team_model_limit_borrows_idle_capacity_from_other_teams on, a team could use everything up to the model's own tpm and rpm while the model was idle. Setting team_model_limit_max_borrow_multiple in the team's metadata caps the team at its model_tpm_limit and model_rpm_limit times that multiple, with a 429 whose rate_limited_by header reads model_per_team_max_borrow_multiple. Unset keeps the previous behavior. Values below 1 or non-numeric are logged and ignored. * fix(rate-limit): reject infinite or oversized team_model_limit_max_borrow_multiple values instead of failing the rate check * fix(team): make team_model_limit_max_borrow_multiple proxy-controlled team metadata so team admins cannot lift their own cap --------- Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../proxy/hooks/dynamic_rate_limiter_v3.py | 116 ++++++++++++---- .../hooks/parallel_request_limiter_v3.py | 44 +++++- .../team_admin_field_permissions.py | 1 + ..._team_model_limit_borrows_idle_capacity.py | 32 ++++- .../hooks/test_dynamic_rate_limiter_v3.py | 129 ++++++++++++++++++ .../test_team_admin_field_permissions.py | 1 + 6 files changed, 296 insertions(+), 27 deletions(-) diff --git a/litellm/proxy/hooks/dynamic_rate_limiter_v3.py b/litellm/proxy/hooks/dynamic_rate_limiter_v3.py index 5a6d6fc4cdf..3654d0b9176 100644 --- a/litellm/proxy/hooks/dynamic_rate_limiter_v3.py +++ b/litellm/proxy/hooks/dynamic_rate_limiter_v3.py @@ -4,6 +4,7 @@ Dynamic rate limiter v3 - Saturation-aware priority-based rate limiting import os from collections.abc import Callable, Mapping +from dataclasses import dataclass from datetime import datetime from typing import TYPE_CHECKING, Final, Literal @@ -27,6 +28,7 @@ from litellm.proxy.common_utils.proxy_rate_limit_error import ( map_v3_rate_limit_type, ) from litellm.proxy.hooks.parallel_request_limiter_v3 import ( # noqa: F401 # legacy module exports + BorrowingTeamModelLimits, PROXY_MaxParallelRequestsHandler_v3, RateLimitDescriptor, RateLimitDescriptorRateLimitObject, @@ -72,6 +74,19 @@ def _usage_of(response_obj: object) -> Usage | None: TEAM_MODEL_LIMIT_AT_SATURATION: Final = "model_per_team_saturated" +TEAM_MODEL_LIMIT_MAX_BORROW_MULTIPLE: Final = "model_per_team_max_borrow_multiple" + + +@dataclass(frozen=True, slots=True) +class _BorrowingTeamShare: + """The team's share and, when the team caps how far it borrows, its ceiling, both on the share's counter.""" + + limits: BorrowingTeamModelLimits + share: RateLimitDescriptor + ceiling: RateLimitDescriptor | None + + def enforced(self, saturated: bool) -> RateLimitDescriptor | None: + return self.share if saturated else self.ceiling def _team_limit_name(rate_limit_type: str) -> str: @@ -392,9 +407,7 @@ class _PROXY_DynamicRateLimitHandlerV3(CustomLogger): return descriptors - def _create_borrowing_team_model_limit_descriptor( - self, model: str, user_api_key_dict: UserAPIKeyAuth - ) -> RateLimitDescriptor | None: + def _borrowing_team_share(self, model: str, user_api_key_dict: UserAPIKeyAuth) -> _BorrowingTeamShare | None: """One counter per team and resolved model group, so an alias and its target share the team's share.""" rate_limited: Final = self.v3_limiter.rate_limited_model(model) or RateLimitedModel( requested=model, group=model @@ -402,9 +415,18 @@ class _PROXY_DynamicRateLimitHandlerV3(CustomLogger): limits: Final = borrowing_team_model_limits(user_api_key_dict, rate_limited) if limits.tpm is None and limits.rpm is None: return None + ceiling: Final = limits.borrow_ceiling() + value: Final = f"{user_api_key_dict.team_id}:{rate_limited.group}" + return _BorrowingTeamShare( + limits=limits, + share=self._team_share_descriptor(value, limits), + ceiling=None if ceiling is None else self._team_share_descriptor(value, ceiling), + ) + + def _team_share_descriptor(self, value: str, limits: BorrowingTeamModelLimits) -> RateLimitDescriptor: return RateLimitDescriptor( key=TEAM_MODEL_LIMIT_AT_SATURATION, - value=f"{user_api_key_dict.team_id}:{rate_limited.group}", + value=value, rate_limit={ "tokens_per_unit": limits.tpm, "requests_per_unit": limits.rpm, @@ -412,6 +434,55 @@ class _PROXY_DynamicRateLimitHandlerV3(CustomLogger): }, ) + def _team_share_error( + self, + model: str, + status: RateLimitStatus, + saturation: float, + priority: str | None, + user_api_key_dict: UserAPIKeyAuth, + team_share: _BorrowingTeamShare, + ) -> ProxyRateLimitError: + if saturation >= _get_priority_settings().saturation_threshold: + return self._team_limit_error(model, status, saturation, priority, user_api_key_dict) + return self._team_borrow_ceiling_error( + model, status, saturation, priority, user_api_key_dict, team_share.limits + ) + + def _team_borrow_ceiling_error( + self, + model: str, + status: RateLimitStatus, + saturation: float, + priority: str | None, + user_api_key_dict: UserAPIKeyAuth, + limits: BorrowingTeamModelLimits, + ) -> ProxyRateLimitError: + resolved_model, llm_provider = resolve_llm_provider_for_rate_limit(model) + limit_name: Final = _team_limit_name(status["rate_limit_type"]) + return ProxyRateLimitError( + detail={ + "error": f"Team {limit_name} borrow cap reached for {model}: the team may use at most " + f"{limits.max_borrow_multiple}x its {limit_name} while borrowing idle capacity. " + f"Team: {user_api_key_dict.team_id}, " + f"Rate limit type: {status['rate_limit_type']}, " + f"Team {limit_name}: {limits.share(status['rate_limit_type'])}, " + f"Borrow cap: {status['current_limit']}, " + f"Remaining: {status['limit_remaining']}, " + f"Model saturation: {saturation:.1%}" + }, + headers={ + "retry-after": str(self.v3_limiter.window_size), + "rate_limit_type": str(status["rate_limit_type"]), + "rate_limited_by": TEAM_MODEL_LIMIT_MAX_BORROW_MULTIPLE, + "x-litellm-priority": priority or "default", + "x-litellm-saturation": f"{saturation:.2%}", + }, + rate_limit_type=map_v3_rate_limit_type(status["rate_limit_type"]), + model=resolved_model, + llm_provider=llm_provider, + ) + def _team_limit_error( self, model: str, @@ -449,28 +520,28 @@ class _PROXY_DynamicRateLimitHandlerV3(CustomLogger): """Hold a Responses WebSocket turn to the team's share once the model saturates, like the regular limiter does.""" if requested_model is None: return - team_limit_descriptor: Final = self._create_borrowing_team_model_limit_descriptor( - model=requested_model, user_api_key_dict=user_api_key_dict - ) + team_share: Final = self._borrowing_team_share(model=requested_model, user_api_key_dict=user_api_key_dict) model_group_info: Final = self.llm_router.registry().group(requested_model) - if team_limit_descriptor is None or model_group_info is None: + if team_share is None or model_group_info is None: return saturation: Final = await self._check_model_saturation(requested_model, model_group_info) - if saturation < _get_priority_settings().saturation_threshold: + enforced: Final = team_share.enforced(saturation >= _get_priority_settings().saturation_threshold) + if enforced is None: return - tpm_windows: Final = _socket_turn_tpm_windows((team_limit_descriptor,)) + tpm_windows: Final = _socket_turn_tpm_windows((enforced,)) response: Final = await self.v3_limiter.should_rate_limit( tpm_windows, parent_otel_span=user_api_key_dict.parent_otel_span, read_only=True, skip_parallel_check=True ) over_limit: Final = next((s for s in response["statuses"] if s["code"] == "OVER_LIMIT"), None) if over_limit is None: return - raise self._team_limit_error( + raise self._team_share_error( requested_model, over_limit, saturation, self._get_priority_from_user_api_key_dict(user_api_key_dict), user_api_key_dict, + team_share, ) def _create_model_tracking_descriptor( @@ -569,18 +640,17 @@ class _PROXY_DynamicRateLimitHandlerV3(CustomLogger): # asyncio.Lock + in-memory fallback for single-process deployments. # All-or-nothing: if any enforced descriptor would exceed its limit, # no counter is modified and the response carries "OVER_LIMIT". - team_limit_descriptor: Final = self._create_borrowing_team_model_limit_descriptor( - model=model, user_api_key_dict=user_api_key_dict - ) - get_or_create_request_stash().borrowing_team_model_limit = team_limit_descriptor - if team_limit_descriptor is not None: - descriptors_to_check.append(team_limit_descriptor) + team_share: Final = self._borrowing_team_share(model=model, user_api_key_dict=user_api_key_dict) + get_or_create_request_stash().borrowing_team_model_limit = None if team_share is None else team_share.share + if team_share is not None: + descriptors_to_check.append(team_share.share) + enforced_team_share: Final = None if team_share is None else team_share.enforced(should_enforce_priority) enforced_descriptors: Final[list[RateLimitDescriptor]] = [model_wide_descriptor] if priority_descriptors and should_enforce_priority: enforced_descriptors.extend(priority_descriptors) - if team_limit_descriptor is not None and should_enforce_priority: - enforced_descriptors.append(team_limit_descriptor) + if enforced_team_share is not None: + enforced_descriptors.append(enforced_team_share) per_request_increment: Final[dict[Literal["requests", "tokens"], int]] = { "requests": 1, @@ -645,8 +715,8 @@ class _PROXY_DynamicRateLimitHandlerV3(CustomLogger): llm_provider=llm_provider, ) - if descriptor_key == TEAM_MODEL_LIMIT_AT_SATURATION: - raise self._team_limit_error(model, status, saturation, priority, user_api_key_dict) + if descriptor_key == TEAM_MODEL_LIMIT_AT_SATURATION and team_share is not None: + raise self._team_share_error(model, status, saturation, priority, user_api_key_dict, team_share) # Fail-closed guard: overall_code says OVER_LIMIT but no status # matched a descriptor key we know how to translate into a 429. @@ -687,9 +757,9 @@ class _PROXY_DynamicRateLimitHandlerV3(CustomLogger): # priority allocation has accurate usage when it later becomes # enforced. The increment-then-check semantics of should_rate_limit # are fine because we ignore the OVER_LIMIT response. - if team_limit_descriptor is not None and not should_enforce_priority: + if team_share is not None and enforced_team_share is None: await self.v3_limiter.atomic_check_and_increment_by_n( - descriptors=[team_limit_descriptor], + descriptors=[team_share.share], increments=[per_request_increment], parent_otel_span=user_api_key_dict.parent_otel_span, ) diff --git a/litellm/proxy/hooks/parallel_request_limiter_v3.py b/litellm/proxy/hooks/parallel_request_limiter_v3.py index 7f266e67b41..6b53e952f98 100644 --- a/litellm/proxy/hooks/parallel_request_limiter_v3.py +++ b/litellm/proxy/hooks/parallel_request_limiter_v3.py @@ -18,6 +18,7 @@ from datetime import datetime, timezone from types import MappingProxyType from typing import ( TYPE_CHECKING, + Annotated, Any, Final, Literal, @@ -27,7 +28,7 @@ from typing import ( ) from fastapi import HTTPException -from pydantic import TypeAdapter, ValidationError +from pydantic import Field, TypeAdapter, ValidationError from starlette.status import HTTP_400_BAD_REQUEST, HTTP_503_SERVICE_UNAVAILABLE from typing_extensions import NotRequired, ReadOnly @@ -199,17 +200,54 @@ def inherited_team_model_limit( return team_limit +TEAM_MODEL_LIMIT_MAX_BORROW_MULTIPLE: Final = "team_model_limit_max_borrow_multiple" +MAX_BORROW_MULTIPLE_CEILING: Final = 1_000_000 +_MAX_BORROW_MULTIPLE: Final[TypeAdapter[float]] = TypeAdapter( + Annotated[float, Field(ge=1, le=MAX_BORROW_MULTIPLE_CEILING, allow_inf_nan=False)] +) + + @dataclass(frozen=True, slots=True) class BorrowingTeamModelLimits: """Inherited team per-model limits the dynamic rate limiter enforces at saturation instead of the regular limiter.""" tpm: int | None rpm: int | None + max_borrow_multiple: float | None = None + + def share(self, rate_limit_type: str) -> int | None: + return self.rpm if rate_limit_type == "requests" else self.tpm + + def borrow_ceiling(self) -> "BorrowingTeamModelLimits | None": + """The most the team may use while borrowing, or None when it may borrow up to the model's own limit.""" + if self.max_borrow_multiple is None: + return None + return BorrowingTeamModelLimits( + tpm=None if self.tpm is None else int(self.tpm * self.max_borrow_multiple), + rpm=None if self.rpm is None else int(self.rpm * self.max_borrow_multiple), + ) NO_BORROWING: Final = BorrowingTeamModelLimits(tpm=None, rpm=None) +def team_model_limit_max_borrow_multiple(user_api_key_dict: UserAPIKeyAuth) -> float | None: + raw: Final[object] = (user_api_key_dict.team_metadata or {}).get(TEAM_MODEL_LIMIT_MAX_BORROW_MULTIPLE) + if raw is None: + return None + try: + return _MAX_BORROW_MULTIPLE.validate_python(raw) + except ValidationError: + verbose_proxy_logger.warning( + "Ignoring %s=%r on team %s: expected a number from 1 to %d, so the team borrows up to the model limit", + TEAM_MODEL_LIMIT_MAX_BORROW_MULTIPLE, + raw, + user_api_key_dict.team_id, + MAX_BORROW_MULTIPLE_CEILING, + ) + return None + + def borrowing_team_model_limits(user_api_key_dict: UserAPIKeyAuth, model: RateLimitedModel) -> BorrowingTeamModelLimits: settings: Final = litellm.priority_reservation_settings if ( @@ -224,7 +262,9 @@ def borrowing_team_model_limits(user_api_key_dict: UserAPIKeyAuth, model: RateLi ) if limits == NO_BORROWING or not dynamic_rate_limiter_covers(model.group): return NO_BORROWING - return limits + return BorrowingTeamModelLimits( + tpm=limits.tpm, rpm=limits.rpm, max_borrow_multiple=team_model_limit_max_borrow_multiple(user_api_key_dict) + ) def _resolve_media_models(router: "Router | None", model: str, team_id: str | None) -> tuple[str, ...]: diff --git a/litellm/proxy/management_endpoints/team_admin_field_permissions.py b/litellm/proxy/management_endpoints/team_admin_field_permissions.py index 08a84ca91c3..f80aa2a2e29 100644 --- a/litellm/proxy/management_endpoints/team_admin_field_permissions.py +++ b/litellm/proxy/management_endpoints/team_admin_field_permissions.py @@ -108,6 +108,7 @@ PROXY_CONTROLLED_TEAM_METADATA_KEYS: Final[frozenset[str]] = frozenset( "phoenix_project_name_override", "phoenix_project_name", "opik", + "team_model_limit_max_borrow_multiple", ) ) _NOT_COLUMNS: Final[frozenset[str]] = frozenset({"team_id", "metadata"}) diff --git a/tests/integration/routing/test_team_model_limit_borrows_idle_capacity.py b/tests/integration/routing/test_team_model_limit_borrows_idle_capacity.py index 03951dfca21..b9f853d7da8 100644 --- a/tests/integration/routing/test_team_model_limit_borrows_idle_capacity.py +++ b/tests/integration/routing/test_team_model_limit_borrows_idle_capacity.py @@ -16,6 +16,7 @@ MODEL_TPM: Final = 1000 MODEL_RPM: Final = 1000 TEAM_TPM: Final = 100 TEAM_RPM: Final = 3 +MAX_BORROW_MULTIPLE: Final = 2 UPSTREAM_REPLY: Final = json.dumps( { "id": "chatcmpl_team_model_limit_borrowing", @@ -68,7 +69,7 @@ def saturated_model(tmp_path_factory: pytest.TempPathFactory) -> Iterator[tuple[ def _team_calls( - owned: Gateway, upstream: str, calls: int, limit: str = "model_tpm_limit" + owned: Gateway, upstream: str, calls: int, limit: str = "model_tpm_limit", max_borrow_multiple: int | None = None ) -> tuple[httpx.Response, ...]: with owned.scenario() as scenario: model: Final = scenario.model( @@ -78,8 +79,13 @@ def _team_calls( tpm=MODEL_TPM, rpm=MODEL_RPM, ) + cap: Final = ( + {} if max_borrow_multiple is None else {"team_model_limit_max_borrow_multiple": max_borrow_multiple} + ) key: Final = scenario.key( - team_id=scenario.team(metadata={limit: {model: TEAM_TPM if limit == "model_tpm_limit" else TEAM_RPM}}) + team_id=scenario.team( + metadata={limit: {model: TEAM_TPM if limit == "model_tpm_limit" else TEAM_RPM}, **cap} + ) ) return tuple( owned.request( @@ -115,3 +121,25 @@ def test_team_is_held_to_its_model_limit_once_the_model_saturates( assert first.headers.get("rate_limit_type") == rate_limit_type, dict(first.headers) assert f"Team {limit} reached" in first.json()["error"]["message"], first.text assert "no idle capacity to borrow" in first.json()["error"]["message"], first.text + + +@pytest.mark.parametrize( + "limit, rate_limit_type", [("model_tpm_limit", "tokens"), ("model_rpm_limit", "requests")], ids=["tpm", "rpm"] +) +def test_team_borrowing_stops_at_its_max_borrow_multiple_while_the_model_is_idle( + idle_model: tuple[Gateway, str], limit: str, rate_limit_type: str +) -> None: + owned, upstream = idle_model + responses: Final = _team_calls(owned, upstream, calls=12, limit=limit, max_borrow_multiple=MAX_BORROW_MULTIPLE) + admitted: Final = tuple(response for response in responses if response.status_code == 200) + blocked: Final = tuple(response for response in responses if response.status_code == 429) + assert blocked, [r.text for r in responses] + assert responses[0].status_code == 200, responses[0].text + share: Final = TEAM_TPM if limit == "model_tpm_limit" else TEAM_RPM + tokens_per_call: Final = 30 if limit == "model_tpm_limit" else 1 + assert len(admitted) * tokens_per_call <= share * MAX_BORROW_MULTIPLE + tokens_per_call, [r.text for r in responses] + first: Final = blocked[0] + assert first.headers.get("rate_limited_by") == "model_per_team_max_borrow_multiple", dict(first.headers) + assert first.headers.get("rate_limit_type") == rate_limit_type, dict(first.headers) + assert f"Team {limit} borrow cap reached" in first.json()["error"]["message"], first.text + assert f"{MAX_BORROW_MULTIPLE}.0x its {limit}" in first.json()["error"]["message"], first.text diff --git a/tests/unit/proxy/hooks/test_dynamic_rate_limiter_v3.py b/tests/unit/proxy/hooks/test_dynamic_rate_limiter_v3.py index 4977e8d5737..a8265680856 100644 --- a/tests/unit/proxy/hooks/test_dynamic_rate_limiter_v3.py +++ b/tests/unit/proxy/hooks/test_dynamic_rate_limiter_v3.py @@ -2080,3 +2080,132 @@ async def test_a_websocket_turn_still_borrows_while_the_model_is_idle(monkeypatc handler = _borrowing_handler(monkeypatch, borrows=True, saturation_threshold=0.5) await _admit_then_settle(handler, _team_key(), 150) await handler.enforce_token_limits_for_frame(user_api_key_dict=_team_key(), requested_model="shared-model") + + +def _capped_team_key(multiple: object, limit: str = "model_tpm_limit", share: int = 100) -> UserAPIKeyAuth: + return _team_key(team_limits={limit: {"shared-model": share}, "team_model_limit_max_borrow_multiple": multiple}) + + +def _held_to_borrow_cap(exc: ProxyRateLimitError, rate_limit_type: str, limit_sentence: str) -> None: + assert exc.status_code == 429 + assert exc.headers is not None + assert exc.headers["rate_limited_by"] == "model_per_team_max_borrow_multiple" + assert exc.headers["rate_limit_type"] == rate_limit_type + assert str(exc.detail).startswith( + "{'error': 'Team " + limit_sentence + " borrow cap reached for shared-model: the team may use at most " + ), exc.detail + + +@pytest.mark.asyncio +async def test_team_borrows_past_its_share_while_it_stays_under_its_max_borrow_multiple( + monkeypatch: pytest.MonkeyPatch, +): + handler = _borrowing_handler(monkeypatch, borrows=True) + await _admit_then_settle(handler, _capped_team_key(2), 190) + await _check(handler, _capped_team_key(2), saturation=0.49) + + +@pytest.mark.asyncio +async def test_team_at_its_max_borrow_multiple_is_refused_while_the_model_is_idle(monkeypatch: pytest.MonkeyPatch): + handler = _borrowing_handler(monkeypatch, borrows=True) + await _admit_then_settle(handler, _capped_team_key(2), 200) + with pytest.raises(ProxyRateLimitError) as exc: + await _check(handler, _capped_team_key(2), saturation=0.0) + _held_to_borrow_cap(exc.value, "tokens", "model_tpm_limit") + assert ( + "2.0x its model_tpm_limit while borrowing idle capacity. Team: team-a, Rate limit type: tokens, " + "Team model_tpm_limit: 100, Borrow cap: 200, Remaining: 0, Model saturation: 0.0%" + ) in str(exc.value.detail), exc.value.detail + + +@pytest.mark.asyncio +async def test_a_fractional_max_borrow_multiple_caps_the_team_at_that_fraction_of_its_share( + monkeypatch: pytest.MonkeyPatch, +): + handler = _borrowing_handler(monkeypatch, borrows=True) + await _admit_then_settle(handler, _capped_team_key(1.5), 149) + await _check(handler, _capped_team_key(1.5), saturation=0.0) + await _settle_tokens(handler, 1) + with pytest.raises(ProxyRateLimitError) as exc: + await _check(handler, _capped_team_key(1.5), saturation=0.0) + _held_to_borrow_cap(exc.value, "tokens", "model_tpm_limit") + assert "Borrow cap: 150," in str(exc.value.detail), exc.value.detail + + +@pytest.mark.asyncio +async def test_the_max_borrow_multiple_caps_requests_as_well_as_tokens(monkeypatch: pytest.MonkeyPatch): + handler = _borrowing_handler(monkeypatch, borrows=True) + auth: Final = _capped_team_key(2, limit="model_rpm_limit", share=2) + for _ in range(4): + await _check(handler, auth, saturation=0.0) + with pytest.raises(ProxyRateLimitError) as exc: + await _check(handler, auth, saturation=0.0) + _held_to_borrow_cap(exc.value, "requests", "model_rpm_limit") + assert "Team model_rpm_limit: 2, Borrow cap: 4, Remaining: 0," in str(exc.value.detail), exc.value.detail + + +@pytest.mark.asyncio +async def test_a_team_under_its_max_borrow_multiple_is_still_held_to_its_share_once_the_model_saturates( + monkeypatch: pytest.MonkeyPatch, +): + handler = _borrowing_handler(monkeypatch, borrows=True) + await _admit_then_settle(handler, _capped_team_key(5), 150) + with pytest.raises(ProxyRateLimitError) as exc: + await _check(handler, _capped_team_key(5), saturation=0.5) + _held_to_team_limit(exc.value, "tokens", "model_tpm_limit") + + +@pytest.mark.asyncio +async def test_a_team_at_its_max_borrow_multiple_is_held_to_its_share_not_its_cap_once_the_model_saturates( + monkeypatch: pytest.MonkeyPatch, +): + handler = _borrowing_handler(monkeypatch, borrows=True) + await _admit_then_settle(handler, _capped_team_key(2), 100) + await _check(handler, _capped_team_key(2), saturation=0.49) + with pytest.raises(ProxyRateLimitError) as exc: + await _check(handler, _capped_team_key(2), saturation=0.5) + _held_to_team_limit(exc.value, "tokens", "model_tpm_limit") + + +@pytest.mark.asyncio +async def test_another_teams_borrowing_does_not_count_against_a_teams_max_borrow_multiple( + monkeypatch: pytest.MonkeyPatch, +): + handler = _borrowing_handler(monkeypatch, borrows=True) + await _admit_then_settle(handler, _team_key("team-b"), 150) + await _admit_then_settle(handler, _capped_team_key(2), 150) + await _check(handler, _capped_team_key(2), saturation=0.0) + + +@pytest.mark.parametrize( + "multiple", + [0.5, 0, -2, "two", None, "Infinity", float("inf"), 1e308], + ids=["below-one", "zero", "negative", "text", "null", "infinity-text", "infinity", "overflows-the-share"], +) +@pytest.mark.asyncio +async def test_an_unusable_max_borrow_multiple_is_ignored_so_the_team_borrows_up_to_the_model_limit( + monkeypatch: pytest.MonkeyPatch, multiple: object +): + handler = _borrowing_handler(monkeypatch, borrows=True) + await _admit_then_settle(handler, _capped_team_key(multiple), 300) + await _check(handler, _capped_team_key(multiple), saturation=0.49) + + +@pytest.mark.asyncio +async def test_a_websocket_turn_is_refused_at_the_teams_max_borrow_multiple_while_the_model_is_idle( + monkeypatch: pytest.MonkeyPatch, +): + handler = _borrowing_handler(monkeypatch, borrows=True, saturation_threshold=0.5) + await _admit_then_settle(handler, _capped_team_key(2), 200) + with pytest.raises(ProxyRateLimitError) as exc: + await handler.enforce_token_limits_for_frame( + user_api_key_dict=_capped_team_key(2), requested_model="shared-model" + ) + _held_to_borrow_cap(exc.value, "tokens", "model_tpm_limit") + + +@pytest.mark.asyncio +async def test_a_websocket_turn_under_the_teams_max_borrow_multiple_still_borrows(monkeypatch: pytest.MonkeyPatch): + handler = _borrowing_handler(monkeypatch, borrows=True, saturation_threshold=0.5) + await _admit_then_settle(handler, _capped_team_key(2), 190) + await handler.enforce_token_limits_for_frame(user_api_key_dict=_capped_team_key(2), requested_model="shared-model") diff --git a/tests/unit/proxy/management_endpoints/test_team_admin_field_permissions.py b/tests/unit/proxy/management_endpoints/test_team_admin_field_permissions.py index 88b02ed1aa1..c3e7ee8ad0d 100644 --- a/tests/unit/proxy/management_endpoints/test_team_admin_field_permissions.py +++ b/tests/unit/proxy/management_endpoints/test_team_admin_field_permissions.py @@ -344,6 +344,7 @@ _PROXY_READ_TEAM_METADATA_KEYS: Final = ( "phoenix_project_name_override", "phoenix_project_name", "opik", + "team_model_limit_max_borrow_multiple", ) _STORED_METADATA: Final = {"cost_center": "eng", "region": "eu", "priority": "high", "guardrails": ["pii"]} From db8ad3b13e7320bf835205cae58be5a58e0ff366 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 12 Nov 2026 06:59:37 +0000 Subject: [PATCH 004/168] fix(gemini): add batch and priority image output token prices to gemini/gemini-2.5-flash-image (#51368) Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/model_prices_and_context_window_backup.json | 2 ++ model_prices_and_context_window.json | 2 ++ 2 files changed, 4 insertions(+) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index f76ec44b0fe..5a7e4e95764 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -31936,6 +31936,8 @@ "mode": "image_generation", "output_cost_per_image": 0.039, "output_cost_per_image_token": 3e-05, + "output_cost_per_image_token_batches": 1.5e-05, + "output_cost_per_image_token_priority": 5.4e-05, "output_cost_per_reasoning_token": 2.5e-06, "output_cost_per_token": 2.5e-06, "output_cost_per_token_batches": 1.25e-06, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index f76ec44b0fe..5a7e4e95764 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -31936,6 +31936,8 @@ "mode": "image_generation", "output_cost_per_image": 0.039, "output_cost_per_image_token": 3e-05, + "output_cost_per_image_token_batches": 1.5e-05, + "output_cost_per_image_token_priority": 5.4e-05, "output_cost_per_reasoning_token": 2.5e-06, "output_cost_per_token": 2.5e-06, "output_cost_per_token_batches": 1.25e-06, From 1677c2250e24697a904dd1750815f574af8040f4 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Wed, 11 Nov 2026 23:52:32 -0800 Subject: [PATCH 005/168] ci: stop PR runs from thrashing the Actions cache and stop smoke compiling from source (#51343) * ci: stop PR runs from writing Actions caches and remove the sccache GitHub cache backend PR runs were saving their own cache copies and sccache was making one cache request per compiled file, about 700 to 2000 per Rust build. Together they pushed the repository past GitHub's cache storage and request limits, so restores failed with 429 or found nothing and jobs rebuilt from scratch. The rust-bridge job that gates every unit shard went from 0% cache misses on Oct 21 to about 65% on Nov 11. Caches are now saved only from main and restored everywhere else. The Rust DB cache gets the same save-if guard the other Rust jobs already have. The lint python and ui-api-types jobs restore the uv cache that the unit workflow saves on main, because lint only runs on pull requests and could never save a cache that another pull request can read. The Terraform provider jobs turn off setup-go's built-in cache, which saves from every ref, and use a new cache-go-build action that follows the existing restore-everywhere and save-on-main pattern. sccache is removed from the unit, Rust and Rust DB workflows. The Cargo target directory caches they already restore stay in place. The smoke diagnostics upload no longer fails a job whose checks passed. * ci(smoke): install the prebuilt Rust bridge and cache wheels built from source Every smoke job compiled the Rust bridge inside uv sync, about 4.5 minutes each in 8 jobs per run, and the Python 3.14 and 3.15 jobs also compiled grpcio-tools, numpy and jiter from source on every run because no wheels exist for those versions yet. Smoke only ran on pull requests, so nothing it built could be saved for the next run. The Linux smoke jobs now restore the bridge that the unit workflow already builds once and saves on main. The bridge is abi3, so the same build serves every Python version. When the restore misses, which happens when a pull request changes a Rust build input, the job restores the Cargo build cache and compiles as before. The cache key and restore step move into a restore-rust-bridge action that the unit workflow's producer job also uses, so the key is defined once. Smoke now also runs on pushes to main and prunes the uv cache down to wheels built from source before setup-uv saves it. Pull requests restore that cache, so 3.14 and 3.15 stop recompiling the same packages. The windows-sdk job drops its Cargo cache step. Nothing ever saved a Windows copy, and saving one from main would add about 2 GB to a cache that is already near its limit. --- .github/actions/cache-go-build/action.yml | 39 ++++++++++++ .../actions/restore-rust-bridge/action.yml | 37 +++++++++++ .github/workflows/test-linting.yml | 8 +-- .github/workflows/test-merge-smoke.yml | 62 +++++++++++++++++-- .github/workflows/test-rust-db.yml | 10 +-- .github/workflows/test-rust.yml | 19 ------ .github/workflows/test-terraform-provider.yml | 18 ++++-- .github/workflows/test-unit.yml | 28 +-------- 8 files changed, 154 insertions(+), 67 deletions(-) create mode 100644 .github/actions/cache-go-build/action.yml create mode 100644 .github/actions/restore-rust-bridge/action.yml diff --git a/.github/actions/cache-go-build/action.yml b/.github/actions/cache-go-build/action.yml new file mode 100644 index 00000000000..f29b144c934 --- /dev/null +++ b/.github/actions/cache-go-build/action.yml @@ -0,0 +1,39 @@ +name: "Cache the Go build" +description: >- + Restore the Go module and build caches on every run and save them only from + main, so pull requests reuse main's cache instead of evicting it with their + own copies. + + actions/setup-go's built-in cache saves from every ref, and a cache saved by a + pull request is readable only by that pull request. Callers must pass + `cache: false` to setup-go and run this action after it. + +inputs: + dependency-path: + description: "Path to the go.sum that keys the cache" + required: true + +runs: + using: composite + steps: + - name: Restore and save the Go module and build caches + if: github.ref == 'refs/heads/main' + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 + with: + path: | + ~/go/pkg/mod + ~/.cache/go-build + key: ${{ runner.os }}-go-${{ hashFiles(inputs.dependency-path) }} + restore-keys: | + ${{ runner.os }}-go- + + - name: Restore the Go module and build caches + if: github.ref != 'refs/heads/main' + uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 + with: + path: | + ~/go/pkg/mod + ~/.cache/go-build + key: ${{ runner.os }}-go-${{ hashFiles(inputs.dependency-path) }} + restore-keys: | + ${{ runner.os }}-go- diff --git a/.github/actions/restore-rust-bridge/action.yml b/.github/actions/restore-rust-bridge/action.yml new file mode 100644 index 00000000000..d050431dbc6 --- /dev/null +++ b/.github/actions/restore-rust-bridge/action.yml @@ -0,0 +1,37 @@ +name: "Restore the Rust bridge" +description: >- + Restore the prebuilt Rust bridge that the unit workflow's rust-bridge job saves + on main, keyed on every input of the Rust build. + + The bridge is abi3, so one build serves every supported Python version. A job + whose Rust inputs match main installs the restored wheel and native library + instead of compiling them inside `uv sync`. On a miss the caller builds from + source as before. + + The restored `rust-bridge-dist` directory holds the editable litellm wheel, the + litellm-proxy gateway wheel, and `litellm/rust_bridge/_native.abi3.so`. + +outputs: + key: + description: "Cache key for the bridge built from this checkout" + value: ${{ steps.key.outputs.key }} + cache-hit: + description: "'true' when the bridge for this exact key was restored" + value: ${{ steps.restore.outputs.cache-hit }} + +runs: + using: composite + steps: + - name: Define the Rust bridge cache key + id: key + shell: bash + env: + RUST_BRIDGE_CACHE_KEY: ${{ runner.os }}-rust-bridge-editable-${{ hashFiles('litellm/rust/Cargo.lock', 'litellm/rust/Cargo.toml', 'litellm/rust/crates/**', 'model_prices_and_context_window.json', 'pyproject.toml', 'litellm/packaging/litellm-proxy/pyproject.toml') }} + run: echo "key=$RUST_BRIDGE_CACHE_KEY" >> "$GITHUB_OUTPUT" + + - name: Restore the Rust bridge + id: restore + uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 + with: + path: rust-bridge-dist + key: ${{ steps.key.outputs.key }} diff --git a/.github/workflows/test-linting.yml b/.github/workflows/test-linting.yml index f401d022858..2358cbfc028 100644 --- a/.github/workflows/test-linting.yml +++ b/.github/workflows/test-linting.yml @@ -73,14 +73,14 @@ jobs: - name: Cache uv dependencies if: steps.changes.outputs.decision != 'skip' - uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 + uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 with: path: | ~/.cache/uv .venv - key: ${{ runner.os }}-uv-lint-${{ hashFiles('uv.lock') }} + key: ${{ runner.os }}-uv-${{ hashFiles('uv.lock') }} restore-keys: | - ${{ runner.os }}-uv-lint- + ${{ runner.os }}-uv- - name: Clean Python cache if: steps.changes.outputs.decision != 'skip' @@ -396,7 +396,7 @@ jobs: - name: Cache uv dependencies if: steps.changes.outputs.relevant == 'true' - uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 + uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 with: path: | ~/.cache/uv diff --git a/.github/workflows/test-merge-smoke.yml b/.github/workflows/test-merge-smoke.yml index 362d56c314e..8ed1876a081 100644 --- a/.github/workflows/test-merge-smoke.yml +++ b/.github/workflows/test-merge-smoke.yml @@ -3,6 +3,8 @@ name: smoke on: pull_request: branches: [main, litellm_internal_staging, litellm_oss_staging, "litellm_**"] + push: + branches: [main] workflow_dispatch: permissions: @@ -52,8 +54,26 @@ jobs: with: version: "0.10.9" + - name: Restore the Rust bridge + id: rust-bridge + uses: ./.github/actions/restore-rust-bridge + + - name: Restore the Cargo build cache + if: steps.rust-bridge.outputs.cache-hit != 'true' + uses: ./.github/actions/cache-cargo-build + - name: Install dependencies - run: .github/scripts/uv_sync_with_retries.sh --frozen --extra proxy --extra cli --group dev --group proxy-dev --python ${{ matrix.python-version }} + env: + RUST_BRIDGE_RESTORED: ${{ steps.rust-bridge.outputs.cache-hit }} + PYTHON_VERSION: ${{ matrix.python-version }} + run: | + if [ "$RUST_BRIDGE_RESTORED" != "true" ]; then + .github/scripts/uv_sync_with_retries.sh --frozen --extra proxy --extra cli --group dev --group proxy-dev --python "$PYTHON_VERSION" + exit 0 + fi + .github/scripts/uv_sync_with_retries.sh --frozen --extra proxy --extra cli --group dev --group proxy-dev --python "$PYTHON_VERSION" --no-install-project + uv pip install --no-deps --python .venv/bin/python rust-bridge-dist/litellm-*.whl + cp rust-bridge-dist/litellm/rust_bridge/_native.abi3.so litellm/rust_bridge/_native.abi3.so - name: Create the loopback-only network namespace run: | @@ -87,6 +107,7 @@ jobs: - name: Upload smoke diagnostics if: always() + continue-on-error: true uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1 with: name: merge-smoke-diagnostics-py${{ matrix.python-version }} @@ -97,6 +118,10 @@ jobs: if: always() run: sudo ip netns delete smoke + - name: Keep only source-built wheels in the uv cache + if: always() && github.ref == 'refs/heads/main' + run: uv cache prune --ci + all-extras: name: all-extras-py${{ matrix.python-version }} runs-on: ubuntu-24.04 @@ -123,12 +148,34 @@ jobs: with: version: "0.10.9" + - name: Restore the Rust bridge + id: rust-bridge + uses: ./.github/actions/restore-rust-bridge + + - name: Restore the Cargo build cache + if: steps.rust-bridge.outputs.cache-hit != 'true' + uses: ./.github/actions/cache-cargo-build + - name: Install every extra and dependency group - run: .github/scripts/uv_sync_with_retries.sh --frozen --all-groups --all-extras --python ${{ matrix.python-version }} + env: + RUST_BRIDGE_RESTORED: ${{ steps.rust-bridge.outputs.cache-hit }} + PYTHON_VERSION: ${{ matrix.python-version }} + run: | + if [ "$RUST_BRIDGE_RESTORED" != "true" ]; then + .github/scripts/uv_sync_with_retries.sh --frozen --all-groups --all-extras --python "$PYTHON_VERSION" + exit 0 + fi + .github/scripts/uv_sync_with_retries.sh --frozen --all-groups --all-extras --python "$PYTHON_VERSION" --no-install-project --no-install-package litellm-proxy + uv pip install --no-deps --python .venv/bin/python rust-bridge-dist/*.whl + cp rust-bridge-dist/litellm/rust_bridge/_native.abi3.so litellm/rust_bridge/_native.abi3.so - name: Import the SDK, the proxy server and the enterprise package run: .venv/bin/python -c "import litellm, litellm.proxy.proxy_server, litellm_enterprise" + - name: Keep only source-built wheels in the uv cache + if: always() && github.ref == 'refs/heads/main' + run: uv cache prune --ci + windows-sdk: name: windows-sdk runs-on: windows-2025 @@ -154,15 +201,16 @@ jobs: with: version: "0.10.9" - - name: Set up the Rust bridge - uses: ./.github/actions/rust-bridge - - name: Install dependencies run: .github/scripts/uv_sync_with_retries.sh --frozen --group dev --python 3.11 - name: Run the Windows tests run: uv run --no-sync python -m pytest --tb=short -v tests/windows_tests + - name: Keep only source-built wheels in the uv cache + if: always() && github.ref == 'refs/heads/main' + run: uv cache prune --ci + proxy-build-from-pip: name: proxy-build-from-pip runs-on: ubuntu-24.04 @@ -298,6 +346,10 @@ jobs: if: always() run: cat /tmp/litellm-pip.log || true + - name: Keep only source-built wheels in the uv cache + if: always() && github.ref == 'refs/heads/main' + run: uv cache prune --ci + smoke-passed: name: smoke passed needs: [dashboard-build, core-checks, all-extras, windows-sdk, proxy-build-from-pip] diff --git a/.github/workflows/test-rust-db.yml b/.github/workflows/test-rust-db.yml index 6d4eba7b25e..116d6785287 100644 --- a/.github/workflows/test-rust-db.yml +++ b/.github/workflows/test-rust-db.yml @@ -42,10 +42,6 @@ concurrency: env: CARGO_TERM_COLOR: always - RUSTC_WRAPPER: sccache - SCCACHE_GHA_ENABLED: "true" - SCCACHE_GHA_VERSION: litellm-ci-v1 - SCCACHE_GHA_RW_MODE: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' && 'READ_WRITE' || 'READ_ONLY' }} CARGO_INCREMENTAL: "0" jobs: @@ -91,11 +87,7 @@ jobs: with: workspaces: litellm/rust cache-on-failure: true - - - name: Set up sccache - uses: mozilla-actions/sccache-action@d651010b8da762cde178750d8eda7b5febfe147a - with: - version: "v0.18.0" + save-if: ${{ github.ref == 'refs/heads/main' }} - name: Install Python dependencies working-directory: . diff --git a/.github/workflows/test-rust.yml b/.github/workflows/test-rust.yml index 74a6957aba4..880e8ea785c 100644 --- a/.github/workflows/test-rust.yml +++ b/.github/workflows/test-rust.yml @@ -78,10 +78,6 @@ concurrency: env: CARGO_TERM_COLOR: always - RUSTC_WRAPPER: sccache - SCCACHE_GHA_ENABLED: "true" - SCCACHE_GHA_VERSION: litellm-ci-v1 - SCCACHE_GHA_RW_MODE: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' && 'READ_WRITE' || 'READ_ONLY' }} CARGO_INCREMENTAL: "0" jobs: @@ -108,11 +104,6 @@ jobs: cache-on-failure: true save-if: ${{ github.ref == 'refs/heads/main' }} - - name: Set up sccache - uses: mozilla-actions/sccache-action@d651010b8da762cde178750d8eda7b5febfe147a - with: - version: "v0.18.0" - - run: cargo clippy --workspace --all-targets --locked -- -D warnings rust-test: @@ -155,11 +146,6 @@ jobs: cache-on-failure: true save-if: ${{ github.ref == 'refs/heads/main' }} - - name: Set up sccache - uses: mozilla-actions/sccache-action@d651010b8da762cde178750d8eda7b5febfe147a - with: - version: "v0.18.0" - - name: Check generated trace contracts working-directory: . run: uv run litellm/scripts/generate_trace_types.py --check @@ -206,11 +192,6 @@ jobs: cache-on-failure: true save-if: ${{ github.ref == 'refs/heads/main' }} - - name: Set up sccache - uses: mozilla-actions/sccache-action@d651010b8da762cde178750d8eda7b5febfe147a - with: - version: "v0.18.0" - - run: uv build --wheel --out-dir dist - run: python .github/scripts/verify_linux_native_wheel.py dist/*.whl diff --git a/.github/workflows/test-terraform-provider.yml b/.github/workflows/test-terraform-provider.yml index 7dc61f52364..34958526c59 100644 --- a/.github/workflows/test-terraform-provider.yml +++ b/.github/workflows/test-terraform-provider.yml @@ -4,6 +4,7 @@ on: push: paths: - "litellm/deploy/terraform/provider/**" + - ".github/actions/cache-go-build/**" - ".github/workflows/test-terraform-provider.yml" pull_request: branches: @@ -13,6 +14,7 @@ on: - "litellm/deploy/terraform/provider/**" - "litellm/proxy/**" - "litellm/enterprise/**" + - ".github/actions/cache-go-build/**" - ".github/workflows/test-terraform-provider.yml" permissions: @@ -38,8 +40,12 @@ jobs: - uses: actions/setup-go@7a3fe6cf4cb3a834922a1244abfce67bcef6a0c5 # v6.2.0 with: go-version-file: litellm/deploy/terraform/provider/go.mod - cache: true - cache-dependency-path: litellm/deploy/terraform/provider/go.sum + cache: false + + - name: Cache the Go build + uses: ./.github/actions/cache-go-build + with: + dependency-path: litellm/deploy/terraform/provider/go.sum - name: gofmt run: | @@ -120,8 +126,12 @@ jobs: - uses: actions/setup-go@7a3fe6cf4cb3a834922a1244abfce67bcef6a0c5 # v6.2.0 with: go-version-file: litellm/deploy/terraform/provider/go.mod - cache: true - cache-dependency-path: litellm/deploy/terraform/provider/go.sum + cache: false + + - name: Cache the Go build + uses: ./.github/actions/cache-go-build + with: + dependency-path: litellm/deploy/terraform/provider/go.sum - name: Audit provider endpoints against the schema working-directory: litellm/deploy/terraform/provider diff --git a/.github/workflows/test-unit.yml b/.github/workflows/test-unit.yml index 4c048ace26a..f27b58149ab 100644 --- a/.github/workflows/test-unit.yml +++ b/.github/workflows/test-unit.yml @@ -41,13 +41,6 @@ jobs: timeout-minutes: 2 uses: ./.github/actions/detect-changes - - name: Define editable Rust bridge cache key - id: rust-bridge-key - if: steps.changes.outputs.decision != 'skip' - env: - RUST_BRIDGE_CACHE_KEY: ${{ runner.os }}-rust-bridge-editable-${{ hashFiles('litellm/rust/Cargo.lock', 'litellm/rust/Cargo.toml', 'litellm/rust/crates/**', 'model_prices_and_context_window.json', 'pyproject.toml', 'litellm/packaging/litellm-proxy/pyproject.toml') }} - run: echo "key=$RUST_BRIDGE_CACHE_KEY" >> "$GITHUB_OUTPUT" - - name: Set up Python if: steps.changes.outputs.decision != 'skip' timeout-minutes: 3 @@ -66,30 +59,17 @@ jobs: id: rust-bridge-cache if: steps.changes.outputs.decision != 'skip' timeout-minutes: 5 - uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 - with: - path: rust-bridge-dist - key: ${{ steps.rust-bridge-key.outputs.key }} + uses: ./.github/actions/restore-rust-bridge - name: Cache the Rust build if: steps.changes.outputs.decision != 'skip' && steps.rust-bridge-cache.outputs.cache-hit != 'true' timeout-minutes: 5 uses: ./.github/actions/cache-cargo-build - - name: Set up sccache - if: steps.changes.outputs.decision != 'skip' && steps.rust-bridge-cache.outputs.cache-hit != 'true' - uses: mozilla-actions/sccache-action@d651010b8da762cde178750d8eda7b5febfe147a - with: - version: "v0.18.0" - - name: Build editable Rust bridge if: steps.changes.outputs.decision != 'skip' && steps.rust-bridge-cache.outputs.cache-hit != 'true' timeout-minutes: 15 env: - RUSTC_WRAPPER: sccache - SCCACHE_GHA_ENABLED: "true" - SCCACHE_GHA_VERSION: litellm-ci-v1 - SCCACHE_GHA_RW_MODE: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' && 'READ_WRITE' || 'READ_ONLY' }} CARGO_INCREMENTAL: "0" run: | mkdir -p rust-bridge-dist @@ -103,10 +83,6 @@ jobs: timeout-minutes: 15 working-directory: litellm/packaging/litellm-proxy env: - RUSTC_WRAPPER: sccache - SCCACHE_GHA_ENABLED: "true" - SCCACHE_GHA_VERSION: litellm-ci-v1 - SCCACHE_GHA_RW_MODE: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' && 'READ_WRITE' || 'READ_ONLY' }} CARGO_INCREMENTAL: "0" run: uv run --no-project --with maturin==1.15.0 maturin build --profile dev --strip --out ../../../rust-bridge-dist @@ -125,7 +101,7 @@ jobs: uses: actions/cache/save@0057852bfaa89a56745cba8c7296529d2fc39830 with: path: rust-bridge-dist - key: ${{ steps.rust-bridge-key.outputs.key }} + key: ${{ steps.rust-bridge-cache.outputs.key }} - name: Expose the Rust bridge artifact id: rust-bridge-artifact From 52d1c4cd80162fbf03d7f92def0cb1631f1edfd4 Mon Sep 17 00:00:00 2001 From: Kerry Lu Date: Thu, 12 Nov 2026 01:10:23 -0800 Subject: [PATCH 006/168] feat(dynamic_rate_limiter_v3): steer flexible teams to a permitted alternate when their preferred model is saturated (#51350) * feat(router): steer flexible teams to a permitted alternate model when their preferred model is saturated Teams opt in with metadata.model_preferences: [{"model": "luna", "alternates": ["opus"]}]. The dynamic rate limiter v3 pre-call hook reads the saturation cache for the preferred model and, once it is at or above saturation_threshold, rewrites the request to the first alternate the team may call that is below the threshold. The response carries x-litellm-model-steered-from with the preferred model. Rate limits and spend are charged to the model that served. Teams without model_preferences are untouched. * fix(dynamic_rate_limiter_v3): type the hook request dicts so the steering rewrite passes the lint gates * fix(dynamic_rate_limiter_v3): authorize alternates against the key and its model budget, send the steering header on streams Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(dynamic_rate_limiter_v3): enforce member and project allowlists on alternates, skip the steering header for non Latin-1 names * fix(dynamic_rate_limiter_v3): check every budget layer for an alternate, fall back to the preferred model when the alternate refuses, merge the served model's guardrails * fix(dynamic_rate_limiter_v3): never steer agent keys, type the steering unit tests * fix(dynamic_rate_limiter_v3): hold alternates to customer grants, access group budgets, guardrails and the zero-cost waiver An alternate now passes every check a budget fallback passes. Customer model grants are enforced, the access groups that grant the alternate are budget checked and stamped on the token and request metadata so spend charges them, a zero-cost preferred model that waived the aggregate budget checks is never steered to a priced alternate, and an alternate whose deployments add model-level guardrails the request did not run is skipped. * fix(dynamic_rate_limiter_v3): stamp the alternate's groups on every bucket and skip alternates the request cannot safely take The served model's access groups now land on every metadata bucket and the auth copy inside it even when the preferred model had none, so group spend is charged. An alternate is skipped when a budget reservation already holds a different group, when it does not serve the request's endpoint, when unpriced models are blocked and it has no price, or when its guardrails differ from the request's by name or settings. A failing alternate check or alternate admission leaves the request on the preferred model instead of failing it. * fix(dynamic_rate_limiter_v3): keep a reserved budget off pricier alternates While the request holds a budget reservation sized for the preferred model, an alternate that costs more per token is skipped, so fail-closed budget enforcement never admits a call the reservation does not cover. * fix(dynamic_rate_limiter_v3): an unpriced alternate never takes a reserved budget While a budget reservation is held, an alternate with no input or output price is skipped rather than treated as free. --------- Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../proxy/hooks/dynamic_rate_limiter_v3.py | 472 ++++++++++++++++-- .../hooks/parallel_request_limiter_v3.py | 3 +- .../test_team_model_preference_steering.py | 144 ++++++ .../hooks/test_dynamic_rate_limiter_v3.py | 285 +++++++++++ 4 files changed, 872 insertions(+), 32 deletions(-) create mode 100644 tests/integration/routing/test_team_model_preference_steering.py diff --git a/litellm/proxy/hooks/dynamic_rate_limiter_v3.py b/litellm/proxy/hooks/dynamic_rate_limiter_v3.py index 3654d0b9176..bd20fbf3ab3 100644 --- a/litellm/proxy/hooks/dynamic_rate_limiter_v3.py +++ b/litellm/proxy/hooks/dynamic_rate_limiter_v3.py @@ -2,14 +2,15 @@ Dynamic rate limiter v3 - Saturation-aware priority-based rate limiting """ +import json import os -from collections.abc import Callable, Mapping +from collections.abc import Awaitable, Callable, Mapping from dataclasses import dataclass from datetime import datetime -from typing import TYPE_CHECKING, Final, Literal +from typing import TYPE_CHECKING, Final, Literal, TypeAlias from fastapi import HTTPException -from pydantic import BaseModel, ConfigDict +from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError import litellm from litellm import Router @@ -22,6 +23,7 @@ from litellm.litellm_core_utils.add_retry_fallback_headers import ( ensure_response_additional_headers, response_has_hidden_params, ) +from litellm.litellm_core_utils.internal_call_metadata import MODEL_ACCESS_GROUP_METADATA_KEY from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.common_utils.proxy_rate_limit_error import ( ProxyRateLimitError, @@ -40,12 +42,13 @@ from litellm.proxy.hooks.parallel_request_limiter_v3 import ( # noqa: F401 # l borrowing_team_model_limits, claim_request_stash_for_data, get_or_create_request_stash, + get_request_stash_for_call, ) from litellm.proxy.hooks.rate_limiter_utils import ( convert_priority_to_percent, resolve_llm_provider_for_rate_limit, ) -from litellm.proxy.utils import InternalUsageCache +from litellm.proxy.utils import InternalUsageCache, check_and_merge_model_level_guardrails from litellm.redis.stores.rate_limits import RATE_LIMITS_TARGET, CounterIncrement, RateLimitCounter, RateLimitScope from litellm.types.router import ModelGroupInfo from litellm.types.utils import CallTypesLiteral, LLMResponseTypes, Usage @@ -54,6 +57,270 @@ if TYPE_CHECKING: from litellm.types.utils import PriorityReservationSettings +STEERED_FROM_HEADER: Final = "x-litellm-model-steered-from" + + +@dataclass(frozen=True, slots=True) +class AlternateGrant: + """What an authorized alternate carries onto the request: the budgeted access groups that granted it.""" + + access_groups: tuple[str, ...] = () + + +AlternateAuthorizer: TypeAlias = Callable[[UserAPIKeyAuth, str], Awaitable[AlternateGrant | None]] + + +class ModelPreference(BaseModel): + model_config = ConfigDict(frozen=True, extra="ignore") + + model: str + alternates: tuple[str, ...] = () + + +MODEL_PREFERENCES: Final = TypeAdapter(tuple[ModelPreference, ...]) + + +@dataclass(frozen=True, slots=True) +class SteeredModel: + model: str + group_info: ModelGroupInfo + access_groups: tuple[str, ...] = () + + +async def alternate_grant_for(user_api_key_dict: UserAPIKeyAuth, model: str) -> AlternateGrant | None: + """The grant a budget fallback to ``model`` would get: the key, team, member, project and customer allowlists, + every model budget layer and the budgets of the access groups that authorize it, else None.""" + from litellm.exceptions import BudgetExceededError + from litellm.proxy.auth.auth_checks import ( + _model_access_group_max_budget_check, # pyright: ignore[reportPrivateUsage] # steering shares the fallback contract + can_customer_access_model, + collect_matched_model_access_groups, + get_end_user_object, + get_project_object, + get_team_membership, + get_team_object, + ) + from litellm.proxy.auth.user_api_key_auth import ( + _is_fallback_model_authorized, # pyright: ignore[reportPrivateUsage] # steering shares the fallback contract + _is_within_every_model_budget, # pyright: ignore[reportPrivateUsage] # steering shares the fallback contract + ) + from litellm.proxy.proxy_server import ( + llm_model_list, # pyright: ignore[reportUnknownVariableType] # proxy global has untyped deployment entries + llm_router, + model_max_budget_limiter, + prisma_client, + proxy_logging_obj, + user_api_key_cache, + ) + + user_id: Final = user_api_key_dict.user_id + team_id: Final = user_api_key_dict.team_id + project_id: Final = user_api_key_dict.project_id + end_user_id: Final = user_api_key_dict.end_user_id + try: + team_object: Final = ( + await get_team_object( + team_id=team_id, + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + proxy_logging_obj=proxy_logging_obj, + ) + if team_id is not None + else None + ) + team_membership: Final = ( + await get_team_membership( + user_id=user_id, + team_id=team_id, + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + proxy_logging_obj=proxy_logging_obj, + ) + if user_id is not None and team_id is not None + else None + ) + project_object: Final = ( + await get_project_object( + project_id=project_id, + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + proxy_logging_obj=proxy_logging_obj, + ) + if project_id is not None + else None + ) + end_user_object: Final = ( + await get_end_user_object( + end_user_id=end_user_id, + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + proxy_logging_obj=proxy_logging_obj, + ) + if end_user_id is not None + else None + ) + if end_user_object is not None and end_user_object.models: + can_customer_access_model( + model=model, end_user_object=end_user_object, llm_router=llm_router, valid_token=user_api_key_dict + ) + except Exception: # noqa: BLE001 # a failed team, member, project or customer lookup must not authorize an alternate + return None + if not await _is_fallback_model_authorized( + model=model, + valid_token=user_api_key_dict, + llm_model_list=llm_model_list, # pyright: ignore[reportUnknownArgumentType] # proxy global has untyped deployment entries + llm_router=llm_router, + prisma_client=prisma_client, + team_membership=team_membership, + project_object=project_object, + refuse_agent_keys=True, + team_object=team_object, + ): + return None + if not await _is_within_every_model_budget( + valid_token=user_api_key_dict, model_max_budget_limiter=model_max_budget_limiter, model=model + ): + return None + try: + access_groups: Final = await collect_matched_model_access_groups( + model=model, + valid_token=user_api_key_dict, + team_object=team_object, + project_object=project_object, + llm_router=llm_router, + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + proxy_logging_obj=proxy_logging_obj, + team_membership=team_membership, + team_membership_loaded=True, + ) + await _model_access_group_max_budget_check( + matched_model_access_groups=access_groups, + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + ) + except BudgetExceededError: + return None + except Exception: # noqa: BLE001 # an unreadable group budget must not authorize an alternate + return None + return AlternateGrant(access_groups=access_groups) + + +REQUEST_BUCKET: Final = TypeAdapter(dict[str, object]) +REQUEST_ENTRIES: Final = TypeAdapter(tuple[object, ...]) +RESERVATION_ENTRIES: Final = TypeAdapter(tuple[Mapping[str, object], ...]) +MODEL_ACCESS_GROUP_ENTITY: Final = "Model access group" + + +def _metadata_bucket(data: Mapping[str, object], key: str) -> Mapping[str, object] | None: + """The request's ``key`` metadata bucket as a typed dict, or None when it is absent or not a mapping.""" + value: Final = data.get(key) + if not isinstance(value, Mapping): + return None + try: + return REQUEST_BUCKET.validate_python(value) + except ValidationError: + return None + + +def _guardrail_entries(data: Mapping[str, object]) -> frozenset[str]: + """The request's guardrail entries, settings included, in a form two lists can be compared by.""" + metadata: Final = _metadata_bucket(data, "metadata") + guardrails: Final = metadata.get("guardrails") if metadata is not None else None + entries: Final = ( + REQUEST_ENTRIES.validate_python(guardrails) + if isinstance(guardrails, list) + else (guardrails,) + if guardrails + else () + ) + return frozenset( + entry if isinstance(entry, str) else json.dumps(entry, sort_keys=True, default=str) for entry in entries + ) + + +def _auth_copy_with_groups(bucket: Mapping[str, object], access_groups: tuple[str, ...]) -> dict[str, object]: + auth: Final = bucket.get("user_api_key_auth") + if isinstance(auth, UserAPIKeyAuth): + return {"user_api_key_auth": auth.model_copy(update={"matched_model_access_groups": list(access_groups)})} + if isinstance(auth, Mapping): + return { + "user_api_key_auth": { + **REQUEST_BUCKET.validate_python(auth), + "matched_model_access_groups": list(access_groups), + } + } + return {} + + +def _with_access_groups(data: Mapping[str, object], access_groups: tuple[str, ...]) -> dict[str, object]: + """``data`` with the served model's access groups stamped on every metadata bucket, the auth copy in it + included, so spend attribution charges the groups that granted the served model and nothing else.""" + stamped: Final = { + key: { + **bucket, + MODEL_ACCESS_GROUP_METADATA_KEY: list(access_groups), + **_auth_copy_with_groups(bucket, access_groups), + } + for key in ("metadata", "litellm_metadata") + if (bucket := _metadata_bucket(data, key)) is not None + } + return {**data, **stamped} + + +def _priced_within(preferred: ModelGroupInfo, alternate: ModelGroupInfo) -> bool: + """Whether ``alternate`` costs no more per token than ``preferred``, so a budget reserved for the preferred + model's estimate still covers the alternate.""" + if alternate.input_cost_per_token is None or alternate.output_cost_per_token is None: + return False + return alternate.input_cost_per_token <= (preferred.input_cost_per_token or 0.0) and ( + alternate.output_cost_per_token <= (preferred.output_cost_per_token or 0.0) + ) + + +def _reservation_covers( + user_api_key_dict: UserAPIKeyAuth, + access_groups: tuple[str, ...], + preferred: ModelGroupInfo, + alternate: ModelGroupInfo, +) -> bool: + """Whether the budget the request already reserved still fits the alternate: the alternate costs no more per + token than the preferred model the estimate was made for, and every model access group reserved against also + grants the alternate, so settlement never charges a group that did not serve.""" + reservation: Final = user_api_key_dict.budget_reservation + if reservation is None: + return True + if not _priced_within(preferred, alternate): + return False + entries: Final = reservation.get("entries") + if not isinstance(entries, list): + return True + try: + reserved: Final = frozenset( + entry.get("entity_id") + for entry in RESERVATION_ENTRIES.validate_python(entries) + if entry.get("entity_type") == MODEL_ACCESS_GROUP_ENTITY + ) + except ValidationError: + return False + return reserved <= frozenset(access_groups) + + +def alternates_for(user_api_key_dict: UserAPIKeyAuth, model: str) -> tuple[str, ...]: + """The team's opt-in alternates for ``model`` from ``team_metadata.model_preferences``, or none.""" + raw: Final = (user_api_key_dict.team_metadata or {}).get("model_preferences") + if raw is None: + return () + try: + preferences: Final = MODEL_PREFERENCES.validate_python(raw) + except ValidationError as error: + verbose_proxy_logger.warning( + "Ignoring malformed team model_preferences for team %s: %s", user_api_key_dict.team_id, error + ) + return () + return next((preference.alternates for preference in preferences if preference.model == model), ()) + + class _SettlementMetadata(BaseModel): """The two fields of a request's standard logging metadata that token settlement reads.""" @@ -155,8 +422,10 @@ class _PROXY_DynamicRateLimitHandlerV3(CustomLogger): self, internal_usage_cache: DualCache, time_provider: Callable[[], datetime] | None = None, + is_model_authorized: AlternateAuthorizer = alternate_grant_for, ): self.internal_usage_cache = InternalUsageCache(dual_cache=internal_usage_cache) + self._is_model_authorized = is_model_authorized self.v3_limiter = PROXY_MaxParallelRequestsHandler_v3(self.internal_usage_cache, time_provider=time_provider) def update_variables(self, llm_router: Router) -> None: @@ -781,7 +1050,7 @@ class _PROXY_DynamicRateLimitHandlerV3(CustomLogger): self, user_api_key_dict: UserAPIKeyAuth, cache: DualCache, - data: dict, + data: Mapping[str, object], call_type: CallTypesLiteral, ) -> Exception | str | dict | None: """ @@ -823,38 +1092,20 @@ class _PROXY_DynamicRateLimitHandlerV3(CustomLogger): claim_request_stash_for_data(data) model: Final = data["model"] - priority: Final = self._get_priority_from_user_api_key_dict(user_api_key_dict=user_api_key_dict) + if not isinstance(model, str): + return None # Get model configuration - model_group_info: Final[ModelGroupInfo | None] = self.llm_router.registry().group(model) - if model_group_info is None: + requested_group_info: Final[ModelGroupInfo | None] = self.llm_router.registry().group(model) + if requested_group_info is None: verbose_proxy_logger.debug("No model group info for %s, allowing request", model) return None + steered: Final = await self._steer_to_alternate(model, requested_group_info, user_api_key_dict, data, call_type) try: - # STEP 1: Check current saturation level - saturation: Final = await self._check_model_saturation(model, model_group_info) - - saturation_threshold: Final = _get_priority_settings().saturation_threshold - - verbose_proxy_logger.debug( - f"[Dynamic Rate Limiter] Model={model}, Saturation={saturation:.1%}, " - f"Threshold={saturation_threshold:.1%}, Priority={priority}" + admitted: Final = await self._admit_steered_or_preferred( + model, requested_group_info, steered, user_api_key_dict ) - - # STEP 2: Check rate limits in THREE phases - # Phase 1: Read-only check of ALL limits (no increments) - # Phase 2: Decide which limits to enforce (based on saturation) - # Phase 3: Increment ALL counters only if request will be allowed - # This prevents partial increments and ensures accurate tracking - await self._check_rate_limits( - model=model, - model_group_info=model_group_info, - user_api_key_dict=user_api_key_dict, - priority=priority, - saturation=saturation, - ) - except HTTPException: raise except Exception as e: @@ -862,11 +1113,155 @@ class _PROXY_DynamicRateLimitHandlerV3(CustomLogger): # Fail open on unexpected errors return None + if admitted is None: + return None + get_or_create_request_stash().steered_from_model = model + groups: Final = list(admitted.access_groups) + user_api_key_dict.matched_model_access_groups = groups # rebind-ok: spend charges the served model's groups + return _with_access_groups(dict(data, model=admitted.model), admitted.access_groups) + + async def _admit_steered_or_preferred( + self, + model: str, + requested_group_info: ModelGroupInfo, + steered: "SteeredModel | None", + user_api_key_dict: UserAPIKeyAuth, + ) -> "SteeredModel | None": + """The alternate once it admits the request, else None after the preferred model's own check.""" + if steered is not None: + try: + await self._admit(steered.model, steered.group_info, user_api_key_dict) + except Exception: # noqa: BLE001 # the alternate's refusal or error keeps the request on its own model + verbose_proxy_logger.info( + "Alternate %s refused team %s, staying on %s", steered.model, user_api_key_dict.team_id, model + ) + else: + return steered + await self._admit(model, requested_group_info, user_api_key_dict) return None + async def _admit(self, model: str, model_group_info: ModelGroupInfo, user_api_key_dict: UserAPIKeyAuth) -> None: + """Read saturation, then the three-phase rate limit check, raising the 429 when ``model`` is out of capacity.""" + priority: Final = self._get_priority_from_user_api_key_dict(user_api_key_dict=user_api_key_dict) + saturation: Final = await self._check_model_saturation(model, model_group_info) + saturation_threshold: Final = _get_priority_settings().saturation_threshold + verbose_proxy_logger.debug( + f"[Dynamic Rate Limiter] Model={model}, Saturation={saturation:.1%}, " + f"Threshold={saturation_threshold:.1%}, Priority={priority}" + ) + await self._check_rate_limits( + model=model, + model_group_info=model_group_info, + user_api_key_dict=user_api_key_dict, + priority=priority, + saturation=saturation, + ) + + async def _steer_to_alternate( + self, + model: str, + model_group_info: ModelGroupInfo, + user_api_key_dict: UserAPIKeyAuth, + data: Mapping[str, object], + call_type: CallTypesLiteral, + ) -> "SteeredModel | None": + """First team alternate below ``saturation_threshold`` once ``model`` is at or above it, else None. A + broken alternate check is logged and leaves the request on ``model``, as the limiter fails open elsewhere.""" + try: + return await self._pick_alternate(model, model_group_info, user_api_key_dict, data, call_type) + except Exception: # noqa: BLE001 # a failing alternate check must not fail a request that can stay put + verbose_proxy_logger.warning( + "Alternate check failed for team %s, staying on %s", user_api_key_dict.team_id, model, exc_info=True + ) + return None + + async def _pick_alternate( + self, + model: str, + model_group_info: ModelGroupInfo, + user_api_key_dict: UserAPIKeyAuth, + data: Mapping[str, object], + call_type: CallTypesLiteral, + ) -> "SteeredModel | None": + alternates: Final = alternates_for(user_api_key_dict, model) + if not alternates: + return None + threshold: Final = _get_priority_settings().saturation_threshold + if await self._check_model_saturation(model, model_group_info) < threshold: + return None + for alternate in alternates: + if alternate == model or not self._steerable(model, alternate, data, call_type, user_api_key_dict.team_id): + continue + if ( + candidate := await self._idle_permitted_alternate( + alternate, user_api_key_dict, threshold, model_group_info + ) + ) is not None: + verbose_proxy_logger.info("Steering team %s from %s to %s", user_api_key_dict.team_id, model, alternate) + return candidate + return None + + def _steerable( + self, model: str, alternate: str, data: Mapping[str, object], call_type: CallTypesLiteral, team_id: str | None + ) -> bool: + """Whether ``alternate`` may serve a request that already passed auth and the guardrail walk as ``model``: a + zero-cost ``model`` waived the aggregate budget checks, so only another zero-cost alternate may take it, an + unpriced alternate is blocked when unpriced models are, the alternate has to serve this endpoint, and an + alternate with a model-level guardrail the request has not run would reach it unchecked.""" + from litellm.proxy.auth.auth_checks import ( + _unpriced_models_in_request, # pyright: ignore[reportPrivateUsage] # steering shares the fallback contract + is_model_cost_zero, + ) + from litellm.proxy.route_llm_request import endpoint_unsupported_by_model + + if is_model_cost_zero(model=model, llm_router=self.llm_router) and not is_model_cost_zero( + model=alternate, llm_router=self.llm_router + ): + verbose_proxy_logger.info("Not steering zero-cost %s to priced %s", model, alternate) + return False + if litellm.block_requests_for_models_without_pricing and _unpriced_models_in_request( + model=alternate, llm_router=self.llm_router + ): + verbose_proxy_logger.info("Not steering %s to unpriced %s", model, alternate) + return False + if endpoint_unsupported_by_model(call_type, dict(data, model=alternate), self.llm_router, team_id): + verbose_proxy_logger.info( + "Not steering %s to %s, the alternate does not serve %s", model, alternate, call_type + ) + return False + probe: Final = REQUEST_BUCKET.validate_python( + check_and_merge_model_level_guardrails( + data=dict(data, model=alternate, metadata=_metadata_bucket(data, "metadata") or {}), + llm_router=self.llm_router, + trust_client_model_info=False, + ) + ) + if _guardrail_entries(probe) != _guardrail_entries(data): + verbose_proxy_logger.info( + "Not steering %s to %s, the alternate adds model-level guardrails", model, alternate + ) + return False + return True + + async def _idle_permitted_alternate( + self, alternate: str, user_api_key_dict: UserAPIKeyAuth, threshold: float, preferred: ModelGroupInfo + ) -> "SteeredModel | None": + group_info: Final = self.llm_router.registry().group(alternate) + if group_info is None: + return None + grant: Final = await self._is_model_authorized(user_api_key_dict, alternate) + if grant is None or not _reservation_covers(user_api_key_dict, grant.access_groups, preferred, group_info): + return None + if await self._check_model_saturation(alternate, group_info) >= threshold: + return None + return SteeredModel(model=alternate, group_info=group_info, access_groups=grant.access_groups) + @with_service_target(RATE_LIMITS_TARGET) async def async_post_call_success_hook( - self, data: dict, user_api_key_dict: UserAPIKeyAuth, response + self, + data: dict[str, object], # mutable-ok: the CustomLogger contract hands the v3 limiter a mutable dict + user_api_key_dict: UserAPIKeyAuth, + response, ) -> LLMResponseTypes: """ Post-call hook to add rate limit headers to response. @@ -896,6 +1291,21 @@ class _PROXY_DynamicRateLimitHandlerV3(CustomLogger): verbose_proxy_logger.exception("Error in dynamic rate limiter v3 post-call hook: %s", e) return response + async def async_post_call_response_headers_hook( + self, + data: Mapping[str, object], + user_api_key_dict: UserAPIKeyAuth, + response: object, + request_headers: Mapping[str, str] | None = None, + litellm_call_info: Mapping[str, object] | None = None, + ) -> dict[str, str] | None: # mutable-ok: the CustomLogger contract returns a dict + """Names the model the caller asked for when steering served another, on streamed and buffered replies.""" + call_id: Final = data.get("litellm_call_id") + stash: Final = get_request_stash_for_call(call_id if isinstance(call_id, str) else None) + if stash is None or stash.steered_from_model is None or not _is_latin1_encodable(stash.steered_from_model): + return None + return {STEERED_FROM_HEADER: stash.steered_from_model} + @with_service_target(RATE_LIMITS_TARGET) async def async_log_success_event(self, kwargs, response_obj: object, start_time, end_time) -> None: """ diff --git a/litellm/proxy/hooks/parallel_request_limiter_v3.py b/litellm/proxy/hooks/parallel_request_limiter_v3.py index 6b53e952f98..4cbe08d0042 100644 --- a/litellm/proxy/hooks/parallel_request_limiter_v3.py +++ b/litellm/proxy/hooks/parallel_request_limiter_v3.py @@ -602,6 +602,7 @@ class RequestRateLimiterStash: reserved_model: RateLimitedModel | None = None reserved_scopes: frozenset[tuple[str, str]] = field(default_factory=frozenset) fallback_models: frozenset[str] = field(default_factory=frozenset) + steered_from_model: str | None = None ptu_ceiling: PTUTeamCeiling | None = None ptu_reserved_tokens: int = 0 ptu_reserved_window_identities: frozenset[ReservationWindowIdentity] = field(default_factory=frozenset) @@ -706,7 +707,7 @@ def get_or_create_request_stash() -> RequestRateLimiterStash: return stash -def claim_request_stash_for_data(data: dict) -> RequestRateLimiterStash: +def claim_request_stash_for_data(data: Mapping[str, object]) -> RequestRateLimiterStash: stash: Final = get_or_create_request_stash() owner_call_id: Final = data.get("litellm_call_id") if isinstance(owner_call_id, str): diff --git a/tests/integration/routing/test_team_model_preference_steering.py b/tests/integration/routing/test_team_model_preference_steering.py new file mode 100644 index 00000000000..c99cfcf660b --- /dev/null +++ b/tests/integration/routing/test_team_model_preference_steering.py @@ -0,0 +1,144 @@ +import json +import uuid +from collections.abc import Iterator +from pathlib import Path +from typing import Final + +import httpx +import pytest +import yaml +from integration._support.client import Gateway, Scenario, eventually, gateway_from_environment +from integration._support.process import owned_proxy +from integration._support.wire import Reply, Request, wire_server + +OPENAI_MODEL: Final = "gpt-5.6" +PREFERRED_RPM: Final = 2 +ALTERNATE_RPM: Final = 1000 +SATURATION_THRESHOLD: Final = 0.5 +STEERED_FROM_HEADER: Final = "x-litellm-model-steered-from" + + +def _reply() -> bytes: + return json.dumps( + { + "id": f"chatcmpl_{uuid.uuid4().hex}", + "object": "chat.completion", + "created": 1700000000, + "model": OPENAI_MODEL, + "choices": [{"index": 0, "message": {"role": "assistant", "content": "steered"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 20, "completion_tokens": 10, "total_tokens": 30}, + } + ).encode() + + +@pytest.fixture(scope="module") +def steering_proxy(tmp_path_factory: pytest.TempPathFactory) -> Iterator[tuple[Gateway, str]]: + directory: Final = tmp_path_factory.mktemp("model-preference-steering") + base: Final = yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text()) + configuration: Final = { + **base, + "litellm_settings": { + **base["litellm_settings"], + "callbacks": ["dynamic_rate_limiter_v3"], + "priority_reservation_settings": { + "saturation_threshold": SATURATION_THRESHOLD, + "saturation_check_cache_ttl": 1, + }, + }, + } + path: Final = directory / "model_preference_steering.yaml" + path.write_text(yaml.safe_dump(configuration)) + + def respond(request: Request) -> Reply: + assert request.method == "POST" and request.target == "/v1/chat/completions", request.target + return Reply(body=_reply()) + + with ( + gateway_from_environment() as rig, + wire_server(respond) as wire, + owned_proxy(rig, directory, {}, config=path) as owned, + ): + yield owned, wire.url + + +def _deployment(owned: Gateway, scenario: Scenario, upstream: str, rpm: int) -> tuple[str, str]: + name: Final = scenario.model( + model=f"openai/{OPENAI_MODEL}", api_base=f"{upstream}/v1", api_key="synthetic-key", rpm=rpm + ) + rows: Final = owned.rows('SELECT model_id FROM "LiteLLM_ProxyModelTable" WHERE model_name = %s', (name,)) + return name, str(rows[0]["model_id"]) + + +def _chat(owned: Gateway, key: str, model: str) -> httpx.Response: + return owned.request( + "POST", + "/v1/chat/completions", + {"model": model, "messages": [{"role": "user", "content": f"model preference steering {uuid.uuid4().hex}"}]}, + key=key, + ) + + +def _spend_row(owned: Gateway, response: httpx.Response) -> list[dict[str, object]]: + return owned.rows( + 'SELECT model_group FROM "LiteLLM_SpendLogs" WHERE request_id = %s', (str(response.json()["id"]),) + ) + + +def test_flexible_team_is_steered_to_its_permitted_alternate_once_the_preferred_model_saturates( + steering_proxy: tuple[Gateway, str], +) -> None: + owned, upstream = steering_proxy + with owned.scenario() as scenario: + preferred, preferred_id = _deployment(owned, scenario, upstream, PREFERRED_RPM) + alternate, alternate_id = _deployment(owned, scenario, upstream, ALTERNATE_RPM) + rigid_key: Final = scenario.key(team_id=scenario.team(models=[preferred, alternate])) + flexible_key: Final = scenario.key( + team_id=scenario.team( + models=[preferred, alternate], + metadata={"model_preferences": [{"model": preferred, "alternates": [alternate]}]}, + ) + ) + + warmup: Final = _chat(owned, rigid_key, preferred) + assert warmup.status_code == 200, warmup.text + assert warmup.headers["x-litellm-model-id"] == preferred_id + assert STEERED_FROM_HEADER not in warmup.headers + + steered: Final = eventually( + lambda: _chat(owned, flexible_key, preferred), + lambda response: STEERED_FROM_HEADER in response.headers, + seconds=15, + return_last_on_timeout=True, + ) + assert steered.status_code == 200, steered.text + assert steered.headers[STEERED_FROM_HEADER] == preferred + assert steered.headers["x-litellm-model-id"] == alternate_id + + rigid: Final = _chat(owned, rigid_key, preferred) + assert rigid.status_code == 200, rigid.text + assert rigid.headers["x-litellm-model-id"] == preferred_id + assert STEERED_FROM_HEADER not in rigid.headers + + charged: Final = eventually(lambda: _spend_row(owned, steered), lambda rows: len(rows) == 1, seconds=30) + assert charged[0]["model_group"] == alternate + + +def test_alternate_the_team_may_not_call_is_never_used(steering_proxy: tuple[Gateway, str]) -> None: + owned, upstream = steering_proxy + with owned.scenario() as scenario: + preferred, _ = _deployment(owned, scenario, upstream, PREFERRED_RPM) + forbidden, _ = _deployment(owned, scenario, upstream, ALTERNATE_RPM) + key: Final = scenario.key( + team_id=scenario.team( + models=[preferred], + metadata={"model_preferences": [{"model": preferred, "alternates": [forbidden]}]}, + ) + ) + + first: Final = _chat(owned, key, preferred) + assert first.status_code == 200, first.text + second: Final = eventually( + lambda: _chat(owned, key, preferred), lambda response: response.status_code != 200, seconds=15 + ) + assert second.status_code == 429, second.text + assert STEERED_FROM_HEADER not in second.headers diff --git a/tests/unit/proxy/hooks/test_dynamic_rate_limiter_v3.py b/tests/unit/proxy/hooks/test_dynamic_rate_limiter_v3.py index a8265680856..0bf0ea4d6ad 100644 --- a/tests/unit/proxy/hooks/test_dynamic_rate_limiter_v3.py +++ b/tests/unit/proxy/hooks/test_dynamic_rate_limiter_v3.py @@ -6,7 +6,9 @@ Core tests to validate that priority weights are respected (0.9/0.1) instead of import asyncio import time +from collections.abc import Mapping from datetime import datetime, timedelta +from types import MappingProxyType from typing import Final from unittest.mock import AsyncMock, patch @@ -14,9 +16,11 @@ import pytest import litellm from litellm import DualCache, Router +from litellm.litellm_core_utils.internal_call_metadata import MODEL_ACCESS_GROUP_METADATA_KEY from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.common_utils.proxy_rate_limit_error import ProxyRateLimitError from litellm.proxy.hooks.dynamic_rate_limiter_v3 import ( + AlternateGrant, PROXY_DynamicRateLimitHandlerV3, ) from litellm.proxy.hooks.dynamic_rate_limiter_v3 import ( @@ -1824,6 +1828,287 @@ async def test_post_call_success_hook_priority_header_is_always_http_encodable(t assert http_response.headers["x-litellm-rate-limiter-version"] == "v3" +def _steering_router( + preferred: str, + alternate: str, + preferred_rpm: int = 2, + alternate_rpm: int = 1000, + extra_params: Mapping[str, Mapping[str, object]] = MappingProxyType({}), + model_info: Mapping[str, Mapping[str, object]] = MappingProxyType({}), +) -> Router: + return Router( + model_list=[ + { + "model_name": name, + "litellm_params": { + "model": "gpt-3.5-turbo", + "api_key": "test-key", + "api_base": "test-base", + "rpm": rpm, + **extra_params.get(name, {}), + }, + "model_info": dict(model_info.get(name, {})), + } + for name, rpm in ((preferred, preferred_rpm), (alternate, alternate_rpm)) + ] + ) + + +def _team_user(team_models: list[str], preferences: object) -> UserAPIKeyAuth: + return UserAPIKeyAuth( + team_id="flexible-team", team_models=team_models, team_metadata={"model_preferences": preferences} + ) + + +async def _team_allowlist(user_api_key_dict: UserAPIKeyAuth, model: str) -> AlternateGrant | None: + return AlternateGrant() if model in user_api_key_dict.team_models else None + + +async def _pooled_team_allowlist(user_api_key_dict: UserAPIKeyAuth, model: str) -> AlternateGrant | None: + return AlternateGrant(access_groups=(f"{model}-pool",)) if model in user_api_key_dict.team_models else None + + +def _steering_handler() -> DynamicRateLimitHandler: + return DynamicRateLimitHandler(internal_usage_cache=DualCache(), is_model_authorized=_team_allowlist) + + +async def _saturate(handler: DynamicRateLimitHandler, model: str, requests: int = 1) -> None: + for _ in range(requests): + assert await handler.async_pre_call_hook(UserAPIKeyAuth(), DualCache(), {"model": model}, "completion") is None + + +@pytest.mark.asyncio +async def test_flexible_team_is_steered_to_its_alternate_once_the_preferred_model_saturates( + monkeypatch: pytest.MonkeyPatch, +): + monkeypatch.setattr(litellm, "priority_reservation", None) + monkeypatch.setattr(litellm.priority_reservation_settings, "saturation_threshold", 0.5) + handler: Final = _steering_handler() + handler.update_variables(llm_router=_steering_router("luna", "opus")) + flexible: Final = _team_user(["luna", "opus"], [{"model": "luna", "alternates": ["opus"]}]) + + assert await handler.async_pre_call_hook(flexible, DualCache(), {"model": "luna"}, "completion") is None + + await _saturate(handler, "luna") + steered: Final = await handler.async_pre_call_hook(flexible, DualCache(), {"model": "luna", "n": 1}, "completion") + assert steered == {"model": "opus", "n": 1} + + headers: Final = await handler.async_post_call_response_headers_hook( + data={"model": "opus", "stream": True}, user_api_key_dict=flexible, response=None + ) + assert headers == {"x-litellm-model-steered-from": "luna"} + + from litellm.proxy.hooks.parallel_request_limiter_v3 import get_or_create_request_stash + + get_or_create_request_stash().steered_from_model = "luna\u2728" + assert ( + await handler.async_post_call_response_headers_hook(data={}, user_api_key_dict=flexible, response=None) is None + ) + + +@pytest.mark.asyncio +async def test_teams_without_a_permitted_idle_alternate_stay_on_the_preferred_model(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setattr(litellm, "priority_reservation", None) + monkeypatch.setattr(litellm.priority_reservation_settings, "saturation_threshold", 0.5) + handler: Final = _steering_handler() + handler.update_variables(llm_router=_steering_router("luna", "opus", preferred_rpm=10, alternate_rpm=1)) + await _saturate(handler, "luna", requests=5) + await _saturate(handler, "opus") + + forbidden: Final = _team_user(["luna"], [{"model": "luna", "alternates": ["opus"]}]) + unknown: Final = _team_user([], [{"model": "luna", "alternates": ["missing-model"]}]) + malformed: Final = _team_user([], {"luna": "opus"}) + both_saturated: Final = _team_user(["luna", "opus"], [{"model": "luna", "alternates": ["opus"]}]) + for team in (forbidden, unknown, malformed, both_saturated): + assert await handler.async_pre_call_hook(team, DualCache(), {"model": "luna"}, "completion") is None + assert ( + await handler.async_post_call_response_headers_hook(data={}, user_api_key_dict=team, response=None) is None + ) + + +@pytest.mark.asyncio +async def test_a_zero_cost_preferred_model_is_never_steered_to_a_priced_alternate(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setattr(litellm, "priority_reservation", None) + monkeypatch.setattr(litellm.priority_reservation_settings, "saturation_threshold", 0.5) + handler: Final = _steering_handler() + free_luna: Final = {"luna": {"input_cost_per_token": 0, "output_cost_per_token": 0}} + handler.update_variables(llm_router=_steering_router("luna", "opus", extra_params=free_luna)) + await _saturate(handler, "luna") + + flexible: Final = _team_user(["luna", "opus"], [{"model": "luna", "alternates": ["opus"]}]) + assert await handler.async_pre_call_hook(flexible, DualCache(), {"model": "luna"}, "completion") is None + assert ( + await handler.async_post_call_response_headers_hook(data={}, user_api_key_dict=flexible, response=None) is None + ) + + +@pytest.mark.asyncio +async def test_an_alternate_that_adds_model_level_guardrails_is_never_used(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setattr(litellm, "priority_reservation", None) + monkeypatch.setattr(litellm.priority_reservation_settings, "saturation_threshold", 0.5) + handler: Final = _steering_handler() + guarded_opus: Final = {"opus": {"guardrails": ["opus-only-input-scan"]}} + handler.update_variables(llm_router=_steering_router("luna", "opus", extra_params=guarded_opus)) + await _saturate(handler, "luna") + + flexible: Final = _team_user(["luna", "opus"], [{"model": "luna", "alternates": ["opus"]}]) + assert await handler.async_pre_call_hook(flexible, DualCache(), {"model": "luna"}, "completion") is None + assert ( + await handler.async_post_call_response_headers_hook(data={}, user_api_key_dict=flexible, response=None) is None + ) + + already_scanned: Final = {"model": "luna", "metadata": {"guardrails": ["opus-only-input-scan"]}} + steered: Final = await handler.async_pre_call_hook(flexible, DualCache(), already_scanned, "completion") + assert isinstance(steered, dict) + assert steered["model"] == "opus" + assert steered["metadata"]["guardrails"] == ["opus-only-input-scan"] + + configured: Final = _steering_handler() + configured_opus: Final = {"opus": {"guardrails": [{"opus-only-input-scan": {"extra_body": {"strict": True}}}]}} + configured.update_variables(llm_router=_steering_router("luna", "opus", extra_params=configured_opus)) + await _saturate(configured, "luna") + plain_scan: Final = {"model": "luna", "metadata": {"guardrails": ["opus-only-input-scan"]}} + assert await configured.async_pre_call_hook(flexible, DualCache(), plain_scan, "completion") is None + + +@pytest.mark.asyncio +async def test_a_steered_request_is_charged_to_the_groups_that_granted_the_alternate(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setattr(litellm, "priority_reservation", None) + monkeypatch.setattr(litellm.priority_reservation_settings, "saturation_threshold", 0.5) + handler: Final = DynamicRateLimitHandler( + internal_usage_cache=DualCache(), is_model_authorized=_pooled_team_allowlist + ) + handler.update_variables(llm_router=_steering_router("luna", "opus")) + await _saturate(handler, "luna") + + flexible: Final = _team_user(["luna", "opus"], [{"model": "luna", "alternates": ["opus"]}]) + auth_copy: Final = flexible.model_copy() + data: Final = {"model": "luna", "metadata": {"user_api_key_auth": auth_copy, "tags": ["t"]}} + steered: Final = await handler.async_pre_call_hook(flexible, DualCache(), data, "completion") + assert isinstance(steered, dict) + assert steered["model"] == "opus" + assert steered["metadata"]["tags"] == ["t"] + assert steered["metadata"][MODEL_ACCESS_GROUP_METADATA_KEY] == ["opus-pool"] + assert steered["metadata"]["user_api_key_auth"].matched_model_access_groups == ["opus-pool"] + assert flexible.matched_model_access_groups == ["opus-pool"] + assert auth_copy.matched_model_access_groups is None + + +@pytest.mark.asyncio +async def test_an_alternate_granted_by_a_group_the_request_did_not_reserve_against_is_never_used( + monkeypatch: pytest.MonkeyPatch, +): + monkeypatch.setattr(litellm, "priority_reservation", None) + monkeypatch.setattr(litellm.priority_reservation_settings, "saturation_threshold", 0.5) + handler: Final = DynamicRateLimitHandler( + internal_usage_cache=DualCache(), is_model_authorized=_pooled_team_allowlist + ) + handler.update_variables(llm_router=_steering_router("luna", "opus")) + await _saturate(handler, "luna") + + def reserved_against(group: str) -> UserAPIKeyAuth: + user: Final = _team_user(["luna", "opus"], [{"model": "luna", "alternates": ["opus"]}]) + user.budget_reservation = { + "entries": [ + { + "counter_key": f"spend:model_access_group:{group}", + "entity_type": "Model access group", + "entity_id": group, + "reserved_cost": 0.01, + "applied_adjustment": 0.0, + } + ] + } + return user + + assert ( + await handler.async_pre_call_hook(reserved_against("luna-pool"), DualCache(), {"model": "luna"}, "completion") + is None + ) + steered: Final = await handler.async_pre_call_hook( + reserved_against("opus-pool"), DualCache(), {"model": "luna"}, "completion" + ) + assert isinstance(steered, dict) + assert steered["model"] == "opus" + + +@pytest.mark.asyncio +async def test_a_pricier_alternate_is_never_used_while_a_budget_reservation_is_held(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setattr(litellm, "priority_reservation", None) + monkeypatch.setattr(litellm.priority_reservation_settings, "saturation_threshold", 0.5) + handler: Final = _steering_handler() + pricier_opus: Final = { + "luna": {"input_cost_per_token": 0.000001, "output_cost_per_token": 0.000002}, + "opus": {"input_cost_per_token": 0.000005, "output_cost_per_token": 0.000002}, + } + handler.update_variables(llm_router=_steering_router("luna", "opus", extra_params=pricier_opus)) + await _saturate(handler, "luna") + + reserved: Final = _team_user(["luna", "opus"], [{"model": "luna", "alternates": ["opus"]}]) + reserved.budget_reservation = {"reserved_cost": 0.0001, "entries": []} + assert await handler.async_pre_call_hook(reserved, DualCache(), {"model": "luna"}, "completion") is None + + unreserved: Final = _team_user(["luna", "opus"], [{"model": "luna", "alternates": ["opus"]}]) + steered: Final = await handler.async_pre_call_hook(unreserved, DualCache(), {"model": "luna"}, "completion") + assert isinstance(steered, dict) + assert steered["model"] == "opus" + + unpriced: Final = _steering_handler() + unpriced_opus: Final = {"opus": {"model": "openai/no-such-model-xyz"}} + unpriced.update_variables(llm_router=_steering_router("luna", "opus", extra_params=unpriced_opus)) + await _saturate(unpriced, "luna") + assert await unpriced.async_pre_call_hook(reserved, DualCache(), {"model": "luna"}, "completion") is None + + +@pytest.mark.asyncio +async def test_an_alternate_that_does_not_serve_the_endpoint_is_never_used(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setattr(litellm, "priority_reservation", None) + monkeypatch.setattr(litellm.priority_reservation_settings, "saturation_threshold", 0.5) + handler: Final = _steering_handler() + embeddings_only: Final = {"opus": {"enforce_supported_endpoints": True, "supported_endpoints": ["/v1/embeddings"]}} + handler.update_variables(llm_router=_steering_router("luna", "opus", model_info=embeddings_only)) + await _saturate(handler, "luna") + + flexible: Final = _team_user(["luna", "opus"], [{"model": "luna", "alternates": ["opus"]}]) + assert await handler.async_pre_call_hook(flexible, DualCache(), {"model": "luna"}, "acompletion") is None + assert ( + await handler.async_post_call_response_headers_hook(data={}, user_api_key_dict=flexible, response=None) is None + ) + + +@pytest.mark.asyncio +async def test_an_unpriced_alternate_is_never_used_while_unpriced_models_are_blocked(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setattr(litellm, "priority_reservation", None) + monkeypatch.setattr(litellm.priority_reservation_settings, "saturation_threshold", 0.5) + monkeypatch.setattr(litellm, "block_requests_for_models_without_pricing", True) + handler: Final = _steering_handler() + unpriced_opus: Final = {"opus": {"model": "openai/no-such-model-xyz"}} + handler.update_variables(llm_router=_steering_router("luna", "opus", extra_params=unpriced_opus)) + await _saturate(handler, "luna") + + flexible: Final = _team_user(["luna", "opus"], [{"model": "luna", "alternates": ["opus"]}]) + assert await handler.async_pre_call_hook(flexible, DualCache(), {"model": "luna"}, "completion") is None + + +@pytest.mark.asyncio +async def test_a_broken_alternate_check_keeps_the_request_on_the_preferred_model(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setattr(litellm, "priority_reservation", None) + monkeypatch.setattr(litellm.priority_reservation_settings, "saturation_threshold", 0.5) + + async def broken(user_api_key_dict: UserAPIKeyAuth, model: str) -> AlternateGrant | None: + raise RuntimeError("membership lookup is down") + + handler: Final = DynamicRateLimitHandler(internal_usage_cache=DualCache(), is_model_authorized=broken) + handler.update_variables(llm_router=_steering_router("luna", "opus")) + await _saturate(handler, "luna") + + flexible: Final = _team_user(["luna", "opus"], [{"model": "luna", "alternates": ["opus"]}]) + assert await handler.async_pre_call_hook(flexible, DualCache(), {"model": "luna"}, "completion") is None + assert ( + await handler.async_post_call_response_headers_hook(data={}, user_api_key_dict=flexible, response=None) is None + ) + + def _borrowing_handler( monkeypatch: pytest.MonkeyPatch, borrows: bool, From c0114e55aa393c54c55dc4b46b90c80373260b78 Mon Sep 17 00:00:00 2001 From: Kerry Lu Date: Thu, 12 Nov 2026 06:38:07 -0800 Subject: [PATCH 007/168] feat(dynamic_rate_limiter_v3): priority_class_share_splits_equally_among_active_teams splits a priority class share among the teams active on a model (#51366) * feat(dynamic_rate_limiter_v3): split a priority class share equally among the teams active on a model Adds priority_reservation_settings.priority_class_share_splits_equally_among_active_teams (default false). When on, the class weights are renormalized over the classes with a team active on the model in the current window, each class share is divided equally among its active teams, and once the model saturates each team is held to its own share with a 429 whose rate_limited_by header reads priority_class_share_saturated. Active teams are one Redis sorted set per model with the window as TTL. * fix(proxy): keep basedpyright counts flat for the priority class share setting PriorityReservationSettings is built with model_validate instead of **value, so a new field no longer adds an unknown-argument diagnostic, and the limiter unit test helpers type the handler by its implementation class. Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(dynamic_rate_limiter_v3): keep team shares usable when Redis, mixed-priority keys, WebSocket turns or tiny shares get in the way A registry failure falls back to the class pool instead of skipping the model limits, a team's counter is keyed by model and team so keys of different priorities spend one share, Responses WebSocket turns are held to the class share like the team model limit, a share is floored at one request and one token, and the registry declares the rate limits key family. Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(dynamic_rate_limiter_v3): catch only Redis and validation errors when the active team registry fails Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(dynamic_rate_limiter_v3): keys without a team keep their class present, team ids may contain the registry separator, breaker-open counts as a registry failure Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(dynamic_rate_limiter_v3): per-class pool markers that take no team slice, collision-free model and team counter keys, local registry pruned by last-seen time Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(dynamic_rate_limiter_v3): settle tokens only on the stash the call owns, keep a configured class named default apart from teams with no priority Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(dynamic_rate_limiter_v3): tag team registry members and count every unenforced request against the team share A team id can no longer spell a pool marker, since real teams are stored under a team prefix. Below saturation the team share counter is bumped through the atomic increment at the model-wide limit, so the local over-limit short-circuit never skips a request and the counter is complete when the share grows. Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(dynamic_rate_limiter_v3): list the caller under its own class and resync the socket stash share on every frame Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(dynamic_rate_limiter_v3): keep the connection share across a registry outage and settle fallback tokens on the served model's team counter Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../proxy/hooks/dynamic_rate_limiter_v3.py | 199 +++++++- .../hooks/parallel_request_limiter_v3.py | 1 + litellm/proxy/hooks/priority_class_share.py | 127 +++++ litellm/proxy/proxy_server.py | 2 +- litellm/types/utils.py | 4 + ...y_class_share_splits_among_active_teams.py | 136 +++++ .../hooks/test_dynamic_rate_limiter_v3.py | 478 +++++++++++++++++- .../proxy/hooks/test_priority_class_share.py | 157 ++++++ 8 files changed, 1074 insertions(+), 30 deletions(-) create mode 100644 litellm/proxy/hooks/priority_class_share.py create mode 100644 tests/integration/routing/test_priority_class_share_splits_among_active_teams.py create mode 100644 tests/unit/proxy/hooks/test_priority_class_share.py diff --git a/litellm/proxy/hooks/dynamic_rate_limiter_v3.py b/litellm/proxy/hooks/dynamic_rate_limiter_v3.py index bd20fbf3ab3..059efe440ee 100644 --- a/litellm/proxy/hooks/dynamic_rate_limiter_v3.py +++ b/litellm/proxy/hooks/dynamic_rate_limiter_v3.py @@ -11,12 +11,14 @@ from typing import TYPE_CHECKING, Final, Literal, TypeAlias from fastapi import HTTPException from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError +from redis.exceptions import RedisError import litellm from litellm import Router from litellm._internal_context import with_service_target from litellm._logging import verbose_proxy_logger from litellm.caching.caching import DualCache +from litellm.caching.redis_cache import RedisCircuitBreakerOpenError from litellm.cost_calculator import get_usage_object from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.add_retry_fallback_headers import ( @@ -37,13 +39,25 @@ from litellm.proxy.hooks.parallel_request_limiter_v3 import ( # noqa: F401 # l RateLimitedModel, RateLimitResponse, RateLimitStatus, + _call_id_from_callback_kwargs, # pyright: ignore[reportPrivateUsage] # same call-ownership check the regular limiter uses + _is_socket_turn, # pyright: ignore[reportPrivateUsage] # same socket-turn check the regular limiter uses _PROXY_MaxParallelRequestsHandler_v3, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export _socket_turn_tpm_windows, # pyright: ignore[reportPrivateUsage] # same per-turn TPM window the regular limiter checks borrowing_team_model_limits, claim_request_stash_for_data, get_or_create_request_stash, + get_request_stash, get_request_stash_for_call, ) +from litellm.proxy.hooks.priority_class_share import ( + DEFAULT_CLASS, + ActiveTeamRegistry, + TeamShare, + counter_value, + pool_member, + team_member, + team_share, +) from litellm.proxy.hooks.rate_limiter_utils import ( convert_priority_to_percent, resolve_llm_provider_for_rate_limit, @@ -341,6 +355,7 @@ def _usage_of(response_obj: object) -> Usage | None: TEAM_MODEL_LIMIT_AT_SATURATION: Final = "model_per_team_saturated" +PRIORITY_CLASS_SHARE_AT_SATURATION: Final = "priority_class_share_saturated" TEAM_MODEL_LIMIT_MAX_BORROW_MULTIPLE: Final = "model_per_team_max_borrow_multiple" @@ -427,6 +442,7 @@ class _PROXY_DynamicRateLimitHandlerV3(CustomLogger): self.internal_usage_cache = InternalUsageCache(dual_cache=internal_usage_cache) self._is_model_authorized = is_model_authorized self.v3_limiter = PROXY_MaxParallelRequestsHandler_v3(self.internal_usage_cache, time_provider=time_provider) + self.active_teams = ActiveTeamRegistry(internal_usage_cache) def update_variables(self, llm_router: Router) -> None: self.llm_router = llm_router @@ -619,11 +635,54 @@ class _PROXY_DynamicRateLimitHandlerV3(CustomLogger): # Fail open: assume not saturated on error return 0.0 + def _priority_class(self, priority: str | None) -> str: + """The class a team is counted under: its explicit priority, or one shared default class.""" + if ( + priority is not None + and litellm.priority_reservation is not None + and priority in litellm.priority_reservation + ): + return priority + return DEFAULT_CLASS + + async def _team_class_share( + self, + model: str, + user_api_key_dict: UserAPIKeyAuth, + priority: str | None, + normalized_weights: Mapping[str, float], + ) -> TeamShare | None: + """The caller's slice of its class share, or None when the split is off or the key has no team. A key with no + team keeps the class pool but still marks its class active, so the pool's capacity is not lent away.""" + if ( + litellm.priority_reservation is None + or not _get_priority_settings().priority_class_share_splits_equally_among_active_teams + ): + return None + priority_class: Final = self._priority_class(priority) + member: Final = ( + pool_member(priority_class) if user_api_key_dict.team_id is None else team_member(user_api_key_dict.team_id) + ) + try: + members: Final = await self.active_teams.touch(model, member, priority_class, self.v3_limiter.window_size) + except (RedisError, RedisCircuitBreakerOpenError, ValidationError) as e: + verbose_proxy_logger.warning( + "Active team registry unavailable for %s, holding team %s to the class pool: %s", + model, + user_api_key_dict.team_id, + e, + ) + return None + if user_api_key_dict.team_id is None: + return None + return team_share(priority_class, members, normalized_weights, _get_priority_settings().default_priority) + def _create_priority_based_descriptors( self, model: str, user_api_key_dict: UserAPIKeyAuth, priority: str | None, + share: TeamShare | None = None, ) -> list[RateLimitDescriptor]: """ Create rate limit descriptors with normalized priority weights. @@ -653,14 +712,16 @@ class _PROXY_DynamicRateLimitHandlerV3(CustomLogger): ) rate_limit_config: Final[RateLimitDescriptorRateLimitObject] = {} + weight: Final = priority_weight if share is None else share.fraction + floor: Final = 0 if share is None else 1 # Apply priority weight to model limits if model_group_info.tpm is not None: - reserved_tpm: Final = int(model_group_info.tpm * priority_weight) + reserved_tpm: Final = max(floor, int(model_group_info.tpm * weight)) rate_limit_config["tokens_per_unit"] = reserved_tpm if model_group_info.rpm is not None: - reserved_rpm: Final = int(model_group_info.rpm * priority_weight) + reserved_rpm: Final = max(floor, int(model_group_info.rpm * weight)) rate_limit_config["requests_per_unit"] = reserved_rpm if rate_limit_config: @@ -668,8 +729,8 @@ class _PROXY_DynamicRateLimitHandlerV3(CustomLogger): descriptors.append( RateLimitDescriptor( - key="priority_model", - value=priority_key, + key="priority_model" if share is None else PRIORITY_CLASS_SHARE_AT_SATURATION, + value=priority_key if share is None else counter_value(model, str(user_api_key_dict.team_id)), rate_limit=rate_limit_config, ) ) @@ -783,35 +844,93 @@ class _PROXY_DynamicRateLimitHandlerV3(CustomLogger): llm_provider=llm_provider, ) + def _class_share_error( + self, + model: str, + status: RateLimitStatus, + saturation: float, + priority: str | None, + user_api_key_dict: UserAPIKeyAuth, + share: TeamShare, + ) -> ProxyRateLimitError: + resolved_model, llm_provider = resolve_llm_provider_for_rate_limit(model) + return ProxyRateLimitError( + detail={ + "error": f"Priority class share reached for {model} and the model is saturated. " + f"Team: {user_api_key_dict.team_id}, " + f"Priority: {priority or 'default'}, " + f"Active teams in class: {share.active_teams_in_class}, " + f"Active classes: {share.active_classes}, " + f"Rate limit type: {status['rate_limit_type']}, " + f"Team share: {status['current_limit']}, " + f"Remaining: {status['limit_remaining']}, " + f"Model saturation: {saturation:.1%}" + }, + headers={ + "retry-after": str(self.v3_limiter.window_size), + "rate_limit_type": str(status["rate_limit_type"]), + "rate_limited_by": PRIORITY_CLASS_SHARE_AT_SATURATION, + "x-litellm-priority": priority or "default", + "x-litellm-saturation": f"{saturation:.2%}", + }, + rate_limit_type=map_v3_rate_limit_type(status["rate_limit_type"]), + model=resolved_model, + llm_provider=llm_provider, + ) + async def enforce_token_limits_for_frame( self, user_api_key_dict: UserAPIKeyAuth, requested_model: str | None ) -> None: - """Hold a Responses WebSocket turn to the team's share once the model saturates, like the regular limiter does.""" + """Hold a Responses WebSocket turn to the team's shares once the model saturates, like the regular limiter does.""" if requested_model is None: return - team_share: Final = self._borrowing_team_share(model=requested_model, user_api_key_dict=user_api_key_dict) model_group_info: Final = self.llm_router.registry().group(requested_model) - if team_share is None or model_group_info is None: + if model_group_info is None: + return + priority: Final = self._get_priority_from_user_api_key_dict(user_api_key_dict) + share: Final = await self._team_class_share( + requested_model, user_api_key_dict, priority, self._normalize_priority_weights(model_group_info) + ) + share_descriptors: Final = ( + () + if share is None + else tuple( + self._create_priority_based_descriptors( + model=requested_model, user_api_key_dict=user_api_key_dict, priority=priority, share=share + ) + ) + ) + frame_share: Final = next( + (d for d in share_descriptors if d["key"] == PRIORITY_CLASS_SHARE_AT_SATURATION), None + ) + connection_stash: Final = get_request_stash() + if connection_stash is not None and frame_share is not None: + connection_stash.priority_class_share = frame_share + borrowing: Final = self._borrowing_team_share(model=requested_model, user_api_key_dict=user_api_key_dict) + if not share_descriptors and borrowing is None: return saturation: Final = await self._check_model_saturation(requested_model, model_group_info) - enforced: Final = team_share.enforced(saturation >= _get_priority_settings().saturation_threshold) - if enforced is None: + saturated: Final = saturation >= _get_priority_settings().saturation_threshold + enforced_team: Final = None if borrowing is None else borrowing.enforced(saturated) + descriptors: Final = (share_descriptors if saturated else ()) + ( + () if enforced_team is None else (enforced_team,) + ) + if not descriptors: return - tpm_windows: Final = _socket_turn_tpm_windows((enforced,)) + tpm_windows: Final = _socket_turn_tpm_windows(descriptors) response: Final = await self.v3_limiter.should_rate_limit( tpm_windows, parent_otel_span=user_api_key_dict.parent_otel_span, read_only=True, skip_parallel_check=True ) over_limit: Final = next((s for s in response["statuses"] if s["code"] == "OVER_LIMIT"), None) if over_limit is None: return - raise self._team_share_error( - requested_model, - over_limit, - saturation, - self._get_priority_from_user_api_key_dict(user_api_key_dict), - user_api_key_dict, - team_share, - ) + over_class_share: Final = over_limit["descriptor_key"] == PRIORITY_CLASS_SHARE_AT_SATURATION + if share is not None and (over_class_share or borrowing is None): + raise self._class_share_error(requested_model, over_limit, saturation, priority, user_api_key_dict, share) + if borrowing is not None: + raise self._team_share_error( + requested_model, over_limit, saturation, priority, user_api_key_dict, borrowing + ) def _create_model_tracking_descriptor( self, @@ -893,10 +1012,17 @@ class _PROXY_DynamicRateLimitHandlerV3(CustomLogger): descriptors_to_check.append(model_wide_descriptor) # Priority descriptors (always track, conditionally enforce) + share: Final = await self._team_class_share( + model, user_api_key_dict, priority, self._normalize_priority_weights(model_group_info) + ) priority_descriptors: Final = self._create_priority_based_descriptors( model=model, user_api_key_dict=user_api_key_dict, priority=priority, + share=share, + ) + get_or_create_request_stash().priority_class_share = next( + (d for d in priority_descriptors if d["key"] == PRIORITY_CLASS_SHARE_AT_SATURATION), None ) if priority_descriptors: descriptors_to_check.extend(priority_descriptors) @@ -986,6 +1112,8 @@ class _PROXY_DynamicRateLimitHandlerV3(CustomLogger): if descriptor_key == TEAM_MODEL_LIMIT_AT_SATURATION and team_share is not None: raise self._team_share_error(model, status, saturation, priority, user_api_key_dict, team_share) + if descriptor_key == PRIORITY_CLASS_SHARE_AT_SATURATION and share is not None: + raise self._class_share_error(model, status, saturation, priority, user_api_key_dict, share) # Fail-closed guard: overall_code says OVER_LIMIT but no status # matched a descriptor key we know how to translate into a 429. @@ -1032,7 +1160,22 @@ class _PROXY_DynamicRateLimitHandlerV3(CustomLogger): increments=[per_request_increment], parent_otel_span=user_api_key_dict.parent_otel_span, ) - if priority_descriptors and not should_enforce_priority: + if priority_descriptors and share is not None and not should_enforce_priority: + tracking_only: Final = tuple( + RateLimitDescriptor( + key=descriptor["key"], + value=descriptor["value"], + rate_limit=model_wide_descriptor["rate_limit"], + ) + for descriptor in priority_descriptors + ) + await self.v3_limiter.atomic_check_and_increment_by_n( + descriptors=list(tracking_only), + increments=[per_request_increment for _ in tracking_only], + parent_otel_span=user_api_key_dict.parent_otel_span, + ) + get_or_create_request_stash().rate_limit_response = atomic_response + elif priority_descriptors and not should_enforce_priority: priority_tracking_response: Final = await self.v3_limiter.should_rate_limit( descriptors=priority_descriptors, parent_otel_span=user_api_key_dict.parent_otel_span, @@ -1378,13 +1521,27 @@ class _PROXY_DynamicRateLimitHandlerV3(CustomLogger): else: priority_key = f"{model_group}:default_pool" + call_stash: Final = get_request_stash_for_call(_call_id_from_callback_kwargs(kwargs)) + stash: Final = ( + call_stash if call_stash is not None else (get_request_stash() if _is_socket_turn(kwargs) else None) + ) team_model_tokens: Final = _team_model_token_increment( - get_or_create_request_stash().borrowing_team_model_limit, total_tokens, self.v3_limiter.window_size + None if stash is None else stash.borrowing_team_model_limit, total_tokens, self.v3_limiter.window_size + ) + class_share: Final = None if stash is None else stash.priority_class_share + settled_team_id: Final = settlement_metadata.user_api_key_team_id + priority_scope: Final = ( + RateLimitScope("priority_model", priority_key) + if class_share is None + else RateLimitScope( + class_share["key"], + class_share["value"] if settled_team_id is None else counter_value(model_group, settled_team_id), + ) ) pipeline_operations: Final = ( model_tokens, CounterIncrement( - counter=RateLimitScope("priority_model", priority_key).counter("tokens"), + counter=priority_scope.counter("tokens"), amount=total_tokens, ttl=self.v3_limiter.window_size, ), diff --git a/litellm/proxy/hooks/parallel_request_limiter_v3.py b/litellm/proxy/hooks/parallel_request_limiter_v3.py index 4cbe08d0042..16e12eb1e03 100644 --- a/litellm/proxy/hooks/parallel_request_limiter_v3.py +++ b/litellm/proxy/hooks/parallel_request_limiter_v3.py @@ -615,6 +615,7 @@ class RequestRateLimiterStash: batch_enqueued_reservation: BatchEnqueuedTokenReservation | None = None batch_tpd_refund_ops: tuple[ReservationOperation, ...] = () borrowing_team_model_limit: RateLimitDescriptor | None = None + priority_class_share: RateLimitDescriptor | None = None reservation_released: bool = False io_reservation_released: bool = False tpm_limited_tags: frozenset[str] = field(default_factory=frozenset) diff --git a/litellm/proxy/hooks/priority_class_share.py b/litellm/proxy/hooks/priority_class_share.py new file mode 100644 index 00000000000..229afe09856 --- /dev/null +++ b/litellm/proxy/hooks/priority_class_share.py @@ -0,0 +1,127 @@ +"""Splits a priority class's share of a model among the teams active on it. + +Enabled by ``priority_reservation_settings.priority_class_share_splits_equally_among_active_teams``. +""" + +import time +from collections.abc import Callable, Mapping +from dataclasses import dataclass +from types import MappingProxyType +from typing import Final + +from pydantic import TypeAdapter + +from litellm._internal_context import with_service_target +from litellm.caching.caching import DualCache +from litellm.redis.stores.rate_limits import RATE_LIMITS_TARGET + +ACTIVE_TEAMS_KEY_PREFIX: Final = "dynamic_rate_limiter:active_teams:" +MEMBER_SEPARATOR: Final = "\x1f" +DEFAULT_CLASS: Final = "\x1edefault" +_POOL_PREFIX: Final = f"{MEMBER_SEPARATOR}pool{MEMBER_SEPARATOR}" + +_TOUCH_LUA: Final = ( + "redis.call('ZREMRANGEBYSCORE', KEYS[1], '-inf', ARGV[1]) " + "redis.call('ZADD', KEYS[1], ARGV[2], ARGV[3]) " + "redis.call('EXPIRE', KEYS[1], ARGV[4]) " + "return redis.call('ZRANGE', KEYS[1], 0, -1)" +) +_MEMBERS: Final = TypeAdapter[tuple[str | bytes, ...]](tuple[str | bytes, ...]) + + +def pool_member(priority: str) -> str: + """The registry id that stands for every key without a team in ``priority``. It keeps the class present without + being a team, and starts with the separator so no team id can spell it.""" + return f"{_POOL_PREFIX}{priority}" + + +def team_member(team_id: str) -> str: + """The registry id of a real team. It never starts with the separator, so it cannot spell a pool marker.""" + return f"team{MEMBER_SEPARATOR}{team_id}" + + +def is_pool_member(member: str) -> bool: + return member.startswith(_POOL_PREFIX) + + +def counter_value(model: str, team_id: str) -> str: + """One counter per model and team. The length prefix keeps ``chat`` + ``prod:victim`` apart from + ``chat:prod`` + ``victim``.""" + return f"{len(model)}:{model}:{team_id}" + + +@dataclass(frozen=True, slots=True) +class TeamShare: + fraction: float + active_teams_in_class: int + active_classes: int + + +def team_share( + priority: str, members: Mapping[str, str], class_weights: Mapping[str, float], default_weight: float +) -> TeamShare: + """``members`` maps every team active on the model, the caller included, to its priority class. Classes with no + active member give their weight away, and each present class is split equally among its active teams. A pool + member keeps its class present but takes no slice, since keys without a team are held to the class pool.""" + present: Final = frozenset(members.values()) + weights: Final = {cls: class_weights.get(cls, default_weight) for cls in present} + total: Final = sum(weights.values()) + class_fraction: Final = weights[priority] / total if total > 0 else 1.0 / len(present) + teams_in_class: Final = sum(1 for team, cls in members.items() if cls == priority and not is_pool_member(team)) + return TeamShare( + fraction=class_fraction / max(1, teams_in_class), + active_teams_in_class=teams_in_class, + active_classes=len(present), + ) + + +def _members(raw: tuple[str | bytes, ...]) -> Mapping[str, str]: + """Oldest first, so a team seen under two classes in one window keeps its most recent one. The class is split off + from the right because a team id may itself contain the separator.""" + decoded: Final = (item.decode() if isinstance(item, bytes) else item for item in raw) + pairs: Final = (member.rsplit(MEMBER_SEPARATOR, 1) for member in decoded) + return MappingProxyType({pair[0]: pair[1] for pair in pairs if len(pair) == 2}) + + +class ActiveTeamRegistry: + """Which teams used a model in the current window, as one sorted set per model scored by last-seen time, in Redis + when the proxy has one and otherwise in process memory.""" + + def __init__(self, dual_cache: DualCache, time_provider: Callable[[], float] = time.time) -> None: + self._dual_cache = dual_cache + self._local: Mapping[str, Mapping[str, float]] = MappingProxyType({}) + self._now = time_provider + + @with_service_target(RATE_LIMITS_TARGET) + async def touch(self, model: str, team_id: str, priority: str, window: int) -> Mapping[str, str]: + """Record ``team_id`` as active on ``model`` and return every team active on it, the caller included. The + caller is always listed under ``priority``, whatever class another worker's clock last saw it in.""" + key: Final = f"{ACTIVE_TEAMS_KEY_PREFIX}{model}" + member: Final = f"{team_id}{MEMBER_SEPARATOR}{priority}" + now: Final = self._now() + redis_cache: Final = self._dual_cache.redis_cache + seen: Final = ( + self._touch_in_memory(key, member, now, window) + if redis_cache is None + else _members( + _MEMBERS.validate_python( + await redis_cache.async_register_script(_TOUCH_LUA)( + keys=(key,), args=(now - window, now, member, window) + ) + ) + ) + ) + return MappingProxyType({**seen, team_id: priority}) + + def _touch_in_memory(self, key: str, member: str, now: float, window: int) -> Mapping[str, str]: + """Prunes by each member's own last-seen time, so the map lives exactly as long as its newest member.""" + cutoff: Final = now - window + alive: Final = {seen_member: seen for seen_member, seen in self._local.get(key, {}).items() if seen > cutoff} + current: Final = MappingProxyType({**alive, member: now}) + others: Final = { + other_key: others_members + for other_key, others_members in self._local.items() + if other_key != key and any(seen > cutoff for seen in others_members.values()) + } + self._local = MappingProxyType({**others, key: current}) + return _members(tuple(sorted(current, key=current.__getitem__))) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 2a9d400458e..7bf9fcacde2 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -7080,7 +7080,7 @@ class ProxyConfig: elif key == "priority_reservation_settings": from litellm.types.utils import PriorityReservationSettings - litellm.priority_reservation_settings = PriorityReservationSettings(**value) + litellm.priority_reservation_settings = PriorityReservationSettings.model_validate(value) elif key == "callbacks": initialize_callbacks_on_proxy( value=value, diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 9121c016ca1..beac372d882 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -5393,6 +5393,10 @@ class PriorityReservationSettings(LiteLLMBaseModel): default=False, description="When true, a team's model_tpm_limit and model_rpm_limit on a model this limiter covers stop being ceilings. The team may run past them on capacity other teams are not using while the model is below saturation_threshold, and is held to them once the model saturates.", ) + priority_class_share_splits_equally_among_active_teams: bool = Field( + default=False, + description="When true, a priority class's share of a model is renormalized over the classes whose teams used the model in the current window and split equally among that class's active teams, and once the model saturates each team is held to its own share instead of the class pool. Keys without a team keep the class pool.", + ) saturation_check_cache_ttl: int = Field( default=60, diff --git a/tests/integration/routing/test_priority_class_share_splits_among_active_teams.py b/tests/integration/routing/test_priority_class_share_splits_among_active_teams.py new file mode 100644 index 00000000000..650ee8af933 --- /dev/null +++ b/tests/integration/routing/test_priority_class_share_splits_among_active_teams.py @@ -0,0 +1,136 @@ +import json +import uuid +from collections.abc import Iterator +from pathlib import Path +from typing import Final + +import httpx +import pytest +import yaml +from integration._support.client import Gateway, Scenario, gateway_from_environment +from integration._support.process import owned_proxy +from integration._support.wire import Reply, Request, wire_server + +OPENAI_MODEL: Final = "gpt-4.1-mini" +MODEL_TPM: Final = 1000 +TOKENS_PER_CALL: Final = 30 +UPSTREAM_REPLY: Final = json.dumps( + { + "id": "chatcmpl_priority_class_share", + "object": "chat.completion", + "created": 1700000000, + "model": OPENAI_MODEL, + "choices": [{"index": 0, "message": {"role": "assistant", "content": "shared"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 20, "completion_tokens": 10, "total_tokens": TOKENS_PER_CALL}, + } +).encode() + + +def _respond(request: Request) -> Reply: + assert request.method == "POST" and request.target == "/v1/chat/completions", request.target + return Reply(body=UPSTREAM_REPLY) + + +@pytest.fixture(scope="module") +def saturated_model(tmp_path_factory: pytest.TempPathFactory) -> Iterator[tuple[Gateway, str]]: + directory: Final = tmp_path_factory.mktemp("priority-class-share") + configuration: Final = yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text()) + configuration["litellm_settings"] = { + **configuration["litellm_settings"], + "callbacks": ["dynamic_rate_limiter_v3"], + "priority_reservation": {"interactive": 0.6, "batch": 0.4}, + "priority_reservation_settings": { + "default_priority": 1.0, + "saturation_threshold": 0.0, + "priority_class_share_splits_equally_among_active_teams": True, + }, + } + path: Final = directory / "priority_class_share.yaml" + path.write_text(yaml.safe_dump(configuration)) + with ( + gateway_from_environment() as rig, + wire_server(_respond) as wire, + owned_proxy(rig, directory, {}, config=path) as owned, + ): + yield owned, wire.url + + +def _team_key(scenario: Scenario, priority: str) -> str: + return scenario.key(team_id=scenario.team(metadata={"priority": priority})) + + +def _call(owned: Gateway, model: str, key: str) -> httpx.Response: + return owned.request( + "POST", + "/v1/chat/completions", + {"model": model, "messages": [{"role": "user", "content": f"class share {uuid.uuid4().hex}"}]}, + key=key, + ) + + +def _spend_until_blocked(owned: Gateway, model: str, key: str, max_calls: int) -> tuple[httpx.Response, ...]: + def calls() -> Iterator[httpx.Response]: + for _ in range(max_calls): + response: Final = _call(owned, model, key) + yield response + if response.status_code == 429: + return + + return tuple(calls()) + + +def test_second_interactive_team_keeps_its_half_of_the_class_share(saturated_model: tuple[Gateway, str]) -> None: + """Two interactive teams, batch idle: the class share is the whole model and each team is held to half of it.""" + owned, upstream = saturated_model + with owned.scenario() as scenario: + model: Final = scenario.model( + model=f"openai/{OPENAI_MODEL}", api_base=f"{upstream}/v1", api_key="synthetic-openai-key", tpm=MODEL_TPM + ) + first: Final = _team_key(scenario, "interactive") + second: Final = _team_key(scenario, "interactive") + assert _call(owned, model, second).status_code == 200 + + responses: Final = _spend_until_blocked(owned, model, first, max_calls=MODEL_TPM // TOKENS_PER_CALL + 1) + blocked: Final = responses[-1] + assert blocked.status_code == 429, blocked.text + assert blocked.headers.get("rate_limited_by") == "priority_class_share_saturated", dict(blocked.headers) + served: Final = (len(responses) - 1) * TOKENS_PER_CALL + assert MODEL_TPM // 2 <= served < MODEL_TPM // 2 + 2 * TOKENS_PER_CALL, served + + still_served: Final = _call(owned, model, second) + assert still_served.status_code == 200, still_served.text + + +def test_only_active_class_gets_the_whole_model(saturated_model: tuple[Gateway, str]) -> None: + """One batch team alone on the model is held to 100% of the model, not to batch's 40%.""" + owned, upstream = saturated_model + with owned.scenario() as scenario: + model: Final = scenario.model( + model=f"openai/{OPENAI_MODEL}", api_base=f"{upstream}/v1", api_key="synthetic-openai-key", tpm=MODEL_TPM + ) + batch: Final = _team_key(scenario, "batch") + responses: Final = _spend_until_blocked(owned, model, batch, max_calls=MODEL_TPM // TOKENS_PER_CALL + 1) + served: Final = sum(1 for response in responses if response.status_code == 200) * TOKENS_PER_CALL + assert served >= MODEL_TPM - 2 * TOKENS_PER_CALL, [r.text for r in responses] + + +def test_class_weights_renormalize_when_batch_wakes_up(saturated_model: tuple[Gateway, str]) -> None: + """Interactive and batch both active: interactive is held to 60% of the model and batch to 40%.""" + owned, upstream = saturated_model + with owned.scenario() as scenario: + model: Final = scenario.model( + model=f"openai/{OPENAI_MODEL}", api_base=f"{upstream}/v1", api_key="synthetic-openai-key", tpm=MODEL_TPM + ) + interactive: Final = _team_key(scenario, "interactive") + batch: Final = _team_key(scenario, "batch") + assert _call(owned, model, batch).status_code == 200 + + responses: Final = _spend_until_blocked(owned, model, interactive, max_calls=MODEL_TPM // TOKENS_PER_CALL + 1) + assert responses[-1].status_code == 429, responses[-1].text + served: Final = (len(responses) - 1) * TOKENS_PER_CALL + assert int(MODEL_TPM * 0.6) <= served < int(MODEL_TPM * 0.6) + 2 * TOKENS_PER_CALL, served + + batch_responses: Final = _spend_until_blocked(owned, model, batch, max_calls=MODEL_TPM // TOKENS_PER_CALL + 1) + batch_served: Final = sum(1 for response in batch_responses if response.status_code == 200) * TOKENS_PER_CALL + assert batch_served >= int(MODEL_TPM * 0.4) - 2 * TOKENS_PER_CALL, [r.text for r in batch_responses] + assert batch_responses[-1].status_code == 429, batch_responses[-1].text diff --git a/tests/unit/proxy/hooks/test_dynamic_rate_limiter_v3.py b/tests/unit/proxy/hooks/test_dynamic_rate_limiter_v3.py index 0bf0ea4d6ad..22e6d5e3146 100644 --- a/tests/unit/proxy/hooks/test_dynamic_rate_limiter_v3.py +++ b/tests/unit/proxy/hooks/test_dynamic_rate_limiter_v3.py @@ -22,6 +22,7 @@ from litellm.proxy.common_utils.proxy_rate_limit_error import ProxyRateLimitErro from litellm.proxy.hooks.dynamic_rate_limiter_v3 import ( AlternateGrant, PROXY_DynamicRateLimitHandlerV3, + _PROXY_DynamicRateLimitHandlerV3, # pyright: ignore[reportPrivateUsage] # the public name is a Final alias, not a type ) from litellm.proxy.hooks.dynamic_rate_limiter_v3 import ( PROXY_DynamicRateLimitHandlerV3 as DynamicRateLimitHandler, @@ -29,6 +30,7 @@ from litellm.proxy.hooks.dynamic_rate_limiter_v3 import ( from litellm.proxy.hooks.parallel_request_limiter_v3 import ( PROXY_MaxParallelRequestsHandler_v3, ) +from litellm.proxy.hooks.priority_class_share import pool_member from litellm.redis.stores.rate_limits import RateLimitScope from litellm.types.llms.openai import ResponseAPIUsage, ResponsesAPIResponse from tests.unit.redis.stores.fake_rate_limits import FakeRateLimits @@ -2114,7 +2116,9 @@ def _borrowing_handler( borrows: bool, saturation_threshold: float = 0.5, model_group_alias: dict[str, str] | None = None, -) -> PROXY_DynamicRateLimitHandlerV3: + splits: bool = False, + rpm: int = 100, +) -> _PROXY_DynamicRateLimitHandlerV3: from litellm.types.utils import PriorityReservationSettings monkeypatch.setenv("LITELLM_LICENSE", "test-license-key") @@ -2123,7 +2127,9 @@ def _borrowing_handler( litellm, "priority_reservation_settings", PriorityReservationSettings( - saturation_threshold=saturation_threshold, team_model_limit_borrows_idle_capacity_from_other_teams=borrows + saturation_threshold=saturation_threshold, + team_model_limit_borrows_idle_capacity_from_other_teams=borrows, + priority_class_share_splits_equally_among_active_teams=splits, ), ) monkeypatch.setattr(litellm, "callbacks", ["dynamic_rate_limiter_v3"]) @@ -2131,8 +2137,20 @@ def _borrowing_handler( model_list=[ { "model_name": "shared-model", + "litellm_params": {"model": "gpt-3.5-turbo", "api_key": "k", "api_base": "b", "tpm": 1000, "rpm": rpm}, + }, + { + "model_name": "other-model", "litellm_params": {"model": "gpt-3.5-turbo", "api_key": "k", "api_base": "b", "tpm": 1000, "rpm": 100}, - } + }, + { + "model_name": "chat", + "litellm_params": {"model": "gpt-3.5-turbo", "api_key": "k", "api_base": "b", "tpm": 1000, "rpm": 100}, + }, + { + "model_name": "chat:prod", + "litellm_params": {"model": "gpt-3.5-turbo", "api_key": "k", "api_base": "b", "tpm": 1000, "rpm": 100}, + }, ], model_group_alias=model_group_alias or {}, ) @@ -2142,12 +2160,14 @@ def _borrowing_handler( return handler -async def _admit_then_settle(handler: PROXY_DynamicRateLimitHandlerV3, auth: UserAPIKeyAuth, tokens: int) -> None: +async def _admit_then_settle(handler: _PROXY_DynamicRateLimitHandlerV3, auth: UserAPIKeyAuth, tokens: int) -> None: await _check(handler, auth, saturation=0.0) await _settle_tokens(handler, tokens) -async def _settle_tokens(handler: PROXY_DynamicRateLimitHandlerV3, tokens: int, response: object | None = None) -> None: +async def _settle_tokens( + handler: _PROXY_DynamicRateLimitHandlerV3, tokens: int, response: object | None = None +) -> None: await handler.async_log_success_event( kwargs={ "litellm_params": {"metadata": {"model_group": "shared-model"}}, @@ -2177,12 +2197,16 @@ def _team_key( async def _check( - handler: PROXY_DynamicRateLimitHandlerV3, auth: UserAPIKeyAuth, saturation: float, model: str = "shared-model" + handler: _PROXY_DynamicRateLimitHandlerV3, + auth: UserAPIKeyAuth, + saturation: float, + model: str = "shared-model", + priority: str | None = None, ) -> None: group: Final = handler.llm_router.registry().group(model) assert group is not None - await handler._check_rate_limits( - model=model, model_group_info=group, user_api_key_dict=auth, priority=None, saturation=saturation + await handler._check_rate_limits( # pyright: ignore[reportPrivateUsage] # saturation is an input here, the public hook reads it from Redis + model=model, model_group_info=group, user_api_key_dict=auth, priority=priority, saturation=saturation ) @@ -2367,6 +2391,273 @@ async def test_a_websocket_turn_still_borrows_while_the_model_is_idle(monkeypatc await handler.enforce_token_limits_for_frame(user_api_key_dict=_team_key(), requested_model="shared-model") +def _class_team(team_id: str, priority: str | None = "interactive") -> UserAPIKeyAuth: + return UserAPIKeyAuth(api_key="sk-class", team_id=team_id, team_metadata={}, metadata={"priority": priority}) + + +async def _burn( + handler: _PROXY_DynamicRateLimitHandlerV3, + auth: UserAPIKeyAuth, + priority: str | None, + tokens: int, + model: str = "shared-model", +): + await _check(handler, auth, saturation=0.0, priority=priority, model=model) + await handler.async_log_success_event( + kwargs={ + "litellm_params": {"metadata": {"model_group": model}}, + "standard_logging_object": { + "metadata": { + "user_api_key_team_id": auth.team_id, + "user_api_key_auth_metadata": {"priority": priority}, + } + }, + }, + response_obj=litellm.ModelResponse( + usage=litellm.Usage(prompt_tokens=tokens, completion_tokens=0, total_tokens=tokens) + ), + start_time=datetime.now(), + end_time=datetime.now(), + ) + + +def _held_to_class_share(exc: ProxyRateLimitError) -> None: + assert exc.status_code == 429 + assert exc.headers is not None + assert exc.headers["rate_limited_by"] == "priority_class_share_saturated" + assert exc.headers["rate_limit_type"] == "tokens" + + +@pytest.mark.asyncio +async def test_two_interactive_teams_each_keep_half_of_the_interactive_share(monkeypatch: pytest.MonkeyPatch): + handler = _borrowing_handler(monkeypatch, borrows=False, splits=True) + await _burn(handler, _class_team("team-a"), "interactive", 20) + await _burn(handler, _class_team("team-b"), "interactive", 300) + await _burn(handler, _class_team("team-b"), "interactive", 200) + with pytest.raises(ProxyRateLimitError) as exc: + await _check(handler, _class_team("team-b"), saturation=0.52, priority="interactive") + _held_to_class_share(exc.value) + assert "Team: team-b, Priority: interactive, Active teams in class: 2, Active classes: 1, " in str(exc.value.detail) + assert "Team share: 500, Remaining: 0, Model saturation: 52.0%" in str(exc.value.detail) + await _check(handler, _class_team("team-a"), saturation=0.52, priority="interactive") + + +@pytest.mark.asyncio +async def test_the_only_active_class_gets_the_whole_model_instead_of_its_weight(monkeypatch: pytest.MonkeyPatch): + handler = _borrowing_handler(monkeypatch, borrows=False, splits=True) + await _burn(handler, _class_team("team-b", "batch"), "batch", 950) + await _check(handler, _class_team("team-b", "batch"), saturation=0.95, priority="batch") + + unsplit = _borrowing_handler(monkeypatch, borrows=False) + await _burn(unsplit, _class_team("team-b", "batch"), "batch", 950) + with pytest.raises(ProxyRateLimitError) as exc: + await _check(unsplit, _class_team("team-b", "batch"), saturation=0.95, priority="batch") + assert "Priority-based rate limit exceeded" in str(exc.value.detail) + + +@pytest.mark.asyncio +async def test_class_weights_apply_again_once_a_team_of_the_other_class_is_active(monkeypatch: pytest.MonkeyPatch): + handler = _borrowing_handler(monkeypatch, borrows=False, splits=True) + await _burn(handler, _class_team("team-b", "batch"), "batch", 10) + await _burn(handler, _class_team("team-a"), "interactive", 600) + with pytest.raises(ProxyRateLimitError) as exc: + await _check(handler, _class_team("team-a"), saturation=0.61, priority="interactive") + assert "Active teams in class: 1, Active classes: 2, Rate limit type: tokens, Team share: 600, " in str( + exc.value.detail + ) + await _check(handler, _class_team("team-b", "batch"), saturation=0.61, priority="batch") + + +@pytest.mark.asyncio +async def test_teams_without_a_priority_split_the_default_share_between_them(monkeypatch: pytest.MonkeyPatch): + handler = _borrowing_handler(monkeypatch, borrows=False, splits=True) + await _burn(handler, _class_team("team-x", None), None, 600) + await _burn(handler, _class_team("team-y", None), None, 1) + with pytest.raises(ProxyRateLimitError) as exc: + await _check(handler, _class_team("team-x", None), saturation=0.6, priority=None) + assert "Priority: default, Active teams in class: 2, Active classes: 1, " in str(exc.value.detail) + assert "Team share: 500, " in str(exc.value.detail) + await _check(handler, _class_team("team-y", None), saturation=0.6, priority=None) + assert exc.value.headers is not None and exc.value.headers["x-litellm-priority"] == "default" + + +@pytest.mark.asyncio +async def test_a_key_without_a_team_keeps_the_shared_class_pool(monkeypatch: pytest.MonkeyPatch): + handler = _borrowing_handler(monkeypatch, borrows=False, splits=True) + await _burn(handler, _class_team("team-a"), "interactive", 100) + no_team: Final = UserAPIKeyAuth(api_key="sk-plain", metadata={"priority": "interactive"}) + await _burn(handler, no_team, "interactive", 600) + with pytest.raises(ProxyRateLimitError) as exc: + await _check(handler, no_team, saturation=0.7, priority="interactive") + assert exc.value.headers is not None and "rate_limited_by" not in exc.value.headers + assert "Priority-based rate limit exceeded" in str(exc.value.detail) + await _check(handler, _class_team("team-a"), saturation=0.7, priority="interactive") + + +@pytest.mark.asyncio +async def test_below_saturation_a_team_may_run_past_its_share(monkeypatch: pytest.MonkeyPatch): + handler = _borrowing_handler(monkeypatch, borrows=False, splits=True) + await _burn(handler, _class_team("team-a"), "interactive", 1) + await _burn(handler, _class_team("team-b"), "interactive", 400) + await _check(handler, _class_team("team-b"), saturation=0.49, priority="interactive") + + +@pytest.mark.asyncio +async def test_the_split_is_off_by_default_so_one_team_can_still_empty_the_class_pool(monkeypatch: pytest.MonkeyPatch): + handler = _borrowing_handler(monkeypatch, borrows=False) + await _burn(handler, _class_team("team-a"), "interactive", 600) + with pytest.raises(ProxyRateLimitError) as exc: + await _check(handler, _class_team("team-b"), saturation=0.6, priority="interactive") + assert exc.value.headers is not None and "rate_limited_by" not in exc.value.headers + assert "Priority-based rate limit exceeded" in str(exc.value.detail) + + +@pytest.mark.asyncio +async def test_the_team_model_limit_and_the_class_share_both_apply_when_both_are_on(monkeypatch: pytest.MonkeyPatch): + handler = _borrowing_handler(monkeypatch, borrows=True, splits=True) + team: Final = UserAPIKeyAuth( + api_key="sk-both", + team_id="team-a", + team_metadata={"model_tpm_limit": {"shared-model": 100}}, + metadata={"priority": "interactive"}, + ) + await _burn(handler, team, "interactive", 150) + with pytest.raises(ProxyRateLimitError) as exc: + await _check(handler, team, saturation=0.5, priority="interactive") + _held_to_team_limit(exc.value, "tokens", "model_tpm_limit") + + +@pytest.mark.asyncio +async def test_a_team_active_on_another_model_does_not_shrink_this_models_shares(monkeypatch: pytest.MonkeyPatch): + handler = _borrowing_handler(monkeypatch, borrows=False, splits=True) + await _check(handler, _class_team("team-b"), saturation=0.0, priority="interactive", model="other-model") + await _burn(handler, _class_team("team-a"), "interactive", 700) + await _check(handler, _class_team("team-a"), saturation=0.7, priority="interactive") + + +class _UnavailableRegistry: + async def touch(self, model: str, team_id: str, priority: str, window: int) -> Mapping[str, str]: + from redis.exceptions import ConnectionError as RedisConnectionError + + raise RedisConnectionError("redis is down") + + +@pytest.mark.asyncio +async def test_a_registry_failure_falls_back_to_the_class_pool_instead_of_skipping_the_limits( + monkeypatch: pytest.MonkeyPatch, +): + handler = _borrowing_handler(monkeypatch, borrows=False, splits=True) + handler.active_teams = _UnavailableRegistry() # pyright: ignore[reportAttributeAccessIssue] # injected failing registry + await _burn(handler, _class_team("team-a"), "interactive", 650) + with pytest.raises(ProxyRateLimitError) as exc: + await _check(handler, _class_team("team-a"), saturation=0.65, priority="interactive") + assert exc.value.headers is not None + assert "rate_limited_by" not in exc.value.headers + assert "Priority-based rate limit exceeded" in str(exc.value.detail) + + +@pytest.mark.asyncio +async def test_one_team_using_keys_of_two_priorities_still_spends_one_share(monkeypatch: pytest.MonkeyPatch): + handler = _borrowing_handler(monkeypatch, borrows=False, splits=True) + await _burn(handler, _class_team("team-b"), "interactive", 20) + await _burn(handler, _class_team("team-a", "interactive"), "interactive", 300) + await _burn(handler, _class_team("team-a", "batch"), "batch", 300) + with pytest.raises(ProxyRateLimitError) as exc: + await _check(handler, _class_team("team-a", "batch"), saturation=0.62, priority="batch") + _held_to_class_share(exc.value) + assert "Team: team-a, Priority: batch, Active teams in class: 1, Active classes: 2, " in str(exc.value.detail) + assert "Team share: 400, Remaining: 0," in str(exc.value.detail) + + +@pytest.mark.asyncio +async def test_a_share_smaller_than_one_request_still_lets_each_team_send_one(monkeypatch: pytest.MonkeyPatch): + handler = _borrowing_handler(monkeypatch, borrows=False, splits=True, rpm=2) + for team in ("team-a", "team-b", "team-c"): + await handler.active_teams.touch("shared-model", team, "interactive", 60) + await _check(handler, _class_team("team-d"), saturation=0.9, priority="interactive") + with pytest.raises(ProxyRateLimitError) as exc: + await _check(handler, _class_team("team-d"), saturation=0.9, priority="interactive") + assert exc.value.headers is not None + assert exc.value.headers["rate_limited_by"] == "priority_class_share_saturated" + assert exc.value.headers["rate_limit_type"] == "requests" + + +@pytest.mark.asyncio +async def test_a_websocket_turn_is_held_to_the_class_share_once_the_model_saturates(monkeypatch: pytest.MonkeyPatch): + handler = _borrowing_handler(monkeypatch, borrows=False, splits=True, saturation_threshold=0.1) + await _burn(handler, _class_team("team-a"), "interactive", 20) + await _burn(handler, _class_team("team-b"), "interactive", 600) + with pytest.raises(ProxyRateLimitError) as exc: + await handler.enforce_token_limits_for_frame( + user_api_key_dict=_class_team("team-b"), requested_model="shared-model" + ) + _held_to_class_share(exc.value) + await handler.enforce_token_limits_for_frame( + user_api_key_dict=_class_team("team-a"), requested_model="shared-model" + ) + + +@pytest.mark.asyncio +async def test_a_websocket_turn_ignores_the_class_share_while_the_split_is_off(monkeypatch: pytest.MonkeyPatch): + handler = _borrowing_handler(monkeypatch, borrows=False, saturation_threshold=0.1) + await _burn(handler, _class_team("team-a"), "interactive", 20) + await _burn(handler, _class_team("team-b"), "interactive", 600) + await handler.enforce_token_limits_for_frame( + user_api_key_dict=_class_team("team-b"), requested_model="shared-model" + ) + + +@pytest.mark.asyncio +async def test_traffic_from_keys_without_a_team_keeps_its_class_present_in_the_split(monkeypatch: pytest.MonkeyPatch): + handler = _borrowing_handler(monkeypatch, borrows=False, splits=True) + no_team: Final = UserAPIKeyAuth(api_key="sk-plain", metadata={"priority": "interactive"}) + await _burn(handler, no_team, "interactive", 20) + await _burn(handler, _class_team("team-c", "batch"), "batch", 450) + with pytest.raises(ProxyRateLimitError) as exc: + await _check(handler, _class_team("team-c", "batch"), saturation=0.6, priority="batch") + _held_to_class_share(exc.value) + assert "Team share: 400, Remaining: 0," in str(exc.value.detail) + + +@pytest.mark.asyncio +async def test_a_team_id_containing_the_registry_separator_is_still_held_to_its_share(monkeypatch: pytest.MonkeyPatch): + handler = _borrowing_handler(monkeypatch, borrows=False, splits=True) + await _burn(handler, _class_team("team-b"), "interactive", 20) + await _burn(handler, _class_team("owned\x1fextra", "batch"), "batch", 450) + with pytest.raises(ProxyRateLimitError) as exc: + await _check(handler, _class_team("owned\x1fextra", "batch"), saturation=0.6, priority="batch") + _held_to_class_share(exc.value) + + +@pytest.mark.asyncio +async def test_a_team_share_is_not_shrunk_by_teamless_traffic_in_its_class(monkeypatch: pytest.MonkeyPatch): + handler = _borrowing_handler(monkeypatch, borrows=False, splits=True) + no_team: Final = UserAPIKeyAuth(api_key="sk-plain", metadata={"priority": "interactive"}) + await _burn(handler, no_team, "interactive", 20) + await _burn(handler, _class_team("team-a"), "interactive", 900) + await _check(handler, _class_team("team-a"), saturation=0.92, priority="interactive") + + +@pytest.mark.asyncio +async def test_teamless_traffic_in_two_classes_keeps_both_classes_present(monkeypatch: pytest.MonkeyPatch): + handler = _borrowing_handler(monkeypatch, borrows=False, splits=True) + await _burn(handler, UserAPIKeyAuth(api_key="sk-i", metadata={"priority": "interactive"}), "interactive", 20) + await _burn(handler, UserAPIKeyAuth(api_key="sk-b", metadata={"priority": "batch"}), "batch", 20) + await _burn(handler, _class_team("team-c", "batch"), "batch", 450) + with pytest.raises(ProxyRateLimitError) as exc: + await _check(handler, _class_team("team-c", "batch"), saturation=0.6, priority="batch") + _held_to_class_share(exc.value) + assert "Team share: 400, Remaining: 0," in str(exc.value.detail) + + +@pytest.mark.asyncio +async def test_team_counters_do_not_collide_across_models_whose_names_share_a_colon(monkeypatch: pytest.MonkeyPatch): + handler = _borrowing_handler(monkeypatch, borrows=False, splits=True) + await _burn(handler, _class_team("prod:victim"), "interactive", 600, model="chat") + await _burn(handler, _class_team("team-x"), "interactive", 20, model="chat:prod") + await _check(handler, _class_team("victim"), saturation=0.6, priority="interactive", model="chat:prod") + + def _capped_team_key(multiple: object, limit: str = "model_tpm_limit", share: int = 100) -> UserAPIKeyAuth: return _team_key(team_limits={limit: {"shared-model": share}, "team_model_limit_max_borrow_multiple": multiple}) @@ -2494,3 +2785,174 @@ async def test_a_websocket_turn_under_the_teams_max_borrow_multiple_still_borrow handler = _borrowing_handler(monkeypatch, borrows=True, saturation_threshold=0.5) await _admit_then_settle(handler, _capped_team_key(2), 190) await handler.enforce_token_limits_for_frame(user_api_key_dict=_capped_team_key(2), requested_model="shared-model") + + +@pytest.mark.asyncio +async def test_a_configured_class_named_default_is_not_merged_with_teams_that_have_no_priority( + monkeypatch: pytest.MonkeyPatch, +): + handler = _borrowing_handler(monkeypatch, borrows=False, splits=True) + monkeypatch.setattr(litellm, "priority_reservation", {"default": 0.6, "batch": 0.4}) + await _burn(handler, _class_team("team-a", "default"), "default", 20) + await _burn(handler, _class_team("team-n", None), None, 20) + await _burn(handler, _class_team("team-a", "default"), "default", 700) + with pytest.raises(ProxyRateLimitError) as exc: + await _check(handler, _class_team("team-a", "default"), saturation=0.74, priority="default") + assert "Priority: default, Active teams in class: 1, Active classes: 2, " in str(exc.value.detail) + + +@pytest.mark.asyncio +async def test_tokens_from_a_nested_call_do_not_land_on_the_outer_requests_team_counter( + monkeypatch: pytest.MonkeyPatch, +): + handler = _borrowing_handler(monkeypatch, borrows=False, splits=True) + await _burn(handler, _class_team("team-b"), "interactive", 20) + await _check(handler, _class_team("team-a"), saturation=0.0, priority="interactive") + from litellm.proxy.hooks.parallel_request_limiter_v3 import get_or_create_request_stash + + stash: Final = get_or_create_request_stash() + stash.owner_litellm_call_id = "outer-call" + await handler.async_log_success_event( + kwargs={ + "litellm_call_id": "judge-call", + "litellm_params": {"metadata": {"model_group": "shared-model"}}, + "standard_logging_object": { + "metadata": { + "user_api_key_team_id": "team-a", + "user_api_key_auth_metadata": {"priority": "interactive"}, + } + }, + }, + response_obj=litellm.ModelResponse( + usage=litellm.Usage(prompt_tokens=600, completion_tokens=0, total_tokens=600) + ), + start_time=datetime.now(), + end_time=datetime.now(), + ) + await _check(handler, _class_team("team-a"), saturation=0.62, priority="interactive") + + +@pytest.mark.asyncio +async def test_a_team_id_spelled_like_a_pool_marker_still_counts_as_a_team(monkeypatch: pytest.MonkeyPatch): + handler = _borrowing_handler(monkeypatch, borrows=False, splits=True) + await _burn(handler, _class_team(pool_member("interactive")), "interactive", 20) + await _burn(handler, _class_team("team-a"), "interactive", 600) + with pytest.raises(ProxyRateLimitError) as exc: + await _check(handler, _class_team("team-a"), saturation=0.62, priority="interactive") + assert "Active teams in class: 2, Active classes: 1, " in str(exc.value.detail) + + +@pytest.mark.asyncio +async def test_requests_sent_while_the_share_was_not_enforced_all_count_once_the_share_grows( + monkeypatch: pytest.MonkeyPatch, +): + handler = _borrowing_handler(monkeypatch, borrows=False, splits=True) + await _check(handler, _class_team("team-b"), saturation=0.0, priority="interactive") + for _ in range(70): + await _check(handler, _class_team("team-a"), saturation=0.0, priority="interactive") + await _check(handler, _class_team("team-b", "batch"), saturation=0.0, priority="batch") + with pytest.raises(ProxyRateLimitError) as exc: + await _check(handler, _class_team("team-a"), saturation=0.6, priority="interactive") + assert "Active teams in class: 1, Active classes: 2, " in str(exc.value.detail) + + +@pytest.mark.asyncio +async def test_a_websocket_turn_settles_on_the_team_share_once_the_registry_recovers(monkeypatch: pytest.MonkeyPatch): + handler = _borrowing_handler(monkeypatch, borrows=False, splits=True, saturation_threshold=0.1) + await _burn(handler, _class_team("team-b"), "interactive", 20) + registry: Final = handler.active_teams + handler.active_teams = _UnavailableRegistry() # pyright: ignore[reportAttributeAccessIssue] # registry down at connect + await _check(handler, _class_team("team-a"), saturation=0.0, priority="interactive") + handler.active_teams = registry + await handler.enforce_token_limits_for_frame( + user_api_key_dict=_class_team("team-a"), requested_model="shared-model" + ) + await handler.async_log_success_event( + kwargs={ + "call_type": "_aresponses_websocket", + "litellm_params": {"metadata": {"model_group": "shared-model"}}, + "standard_logging_object": { + "metadata": { + "user_api_key_team_id": "team-a", + "user_api_key_auth_metadata": {"priority": "interactive"}, + } + }, + }, + response_obj=litellm.ModelResponse( + usage=litellm.Usage(prompt_tokens=550, completion_tokens=0, total_tokens=550) + ), + start_time=datetime.now(), + end_time=datetime.now(), + ) + with pytest.raises(ProxyRateLimitError) as exc: + await handler.enforce_token_limits_for_frame( + user_api_key_dict=_class_team("team-a"), requested_model="shared-model" + ) + _held_to_class_share(exc.value) + + +@pytest.mark.asyncio +async def test_a_frame_checked_while_the_registry_is_down_does_not_erase_the_connections_team_share( + monkeypatch: pytest.MonkeyPatch, +): + handler = _borrowing_handler(monkeypatch, borrows=False, splits=True, saturation_threshold=0.1) + await _burn(handler, _class_team("team-b"), "interactive", 20) + await _check(handler, _class_team("team-a"), saturation=0.0, priority="interactive") + registry: Final = handler.active_teams + handler.active_teams = _UnavailableRegistry() # pyright: ignore[reportAttributeAccessIssue] # registry down mid-connection + await handler.enforce_token_limits_for_frame( + user_api_key_dict=_class_team("team-a"), requested_model="shared-model" + ) + handler.active_teams = registry + await handler.async_log_success_event( + kwargs={ + "call_type": "_aresponses_websocket", + "litellm_params": {"metadata": {"model_group": "shared-model"}}, + "standard_logging_object": { + "metadata": { + "user_api_key_team_id": "team-a", + "user_api_key_auth_metadata": {"priority": "interactive"}, + } + }, + }, + response_obj=litellm.ModelResponse( + usage=litellm.Usage(prompt_tokens=550, completion_tokens=0, total_tokens=550) + ), + start_time=datetime.now(), + end_time=datetime.now(), + ) + with pytest.raises(ProxyRateLimitError) as exc: + await handler.enforce_token_limits_for_frame( + user_api_key_dict=_class_team("team-a"), requested_model="shared-model" + ) + _held_to_class_share(exc.value) + + +@pytest.mark.asyncio +async def test_tokens_served_by_a_fallback_model_land_on_the_teams_counter_for_that_model( + monkeypatch: pytest.MonkeyPatch, +): + handler = _borrowing_handler(monkeypatch, borrows=False, splits=True) + await _burn(handler, _class_team("team-b"), "interactive", 20, model="other-model") + await _burn(handler, _class_team("team-a"), "interactive", 10, model="other-model") + await _check(handler, _class_team("team-a"), saturation=0.0, priority="interactive") + await handler.async_log_success_event( + kwargs={ + "litellm_params": {"metadata": {"model_group": "other-model"}}, + "standard_logging_object": { + "metadata": { + "user_api_key_team_id": "team-a", + "user_api_key_auth_metadata": {"priority": "interactive"}, + } + }, + }, + response_obj=litellm.ModelResponse( + usage=litellm.Usage(prompt_tokens=550, completion_tokens=0, total_tokens=550) + ), + start_time=datetime.now(), + end_time=datetime.now(), + ) + await _check(handler, _class_team("team-a"), saturation=0.62, priority="interactive") + with pytest.raises(ProxyRateLimitError) as exc: + await _check(handler, _class_team("team-a"), saturation=0.62, priority="interactive", model="other-model") + _held_to_class_share(exc.value) diff --git a/tests/unit/proxy/hooks/test_priority_class_share.py b/tests/unit/proxy/hooks/test_priority_class_share.py new file mode 100644 index 00000000000..136a5d7ed87 --- /dev/null +++ b/tests/unit/proxy/hooks/test_priority_class_share.py @@ -0,0 +1,157 @@ +"""Share math and the active-team registry behind +``priority_reservation_settings.priority_class_share_splits_equally_among_active_teams``.""" + +from typing import Final + +import pytest + +from litellm.caching.caching import DualCache +from litellm.proxy.hooks.priority_class_share import ActiveTeamRegistry, pool_member, team_share + +WEIGHTS: Final = {"interactive": 0.6, "batch": 0.4} + + +def test_two_interactive_and_four_batch_teams_split_the_model_sixty_forty(): + members: Final = { + "i1": "interactive", + "i2": "interactive", + "b1": "batch", + "b2": "batch", + "b3": "batch", + "b4": "batch", + } + interactive: Final = team_share("interactive", members, WEIGHTS, default_weight=0.2) + batch: Final = team_share("batch", members, WEIGHTS, default_weight=0.2) + assert int(500_000_000 * interactive.fraction) == 150_000_000 + assert int(500_000_000 * batch.fraction) == 50_000_000 + assert (interactive.active_teams_in_class, interactive.active_classes) == (2, 2) + + +def test_only_active_class_takes_the_whole_model(): + members: Final = {"b1": "batch", "b2": "batch", "b3": "batch", "b4": "batch"} + assert int(500_000_000 * team_share("batch", members, WEIGHTS, default_weight=0.2).fraction) == 125_000_000 + + +def test_a_single_active_team_gets_everything(): + share: Final = team_share("interactive", {"i1": "interactive"}, WEIGHTS, default_weight=0.2) + assert share.fraction == 1.0 + assert (share.active_teams_in_class, share.active_classes) == (1, 1) + + +def test_a_class_without_a_configured_weight_uses_the_default_weight(): + members: Final = {"i1": "interactive", "d1": "default"} + assert team_share("default", members, WEIGHTS, default_weight=0.2).fraction == pytest.approx(0.2 / 0.8) + assert team_share("interactive", members, WEIGHTS, default_weight=0.2).fraction == pytest.approx(0.6 / 0.8) + + +def test_active_team_fractions_always_sum_to_one(): + members: Final = {"i1": "interactive", "i2": "interactive", "b1": "batch", "d1": "default", "d2": "default"} + total: Final = sum(team_share(cls, members, WEIGHTS, default_weight=0.2).fraction for cls in members.values()) + assert total == pytest.approx(1.0) + + +def test_zero_weights_split_the_model_equally_between_classes(): + members: Final = {"a": "interactive", "b": "batch"} + assert team_share("batch", members, {"interactive": 0.0, "batch": 0.0}, default_weight=0.0).fraction == 0.5 + + +class _Clock: + def __init__(self) -> None: + self.now = 1_000.0 + + def __call__(self) -> float: + return self.now + + +@pytest.mark.asyncio +async def test_registry_lists_every_team_seen_in_the_window_oldest_first(): + clock: Final = _Clock() + registry: Final = ActiveTeamRegistry(DualCache(), time_provider=clock) + assert dict(await registry.touch("m", "a", "interactive", window=60)) == {"a": "interactive"} + clock.now += 10 + assert dict(await registry.touch("m", "b", "batch", window=60)) == {"a": "interactive", "b": "batch"} + + +@pytest.mark.asyncio +async def test_registry_forgets_a_team_once_its_last_request_leaves_the_window(): + clock: Final = _Clock() + registry: Final = ActiveTeamRegistry(DualCache(), time_provider=clock) + await registry.touch("m", "a", "interactive", window=60) + clock.now += 59 + assert "a" in await registry.touch("m", "b", "batch", window=60) + clock.now += 1 + assert dict(await registry.touch("m", "b", "batch", window=60)) == {"b": "batch"} + + +@pytest.mark.asyncio +async def test_registry_refreshes_a_teams_last_seen_time_on_every_request(): + clock: Final = _Clock() + registry: Final = ActiveTeamRegistry(DualCache(), time_provider=clock) + await registry.touch("m", "a", "interactive", window=60) + clock.now += 40 + await registry.touch("m", "a", "interactive", window=60) + clock.now += 40 + assert "a" in await registry.touch("m", "b", "batch", window=60) + + +@pytest.mark.asyncio +async def test_registry_keeps_models_apart(): + registry: Final = ActiveTeamRegistry(DualCache(), time_provider=_Clock()) + await registry.touch("m1", "a", "interactive", window=60) + assert dict(await registry.touch("m2", "b", "batch", window=60)) == {"b": "batch"} + + +@pytest.mark.asyncio +async def test_registry_keeps_a_teams_newest_class_when_it_changed_class_mid_window(): + clock: Final = _Clock() + registry: Final = ActiveTeamRegistry(DualCache(), time_provider=clock) + await registry.touch("m", "a", "interactive", window=60) + clock.now += 1 + assert dict(await registry.touch("m", "a", "batch", window=60)) == {"a": "batch"} + + +@pytest.mark.asyncio +async def test_registry_lists_the_caller_under_the_class_it_just_sent_with_even_when_another_worker_wrote_later(): + clock: Final = _Clock() + registry: Final = ActiveTeamRegistry(DualCache(), time_provider=clock) + clock.now += 5 + await registry.touch("m", "a", "batch", window=60) + clock.now -= 5 + members: Final = await registry.touch("m", "a", "interactive", window=60) + assert dict(members) == {"a": "interactive"} + assert team_share("interactive", members, WEIGHTS, 1.0).fraction == 1.0 + + +@pytest.mark.asyncio +async def test_registry_keeps_a_team_id_that_contains_the_separator_whole(): + registry: Final = ActiveTeamRegistry(DualCache(), time_provider=_Clock()) + members: Final = await registry.touch("m", "owned\x1fextra", "batch", window=60) + assert dict(members) == {"owned\x1fextra": "batch"} + assert team_share("batch", members, {"interactive": 0.6, "batch": 0.4}, 1.0).fraction == 1.0 + + +def test_a_pool_member_keeps_its_class_present_without_taking_a_team_slice(): + members: Final = {pool_member("interactive"): "interactive", "a": "interactive", "c": "batch"} + share: Final = team_share("interactive", members, {"interactive": 0.6, "batch": 0.4}, 1.0) + assert share.fraction == 0.6 + assert share.active_teams_in_class == 1 + assert share.active_classes == 2 + + +@pytest.mark.asyncio +async def test_registry_keeps_pool_markers_of_different_classes_apart(): + registry: Final = ActiveTeamRegistry(DualCache(), time_provider=_Clock()) + await registry.touch("m", pool_member("interactive"), "interactive", window=60) + members: Final = await registry.touch("m", pool_member("batch"), "batch", window=60) + assert set(members.values()) == {"interactive", "batch"} + + +@pytest.mark.asyncio +async def test_registry_keeps_a_team_that_was_refreshed_after_the_first_window_opened(): + clock: Final = _Clock() + registry: Final = ActiveTeamRegistry(DualCache(), time_provider=clock) + await registry.touch("m", "a", "interactive", window=60) + clock.now += 50 + await registry.touch("m", "a", "interactive", window=60) + clock.now += 50 + assert dict(await registry.touch("m", "b", "batch", window=60)) == {"a": "interactive", "b": "batch"} From 2c7a36abe9c6ebed5be16be04d5946069f85b04c Mon Sep 17 00:00:00 2001 From: Kerry Lu Date: Thu, 12 Nov 2026 07:52:01 -0800 Subject: [PATCH 008/168] fix(rate-limit): hold router fallbacks, nested calls, WebSocket turns and oversized prompts to a borrowing team's model share (#51367) --- litellm/proxy/auth/fallback_rate_limit.py | 15 +- .../proxy/hooks/dynamic_rate_limiter_v3.py | 405 ++++++++++++++--- .../hooks/parallel_request_limiter_v3.py | 64 +++ ..._team_model_limit_borrows_idle_capacity.py | 38 +- .../hooks/test_dynamic_rate_limiter_v3.py | 417 +++++++++++++++++- 5 files changed, 876 insertions(+), 63 deletions(-) diff --git a/litellm/proxy/auth/fallback_rate_limit.py b/litellm/proxy/auth/fallback_rate_limit.py index 1d4a33b2994..d2f6cf9498a 100644 --- a/litellm/proxy/auth/fallback_rate_limit.py +++ b/litellm/proxy/auth/fallback_rate_limit.py @@ -13,10 +13,12 @@ from collections.abc import Callable, Mapping from dataclasses import dataclass from typing import Final +import litellm from litellm._logging import verbose_proxy_logger from litellm.proxy.auth.fallback_budget import ( _user_api_key_auth_from_request, # pyright: ignore[reportPrivateUsage] # shared auth lookup for fallback checks ) +from litellm.proxy.hooks.dynamic_rate_limiter_v3 import PROXY_DynamicRateLimitHandlerV3 from litellm.proxy.hooks.parallel_request_limiter_v3 import ( PROXY_MaxParallelRequestsHandler_v3, # pyright: ignore[reportPrivateUsage] # the proxy's limiter type ) @@ -42,7 +44,18 @@ class RouterFallbackRateLimitCheck: if valid_token is None: return True try: - return await limiter.async_fallback_target_allowed(valid_token, model, request_kwargs) + dynamic_limiter: Final = next( + (cb for cb in litellm.callbacks if isinstance(cb, PROXY_DynamicRateLimitHandlerV3)), None + ) + if dynamic_limiter is not None and not await dynamic_limiter.async_fallback_target_allowed( + valid_token, model, request_kwargs, commit=False + ): + return False + if not await limiter.async_fallback_target_allowed(valid_token, model, request_kwargs): + return False + return dynamic_limiter is None or await dynamic_limiter.async_fallback_target_allowed( + valid_token, model, request_kwargs + ) except Exception as e: # noqa: BLE001 # fail closed: a limiter failure must not hand out unchecked capacity verbose_proxy_logger.warning("Skipping fallback to model=%s: rate limit check failed: %s", model, e) return False diff --git a/litellm/proxy/hooks/dynamic_rate_limiter_v3.py b/litellm/proxy/hooks/dynamic_rate_limiter_v3.py index 059efe440ee..89818c21850 100644 --- a/litellm/proxy/hooks/dynamic_rate_limiter_v3.py +++ b/litellm/proxy/hooks/dynamic_rate_limiter_v3.py @@ -7,6 +7,7 @@ import os from collections.abc import Awaitable, Callable, Mapping from dataclasses import dataclass from datetime import datetime +from types import MappingProxyType from typing import TYPE_CHECKING, Final, Literal, TypeAlias from fastapi import HTTPException @@ -20,18 +21,22 @@ from litellm._logging import verbose_proxy_logger from litellm.caching.caching import DualCache from litellm.caching.redis_cache import RedisCircuitBreakerOpenError from litellm.cost_calculator import get_usage_object +from litellm.exceptions import ServiceUnavailableError from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.add_retry_fallback_headers import ( ensure_response_additional_headers, response_has_hidden_params, ) from litellm.litellm_core_utils.internal_call_metadata import MODEL_ACCESS_GROUP_METADATA_KEY +from litellm.litellm_core_utils.token_counter import offload_token_count from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.common_utils.proxy_rate_limit_error import ( ProxyRateLimitError, map_v3_rate_limit_type, ) from litellm.proxy.hooks.parallel_request_limiter_v3 import ( # noqa: F401 # legacy module exports + RATE_LIMIT_UNVERIFIABLE_MESSAGE, + BorrowingTeamModelHold, BorrowingTeamModelLimits, PROXY_MaxParallelRequestsHandler_v3, RateLimitDescriptor, @@ -39,15 +44,20 @@ from litellm.proxy.hooks.parallel_request_limiter_v3 import ( # noqa: F401 # l RateLimitedModel, RateLimitResponse, RateLimitStatus, - _call_id_from_callback_kwargs, # pyright: ignore[reportPrivateUsage] # same call-ownership check the regular limiter uses - _is_socket_turn, # pyright: ignore[reportPrivateUsage] # same socket-turn check the regular limiter uses + RateLimitUnverifiableError, + RequestRateLimiterStash, + ReservationOperation, + ReservationWindowIdentity, + WindowGuardedIncrement, _PROXY_MaxParallelRequestsHandler_v3, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export _socket_turn_tpm_windows, # pyright: ignore[reportPrivateUsage] # same per-turn TPM window the regular limiter checks borrowing_team_model_limits, + callback_kwargs_mapping, claim_request_stash_for_data, get_or_create_request_stash, get_request_stash, get_request_stash_for_call, + request_stash_for_callback, ) from litellm.proxy.hooks.priority_class_share import ( DEFAULT_CLASS, @@ -61,6 +71,7 @@ from litellm.proxy.hooks.priority_class_share import ( from litellm.proxy.hooks.rate_limiter_utils import ( convert_priority_to_percent, resolve_llm_provider_for_rate_limit, + socket_session_usage, ) from litellm.proxy.utils import InternalUsageCache, check_and_merge_model_level_guardrails from litellm.redis.stores.rate_limits import RATE_LIMITS_TARGET, CounterIncrement, RateLimitCounter, RateLimitScope @@ -349,9 +360,36 @@ class _SettlementMetadata(BaseModel): return value if isinstance(value, str) else None -def _usage_of(response_obj: object) -> Usage | None: +class _SettlementLoggingObject(BaseModel): + """The slice of a request's standard logging object that token settlement reads.""" + + model_config = ConfigDict(frozen=True) + + metadata: _SettlementMetadata | None = None + + +def _usage_of(kwargs: Mapping[str, object], response_obj: object) -> Usage | None: + """Usage of a modeled response, a Responses WebSocket turn's combined usage, or its raw event list.""" usage: Final = getattr(response_obj, "usage", None) - return usage if isinstance(usage, Usage) else get_usage_object(response_obj) + if isinstance(usage, Usage): + return usage + combined_usage: Final = kwargs.get("combined_usage_object") + if isinstance(combined_usage, Usage): + return combined_usage + socket_usage: Final = socket_session_usage(kwargs, response_obj) + return socket_usage if socket_usage is not None else get_usage_object(response_obj) + + +def _settled_tokens(usage: Usage | None, rate_limit_type: str) -> int: + if usage is None: + return 0 + match rate_limit_type: + case "output": + return usage.completion_tokens + case "input": + return usage.prompt_tokens + case _: + return usage.total_tokens TEAM_MODEL_LIMIT_AT_SATURATION: Final = "model_per_team_saturated" @@ -376,10 +414,10 @@ def _team_limit_name(rate_limit_type: str) -> str: def _team_model_token_increment( - descriptor: RateLimitDescriptor | None, total_tokens: int, ttl: int + descriptor: RateLimitDescriptor | None, amount: int, ttl: int ) -> tuple[CounterIncrement, ...]: - """Charge the team counter the pre-call hook tracked, and only when that team limit has a TPM share.""" - if descriptor is None: + """Move the team counter the pre-call hook tracked by ``amount``, and only when that team limit has a TPM share.""" + if descriptor is None or amount == 0: return () rate_limit: Final = descriptor["rate_limit"] if rate_limit is None or rate_limit.get("tokens_per_unit") is None: @@ -387,12 +425,60 @@ def _team_model_token_increment( return ( CounterIncrement( counter=RateLimitScope(descriptor["key"], descriptor["value"]).counter("tokens"), - amount=total_tokens, + amount=amount, ttl=ttl, ), ) +def _window_guarded( + operations: tuple[CounterIncrement, ...], window: ReservationWindowIdentity | None +) -> tuple[ReservationOperation, ...]: + """Apply a reservation's settlement only while the window it was held in is still open, never to a newer window.""" + if window is None: + return operations + return tuple( + WindowGuardedIncrement(increment=op, expected_window_start=window[1], reservation_backend=window[2]) + if op.counter == window[0] + else op + for op in operations + ) + + +def _team_model_settlement( + stash: RequestRateLimiterStash | None, model_group: str | None, total_tokens: int, ttl: int +) -> tuple[ReservationOperation, ...]: + """Settle the counter that served the call to actual usage and return the admitting reservation it did not use. + + The admitting team counter serves unless a fallback group did, in which case that fallback's hold serves and + the admitting reservation is returned. A hold on a fallback target that did not serve stays for that attempt's + own failure callback, which may still be reporting the usage it recorded. + """ + if stash is None: + return () + served: Final = stash.borrowing_team_model_fallback_limits.get(model_group) if model_group is not None else None + admitted: Final = stash.borrowing_team_model_limit + reserved_tokens: Final = stash.borrowing_team_model_reserved_tokens + window: Final = stash.borrowing_team_model_reserved_window + if served is None: + return _window_guarded(_team_model_token_increment(admitted, total_tokens - reserved_tokens, ttl), window) + return ( + *_window_guarded(_team_model_token_increment(admitted, -reserved_tokens, ttl), window), + *_window_guarded( + _team_model_token_increment(served.descriptor, total_tokens - served.reserved_tokens, ttl), served.window + ), + ) + + +def _reserved_window( + response: RateLimitResponse, descriptor: RateLimitDescriptor | None +) -> ReservationWindowIdentity | None: + if descriptor is None: + return None + counter: Final = RateLimitScope(descriptor["key"], descriptor["value"]).counter("tokens") + return next((w for w in response.get("reservation_windows", frozenset()) if w[0] == counter), None) + + def _get_priority_settings() -> "PriorityReservationSettings": """ Get the priority reservation settings, guaranteed to be non-None. @@ -918,9 +1004,17 @@ class _PROXY_DynamicRateLimitHandlerV3(CustomLogger): if not descriptors: return tpm_windows: Final = _socket_turn_tpm_windows(descriptors) - response: Final = await self.v3_limiter.should_rate_limit( - tpm_windows, parent_otel_span=user_api_key_dict.parent_otel_span, read_only=True, skip_parallel_check=True - ) + try: + response: Final = await self.v3_limiter.should_rate_limit( + tpm_windows, + parent_otel_span=user_api_key_dict.parent_otel_span, + read_only=True, + skip_parallel_check=True, + ) + except RateLimitUnverifiableError as unverifiable: + raise ServiceUnavailableError( + message=RATE_LIMIT_UNVERIFIABLE_MESSAGE, llm_provider="litellm_proxy", model=requested_model + ) from unverifiable over_limit: Final = next((s for s in response["statuses"] if s["code"] == "OVER_LIMIT"), None) if over_limit is None: return @@ -966,6 +1060,8 @@ class _PROXY_DynamicRateLimitHandlerV3(CustomLogger): user_api_key_dict: UserAPIKeyAuth, priority: str | None, saturation: float, + data: Mapping[str, object] = MappingProxyType({}), + call_type: str | None = None, ) -> None: """ Check rate limits using THREE-PHASE approach to prevent partial increments. @@ -992,6 +1088,8 @@ class _PROXY_DynamicRateLimitHandlerV3(CustomLogger): user_api_key_dict: User authentication info priority: User's priority level saturation: Current saturation level + data: Request body, used to size the token reservation a saturated team limit holds + call_type: The endpoint's call type, used to read the body's prompt fields Raises: HTTPException: If any limit is exceeded @@ -1051,9 +1149,19 @@ class _PROXY_DynamicRateLimitHandlerV3(CustomLogger): "requests": 1, "tokens": 0, } + team_reservation: Final = ( + await self._team_token_reservation(enforced_team_share, data, model, call_type, user_api_key_dict) + if enforced_team_share is not None + else 0 + ) atomic_response: Final = await self.v3_limiter.atomic_check_and_increment_by_n( descriptors=enforced_descriptors, - increments=[per_request_increment for _ in enforced_descriptors], + increments=[ + {"requests": 1, "tokens": team_reservation} + if descriptor is enforced_team_share + else per_request_increment + for descriptor in enforced_descriptors + ], parent_otel_span=user_api_key_dict.parent_otel_span, ) @@ -1154,6 +1262,12 @@ class _PROXY_DynamicRateLimitHandlerV3(CustomLogger): # priority allocation has accurate usage when it later becomes # enforced. The increment-then-check semantics of should_rate_limit # are fine because we ignore the OVER_LIMIT response. + stash: Final = get_or_create_request_stash() + stash.borrowing_team_model_reserved_tokens = team_reservation + stash.borrowing_team_model_reserved_window = ( + _reserved_window(atomic_response, enforced_team_share) if team_reservation else None + ) + self._cap_admitted_output(enforced_team_share, team_reservation, data, call_type) if team_share is not None and enforced_team_share is None: await self.v3_limiter.atomic_check_and_increment_by_n( descriptors=[team_share.share], @@ -1188,6 +1302,124 @@ class _PROXY_DynamicRateLimitHandlerV3(CustomLogger): else: get_or_create_request_stash().rate_limit_response = atomic_response + def _cap_admitted_output( + self, + enforced_team_share: RateLimitDescriptor | None, + team_reservation: int, + data: Mapping[str, object], + call_type: str | None, + ) -> None: + """Write the output cap the hold was estimated with onto the admitted request, unless only input tokens count.""" + tpm_share: Final = ( + None if enforced_team_share is None else (enforced_team_share["rate_limit"] or {}).get("tokens_per_unit") + ) + if team_reservation and tpm_share is not None and self.v3_limiter.get_rate_limit_type() != "input": + self.v3_limiter.bound_output_for_limit(data, tpm_share, call_type) + + async def _team_token_reservation( + self, + descriptor: RateLimitDescriptor, + data: Mapping[str, object], + model: str, + call_type: str | None, + user_api_key_dict: UserAPIKeyAuth, + ) -> int: + """Tokens to hold against the team's share before the call, as the regular limiter does for a hard limit.""" + tpm_share: Final = (descriptor["rate_limit"] or {}).get("tokens_per_unit") + if tpm_share is None or not data: + return 0 + as_dispatched: Final = dict(data, model=model) + if self.v3_limiter.get_rate_limit_type() != "input": + self.v3_limiter.bound_output_for_limit(as_dispatched, tpm_share, call_type) + try: + estimate: Final = await offload_token_count(self.v3_limiter.estimate_request_tokens)( + data=as_dispatched, + model=model, + min_configured_tpm_limit=tpm_share, + call_type=call_type, + team_id=user_api_key_dict.team_id, + rate_limit_type=self.v3_limiter.get_rate_limit_type(), + ) + except Exception as e: # noqa: BLE001 # an unreadable prompt still holds the minimum so admission runs + verbose_proxy_logger.warning( + "Dynamic rate limiter: token estimate failed for %s, holding one token: %s", model, e + ) + return 1 + return max(estimate, 1) + + async def async_fallback_target_allowed( + self, user_api_key_dict: UserAPIKeyAuth, model: str, request_data: Mapping[str, object], *, commit: bool = True + ) -> bool: + """Hold a router fallback target to the team's share on it once it saturates, and to its borrow ceiling before. + + With ``commit`` the attempt is counted toward the target's saturation and the team's counter, holding the + prompt's estimated tokens when the share is enforced, otherwise only checked. + """ + team_share: Final = self._borrowing_team_share(model=model, user_api_key_dict=user_api_key_dict) + model_group_info: Final = self.llm_router.registry().group(model) + if team_share is None or model_group_info is None: + return True + saturation: Final = await self._check_model_saturation(model, model_group_info) + enforced: Final = team_share.enforced(saturation >= _get_priority_settings().saturation_threshold) + reservation: Final = ( + await self._team_token_reservation( + enforced, + {**request_data, "model": model}, + model, + get_or_create_request_stash().call_type, + user_api_key_dict, + ) + if enforced is not None + else 0 + ) + if not commit: + return enforced is None or await self._share_has_room(enforced, reservation, user_api_key_dict) + model_wide_descriptor: Final = self._create_model_tracking_descriptor( + model=model, model_group_info=model_group_info, high_limit_multiplier=1 + ) + response: Final = await self.v3_limiter.atomic_check_and_increment_by_n( + descriptors=[model_wide_descriptor, *(() if enforced is None else (enforced,))], + increments=[ + {"requests": 1, "tokens": 0}, + *(() if enforced is None else ({"requests": 1, "tokens": reservation},)), + ], + parent_otel_span=user_api_key_dict.parent_otel_span, + ) + if response["overall_code"] == "OVER_LIMIT": + return False + if enforced is None: + await self.v3_limiter.atomic_check_and_increment_by_n( + descriptors=[team_share.share], + increments=[{"requests": 1, "tokens": reservation}], + parent_otel_span=user_api_key_dict.parent_otel_span, + ) + self._hold_fallback_share(model, team_share.share, reservation, _reserved_window(response, enforced)) + return True + + async def _share_has_room( + self, descriptor: RateLimitDescriptor, tokens: int, user_api_key_dict: UserAPIKeyAuth + ) -> bool: + checked: Final = await self.v3_limiter.should_rate_limit( + [descriptor], parent_otel_span=user_api_key_dict.parent_otel_span, read_only=True + ) + return all( + status["code"] != "OVER_LIMIT" + and (status["rate_limit_type"] != "tokens" or (status["limit_remaining"] or 0) >= tokens) + for status in checked["statuses"] + ) + + def _hold_fallback_share( + self, model: str, share: RateLimitDescriptor, reserved_tokens: int, window: ReservationWindowIdentity | None + ) -> None: + """Remember the hold under the requested name and the resolved group, so settlement finds it either way.""" + stash: Final = get_or_create_request_stash() + earlier: Final = stash.borrowing_team_model_fallback_limits.get(model) + carried: Final = earlier.reserved_tokens if earlier is not None and earlier.window == window else 0 + hold: Final = BorrowingTeamModelHold(descriptor=share, reserved_tokens=reserved_tokens + carried, window=window) + stash.borrowing_team_model_fallback_limits = MappingProxyType( + {**stash.borrowing_team_model_fallback_limits, model: hold, share["value"].rpartition(":")[2]: hold} + ) + @with_service_target(RATE_LIMITS_TARGET) async def async_pre_call_hook( self, @@ -1247,7 +1479,7 @@ class _PROXY_DynamicRateLimitHandlerV3(CustomLogger): steered: Final = await self._steer_to_alternate(model, requested_group_info, user_api_key_dict, data, call_type) try: admitted: Final = await self._admit_steered_or_preferred( - model, requested_group_info, steered, user_api_key_dict + model, requested_group_info, steered, user_api_key_dict, data, call_type ) except HTTPException: raise @@ -1269,21 +1501,30 @@ class _PROXY_DynamicRateLimitHandlerV3(CustomLogger): requested_group_info: ModelGroupInfo, steered: "SteeredModel | None", user_api_key_dict: UserAPIKeyAuth, + data: Mapping[str, object], + call_type: str | None, ) -> "SteeredModel | None": """The alternate once it admits the request, else None after the preferred model's own check.""" if steered is not None: try: - await self._admit(steered.model, steered.group_info, user_api_key_dict) + await self._admit(steered.model, steered.group_info, user_api_key_dict, data, call_type) except Exception: # noqa: BLE001 # the alternate's refusal or error keeps the request on its own model verbose_proxy_logger.info( "Alternate %s refused team %s, staying on %s", steered.model, user_api_key_dict.team_id, model ) else: return steered - await self._admit(model, requested_group_info, user_api_key_dict) + await self._admit(model, requested_group_info, user_api_key_dict, data, call_type) return None - async def _admit(self, model: str, model_group_info: ModelGroupInfo, user_api_key_dict: UserAPIKeyAuth) -> None: + async def _admit( + self, + model: str, + model_group_info: ModelGroupInfo, + user_api_key_dict: UserAPIKeyAuth, + data: Mapping[str, object], + call_type: str | None, + ) -> None: """Read saturation, then the three-phase rate limit check, raising the 429 when ``model`` is out of capacity.""" priority: Final = self._get_priority_from_user_api_key_dict(user_api_key_dict=user_api_key_dict) saturation: Final = await self._check_model_saturation(model, model_group_info) @@ -1298,6 +1539,8 @@ class _PROXY_DynamicRateLimitHandlerV3(CustomLogger): user_api_key_dict=user_api_key_dict, priority=priority, saturation=saturation, + data=data, + call_type=call_type, ) async def _steer_to_alternate( @@ -1450,7 +1693,9 @@ class _PROXY_DynamicRateLimitHandlerV3(CustomLogger): return {STEERED_FROM_HEADER: stash.steered_from_model} @with_service_target(RATE_LIMITS_TARGET) - async def async_log_success_event(self, kwargs, response_obj: object, start_time, end_time) -> None: + async def async_log_success_event( + self, kwargs: Mapping[str, object], response_obj: object, start_time: object, end_time: object + ) -> None: """ Update token usage for priority-based rate limiting after successful API calls. @@ -1469,36 +1714,41 @@ class _PROXY_DynamicRateLimitHandlerV3(CustomLogger): verbose_proxy_logger.debug("INSIDE dynamic rate limiter ASYNC SUCCESS LOGGING") litellm_parent_otel_span: Final = get_parent_otel_span_from_kwargs(kwargs) - - # Get metadata from standard_logging_object - standard_logging_object: Final = kwargs.get("standard_logging_object") or {} - settlement_metadata: Final = _SettlementMetadata.model_validate( - standard_logging_object.get("metadata") or {} + callback_kwargs: Final = callback_kwargs_mapping(kwargs) + settlement_metadata: Final = ( + _SettlementLoggingObject.model_validate(callback_kwargs.get("standard_logging_object") or {}).metadata + or _SettlementMetadata() ) - # Get model and priority - model_group: Final = get_model_group_from_litellm_kwargs(kwargs) + model_group: Final = get_model_group_from_litellm_kwargs(dict(kwargs)) + total_tokens: Final = _settled_tokens( + _usage_of(callback_kwargs, response_obj), self.v3_limiter.get_rate_limit_type() + ) + stash: Final = request_stash_for_callback(callback_kwargs) + team_model_tokens: Final = _team_model_settlement( + stash, model_group, total_tokens, self.v3_limiter.window_size + ) + if stash is not None: + served: Final = ( + stash.borrowing_team_model_fallback_limits.get(model_group) if model_group is not None else None + ) + stash.borrowing_team_model_reserved_tokens = 0 + stash.borrowing_team_model_reserved_window = None + stash.borrowing_team_model_fallback_limits = MappingProxyType( + {k: h for k, h in stash.borrowing_team_model_fallback_limits.items() if h is not served} + ) if not model_group: + if team_model_tokens: + await self.v3_limiter.async_increment_reservation_aware_tokens( + pipeline_operations=team_model_tokens, parent_otel_span=litellm_parent_otel_span + ) return # Get priority from user_api_key_auth_metadata in standard_logging_metadata # This is where user_api_key_dict.metadata is stored during pre-call key_priority: Final = settlement_metadata.priority - # Get total tokens from response - total_tokens = 0 - rate_limit_type: Final = self.v3_limiter.get_rate_limit_type() - - _usage: Final = _usage_of(response_obj) - if _usage is not None: - if rate_limit_type == "output": - total_tokens = _usage.completion_tokens - elif rate_limit_type == "input": - total_tokens = _usage.prompt_tokens - elif rate_limit_type == "total": - total_tokens = _usage.total_tokens - - if total_tokens == 0: + if total_tokens == 0 and not team_model_tokens: return # Model-wide token tracking (model_saturation_check) @@ -1521,13 +1771,6 @@ class _PROXY_DynamicRateLimitHandlerV3(CustomLogger): else: priority_key = f"{model_group}:default_pool" - call_stash: Final = get_request_stash_for_call(_call_id_from_callback_kwargs(kwargs)) - stash: Final = ( - call_stash if call_stash is not None else (get_request_stash() if _is_socket_turn(kwargs) else None) - ) - team_model_tokens: Final = _team_model_token_increment( - None if stash is None else stash.borrowing_team_model_limit, total_tokens, self.v3_limiter.window_size - ) class_share: Final = None if stash is None else stash.priority_class_share settled_team_id: Final = settlement_metadata.user_api_key_team_id priority_scope: Final = ( @@ -1538,19 +1781,23 @@ class _PROXY_DynamicRateLimitHandlerV3(CustomLogger): class_share["value"] if settled_team_id is None else counter_value(model_group, settled_team_id), ) ) - pipeline_operations: Final = ( - model_tokens, - CounterIncrement( - counter=priority_scope.counter("tokens"), - amount=total_tokens, - ttl=self.v3_limiter.window_size, - ), - *team_model_tokens, + usage_operations: Final = ( + () + if total_tokens == 0 + else ( + model_tokens, + CounterIncrement( + counter=priority_scope.counter("tokens"), + amount=total_tokens, + ttl=self.v3_limiter.window_size, + ), + ) ) + pipeline_operations: Final[tuple[ReservationOperation, ...]] = (*usage_operations, *team_model_tokens) # Execute token increments with TTL preservation if pipeline_operations: - await self.v3_limiter.async_increment_tokens_with_ttl_preservation( + await self.v3_limiter.async_increment_reservation_aware_tokens( pipeline_operations=pipeline_operations, parent_otel_span=litellm_parent_otel_span, ) @@ -1568,5 +1815,55 @@ class _PROXY_DynamicRateLimitHandlerV3(CustomLogger): except Exception as e: verbose_proxy_logger.exception("Error in dynamic rate limiter success event: %s", e) + @with_service_target(RATE_LIMITS_TARGET) + async def async_log_failure_event( + self, kwargs: object, response_obj: object, start_time: object, end_time: object + ) -> None: + """Settle the tokens held for a failed attempt to the usage it recorded, or give them all back when it recorded + none, on the admitting team counter or the fallback target that failed, leaving holds on other fallback + targets to the attempts still running.""" + from litellm.proxy.common_utils.callback_utils import get_model_group_from_litellm_kwargs + + callback_kwargs: Final = callback_kwargs_mapping(kwargs) + stash: Final = request_stash_for_callback(callback_kwargs) + if stash is None: + return + failed_group: Final = get_model_group_from_litellm_kwargs(dict(callback_kwargs)) + failed_hold: Final = ( + stash.borrowing_team_model_fallback_limits.get(failed_group) if failed_group is not None else None + ) + used: Final = _settled_tokens( + _usage_of(callback_kwargs, None if isinstance(response_obj, BaseException) else response_obj), + self.v3_limiter.get_rate_limit_type(), + ) + refund: Final = ( + *_window_guarded( + _team_model_token_increment( + stash.borrowing_team_model_limit, + (used if failed_hold is None else 0) - stash.borrowing_team_model_reserved_tokens, + self.v3_limiter.window_size, + ), + stash.borrowing_team_model_reserved_window, + ), + *( + () + if failed_hold is None + else _window_guarded( + _team_model_token_increment( + failed_hold.descriptor, used - failed_hold.reserved_tokens, self.v3_limiter.window_size + ), + failed_hold.window, + ) + ), + ) + stash.borrowing_team_model_reserved_tokens = 0 + stash.borrowing_team_model_reserved_window = None + if failed_hold is not None: + stash.borrowing_team_model_fallback_limits = MappingProxyType( + {k: h for k, h in stash.borrowing_team_model_fallback_limits.items() if h is not failed_hold} + ) + if refund: + await self.v3_limiter.async_increment_reservation_aware_tokens(pipeline_operations=refund) + PROXY_DynamicRateLimitHandlerV3: Final = _PROXY_DynamicRateLimitHandlerV3 diff --git a/litellm/proxy/hooks/parallel_request_limiter_v3.py b/litellm/proxy/hooks/parallel_request_limiter_v3.py index 16e12eb1e03..39409002058 100644 --- a/litellm/proxy/hooks/parallel_request_limiter_v3.py +++ b/litellm/proxy/hooks/parallel_request_limiter_v3.py @@ -570,6 +570,15 @@ class CallTypeRateLimiter(Protocol): ) -> Exception | str | dict[str, object] | None: ... +@dataclass(frozen=True, slots=True) +class BorrowingTeamModelHold: + """A router fallback target's team share with the tokens held on it, settled to actual usage or given back.""" + + descriptor: RateLimitDescriptor + reserved_tokens: int = 0 + window: ReservationWindowIdentity | None = None + + @dataclass(slots=True) class RequestRateLimiterStash: """ @@ -594,6 +603,7 @@ class RequestRateLimiterStash: """ owner_litellm_call_id: str | None = None + call_type: str | None = None rate_limit_response: RateLimitResponse | None = None parallel_slot: ParallelSlotAcquisition | None = None parallel_slot_reused_at: datetime | None = None @@ -615,6 +625,11 @@ class RequestRateLimiterStash: batch_enqueued_reservation: BatchEnqueuedTokenReservation | None = None batch_tpd_refund_ops: tuple[ReservationOperation, ...] = () borrowing_team_model_limit: RateLimitDescriptor | None = None + borrowing_team_model_reserved_tokens: int = 0 + borrowing_team_model_reserved_window: ReservationWindowIdentity | None = None + borrowing_team_model_fallback_limits: Mapping[str, BorrowingTeamModelHold] = field( + default_factory=lambda: MappingProxyType({}) + ) priority_class_share: RateLimitDescriptor | None = None reservation_released: bool = False io_reservation_released: bool = False @@ -763,6 +778,23 @@ def _is_socket_turn(kwargs: Mapping[str, object]) -> bool: ) +def callback_kwargs_mapping(kwargs: object) -> Mapping[str, object]: + """A logging callback's kwargs as a typed mapping, empty when they are not a mapping.""" + return _callback_kwargs_mapping(kwargs) + + +def request_stash_for_callback(kwargs: Mapping[str, object]) -> RequestRateLimiterStash | None: + """The stash of the request a logging callback settles, or the connection's for a Responses WebSocket turn. + + A nested call (an LLM judge, a silent experiment) mints its own call id and gets None. + """ + call_id: Final = kwargs.get("litellm_call_id") + call_stash: Final = get_request_stash_for_call(call_id if isinstance(call_id, str) else None) + if call_stash is not None: + return call_stash + return get_request_stash() if _is_socket_turn(kwargs) else None + + _AGENT_DESCRIPTOR_KEYS: Final = frozenset({"agent", "agent_session"}) _SOCKET_TURN_UNCOUNTED_DESCRIPTOR_KEYS: Final = frozenset( @@ -1060,6 +1092,37 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): if existing_cap is None or effective_cap < existing_cap: data[cap_field] = effective_cap # rebind-ok: downstream routing requires the bounded output cap + def estimate_request_tokens( + self, + data: object, + model: str, + min_configured_tpm_limit: int, + call_type: str | None, + team_id: str | None, + rate_limit_type: str = "total", + ) -> int: + """The reservation this limiter would hold for the request, for another limiter settling the same call.""" + if rate_limit_type == "total": + return self._estimate_tokens_for_request( + data=data, + model=model, + min_configured_tpm_limit=min_configured_tpm_limit, + call_type=call_type, + team_id=team_id, + ) + estimated_input, output_budget = self._estimate_input_and_output_tokens( + data=data, + min_configured_tpm_limit=min_configured_tpm_limit, + call_type=call_type, + team_id=team_id, + media_model_resolver=self._media_model_resolver, + ) + return output_budget if rate_limit_type == "output" else estimated_input + + def bound_output_for_limit(self, data: object, min_configured_limit: int, call_type: str | None) -> None: + """Cap a request without an explicit output cap the way this limiter does for its own small TPM limits.""" + self._apply_implicit_output_cap(data=data, min_configured_limit=min_configured_limit, call_type=call_type) + def _estimate_tokens_for_request( self, data: object, @@ -3988,6 +4051,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): verbose_proxy_logger.debug("Inside Rate Limit Pre-Call Hook") stash: Final = claim_request_stash_for_data(data) + stash.call_type = call_type reuses_request_slot: Final = call_type == CallTypes.call_mcp_tool.value and stash.parallel_slot is not None if reuses_request_slot: stash.parallel_slot_reused_at = self._get_current_time() diff --git a/tests/integration/routing/test_team_model_limit_borrows_idle_capacity.py b/tests/integration/routing/test_team_model_limit_borrows_idle_capacity.py index b9f853d7da8..ee0fcbea41b 100644 --- a/tests/integration/routing/test_team_model_limit_borrows_idle_capacity.py +++ b/tests/integration/routing/test_team_model_limit_borrows_idle_capacity.py @@ -30,8 +30,10 @@ UPSTREAM_REPLY: Final = json.dumps( def _respond(request: Request) -> Reply: - assert request.method == "POST" and request.target == "/v1/chat/completions", request.target + assert request.method == "POST" and request.target.endswith("/v1/chat/completions"), request.target assert request.headers["authorization"] == "Bearer synthetic-openai-key" + if request.target.startswith("/down/"): + return Reply(status=500, body=b'{"error": {"message": "upstream down"}}') return Reply(body=UPSTREAM_REPLY) @@ -123,6 +125,40 @@ def test_team_is_held_to_its_model_limit_once_the_model_saturates( assert "no idle capacity to borrow" in first.json()["error"]["message"], first.text +def test_a_fallback_target_is_held_to_the_teams_share_on_it_once_it_saturates( + saturated_model: tuple[Gateway, str], +) -> None: + owned, upstream = saturated_model + with owned.scenario() as scenario: + source: Final = scenario.model( + name=f"down-{uuid.uuid4().hex[:8]}", + model=f"openai/{OPENAI_MODEL}", + api_base=f"{upstream}/down/v1", + api_key="synthetic-openai-key", + ) + target: Final = scenario.model( + model=f"openai/{OPENAI_MODEL}", + api_base=f"{upstream}/v1", + api_key="synthetic-openai-key", + tpm=MODEL_TPM, + rpm=MODEL_RPM, + ) + owned.post("/config/update", {"router_settings": {"num_retries": 0, "fallbacks": [{source: [target]}]}}) + key: Final = scenario.key(team_id=scenario.team(metadata={"model_rpm_limit": {target: 1}})) + responses: Final = tuple( + owned.request( + "POST", + "/v1/chat/completions", + {"model": source, "messages": [{"role": "user", "content": f"fallback borrowing {uuid.uuid4().hex}"}]}, + key=key, + ) + for _ in range(2) + ) + assert responses[0].status_code == 200, responses[0].text + assert responses[0].headers.get("x-litellm-attempted-fallbacks") == "1", dict(responses[0].headers) + assert responses[1].status_code != 200, responses[1].text + + @pytest.mark.parametrize( "limit, rate_limit_type", [("model_tpm_limit", "tokens"), ("model_rpm_limit", "requests")], ids=["tpm", "rpm"] ) diff --git a/tests/unit/proxy/hooks/test_dynamic_rate_limiter_v3.py b/tests/unit/proxy/hooks/test_dynamic_rate_limiter_v3.py index 22e6d5e3146..efd2e9e7e18 100644 --- a/tests/unit/proxy/hooks/test_dynamic_rate_limiter_v3.py +++ b/tests/unit/proxy/hooks/test_dynamic_rate_limiter_v3.py @@ -1340,7 +1340,7 @@ async def test_async_log_success_event_increments_by_actual_tokens(monkeypatch): } ) - handler.v3_limiter.async_increment_tokens_with_ttl_preservation = mock_increment + handler.v3_limiter.async_increment_reservation_aware_tokens = mock_increment # Create mock response with 50 completion tokens mock_response = MagicMock(spec=ModelResponse) @@ -1514,7 +1514,7 @@ async def test_async_log_success_event_uses_team_priority_from_auth_metadata(mon for op in pipeline_operations: incremented_keys.append(op.counter.scope) - handler.v3_limiter.async_increment_tokens_with_ttl_preservation = mock_increment + handler.v3_limiter.async_increment_reservation_aware_tokens = mock_increment # Create mock response mock_response = MagicMock(spec=ModelResponse) @@ -2166,14 +2166,21 @@ async def _admit_then_settle(handler: _PROXY_DynamicRateLimitHandlerV3, auth: Us async def _settle_tokens( - handler: _PROXY_DynamicRateLimitHandlerV3, tokens: int, response: object | None = None + handler: PROXY_DynamicRateLimitHandlerV3, + tokens: int, + response: object | None = None, + call_id: str | None = None, + model_group: str = "shared-model", + **extra_kwargs: object, ) -> None: await handler.async_log_success_event( kwargs={ - "litellm_params": {"metadata": {"model_group": "shared-model"}}, + "litellm_params": {"metadata": {"model_group": model_group}}, "standard_logging_object": { "metadata": {"user_api_key_team_id": "team-a", "user_api_key_auth_metadata": {}} }, + **({"litellm_call_id": call_id} if call_id is not None else {}), + **extra_kwargs, }, response_obj=response if response is not None @@ -2197,19 +2204,45 @@ def _team_key( async def _check( - handler: _PROXY_DynamicRateLimitHandlerV3, + handler: PROXY_DynamicRateLimitHandlerV3, auth: UserAPIKeyAuth, saturation: float, model: str = "shared-model", + data: dict[str, object] | None = None, priority: str | None = None, ) -> None: group: Final = handler.llm_router.registry().group(model) assert group is not None - await handler._check_rate_limits( # pyright: ignore[reportPrivateUsage] # saturation is an input here, the public hook reads it from Redis - model=model, model_group_info=group, user_api_key_dict=auth, priority=priority, saturation=saturation + await handler._check_rate_limits( + model=model, + model_group_info=group, + user_api_key_dict=auth, + priority=priority, + saturation=saturation, + data=data or {}, + call_type="acompletion" if data else None, ) +async def _team_tokens_remaining( + handler: PROXY_DynamicRateLimitHandlerV3, auth: UserAPIKeyAuth | None = None, model: str = "shared-model" +) -> int: + share: Final = handler._borrowing_team_share(model, auth if auth is not None else _team_key()) + assert share is not None + response: Final = await handler.v3_limiter.should_rate_limit( + [share.share], read_only=True, skip_parallel_check=True + ) + return next(s["limit_remaining"] for s in response["statuses"] if s["rate_limit_type"] == "tokens") + + +def _prompt(words: int, max_tokens: int = 10) -> dict[str, object]: + return { + "model": "shared-model", + "messages": [{"role": "user", "content": "word " * words}], + "max_tokens": max_tokens, + } + + def _held_to_team_limit(exc: ProxyRateLimitError, rate_limit_type: str, limit_sentence: str) -> None: assert exc.status_code == 429 assert exc.headers is not None @@ -2391,6 +2424,376 @@ async def test_a_websocket_turn_still_borrows_while_the_model_is_idle(monkeypatc await handler.enforce_token_limits_for_frame(user_api_key_dict=_team_key(), requested_model="shared-model") +@pytest.mark.asyncio +async def test_a_nested_calls_tokens_are_not_charged_to_the_requests_team_share(monkeypatch: pytest.MonkeyPatch): + from litellm.proxy.hooks.parallel_request_limiter_v3 import claim_request_stash_for_data + + handler = _borrowing_handler(monkeypatch, borrows=True) + claim_request_stash_for_data({"litellm_call_id": "outer-call"}) + await _check(handler, _team_key(), saturation=0.0) + await _settle_tokens(handler, 60, call_id="llm-judge-call") + assert await _team_tokens_remaining(handler) == 100 + await _settle_tokens(handler, 60, call_id="outer-call") + assert await _team_tokens_remaining(handler) == 40 + + +@pytest.mark.asyncio +async def test_a_websocket_turns_tokens_count_against_the_team_limit(monkeypatch: pytest.MonkeyPatch): + handler = _borrowing_handler(monkeypatch, borrows=True) + await _check(handler, _team_key(), saturation=0.0) + await _settle_tokens( + handler, + 150, + response=[], + combined_usage_object=litellm.Usage(prompt_tokens=100, completion_tokens=50, total_tokens=150), + ) + with pytest.raises(ProxyRateLimitError) as exc: + await _check(handler, _team_key(), saturation=0.5) + _held_to_team_limit(exc.value, "tokens", "model_tpm_limit") + + +@pytest.mark.asyncio +async def test_a_websocket_turn_gets_a_503_when_the_team_counter_cannot_be_read(monkeypatch: pytest.MonkeyPatch): + from litellm.exceptions import ServiceUnavailableError + from litellm.proxy.hooks.parallel_request_limiter_v3 import RateLimitUnverifiableError + + handler = _borrowing_handler(monkeypatch, borrows=True, saturation_threshold=0.1) + await _admit_then_settle(handler, _team_key(), 150) + monkeypatch.setattr(handler.v3_limiter, "should_rate_limit", AsyncMock(side_effect=RateLimitUnverifiableError())) + with pytest.raises(ServiceUnavailableError): + await handler.enforce_token_limits_for_frame(user_api_key_dict=_team_key(), requested_model="shared-model") + + +@pytest.mark.asyncio +async def test_a_prompt_larger_than_the_team_share_is_refused_before_the_call_once_the_model_saturates( + monkeypatch: pytest.MonkeyPatch, +): + handler = _borrowing_handler(monkeypatch, borrows=True) + await _check(handler, _team_key(), saturation=0.0, data=_prompt(words=400)) + with pytest.raises(ProxyRateLimitError) as exc: + await _check(handler, _team_key(), saturation=0.5, data=_prompt(words=400)) + _held_to_team_limit(exc.value, "tokens", "model_tpm_limit") + + +@pytest.mark.asyncio +async def test_a_reservation_at_saturation_settles_to_the_tokens_actually_used(monkeypatch: pytest.MonkeyPatch): + handler = _borrowing_handler(monkeypatch, borrows=True) + await _check(handler, _team_key(), saturation=0.5, data=_prompt(words=30)) + assert await _team_tokens_remaining(handler) < 70 + await _settle_tokens(handler, 5) + assert await _team_tokens_remaining(handler) == 95 + + +@pytest.mark.asyncio +async def test_a_reservation_at_saturation_is_refunded_when_the_call_fails(monkeypatch: pytest.MonkeyPatch): + handler = _borrowing_handler(monkeypatch, borrows=True) + await _check(handler, _team_key(), saturation=0.5, data=_prompt(words=30)) + assert await _team_tokens_remaining(handler) < 70 + await handler.async_log_failure_event( + kwargs={"litellm_params": {"metadata": {"model_group": "shared-model"}}}, + response_obj=None, + start_time=datetime.now(), + end_time=datetime.now(), + ) + assert await _team_tokens_remaining(handler) == 100 + + +@pytest.mark.asyncio +async def test_a_fallback_target_is_refused_once_it_saturates_and_the_teams_share_on_it_is_used( + monkeypatch: pytest.MonkeyPatch, +): + handler = _borrowing_handler(monkeypatch, borrows=True, saturation_threshold=0.1) + key: Final = _team_key( + team_limits={"model_tpm_limit": {"shared-model": 100}, "model_rpm_limit": {"shared-model": 1}} + ) + await _admit_then_settle(handler, key, 150) + assert await handler.async_fallback_target_allowed(key, "shared-model", {}) is False + + +@pytest.mark.asyncio +async def test_a_fallback_target_still_borrows_while_idle_and_its_tokens_count_against_that_teams_share( + monkeypatch: pytest.MonkeyPatch, +): + handler = _borrowing_handler(monkeypatch, borrows=True) + assert await handler.async_fallback_target_allowed(_team_key(), "shared-model", {}) is True + await _settle_tokens(handler, 150) + with pytest.raises(ProxyRateLimitError) as exc: + await _check(handler, _team_key(), saturation=0.5) + _held_to_team_limit(exc.value, "tokens", "model_tpm_limit") + + +@pytest.mark.asyncio +async def test_a_fallback_attempt_counts_toward_the_targets_saturation(monkeypatch: pytest.MonkeyPatch): + handler = _borrowing_handler(monkeypatch, borrows=True) + group: Final = handler.llm_router.registry().group("shared-model") + assert group is not None + assert await handler.async_fallback_target_allowed(_team_key(), "shared-model", {}) is True + assert await handler._check_model_saturation("shared-model", group) > 0 + + +@pytest.mark.asyncio +async def test_a_read_only_fallback_check_does_not_use_the_teams_share(monkeypatch: pytest.MonkeyPatch): + handler = _borrowing_handler(monkeypatch, borrows=True, saturation_threshold=0.0) + key: Final = _team_key( + team_limits={"model_tpm_limit": {"shared-model": 100}, "model_rpm_limit": {"shared-model": 2}} + ) + await _admit_then_settle(handler, key, 50) + assert await handler.async_fallback_target_allowed(key, "shared-model", {}, commit=False) is True + assert await handler.async_fallback_target_allowed(key, "shared-model", {}, commit=False) is True + assert await handler.async_fallback_target_allowed(key, "shared-model", {}) is True + assert await handler.async_fallback_target_allowed(key, "shared-model", {}) is False + + +async def _model_requests_remaining(handler: PROXY_DynamicRateLimitHandlerV3) -> int: + group: Final = handler.llm_router.registry().group("shared-model") + assert group is not None + descriptor: Final = handler._create_model_tracking_descriptor("shared-model", group, high_limit_multiplier=1) + response: Final = await handler.v3_limiter.should_rate_limit([descriptor], read_only=True, skip_parallel_check=True) + return next(s["limit_remaining"] for s in response["statuses"] if s["rate_limit_type"] == "requests") + + +def _fallback_prompt(words: int) -> dict[str, object]: + return {"messages": [{"role": "user", "content": "word " * words}], "max_tokens": 10} + + +@pytest.mark.asyncio +async def test_idle_fallbacks_past_the_teams_rpm_share_still_count_toward_the_targets_saturation( + monkeypatch: pytest.MonkeyPatch, +): + handler = _borrowing_handler(monkeypatch, borrows=True) + key: Final = _team_key(team_limits={"model_rpm_limit": {"shared-model": 1}}) + assert await handler.async_fallback_target_allowed(key, "shared-model", {}) is True + assert await handler.async_fallback_target_allowed(key, "shared-model", {}) is True + assert await handler.async_fallback_target_allowed(key, "shared-model", {}) is True + assert await _model_requests_remaining(handler) == 97 + + +@pytest.mark.asyncio +async def test_a_saturated_fallback_target_refuses_a_prompt_larger_than_the_teams_remaining_tokens( + monkeypatch: pytest.MonkeyPatch, +): + handler = _borrowing_handler(monkeypatch, borrows=True, saturation_threshold=0.0) + assert await handler.async_fallback_target_allowed(_team_key(), "shared-model", _fallback_prompt(300)) is False + assert ( + await handler.async_fallback_target_allowed(_team_key(), "shared-model", _fallback_prompt(300), commit=False) + is False + ) + assert await _team_tokens_remaining(handler) == 100 + assert ( + await handler.async_fallback_target_allowed(_team_key(), "shared-model", _fallback_prompt(30), commit=False) + is True + ) + assert await _team_tokens_remaining(handler) == 100 + assert await handler.async_fallback_target_allowed(_team_key(), "shared-model", _fallback_prompt(30)) is True + assert await _team_tokens_remaining(handler) < 70 + await _settle_tokens(handler, 5) + assert await _team_tokens_remaining(handler) == 95 + + +@pytest.mark.asyncio +async def test_a_saturated_responses_fallback_holds_the_output_budget_as_well_as_the_input( + monkeypatch: pytest.MonkeyPatch, +): + from litellm.proxy.hooks.parallel_request_limiter_v3 import get_or_create_request_stash + + handler = _borrowing_handler(monkeypatch, borrows=True, saturation_threshold=0.0) + get_or_create_request_stash().call_type = "aresponses" + prompt: Final = {"input": "hi", "max_output_tokens": 500} + assert await handler.async_fallback_target_allowed(_team_key(), "shared-model", prompt) is False + assert await _team_tokens_remaining(handler) == 100 + + +@pytest.mark.asyncio +async def test_an_output_token_limit_holds_only_the_prompts_output_budget(monkeypatch: pytest.MonkeyPatch): + from litellm.proxy import proxy_server + + monkeypatch.setitem(proxy_server.general_settings, "token_rate_limit_type", "output") + handler = _borrowing_handler(monkeypatch, borrows=True, saturation_threshold=0.0) + await _check(handler, _team_key(), saturation=1.0, data=_fallback_prompt(300)) + assert await _team_tokens_remaining(handler) == 90 + + +@pytest.mark.asyncio +async def test_a_request_without_max_tokens_leaves_admission_capped_to_the_teams_share( + monkeypatch: pytest.MonkeyPatch, +): + handler = _borrowing_handler(monkeypatch, borrows=True, saturation_threshold=0.0) + prompt: Final[dict[str, object]] = {"messages": [{"role": "user", "content": "hi"}]} + await _check(handler, _team_key(), saturation=1.0, data=prompt) + assert prompt["max_tokens"] == handler.v3_limiter.no_max_tokens_output_floor(100) + + +@pytest.mark.asyncio +async def test_a_prompt_that_fits_once_its_output_is_capped_is_admitted(monkeypatch: pytest.MonkeyPatch): + handler = _borrowing_handler(monkeypatch, borrows=True, saturation_threshold=0.0) + prompt: Final[dict[str, object]] = {"messages": [{"role": "user", "content": "word " * 60}]} + await _check(handler, _team_key(), saturation=1.0, data=prompt) + held: Final = handler.v3_limiter.estimate_request_tokens( + data={**prompt, "model": "shared-model"}, + model="shared-model", + min_configured_tpm_limit=100, + call_type="acompletion", + team_id="team-a", + ) + assert prompt["max_tokens"] == handler.v3_limiter.no_max_tokens_output_floor(100) + assert await _team_tokens_remaining(handler) == 100 - held + + +@pytest.mark.asyncio +async def test_a_refused_admission_leaves_the_requests_output_uncapped(monkeypatch: pytest.MonkeyPatch): + handler = _borrowing_handler(monkeypatch, borrows=True, saturation_threshold=0.0) + prompt: Final[dict[str, object]] = {"messages": [{"role": "user", "content": "word " * 300}]} + with pytest.raises(ProxyRateLimitError): + await _check(handler, _team_key(), saturation=1.0, data=prompt) + assert "max_tokens" not in prompt + + +@pytest.mark.asyncio +async def test_an_input_token_limit_leaves_the_reply_length_alone(monkeypatch: pytest.MonkeyPatch): + from litellm.proxy import proxy_server + + monkeypatch.setitem(proxy_server.general_settings, "token_rate_limit_type", "input") + handler = _borrowing_handler(monkeypatch, borrows=True, saturation_threshold=0.0) + prompt: Final[dict[str, object]] = {"messages": [{"role": "user", "content": "hi"}]} + await _check(handler, _team_key(), saturation=1.0, data=prompt) + assert "max_tokens" not in prompt + + +@pytest.mark.asyncio +async def test_a_late_failure_from_a_fallback_that_did_not_serve_is_charged_to_that_target( + monkeypatch: pytest.MonkeyPatch, +): + handler = _borrowing_handler(monkeypatch, borrows=True, saturation_threshold=0.0) + key: Final = _team_key(team_limits={"model_tpm_limit": {"shared-model": 100, "other-model": 100, "chat": 100}}) + await _check(handler, key, saturation=1.0, data=_fallback_prompt(30)) + assert await handler.async_fallback_target_allowed(key, "other-model", _fallback_prompt(30)) is True + assert await handler.async_fallback_target_allowed(key, "chat", _fallback_prompt(30)) is True + await _settle_tokens(handler, 7, model_group="chat") + await handler.async_log_failure_event( + kwargs={ + "litellm_params": {"metadata": {"model_group": "other-model"}}, + "combined_usage_object": litellm.Usage(prompt_tokens=20, completion_tokens=5, total_tokens=25), + }, + response_obj=Exception("stream cut"), + start_time=datetime.now(), + end_time=datetime.now(), + ) + assert await _team_tokens_remaining(handler, key, "shared-model") == 100 + assert await _team_tokens_remaining(handler, key, "chat") == 93 + assert await _team_tokens_remaining(handler, key, "other-model") == 75 + + +@pytest.mark.asyncio +async def test_a_served_fallbacks_hold_is_not_refunded_again_under_the_name_the_request_used( + monkeypatch: pytest.MonkeyPatch, +): + handler = _borrowing_handler( + monkeypatch, borrows=True, saturation_threshold=0.0, model_group_alias={"alias-model": "other-model"} + ) + key: Final = _team_key(team_limits={"model_tpm_limit": {"shared-model": 100, "other-model": 100}}) + await _check(handler, key, saturation=1.0, data=_fallback_prompt(30)) + assert await handler.async_fallback_target_allowed(key, "alias-model", _fallback_prompt(30)) is True + await _settle_tokens(handler, 7, model_group="other-model") + assert await _team_tokens_remaining(handler, key, "other-model") == 93 + await handler.async_log_failure_event( + kwargs={"litellm_params": {"metadata": {"model_group": "alias-model"}}}, + response_obj=Exception("late"), + start_time=datetime.now(), + end_time=datetime.now(), + ) + assert await _team_tokens_remaining(handler, key, "other-model") == 93 + + +@pytest.mark.asyncio +async def test_a_steered_requests_images_are_counted_for_the_model_that_serves_it(monkeypatch: pytest.MonkeyPatch): + from litellm.proxy.hooks.parallel_request_limiter_v3 import PROXY_MaxParallelRequestsHandler_v3 + + handler = _borrowing_handler(monkeypatch, borrows=True, saturation_threshold=0.0) + asked: Final = [] # mutable-ok: records the models whose image cost was looked up + + def resolver(model: str, team_id: str | None) -> tuple[str, ...]: + asked.append(model) + return (model,) + + handler.v3_limiter = PROXY_MaxParallelRequestsHandler_v3( + handler.internal_usage_cache, media_model_resolver=resolver + ) + key: Final = _team_key(team_limits={"model_tpm_limit": {"other-model": 10000}}) + pixel: Final = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNkYAAAAAYAAjCB0C8AAAAASUVORK5CYII=" + steered: Final[dict[str, object]] = { + "model": "shared-model", + "messages": [{"role": "user", "content": [{"type": "image_url", "image_url": {"url": pixel}}]}], + "max_tokens": 5, + } + await _check(handler, key, saturation=1.0, model="other-model", data=steered) + assert asked and set(asked) == {"other-model"} + + +@pytest.mark.asyncio +async def test_a_failed_attempt_that_recorded_usage_is_charged_that_usage_instead_of_refunded( + monkeypatch: pytest.MonkeyPatch, +): + handler = _borrowing_handler(monkeypatch, borrows=True, saturation_threshold=0.0) + await _check(handler, _team_key(), saturation=1.0, data=_fallback_prompt(30)) + assert await _team_tokens_remaining(handler) < 70 + await handler.async_log_failure_event( + kwargs={ + "litellm_params": {"metadata": {"model_group": "shared-model"}}, + "combined_usage_object": litellm.Usage(prompt_tokens=20, completion_tokens=5, total_tokens=25), + }, + response_obj=Exception("stream cut"), + start_time=datetime.now(), + end_time=datetime.now(), + ) + assert await _team_tokens_remaining(handler) == 75 + + +@pytest.mark.asyncio +async def test_a_fallback_attempt_that_fails_gives_back_the_tokens_it_held(monkeypatch: pytest.MonkeyPatch): + handler = _borrowing_handler(monkeypatch, borrows=True, saturation_threshold=0.0) + assert await handler.async_fallback_target_allowed(_team_key(), "shared-model", _fallback_prompt(30)) is True + assert await _team_tokens_remaining(handler) < 70 + await handler.async_log_failure_event( + kwargs={"litellm_params": {"metadata": {"model_group": "shared-model"}}}, + response_obj=None, + start_time=datetime.now(), + end_time=datetime.now(), + ) + assert await _team_tokens_remaining(handler) == 100 + + +@pytest.mark.asyncio +async def test_a_prompt_the_token_estimator_cannot_read_is_still_held_to_the_teams_share( + monkeypatch: pytest.MonkeyPatch, +): + handler = _borrowing_handler(monkeypatch, borrows=True) + key: Final = _team_key( + team_limits={"model_tpm_limit": {"shared-model": 100}, "model_rpm_limit": {"shared-model": 1}} + ) + unreadable: Final = { + "model": "shared-model", + "messages": [{"role": "user", "content": [{"type": "text", "text": 1}, {"type": "text", "text": "hi"}]}], + } + await _check(handler, key, saturation=0.5, data=unreadable) + with pytest.raises(ProxyRateLimitError) as exc: + await _check(handler, key, saturation=0.5, data=unreadable) + _held_to_team_limit(exc.value, "requests", "model_rpm_limit") + + +@pytest.mark.asyncio +async def test_a_success_without_a_model_group_still_releases_the_reservation(monkeypatch: pytest.MonkeyPatch): + handler = _borrowing_handler(monkeypatch, borrows=True) + await _check(handler, _team_key(), saturation=0.5, data=_prompt(words=30)) + assert await _team_tokens_remaining(handler) < 70 + await handler.async_log_success_event( + kwargs={"litellm_params": {"metadata": {}}}, + response_obj=litellm.ModelResponse(usage=litellm.Usage(prompt_tokens=5, completion_tokens=0, total_tokens=5)), + start_time=datetime.now(), + end_time=datetime.now(), + ) + assert await _team_tokens_remaining(handler) == 95 + + def _class_team(team_id: str, priority: str | None = "interactive") -> UserAPIKeyAuth: return UserAPIKeyAuth(api_key="sk-class", team_id=team_id, team_metadata={}, metadata={"priority": priority}) From 6d17f8bff5dd43dced619b17df318408bb4f224e Mon Sep 17 00:00:00 2001 From: Kerry Lu Date: Thu, 12 Nov 2026 07:59:03 -0800 Subject: [PATCH 009/168] feat(model-catalog): read output_cost_per_billable_unit in the rust registry check (#51365) * feat(model-catalog): read output_cost_per_billable_unit in the rust registry check * fix(model-catalog): add billable_unit to the flat key schema pattern --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/rust/crates/cost/src/price/meter.rs | 1 + litellm/rust/crates/model-catalog/src/model_info.rs | 1 + litellm/rust/crates/model-catalog/src/schema.rs | 4 +++- litellm/rust/crates/model-catalog/src/wire/flat_key.rs | 5 +++++ 4 files changed, 10 insertions(+), 1 deletion(-) diff --git a/litellm/rust/crates/cost/src/price/meter.rs b/litellm/rust/crates/cost/src/price/meter.rs index 09fd23e42ed..705ac97894c 100644 --- a/litellm/rust/crates/cost/src/price/meter.rs +++ b/litellm/rust/crates/cost/src/price/meter.rs @@ -50,6 +50,7 @@ pub enum OutputUnit { Pixel, Second(Option), VideoSecond, + BillableUnit, } /// `Explicit` is a cache the caller created ahead of time, spelled `explicit_cache_read_*`. `Implicit` is every other read diff --git a/litellm/rust/crates/model-catalog/src/model_info.rs b/litellm/rust/crates/model-catalog/src/model_info.rs index fcd0c027323..cee4ebbf9bd 100644 --- a/litellm/rust/crates/model-catalog/src/model_info.rs +++ b/litellm/rust/crates/model-catalog/src/model_info.rs @@ -122,6 +122,7 @@ pub struct ModelInfoFields { pub output_cost_per_second_4k: Option, pub output_cost_per_video_per_second: Option, pub output_cost_per_image: Option, + pub output_cost_per_billable_unit: Option, pub output_cost_per_pixel: Option, pub output_cost_per_image_token: Option, pub output_cost_per_image_token_batches: Option, diff --git a/litellm/rust/crates/model-catalog/src/schema.rs b/litellm/rust/crates/model-catalog/src/schema.rs index 60c755fe94f..2fc0aa70184 100644 --- a/litellm/rust/crates/model-catalog/src/schema.rs +++ b/litellm/rust/crates/model-catalog/src/schema.rs @@ -64,7 +64,7 @@ struct RegistryDocument { pub(crate) const FLAT_KEY_PATTERN: &str = concat!( r"^(", r"input_cost_per_((audio_|image_|video_)?token|character|image|pixel|query|request|second|text_message|audio_per_second|video_per_second(_above_\d+s_interval)?)", - r"|output_cost_per_((audio_|image_|video_|reasoning_)?token|character|image(_\d+(\.\d+)?K?)?|pixel|second(_\d+[pk])?|video_per_second)", + r"|output_cost_per_((audio_|image_|video_|reasoning_)?token|character|image(_\d+(\.\d+)?K?)?|pixel|second(_\d+[pk])?|video_per_second|billable_unit)", r"|cache_read_input_(audio_|image_|video_)?token_cost", r"|explicit_cache_read_input_token_cost", r"|cache_creation_input_(audio_|image_|video_)?token_cost(_above_1hr|_above_12hr)?", @@ -202,6 +202,8 @@ mod tests { #[case::feature_batch("annotation_cost_per_page_batches")] #[case::video_interval("input_cost_per_video_per_second_above_8s_interval")] #[case::explicit_cache_read("explicit_cache_read_input_token_cost")] + #[case::billable_unit("output_cost_per_billable_unit")] + #[case::billable_unit_is_output_only("input_cost_per_billable_unit")] #[case::wall_clock("cost_per_second")] #[case::unused_meter("cost_per_token")] #[case::other_currency("input_dbu_cost_per_token")] diff --git a/litellm/rust/crates/model-catalog/src/wire/flat_key.rs b/litellm/rust/crates/model-catalog/src/wire/flat_key.rs index 906d917b487..638a5349f05 100644 --- a/litellm/rust/crates/model-catalog/src/wire/flat_key.rs +++ b/litellm/rust/crates/model-catalog/src/wire/flat_key.rs @@ -210,6 +210,7 @@ fn output_unit(unit: &str) -> Option { "pixel" => Some(OutputUnit::Pixel), "second" => Some(OutputUnit::Second(None)), "video_per_second" => Some(OutputUnit::VideoSecond), + "billable_unit" => Some(OutputUnit::BillableUnit), _ => None, } } @@ -285,6 +286,10 @@ mod tests { None ) )] + #[case::billable_unit( + "output_cost_per_billable_unit", + key(Meter::Output(OutputUnit::BillableUnit), Band::Base, None) + )] #[case::image_token_is_a_token( "output_cost_per_image_token", key(Meter::Output(OutputUnit::Token(TokenKind::Image)), Band::Base, None) From 5a511b6fcc4c267317b48e70dac6541ad7f86b3f Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 12 Nov 2026 08:03:26 -0800 Subject: [PATCH 010/168] chore(cost-map): rolling registry audit 2026-11-12 (#51395) Absorb #51347, #51222, #51357 and #51275 with the GMI large-context tier. Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- ...odel_prices_and_context_window_backup.json | 82 +++++++++++++++++-- model_prices_and_context_window.json | 82 +++++++++++++++++-- .../test_model_latest_aliases.py | 2 +- 3 files changed, 149 insertions(+), 17 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 5a7e4e95764..66bffc90701 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -26230,6 +26230,9 @@ "video" ] }, + "fal_ai/bytedance/seedance-latest/text-to-video": { + "alias_of": "fal_ai/bytedance/seedance-2.5/text-to-video" + }, "fal_ai/bytedance/seedance-2.5/image-to-video": { "litellm_provider": "fal_ai", "mode": "video_generation", @@ -26248,6 +26251,9 @@ "video" ] }, + "fal_ai/bytedance/seedance-latest/image-to-video": { + "alias_of": "fal_ai/bytedance/seedance-2.5/image-to-video" + }, "fal_ai/bytedance/seedance-2.5/reference-to-video": { "litellm_provider": "fal_ai", "mode": "video_generation", @@ -26266,6 +26272,9 @@ "video" ] }, + "fal_ai/bytedance/seedance-latest/reference-to-video": { + "alias_of": "fal_ai/bytedance/seedance-2.5/reference-to-video" + }, "fal_ai/minimax/h3/text-to-video": { "litellm_provider": "fal_ai", "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 2k rate"}, @@ -56747,6 +56756,9 @@ "comment": "Output: 20/30/68 credits per second at 480p/720p/1080p @ $0.01 per credit; input video billed additionally at 10/15/34 credits per input second and the 80 credit minimum per task are not modeled" } }, + "runwayml/seedance-latest": { + "alias_of": "runwayml/seedance2_5" + }, "runwayml/hailuo3": { "litellm_provider": "runwayml", "mode": "video_generation", @@ -68564,10 +68576,12 @@ "xai/grok-imagine-image-quality": { "deprecation_date": "2026-11-02", "deprecation_source": "https://docs.x.ai/developers/migration/imagine-image-quality-nov-2", - "input_cost_per_image": 0.05, + "input_cost_per_image": 0.04, "litellm_provider": "xai", "mode": "image_generation", - "source": "https://docs.x.ai/docs/models", + "output_cost_per_image_1K": 0.04, + "output_cost_per_image_2K": 0.06, + "source": "https://docs.x.ai/developers/migration/imagine-image-quality-nov-2", "successor_model": "xai/grok-imagine-image-2.0", "supported_endpoints": [ "/v1/images/generations" @@ -68583,10 +68597,12 @@ "xai/grok-imagine-image-quality-20260403": { "deprecation_date": "2026-11-02", "deprecation_source": "https://docs.x.ai/developers/migration/imagine-image-quality-nov-2", - "input_cost_per_image": 0.05, + "input_cost_per_image": 0.04, "litellm_provider": "xai", "mode": "image_generation", - "source": "https://docs.x.ai/docs/models", + "output_cost_per_image_1K": 0.04, + "output_cost_per_image_2K": 0.06, + "source": "https://docs.x.ai/developers/migration/imagine-image-quality-nov-2", "successor_model": "xai/grok-imagine-image-2.0", "supported_endpoints": [ "/v1/images/generations" @@ -68602,10 +68618,12 @@ "xai/grok-imagine-image-quality-latest": { "deprecation_date": "2026-11-02", "deprecation_source": "https://docs.x.ai/developers/migration/imagine-image-quality-nov-2", - "input_cost_per_image": 0.05, + "input_cost_per_image": 0.04, "litellm_provider": "xai", "mode": "image_generation", - "source": "https://docs.x.ai/docs/models", + "output_cost_per_image_1K": 0.04, + "output_cost_per_image_2K": 0.06, + "source": "https://docs.x.ai/developers/migration/imagine-image-quality-nov-2", "successor_model": "xai/grok-imagine-image-2.0", "supported_endpoints": [ "/v1/images/generations" @@ -68621,10 +68639,12 @@ "xai/grok-imagine-image-pro": { "deprecation_date": "2026-05-15", "deprecation_source": "https://docs.x.ai/developers/migration/may-15-retirement", - "input_cost_per_image": 0.05, + "input_cost_per_image": 0.04, "litellm_provider": "xai", "mode": "image_generation", - "source": "https://docs.x.ai/docs/models", + "output_cost_per_image_1K": 0.04, + "output_cost_per_image_2K": 0.06, + "source": "https://docs.x.ai/developers/migration/imagine-image-quality-nov-2", "successor_model": "xai/grok-imagine-image-2.0", "supported_endpoints": [ "/v1/images/generations" @@ -77928,6 +77948,9 @@ "tpm": 4000000.0, "web_search_billing_unit": "per_query" }, + "gemini/nano-banana-latest": { + "alias_of": "gemini/gemini-nano-banana-2.1" + }, "gemini-nano-banana-2.1": { "cache_read_input_token_cost": 1.5e-07, "cache_read_input_token_cost_batches": 7.5e-08, @@ -78022,6 +78045,9 @@ "supports_web_search": true, "web_search_billing_unit": "per_query" }, + "vertex_ai/nano-banana-latest": { + "alias_of": "vertex_ai/gemini-nano-banana-2.1" + }, "azure_ai/grok-4.7": { "cache_read_input_token_cost": 5e-07, "cache_read_input_token_cost_above_200k_tokens": 1e-06, @@ -86829,6 +86855,15 @@ "output_cost_per_token": 1e-05, "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" }, + "gmi/google/gemini-4-carbon": { + "cache_read_input_token_cost": 1e-07, + "input_cost_per_token": 2e-06, + "litellm_provider": "gmi", + "max_input_tokens": 1048576, + "mode": "chat", + "output_cost_per_token": 1e-05, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, "gmi/google/gemma-4-26b-a4b-it": { "input_cost_per_token": 1.3e-07, "litellm_provider": "gmi", @@ -87162,6 +87197,18 @@ "output_cost_per_token": 6e-06, "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" }, + "gmi/xai/grok4.7": { + "cache_read_input_token_cost": 5e-07, + "cache_read_input_token_cost_above_200k_tokens": 1e-06, + "input_cost_per_token": 2e-06, + "input_cost_per_token_above_200k_tokens": 4e-06, + "litellm_provider": "gmi", + "max_input_tokens": 500000, + "mode": "chat", + "output_cost_per_token": 6e-06, + "output_cost_per_token_above_200k_tokens": 1.2e-05, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, "gmi/zai-org/GLM-4.5-Air-FP8": { "input_cost_per_token": 2e-07, "litellm_provider": "gmi", @@ -97755,6 +97802,9 @@ "output_cost_per_image_4K": 0.1134, "source": "https://replicate.com/google/nano-banana-2.1" }, + "replicate/google/nano-banana-latest": { + "alias_of": "replicate/google/nano-banana-2.1" + }, "replicate/heygen/avatar-iv": { "litellm_provider": "replicate", "mode": "video_generation", @@ -100827,5 +100877,21 @@ "supports_vision": true, "supports_web_search": false, "tpm": 1000000 + }, + "runwayml/kling_4": { + "litellm_provider": "runwayml", + "metadata": {"comment":"Promo rate through 2026-11-30 UTC: 12.56 credits per second (720p) and 18.8 (1080p) without input video @ $0.01 per credit = $0.1256 and $0.188 per second; list rate 15.7/23.5 credits per second applies from 2026-12-01 UTC (ruling runwayml-kling-4-promo-rate). The with-input-video rate (7.52/11.28 credits per output second plus the same per second of the greater of input and output duration) and rounding up to the next credit are not modeled"}, + "mode": "video_generation", + "output_cost_per_second": 0.1256, + "output_cost_per_second_1080p": 0.188, + "source": "https://docs.dev.runwayml.com/guides/pricing/", + "supported_modalities": [ + "text", + "image", + "video" + ], + "supported_output_modalities": [ + "video" + ] } } diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 5a7e4e95764..66bffc90701 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -26230,6 +26230,9 @@ "video" ] }, + "fal_ai/bytedance/seedance-latest/text-to-video": { + "alias_of": "fal_ai/bytedance/seedance-2.5/text-to-video" + }, "fal_ai/bytedance/seedance-2.5/image-to-video": { "litellm_provider": "fal_ai", "mode": "video_generation", @@ -26248,6 +26251,9 @@ "video" ] }, + "fal_ai/bytedance/seedance-latest/image-to-video": { + "alias_of": "fal_ai/bytedance/seedance-2.5/image-to-video" + }, "fal_ai/bytedance/seedance-2.5/reference-to-video": { "litellm_provider": "fal_ai", "mode": "video_generation", @@ -26266,6 +26272,9 @@ "video" ] }, + "fal_ai/bytedance/seedance-latest/reference-to-video": { + "alias_of": "fal_ai/bytedance/seedance-2.5/reference-to-video" + }, "fal_ai/minimax/h3/text-to-video": { "litellm_provider": "fal_ai", "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 2k rate"}, @@ -56747,6 +56756,9 @@ "comment": "Output: 20/30/68 credits per second at 480p/720p/1080p @ $0.01 per credit; input video billed additionally at 10/15/34 credits per input second and the 80 credit minimum per task are not modeled" } }, + "runwayml/seedance-latest": { + "alias_of": "runwayml/seedance2_5" + }, "runwayml/hailuo3": { "litellm_provider": "runwayml", "mode": "video_generation", @@ -68564,10 +68576,12 @@ "xai/grok-imagine-image-quality": { "deprecation_date": "2026-11-02", "deprecation_source": "https://docs.x.ai/developers/migration/imagine-image-quality-nov-2", - "input_cost_per_image": 0.05, + "input_cost_per_image": 0.04, "litellm_provider": "xai", "mode": "image_generation", - "source": "https://docs.x.ai/docs/models", + "output_cost_per_image_1K": 0.04, + "output_cost_per_image_2K": 0.06, + "source": "https://docs.x.ai/developers/migration/imagine-image-quality-nov-2", "successor_model": "xai/grok-imagine-image-2.0", "supported_endpoints": [ "/v1/images/generations" @@ -68583,10 +68597,12 @@ "xai/grok-imagine-image-quality-20260403": { "deprecation_date": "2026-11-02", "deprecation_source": "https://docs.x.ai/developers/migration/imagine-image-quality-nov-2", - "input_cost_per_image": 0.05, + "input_cost_per_image": 0.04, "litellm_provider": "xai", "mode": "image_generation", - "source": "https://docs.x.ai/docs/models", + "output_cost_per_image_1K": 0.04, + "output_cost_per_image_2K": 0.06, + "source": "https://docs.x.ai/developers/migration/imagine-image-quality-nov-2", "successor_model": "xai/grok-imagine-image-2.0", "supported_endpoints": [ "/v1/images/generations" @@ -68602,10 +68618,12 @@ "xai/grok-imagine-image-quality-latest": { "deprecation_date": "2026-11-02", "deprecation_source": "https://docs.x.ai/developers/migration/imagine-image-quality-nov-2", - "input_cost_per_image": 0.05, + "input_cost_per_image": 0.04, "litellm_provider": "xai", "mode": "image_generation", - "source": "https://docs.x.ai/docs/models", + "output_cost_per_image_1K": 0.04, + "output_cost_per_image_2K": 0.06, + "source": "https://docs.x.ai/developers/migration/imagine-image-quality-nov-2", "successor_model": "xai/grok-imagine-image-2.0", "supported_endpoints": [ "/v1/images/generations" @@ -68621,10 +68639,12 @@ "xai/grok-imagine-image-pro": { "deprecation_date": "2026-05-15", "deprecation_source": "https://docs.x.ai/developers/migration/may-15-retirement", - "input_cost_per_image": 0.05, + "input_cost_per_image": 0.04, "litellm_provider": "xai", "mode": "image_generation", - "source": "https://docs.x.ai/docs/models", + "output_cost_per_image_1K": 0.04, + "output_cost_per_image_2K": 0.06, + "source": "https://docs.x.ai/developers/migration/imagine-image-quality-nov-2", "successor_model": "xai/grok-imagine-image-2.0", "supported_endpoints": [ "/v1/images/generations" @@ -77928,6 +77948,9 @@ "tpm": 4000000.0, "web_search_billing_unit": "per_query" }, + "gemini/nano-banana-latest": { + "alias_of": "gemini/gemini-nano-banana-2.1" + }, "gemini-nano-banana-2.1": { "cache_read_input_token_cost": 1.5e-07, "cache_read_input_token_cost_batches": 7.5e-08, @@ -78022,6 +78045,9 @@ "supports_web_search": true, "web_search_billing_unit": "per_query" }, + "vertex_ai/nano-banana-latest": { + "alias_of": "vertex_ai/gemini-nano-banana-2.1" + }, "azure_ai/grok-4.7": { "cache_read_input_token_cost": 5e-07, "cache_read_input_token_cost_above_200k_tokens": 1e-06, @@ -86829,6 +86855,15 @@ "output_cost_per_token": 1e-05, "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" }, + "gmi/google/gemini-4-carbon": { + "cache_read_input_token_cost": 1e-07, + "input_cost_per_token": 2e-06, + "litellm_provider": "gmi", + "max_input_tokens": 1048576, + "mode": "chat", + "output_cost_per_token": 1e-05, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, "gmi/google/gemma-4-26b-a4b-it": { "input_cost_per_token": 1.3e-07, "litellm_provider": "gmi", @@ -87162,6 +87197,18 @@ "output_cost_per_token": 6e-06, "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" }, + "gmi/xai/grok4.7": { + "cache_read_input_token_cost": 5e-07, + "cache_read_input_token_cost_above_200k_tokens": 1e-06, + "input_cost_per_token": 2e-06, + "input_cost_per_token_above_200k_tokens": 4e-06, + "litellm_provider": "gmi", + "max_input_tokens": 500000, + "mode": "chat", + "output_cost_per_token": 6e-06, + "output_cost_per_token_above_200k_tokens": 1.2e-05, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, "gmi/zai-org/GLM-4.5-Air-FP8": { "input_cost_per_token": 2e-07, "litellm_provider": "gmi", @@ -97755,6 +97802,9 @@ "output_cost_per_image_4K": 0.1134, "source": "https://replicate.com/google/nano-banana-2.1" }, + "replicate/google/nano-banana-latest": { + "alias_of": "replicate/google/nano-banana-2.1" + }, "replicate/heygen/avatar-iv": { "litellm_provider": "replicate", "mode": "video_generation", @@ -100827,5 +100877,21 @@ "supports_vision": true, "supports_web_search": false, "tpm": 1000000 + }, + "runwayml/kling_4": { + "litellm_provider": "runwayml", + "metadata": {"comment":"Promo rate through 2026-11-30 UTC: 12.56 credits per second (720p) and 18.8 (1080p) without input video @ $0.01 per credit = $0.1256 and $0.188 per second; list rate 15.7/23.5 credits per second applies from 2026-12-01 UTC (ruling runwayml-kling-4-promo-rate). The with-input-video rate (7.52/11.28 credits per output second plus the same per second of the greater of input and output duration) and rounding up to the next credit are not modeled"}, + "mode": "video_generation", + "output_cost_per_second": 0.1256, + "output_cost_per_second_1080p": 0.188, + "source": "https://docs.dev.runwayml.com/guides/pricing/", + "supported_modalities": [ + "text", + "image", + "video" + ], + "supported_output_modalities": [ + "video" + ] } } diff --git a/tests/unit/litellm_core_utils/test_model_latest_aliases.py b/tests/unit/litellm_core_utils/test_model_latest_aliases.py index c6def915640..a6bb3f23a1d 100644 --- a/tests/unit/litellm_core_utils/test_model_latest_aliases.py +++ b/tests/unit/litellm_core_utils/test_model_latest_aliases.py @@ -38,7 +38,7 @@ def test_latest_aliases_price_and_route_like_their_targets() -> None: alias_key: Final = prefixed if prefixed in litellm.model_cost else alias target_key: Final = f"{provider}/{target}" if f"{provider}/{target}" in litellm.model_cost else target entry: Final = litellm.model_cost[alias_key] - assert entry["mode"] in ("chat", "responses") + assert entry["mode"] in ("chat", "responses", "video_generation", "image_generation") assert entry == litellm.model_cost[target_key] assert resolve_model_latest_alias(alias, provider) == target if provider in PROVIDERS_THAT_AUTHENTICATE_ON_PROVIDER_INFO: From d076cee903deae43a92688100a38924916e68d27 Mon Sep 17 00:00:00 2001 From: tin-berri Date: Thu, 12 Nov 2026 08:30:10 -0800 Subject: [PATCH 011/168] feat(skills): let authors edit and withdraw pending submissions (#51312) --- litellm/models/skill_submission.py | 12 +- litellm/proxy/_lazy_openapi_snapshot.json | 359 +++++++++++++++++- litellm/proxy/_types.py | 2 + .../skill_hub_mutations.py | 17 +- .../skill_hub_submissions.py | 181 ++++++++- litellm/proxy/proxy_server.py | 1 + .../skill_submission_repository.py | 88 ++++- litellm/types/proxy/skill_hub.py | 13 + ...laudeCodePluginsPanel.integration.test.tsx | 101 ++++- .../_components/ClaudeCodePluginsPanel.tsx | 1 + .../skills/_components/SkillSubmissions.tsx | 308 +++++++++------ .../skills/_components/add_plugin_form.tsx | 84 +++- .../src/components/networking.tsx | 32 ++ .../src/i18n/locales/en.json | 13 +- .../src/lib/http/schema.d.ts | 223 ++++++++++- .../database/test_skill_submissions.py | 57 +++ .../test_skill_hub_endpoints.py | 121 ++++++ ...t_request_body_decompression_middleware.py | 1 + .../test_request_size_limit_middleware.py | 1 + 19 files changed, 1468 insertions(+), 147 deletions(-) diff --git a/litellm/models/skill_submission.py b/litellm/models/skill_submission.py index 09d722fa69f..59587588a35 100644 --- a/litellm/models/skill_submission.py +++ b/litellm/models/skill_submission.py @@ -11,7 +11,7 @@ from litellm.models.plugin_upload import ( ) from litellm.types.proxy.skill_scanning import SkillScan -SubmissionStatus: TypeAlias = Literal["pending_review", "approved", "rejected"] +SubmissionStatus: TypeAlias = Literal["pending_review", "approved", "rejected", "withdrawn"] SubmissionOperation: TypeAlias = Literal["create", "update"] @@ -38,6 +38,7 @@ class SkillSubmission: reviewed_by: str | None reviewed_at: datetime | None rejection_reason: str | None + revision: int = 0 @dataclass(frozen=True, slots=True) @@ -99,6 +100,14 @@ class SkillSubmissionStore(Protocol): async def get_archive(self, submission_id: str, submitted_by: str | None) -> bytes | None: ... + async def edit( + self, submission_id: str, owner: str, revision: int, upload: PluginPublication + ) -> SkillSubmission | SubmissionMissing | SubmissionConflict | SubmissionLimitExceeded: ... + + async def withdraw( + self, submission_id: str, owner: str, revision: int + ) -> SkillSubmission | SubmissionMissing | SubmissionConflict: ... + async def review( self, submission_id: str, @@ -107,4 +116,5 @@ class SkillSubmissionStore(Protocol): reason: str | None, *, publication_base_url: str | None = None, + expected_revision: int | None = None, ) -> SubmissionReviewResult: ... diff --git a/litellm/proxy/_lazy_openapi_snapshot.json b/litellm/proxy/_lazy_openapi_snapshot.json index b99fc8b61e1..fa0c423fc25 100644 --- a/litellm/proxy/_lazy_openapi_snapshot.json +++ b/litellm/proxy/_lazy_openapi_snapshot.json @@ -59670,6 +59670,38 @@ "skill_hub": { "components": { "schemas": { + "Body_edit_submission_archive_skill_hub_submissions__submission_id__upload_put": { + "properties": { + "expected_revision": { + "minimum": 0.0, + "title": "Expected Revision", + "type": "integer" + }, + "file": { + "contentMediaType": "application/octet-stream", + "description": "Replacement Agent Skills ZIP with the same declared skill name", + "title": "File", + "type": "string" + }, + "version": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Version" + } + }, + "required": [ + "file", + "expected_revision" + ], + "title": "Body_edit_submission_archive_skill_hub_submissions__submission_id__upload_put", + "type": "object" + }, "Body_replace_skill_archive_skill_hub_skills__name__upload_put": { "properties": { "expected_updated_at": { @@ -60227,6 +60259,18 @@ "SkillHubRejection": { "additionalProperties": false, "properties": { + "expected_revision": { + "anyOf": [ + { + "minimum": 0.0, + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Expected Revision" + }, "reason": { "anyOf": [ { @@ -60536,6 +60580,12 @@ ], "title": "Base Updated At" }, + "can_edit": { + "default": false, + "description": "Whether the caller may edit or withdraw this pending submission", + "title": "Can Edit", + "type": "boolean" + }, "created_at": { "format": "date-time", "title": "Created At", @@ -60594,6 +60644,13 @@ ], "title": "Reviewed By" }, + "revision": { + "default": 0, + "description": "Proposal revision used to prevent stale edits and reviews", + "minimum": 0.0, + "title": "Revision", + "type": "integer" + }, "skill_name": { "title": "Skill Name", "type": "string" @@ -60602,7 +60659,8 @@ "enum": [ "pending_review", "approved", - "rejected" + "rejected", + "withdrawn" ], "title": "Status", "type": "string" @@ -60653,6 +60711,95 @@ "title": "SkillHubSubmissionDetail", "type": "object" }, + "SkillHubSubmissionEdit": { + "additionalProperties": false, + "properties": { + "description": { + "anyOf": [ + { + "maxLength": 1024, + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Description" + }, + "domain": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Domain" + }, + "expected_revision": { + "description": "Revision read from the submission detail", + "minimum": 0.0, + "title": "Expected Revision", + "type": "integer" + }, + "expected_updated_at": { + "anyOf": [ + { + "format": "date-time", + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Published updated_at read by the editor. A stale value returns 409", + "title": "Expected Updated At" + }, + "keywords": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Keywords" + }, + "namespace": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Namespace" + }, + "version": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Explicit higher SemVer, or omit/null for an automatic patch bump", + "title": "Version" + } + }, + "required": [ + "expected_revision" + ], + "title": "SkillHubSubmissionEdit", + "type": "object" + }, "SkillHubSubmissionList": { "properties": { "count": { @@ -60679,6 +60826,22 @@ "title": "SkillHubSubmissionList", "type": "object" }, + "SkillHubSubmissionRevision": { + "additionalProperties": false, + "properties": { + "expected_revision": { + "description": "Revision read from the submission detail", + "minimum": 0.0, + "title": "Expected Revision", + "type": "integer" + } + }, + "required": [ + "expected_revision" + ], + "title": "SkillHubSubmissionRevision", + "type": "object" + }, "SkillHubVersion": { "properties": { "content_kind": { @@ -62118,7 +62281,8 @@ "enum": [ "pending_review", "approved", - "rejected" + "rejected", + "withdrawn" ], "type": "string" }, @@ -62188,7 +62352,7 @@ }, "/skill-hub/submissions/{submission_id}": { "get": { - "description": "Read an immutable proposal and its currently visible published skill.", + "description": "Read a proposal revision and its currently visible published skill.", "operationId": "get_submission_skill_hub_submissions__submission_id__get", "parameters": [ { @@ -62232,6 +62396,62 @@ "tags": [ "skill_hub" ] + }, + "put": { + "description": "Replace a pending proposal's metadata, preserving files. Only its original submitting user may edit.", + "operationId": "edit_submission_skill_hub_submissions__submission_id__put", + "parameters": [ + { + "in": "path", + "name": "submission_id", + "required": true, + "schema": { + "title": "Submission Id", + "type": "string" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SkillHubSubmissionEdit" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SkillHubSubmission" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Edit Submission", + "tags": [ + "skill_hub" + ] } }, "/skill-hub/submissions/{submission_id}/approve": { @@ -62249,6 +62469,23 @@ } } ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "$ref": "#/components/schemas/SkillHubSubmissionRevision" + }, + { + "type": "null" + } + ], + "title": "Body" + } + } + } + }, "responses": { "200": { "content": { @@ -62506,6 +62743,122 @@ "skill_hub" ] } + }, + "/skill-hub/submissions/{submission_id}/upload": { + "put": { + "description": "Replace a pending proposal's files using the upload limits. Only its original submitting user may edit.", + "operationId": "edit_submission_archive_skill_hub_submissions__submission_id__upload_put", + "parameters": [ + { + "in": "path", + "name": "submission_id", + "required": true, + "schema": { + "title": "Submission Id", + "type": "string" + } + } + ], + "requestBody": { + "content": { + "multipart/form-data": { + "schema": { + "$ref": "#/components/schemas/Body_edit_submission_archive_skill_hub_submissions__submission_id__upload_put" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SkillHubSubmission" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Edit Submission Archive", + "tags": [ + "skill_hub" + ] + } + }, + "/skill-hub/submissions/{submission_id}/withdraw": { + "put": { + "description": "Withdraw a pending proposal without publishing it. Only its original submitting user may withdraw.", + "operationId": "withdraw_submission_skill_hub_submissions__submission_id__withdraw_put", + "parameters": [ + { + "in": "path", + "name": "submission_id", + "required": true, + "schema": { + "title": "Submission Id", + "type": "string" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SkillHubSubmissionRevision" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SkillHubSubmission" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Withdraw Submission", + "tags": [ + "skill_hub" + ] + } } } }, diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 1eab9d80484..804ad744989 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -470,6 +470,8 @@ class LiteLLMRoutes(enum.Enum): "/skill-hub/submissions", "/skill-hub/submissions/{submission_id}", "/skill-hub/submissions/{submission_id}/archive", + "/skill-hub/submissions/{submission_id}/upload", + "/skill-hub/submissions/{submission_id}/withdraw", "/skill-hub/submissions/{submission_id}/approve", "/skill-hub/submissions/{submission_id}/reject", "/skill-hub/submissions/{submission_id}/scan", diff --git a/litellm/proxy/discovery_endpoints/skill_hub_mutations.py b/litellm/proxy/discovery_endpoints/skill_hub_mutations.py index 63c519923c6..fff81b8a90c 100644 --- a/litellm/proxy/discovery_endpoints/skill_hub_mutations.py +++ b/litellm/proxy/discovery_endpoints/skill_hub_mutations.py @@ -58,6 +58,15 @@ def submission_owner(caller: UserAPIKeyAuth) -> str | None | SkillMutationError: return caller.user_id +def submission_editor(caller: UserAPIKeyAuth) -> str | SkillMutationError: + error: Final = writer_error(caller) + if error is not None: + return error + if not caller.user_id or caller.user_id == DB_UNAVAILABLE_FALLBACK_USER_ID: + return SkillMutationError(403, "Editing submissions requires the original submitting user's identity.") + return caller.user_id + + async def archive_writer(caller: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)]) -> UserAPIKeyAuth: from litellm.proxy.proxy_server import proxy_logging_obj @@ -149,13 +158,19 @@ async def review_skill( approve: bool, reason: str | None, publication_base_url: str | None = None, + expected_revision: int | None = None, ) -> SubmissionReviewed | SkillMutationError: if not is_proxy_admin(caller): return SkillMutationError(403, "Only proxy admins may review skill submissions.") if submissions is None: return SkillMutationError(503, "Skills Hub requires a connected database") result: Final = await submissions.review( - submission_id, caller.user_id or "proxy-admin", approve, reason, publication_base_url=publication_base_url + submission_id, + caller.user_id or "proxy-admin", + approve, + reason, + publication_base_url=publication_base_url, + expected_revision=expected_revision, ) match result: case SubmissionMissing(): diff --git a/litellm/proxy/discovery_endpoints/skill_hub_submissions.py b/litellm/proxy/discovery_endpoints/skill_hub_submissions.py index c1bb422b2a0..884e15e72a0 100644 --- a/litellm/proxy/discovery_endpoints/skill_hub_submissions.py +++ b/litellm/proxy/discovery_endpoints/skill_hub_submissions.py @@ -1,27 +1,49 @@ import asyncio +from dataclasses import replace from typing import Annotated, Final -from fastapi import APIRouter, Depends, HTTPException, Query, Request, Response +from fastapi import APIRouter, Depends, File, Form, HTTPException, Query, Request, Response, UploadFile -from litellm.models.plugin_upload import PluginPublication -from litellm.models.skill_submission import SkillSubmission, SkillSubmissionStore, SubmissionStatus +from litellm.models.plugin_upload import PluginPublication, PluginPublicationError, PluginUploadStore +from litellm.models.skill_submission import ( + SkillSubmission, + SkillSubmissionStore, + SubmissionConflict, + SubmissionLimitExceeded, + SubmissionMissing, + SubmissionStatus, +) from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.anthropic_endpoints.claude_code_endpoints.claude_code_skill_access import resolve_skill_visibility from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.deps.repositories import PluginUploadRepositoryDep, SkillSubmissionRepositoryDep from litellm.proxy.discovery_endpoints.agent_skills_archive import build_agent_skill_archive from litellm.proxy.discovery_endpoints.skill_hub_endpoints import ( + prepare_skill_publication, skill_metadata_response, skill_proposal_response, skill_publication_response, ) -from litellm.proxy.discovery_endpoints.skill_hub_mutations import SkillMutationError, review_skill, submission_owner +from litellm.proxy.discovery_endpoints.skill_hub_mutations import ( + SkillMutationError, + archive_writer, + publication_error, + review_skill, + submission_editor, + submission_owner, + write_target, +) +from litellm.proxy.discovery_endpoints.skill_import import MAX_IMPORT_BYTES +from litellm.repositories.plugin_publication import resolve_plugin_version from litellm.types.proxy.skill_hub import ( + SkillHubMetadata, SkillHubRejection, SkillHubReviewResponse, SkillHubSubmission, SkillHubSubmissionDetail, + SkillHubSubmissionEdit, SkillHubSubmissionList, + SkillHubSubmissionRevision, ) router: Final = APIRouter(prefix="/skill-hub/submissions", tags=["Skills Hub"]) @@ -40,7 +62,7 @@ def _owner(caller: UserAPIKeyAuth) -> str | None: return owner -def _response(request: Request, submission: SkillSubmission) -> SkillHubSubmission: +def _response(request: Request, submission: SkillSubmission, caller: UserAPIKeyAuth) -> SkillHubSubmission: return SkillHubSubmission( id=submission.id, skill_name=submission.plugin_name, @@ -55,6 +77,8 @@ def _response(request: Request, submission: SkillSubmission) -> SkillHubSubmissi reviewed_by=submission.reviewed_by, reviewed_at=submission.reviewed_at, rejection_reason=submission.rejection_reason, + revision=submission.revision, + can_edit=submission.status == "pending_review" and submission_editor(caller) == submission.submitted_by, ) @@ -71,7 +95,7 @@ async def list_submissions( owner: Final = _owner(caller) page: Final = await _store(submissions).list(owner, status, limit, offset) return SkillHubSubmissionList( - submissions=tuple(_response(request, row) for row in page.submissions), + submissions=tuple(_response(request, row, caller) for row in page.submissions), count=len(page.submissions), has_more=page.has_more, ) @@ -85,7 +109,7 @@ async def get_submission( uploads: PluginUploadRepositoryDep, caller: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)], ) -> SkillHubSubmissionDetail: - """Read an immutable proposal and its currently visible published skill.""" + """Read a proposal revision and its currently visible published skill.""" owner: Final = _owner(caller) submission: Final = await _store(submissions).get(submission_id, owner) if submission is None: @@ -95,7 +119,7 @@ async def get_submission( current: Final = await uploads.get_metadata(submission.plugin_name) visible: Final = current is not None and (owner is None or (await resolve_skill_visibility(caller)).allows(current)) return SkillHubSubmissionDetail( - submission=_response(request, submission), + submission=_response(request, submission, caller), current_skill=skill_metadata_response(request, current) if current is not None and visible else None, ) @@ -143,11 +167,12 @@ async def _review( caller: UserAPIKeyAuth, approve: bool, reason: str | None, + expected_revision: int | None, ) -> SkillHubReviewResponse: from litellm.proxy.utils import get_custom_url result: Final = await review_skill( - submission_id, submissions, caller, approve, reason, get_custom_url(str(request.base_url)) + submission_id, submissions, caller, approve, reason, get_custom_url(str(request.base_url)), expected_revision ) if isinstance(result, SkillMutationError): raise HTTPException(result.status, result.message) @@ -164,7 +189,7 @@ async def _review( ) return SkillHubReviewResponse( action="approved" if approve else "rejected", - submission=_response(request, result.submission), + submission=_response(request, result.submission, caller), skill=None if result.plugin is None else skill_publication_response(request, publication, result.plugin), ) @@ -175,9 +200,12 @@ async def approve_submission( submission_id: str, submissions: SkillSubmissionRepositoryDep, caller: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)], + body: SkillHubSubmissionRevision | None = None, ) -> SkillHubReviewResponse: """Publish a pending proposal and record its approval atomically. Only proxy admins may review.""" - return await _review(request, submission_id, submissions, caller, True, None) + return await _review( + request, submission_id, submissions, caller, True, None, None if body is None else body.expected_revision + ) @router.put("/{submission_id}/reject", response_model=SkillHubReviewResponse) @@ -189,4 +217,133 @@ async def reject_submission( body: SkillHubRejection | None = None, ) -> SkillHubReviewResponse: """Reject a pending proposal without changing the published skill. Only proxy admins may review.""" - return await _review(request, submission_id, submissions, caller, False, None if body is None else body.reason) + return await _review( + request, + submission_id, + submissions, + caller, + False, + None if body is None else body.reason, + None if body is None else body.expected_revision, + ) + + +def _editor(caller: UserAPIKeyAuth) -> str: + owner: Final = submission_editor(caller) + if isinstance(owner, SkillMutationError): + raise HTTPException(owner.status, owner.message) + return owner + + +def _changed( + result: SkillSubmission | SubmissionMissing | SubmissionConflict | SubmissionLimitExceeded, +) -> SkillSubmission: + if isinstance(result, SubmissionMissing): + raise HTTPException(404, "Skill submission not found") + if isinstance(result, SubmissionConflict): + raise HTTPException(409, result.message) + if isinstance(result, SubmissionLimitExceeded): + raise HTTPException(429, result.message) + return result + + +async def _editable( + submission_id: str, + owner: str, + store: SkillSubmissionStore, + revision: int, + version: str | None, + uploads: PluginUploadStore | None, + caller: UserAPIKeyAuth, +) -> SkillSubmission: + submission: Final = await store.get(submission_id, owner) + if submission is None: + raise HTTPException(404, "Skill submission not found") + if submission.status != "pending_review" or submission.revision != revision: + raise HTTPException(409, "This submission changed or is no longer pending. Reload it before trying again.") + existing: Final = await write_target( + submission.plugin_name, submission.operation == "update", uploads, caller, submission.base_updated_at + ) + if isinstance(existing, SkillMutationError): + raise HTTPException(existing.status, existing.message) + validation: Final = resolve_plugin_version( + version, None if existing is None else existing.version, updating=existing is not None + ) + if isinstance(validation, PluginPublicationError): + error: Final = publication_error(validation) + raise HTTPException(error.status, error.message) + return submission + + +@router.put("/{submission_id}", response_model=SkillHubSubmission) +async def edit_submission( + request: Request, + submission_id: str, + body: SkillHubSubmissionEdit, + submissions: SkillSubmissionRepositoryDep, + uploads: PluginUploadRepositoryDep, + caller: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)], +) -> SkillHubSubmission: + """Replace a pending proposal's metadata, preserving files. Only its original submitting user may edit.""" + owner: Final = _editor(caller) + store: Final = _store(submissions) + submission: Final = await _editable( + submission_id, owner, store, body.expected_revision, body.version, uploads, caller + ) + publication: Final = PluginPublication( + name=submission.plugin_name, + version=body.version, + description=body.description, + manifest={ + **{ + key: value + for key, value in submission.proposal.manifest.items() + if key not in SkillHubMetadata.model_fields + }, + **body.model_dump(exclude={"expected_revision"}, exclude_none=True), + }, + created_by=owner, + ) + result: Final = await store.edit(submission_id, owner, body.expected_revision, publication) + return _response(request, _changed(result), caller) + + +@router.put("/{submission_id}/upload", response_model=SkillHubSubmission) +async def edit_submission_archive( + request: Request, + submission_id: str, + submissions: SkillSubmissionRepositoryDep, + uploads: PluginUploadRepositoryDep, + caller: Annotated[UserAPIKeyAuth, Depends(archive_writer)], + file: Annotated[UploadFile, File(description="Replacement Agent Skills ZIP with the same declared skill name")], + expected_revision: Annotated[int, Form(ge=0)], + version: Annotated[str | None, Form()] = None, +) -> SkillHubSubmission: + """Replace a pending proposal's files using the upload limits. Only its original submitting user may edit.""" + owner: Final = _editor(caller) + store: Final = _store(submissions) + submission: Final = await _editable(submission_id, owner, store, expected_revision, version, uploads, caller) + content: Final = await file.read(MAX_IMPORT_BYTES + 1) + if len(content) > MAX_IMPORT_BYTES: + raise HTTPException(413, "Skill ZIP must be 10 MiB or smaller") + archive: Final = await asyncio.to_thread(build_agent_skill_archive, content) + if archive is None or archive.declared_name != submission.plugin_name: + raise HTTPException(422, "Replacement SKILL.md must declare the existing skill name and a description") + prepared: Final = await prepare_skill_publication(request, archive, version, None, owner) + publication: Final = replace(prepared, manifest={**submission.proposal.manifest, **prepared.manifest}) + result: Final = await store.edit(submission_id, owner, expected_revision, publication) + return _response(request, _changed(result), caller) + + +@router.put("/{submission_id}/withdraw", response_model=SkillHubSubmission) +async def withdraw_submission( + request: Request, + submission_id: str, + body: SkillHubSubmissionRevision, + submissions: SkillSubmissionRepositoryDep, + caller: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)], +) -> SkillHubSubmission: + """Withdraw a pending proposal without publishing it. Only its original submitting user may withdraw.""" + owner: Final = _editor(caller) + result: Final = await _store(submissions).withdraw(submission_id, owner, body.expected_revision) + return _response(request, _changed(result), caller) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 7bf9fcacde2..7fd114f9838 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -2993,6 +2993,7 @@ _ROUTE_MAX_REQUEST_SIZE_MB: Final[Mapping[tuple[str, str], float]] = MappingProx ("POST", "/claude-code/plugins/upload"): 11, ("POST", "/skill-hub/skills/upload"): 11, ("PUT", "/skill-hub/skills/{name}/upload"): 11, + ("PUT", "/skill-hub/submissions/{submission_id}/upload"): 11, } ) diff --git a/litellm/repositories/skill_submission_repository.py b/litellm/repositories/skill_submission_repository.py index 03821da042a..673fd7cca8f 100644 --- a/litellm/repositories/skill_submission_repository.py +++ b/litellm/repositories/skill_submission_repository.py @@ -7,7 +7,7 @@ from datetime import datetime, timezone from typing import TYPE_CHECKING, Final, Protocol, cast # noqa: TID251 # Prisma's generated delegates are dynamic from urllib.parse import urlsplit -from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError +from pydantic import BaseModel, ConfigDict, Field, TypeAdapter, ValidationError from litellm.models.plugin_upload import ( PluginPublication, @@ -45,6 +45,7 @@ class _StoredProposal(BaseModel): manifest: Mapping[str, object] version: str | None + revision: int = Field(default=0, ge=0) def proposal(self, name: str) -> SkillProposal: return SkillProposal( @@ -92,6 +93,7 @@ class _SubmissionRow(BaseModel): reviewed_by=self.reviewed_by, reviewed_at=self.reviewed_at, rejection_reason=self.rejection_reason, + revision=stored.revision, ) @@ -283,6 +285,71 @@ class SkillSubmissionRepository: ) return None if row is None or row.archive_content is None else row.archive_content.decode() + async def edit( + self, submission_id: str, owner: str, revision: int, upload: PluginPublication + ) -> SkillSubmission | SubmissionMissing | SubmissionConflict | SubmissionLimitExceeded: + from prisma import Base64 + + async with self._database.tx() as transaction: + await transaction.query_raw( + "SELECT 1 AS locked FROM pg_advisory_xact_lock(hashtext('litellm.skill_submissions'), hashtext($1))", + owner, + ) + row: Final = await _owned_pending(transaction, submission_id, owner, revision) + if isinstance(row, (SubmissionMissing, SubmissionConflict)): + return row + if upload.name != row.plugin_name: + return SubmissionConflict("A submission's skill name cannot change.") + proposal_json: Final = _StoredProposal( + manifest=upload.manifest, version=upload.version, revision=revision + 1 + ).model_dump_json() + archive: Final = row.archive_content if upload.content is None else Base64.encode(upload.content) + usage: Final = _USAGE.validate_python( + await transaction.query_raw( + """SELECT COUNT(*) AS count, + COALESCE(SUM(octet_length(proposal_json) + + COALESCE(octet_length(archive_content), 0)), 0) AS storage_bytes + FROM "LiteLLM_SkillSubmission" WHERE submitted_by = $1 AND status = 'pending_review'""", + owner, + ) + )[0] + previous_bytes: Final = len(row.proposal_json.encode()) + ( + 0 if row.archive_content is None else len(row.archive_content.decode()) + ) + incoming_bytes: Final = len(proposal_json.encode()) + (0 if archive is None else len(archive.decode())) + if usage.storage_bytes - previous_bytes + incoming_bytes > _MAX_PENDING_BYTES: + return SubmissionLimitExceeded( + "Pending skill submissions cannot exceed 50 MiB of stored metadata and archives per user." + ) + updated: Final = await transaction.litellm_skillsubmission.update( + where={"id": submission_id}, + data={"proposal_json": proposal_json, "archive_content": archive, "scan_json": None}, + ) + return SubmissionMissing() if updated is None else _submission(updated) + + async def withdraw( + self, submission_id: str, owner: str, revision: int + ) -> SkillSubmission | SubmissionMissing | SubmissionConflict: + async with self._database.tx() as transaction: + row: Final = await _owned_pending(transaction, submission_id, owner, revision) + if isinstance(row, (SubmissionMissing, SubmissionConflict)): + return row + receipt: Final = _StoredProposal( + manifest={"name": row.plugin_name, "source": {}}, + version=None, + revision=revision, + ) + updated: Final = await transaction.litellm_skillsubmission.update( + where={"id": submission_id}, + data={ + "status": "withdrawn", + "proposal_json": receipt.model_dump_json(), + "archive_content": None, + "scan_json": None, + }, + ) + return SubmissionMissing() if updated is None else _submission(updated) + async def review( self, submission_id: str, @@ -291,6 +358,7 @@ class SkillSubmissionRepository: reason: str | None, *, publication_base_url: str | None = None, + expected_revision: int | None = None, ) -> SubmissionReviewResult: from prisma.errors import UniqueViolationError @@ -304,7 +372,9 @@ class SkillSubmissionRepository: return SubmissionMissing() submission: Final = _submission(row) if submission.status != "pending_review": - return SubmissionConflict("This submission has already been reviewed.") + return SubmissionConflict("This submission is no longer pending review.") + if submission.revision != (expected_revision if expected_revision is not None else 0): + return SubmissionConflict("This submission changed. Reload it before reviewing.") content: Final = None if row.archive_content is None else row.archive_content.decode() if approve and content is not None and not publication_base_url: return SubmissionConflict("Uploaded skill approval requires a trusted publication URL.") @@ -344,6 +414,20 @@ class SkillSubmissionRepository: return SubmissionConflict("A skill with this name is already published. Submit a new update proposal.") +async def _owned_pending( + transaction: SubmissionTransaction, submission_id: str, owner: str, revision: int +) -> "prisma_models.LiteLLM_SkillSubmission | SubmissionMissing | SubmissionConflict": + await transaction.query_raw('SELECT id FROM "LiteLLM_SkillSubmission" WHERE id = $1 FOR UPDATE', submission_id) + row: Final = await transaction.litellm_skillsubmission.find_unique(where={"id": submission_id}) + if row is None or not owner or row.submitted_by != owner: + return SubmissionMissing() + if row.status != "pending_review": + return SubmissionConflict("Only pending submissions can be edited or withdrawn.") + if _StoredProposal.model_validate_json(row.proposal_json).revision != revision: + return SubmissionConflict("This submission changed. Reload it before trying again.") + return row + + async def _publish_submission( transaction: PluginPublicationTransaction, row: "prisma_models.LiteLLM_SkillSubmission", diff --git a/litellm/types/proxy/skill_hub.py b/litellm/types/proxy/skill_hub.py index 9d866f51feb..e045914ead2 100644 --- a/litellm/types/proxy/skill_hub.py +++ b/litellm/types/proxy/skill_hub.py @@ -156,6 +156,8 @@ class SkillHubSubmission(LiteLLMBaseModel): reviewed_by: str | None reviewed_at: datetime | None rejection_reason: str | None + revision: int = Field(0, ge=0, description="Proposal revision used to prevent stale edits and reviews") + can_edit: bool = Field(False, description="Whether the caller may edit or withdraw this pending submission") class SkillHubSubmissionList(LiteLLMBaseModel): @@ -173,6 +175,16 @@ class SkillHubSubmissionDetail(LiteLLMBaseModel): current_skill: SkillHubSkill | None +class SkillHubSubmissionRevision(LiteLLMBaseModel): + model_config = ConfigDict(frozen=True, extra="forbid") + + expected_revision: int = Field(ge=0, description="Revision read from the submission detail") + + +class SkillHubSubmissionEdit(SkillHubMetadata): + expected_revision: int = Field(ge=0, description="Revision read from the submission detail") + + class SkillHubReviewResponse(LiteLLMBaseModel): model_config = ConfigDict(frozen=True) @@ -186,3 +198,4 @@ class SkillHubRejection(LiteLLMBaseModel): model_config = ConfigDict(frozen=True, extra="forbid") reason: str | None = Field(None, max_length=2000) + expected_revision: int | None = Field(None, ge=0) diff --git a/litellm/ui/litellm-dashboard/src/app/(dashboard)/skills/_components/ClaudeCodePluginsPanel.integration.test.tsx b/litellm/ui/litellm-dashboard/src/app/(dashboard)/skills/_components/ClaudeCodePluginsPanel.integration.test.tsx index 9a260bdfc65..6ee327da914 100644 --- a/litellm/ui/litellm-dashboard/src/app/(dashboard)/skills/_components/ClaudeCodePluginsPanel.integration.test.tsx +++ b/litellm/ui/litellm-dashboard/src/app/(dashboard)/skills/_components/ClaudeCodePluginsPanel.integration.test.tsx @@ -31,6 +31,8 @@ const published: SkillHubSkill = { const pending: SkillSubmission = { id: "submission-1", + revision: 0, + can_edit: false, skill_name: published.name, operation: "update", status: "pending_review", @@ -70,8 +72,9 @@ let reviewFailure: string | null; let directPublication: boolean; const respondToWrite = (path: string, body: RequestInit["body"]) => { + if (reviewFailure && path.startsWith("/skill-hub/submissions/")) + return Response.json({ detail: { error: reviewFailure } }, { status: 409 }); if (path.endsWith("/approve") || path.endsWith("/reject")) { - if (reviewFailure) return Response.json({ detail: { error: reviewFailure } }, { status: 409 }); const approved = path.endsWith("/approve"); const reason = typeof body === "string" ? JSON.parse(body).reason : null; submissions = submissions.map((submission) => @@ -88,6 +91,19 @@ const respondToWrite = (path: string, body: RequestInit["body"]) => { }; return Response.json(result); } + if (path === `/skill-hub/submissions/${pending.id}`) { + const metadata = typeof body === "string" ? JSON.parse(body) : {}; + submissions = submissions.map((item) => ({ + ...item, + revision: (item.revision ?? 0) + 1, + proposal: { ...item.proposal, ...metadata }, + })); + return Response.json(submissions[0]); + } + if (path.endsWith("/withdraw")) { + submissions = submissions.map((item) => ({ ...item, status: "withdrawn", can_edit: false })); + return Response.json(submissions[0]); + } if (directPublication) { catalog = [{ ...published, version: "3.0.0", description: "Admin-published instructions" }]; return Response.json(catalog[0]); @@ -573,7 +589,7 @@ it("saves an admin rejection reason for the author and keeps the published metad expect(writes[0]).toMatchObject({ path: `/skill-hub/submissions/${pending.id}/reject`, method: "PUT", - body: JSON.stringify({ reason: "Please include setup instructions" }), + body: JSON.stringify({ expected_revision: 0, reason: "Please include setup instructions" }), }); fireEvent.click(screen.getByRole("tab", { name: "Published skills" })); await waitFor(() => @@ -792,3 +808,84 @@ it("applies tag and scan filters across the catalog and resets pagination when s expect(catalogRequests.at(-1)?.searchParams.get("tag")).toBe("engineering"); expect(catalogRequests.at(-1)?.searchParams.get("security_scan_passed")).toBe("true"); }); + +it("lets an owner edit the viewed proposal and withdraw it with its current revision", async () => { + submissions = [{ ...pending, can_edit: true, revision: 3, proposal: { ...pending.proposal, version: "3.0.0" } }]; + renderWithProviders(, { + searchParams: "tab=submissions", + }); + fireEvent.click(await screen.findByRole("button", { name: "View details" })); + fireEvent.click(await screen.findByRole("button", { name: "Edit submission" })); + expect(screen.getByLabelText("Version override (optional)")).toHaveValue("3.0.0"); + expect( + screen.getByText( + `Current version ${published.version}. Leave blank for the next patch version, or enter a higher version`, + ), + ).toBeInTheDocument(); + fireEvent.change(await screen.findByLabelText("Description (Optional)"), { + target: { value: "Owner revised instructions" }, + }); + fireEvent.click(screen.getByRole("button", { name: "Save submission" })); + expect(await screen.findByRole("cell", { name: "Owner revised instructions" })).toBeInTheDocument(); + expect(JSON.parse(String(writes[0].body))).toMatchObject({ + expected_revision: 3, + description: "Owner revised instructions", + }); + expect(catalog).toEqual([published]); + fireEvent.click(screen.getByRole("button", { name: "Withdraw submission" })); + expect(writes).toHaveLength(1); + fireEvent.click(await screen.findByRole("button", { name: "Confirm withdrawal" })); + expect(await screen.findByText("Withdrawn")).toBeInTheDocument(); + expect(writes[1]).toMatchObject({ + path: `/skill-hub/submissions/${pending.id}/withdraw`, + body: JSON.stringify({ expected_revision: 4 }), + }); + fireEvent.click(screen.getByRole("button", { name: "View details" })); + expect(screen.queryByRole("button", { name: "Edit submission" })).not.toBeInTheDocument(); + expect(screen.queryByRole("button", { name: "Withdraw submission" })).not.toBeInTheDocument(); +}); + +it("keeps admin review separate from ownership and sends the displayed revision", async () => { + submissions = [{ ...pending, revision: 7, can_edit: false }]; + renderSubmissions(); + fireEvent.click(await screen.findByRole("button", { name: "Review" })); + expect(screen.queryByRole("button", { name: "Edit submission" })).not.toBeInTheDocument(); + expect(screen.queryByRole("button", { name: "Withdraw submission" })).not.toBeInTheDocument(); + fireEvent.click(await screen.findByRole("button", { name: "Approve and publish" })); + expect(await screen.findByText("Approved")).toBeInTheDocument(); + expect(JSON.parse(String(writes[0].body))).toEqual({ expected_revision: 7 }); +}); + +it.each(["metadata", "upload"])( + "reloads a conflicted %s edit only when the author discards the draft", + async (mode) => { + submissions = [{ ...pending, can_edit: true }]; + renderWithProviders(, { + searchParams: "tab=submissions", + }); + fireEvent.click(await screen.findByRole("button", { name: "View details" })); + fireEvent.click(await screen.findByRole("button", { name: "Edit submission" })); + fireEvent.change(screen.getByLabelText("Description (Optional)"), { target: { value: "Unsaved draft" } }); + if (mode === "upload") { + fireEvent.click(screen.getByRole("radio", { name: "Upload ZIP" })); + fireEvent.change(screen.getByLabelText("Skill ZIP file"), { + target: { files: [new File(["replacement"], "skill.zip", { type: "application/zip" })] }, + }); + } + submissions = [ + { ...pending, can_edit: true, revision: 1, proposal: { ...pending.proposal, description: "Latest edit" } }, + ]; + reviewFailure = "This submission changed. Reload it before trying again."; + fireEvent.click(screen.getByRole("button", { name: "Save submission" })); + expect(await screen.findByRole("alert")).toHaveTextContent(reviewFailure); + expect(screen.getByRole("dialog", { name: "Edit submission" })).toBeInTheDocument(); + reviewFailure = null; + fireEvent.click(screen.getByRole("button", { name: "Discard draft and reload submission" })); + expect(await screen.findByRole("cell", { name: "Latest edit" })).toBeInTheDocument(); + fireEvent.click(screen.getByRole("button", { name: "Edit submission" })); + expect(screen.getByLabelText("Description (Optional)")).toHaveValue("Latest edit"); + fireEvent.click(screen.getByRole("button", { name: "Save submission" })); + await waitFor(() => expect(writes).toHaveLength(2)); + expect(JSON.parse(String(writes[1].body))).toMatchObject({ expected_revision: 1, description: "Latest edit" }); + }, +); diff --git a/litellm/ui/litellm-dashboard/src/app/(dashboard)/skills/_components/ClaudeCodePluginsPanel.tsx b/litellm/ui/litellm-dashboard/src/app/(dashboard)/skills/_components/ClaudeCodePluginsPanel.tsx index d0ca11f829a..c17246fbe2e 100644 --- a/litellm/ui/litellm-dashboard/src/app/(dashboard)/skills/_components/ClaudeCodePluginsPanel.tsx +++ b/litellm/ui/litellm-dashboard/src/app/(dashboard)/skills/_components/ClaudeCodePluginsPanel.tsx @@ -245,6 +245,7 @@ const SkillsPanelContent: React.FC = ({ accessToken={accessToken} queryScope={queryScope} canReview={isAdmin} + canWrite={canWrite} adminView={adminView} onChanged={refreshSkills} /> diff --git a/litellm/ui/litellm-dashboard/src/app/(dashboard)/skills/_components/SkillSubmissions.tsx b/litellm/ui/litellm-dashboard/src/app/(dashboard)/skills/_components/SkillSubmissions.tsx index 8d20855e4f4..9865610db4c 100644 --- a/litellm/ui/litellm-dashboard/src/app/(dashboard)/skills/_components/SkillSubmissions.tsx +++ b/litellm/ui/litellm-dashboard/src/app/(dashboard)/skills/_components/SkillSubmissions.tsx @@ -7,6 +7,7 @@ import { getSkillSubmission, listSkillSubmissions, reviewSkillSubmission, + withdrawSkillSubmission, } from "@/components/networking"; import type { SkillHubSkill, SkillSubmission } from "@/components/claude_code_plugins/types"; import { Badge } from "@/components/ui/badge"; @@ -16,6 +17,16 @@ import { Textarea } from "@/components/ui/textarea"; import { useUrlTableState, type UrlTableStateOptions } from "@/components/shared/DataTable/useUrlTableState"; import SkillScanPanel from "./SkillScanPanel"; import { toast } from "@/lib/toast"; +import AddPluginForm from "./add_plugin_form"; +import { + AlertDialog, + AlertDialogCancel, + AlertDialogContent, + AlertDialogDescription, + AlertDialogFooter, + AlertDialogHeader, + AlertDialogTitle, +} from "@/components/ui/alert-dialog"; const METADATA_FIELDS = ["name", "version", "description", "source", "keywords", "domain", "namespace"] as const; @@ -28,6 +39,7 @@ const STATUS_CLASSES = { pending_review: "border-warning/30 text-warning", approved: "border-success/30 text-success", rejected: "border-destructive/30 text-destructive", + withdrawn: "border-muted text-muted-foreground", }; const SUBMISSION_TABLE_STATE: UrlTableStateOptions = { @@ -82,51 +94,63 @@ function SubmissionSummary({ submission }: { submission: SkillSubmission }) { function SubmissionComparison({ submission, current, + onDownload, + downloading, }: { submission: SkillSubmission; current?: SkillHubSkill | null; + onDownload: () => Promise; + downloading: boolean; }) { const { t } = useTranslation(); return ( -
- - - - - - - - - - {METADATA_FIELDS.map((field) => ( - - + <> + {!current &&

{t("skills.submissions.noCurrent")}

} +
+
{t("skills.submissions.field")}{t("skills.submissions.current")}{t("skills.submissions.proposed")}
{t(`skills.submissions.fields.${field}`)}
+ + + + + + + + + {METADATA_FIELDS.map((field) => ( + + + + + + ))} + + + - - ))} - - - - - - -
{t("skills.submissions.field")}{t("skills.submissions.current")}{t("skills.submissions.proposed")}
{t(`skills.submissions.fields.${field}`)} +
+                    {displayValue(current?.[field]) || t("skills.submissions.notSet")}
+                  
+
+
+                    {field === "version" && submission.operation === "update" && !submission.proposal.version
+                      ? t("skills.submissions.nextPatchVersion")
+                      : displayValue(submission.proposal[field]) || t("skills.submissions.notSet")}
+                  
+
{t("skills.submissions.fields.agent_skill")}{t("skills.submissions.descriptorUnavailable")}
-                  {displayValue(current?.[field]) || t("skills.submissions.notSet")}
-                
-
-
-                  {field === "version" && submission.operation === "update" && !submission.proposal.version
-                    ? t("skills.submissions.nextPatchVersion")
-                    : displayValue(submission.proposal[field]) || t("skills.submissions.notSet")}
+                  {displayValue(submission.proposal.agent_skill) || t("skills.submissions.notSet")}
                 
{t("skills.submissions.fields.agent_skill")}{t("skills.submissions.descriptorUnavailable")} -
-                {displayValue(submission.proposal.agent_skill) || t("skills.submissions.notSet")}
-              
-
-
+ + + + {submission.has_archive && submission.proposal.agent_skill && ( + + )} + ); } @@ -135,33 +159,46 @@ interface SubmissionReviewProps { accessToken: string; queryScope: QueryKey; canReview: boolean; + canWrite: boolean; onClose: () => void; onChanged: () => Promise; } -function SubmissionReview({ id, accessToken, queryScope, canReview, onClose, onChanged }: SubmissionReviewProps) { +function SubmissionReview({ + id, + accessToken, + queryScope, + canReview, + canWrite, + onClose, + onChanged, +}: SubmissionReviewProps) { const { t } = useTranslation(); const [reason, setReason] = useState(""); const [busy, setBusy] = useState(false); const [downloading, setDownloading] = useState(false); const [error, setError] = useState(null); - const detail = useQuery({ + const [editing, setEditing] = useState(null); + const [withdrawing, setWithdrawing] = useState(null); + const detailOptions = { queryKey: [...queryScope, "submission", id], queryFn: () => getSkillSubmission(accessToken, id), - }); + refetchOnWindowFocus: false, + refetchOnReconnect: false, + }; + const detail = useQuery(detailOptions); const { submission, current_skill: current } = detail.data ?? {}; const canDecide = canReview && submission?.status === "pending_review"; + const actionsDisabled = busy || detail.isFetching || detail.isError; const review = async (action: "approve" | "reject") => { - if (!canDecide || busy) return; + if (!canDecide || !submission || busy) return; setBusy(true); setError(null); try { - await reviewSkillSubmission( - accessToken, - id, - action, - action === "reject" ? { reason: reason.trim() || null } : undefined, - ); + await reviewSkillSubmission(accessToken, id, action, { + expected_revision: submission.revision ?? 0, + ...(action === "reject" ? { reason: reason.trim() || null } : {}), + }); onClose(); await onChanged(); toast.success(t(action === "approve" ? "skills.submissions.approved" : "skills.submissions.rejected")); @@ -172,6 +209,24 @@ function SubmissionReview({ id, accessToken, queryScope, canReview, onClose, onC setBusy(false); } }; + const withdraw = async () => { + if (!withdrawing || busy) return; + setBusy(true); + setError(null); + try { + await withdrawSkillSubmission(accessToken, withdrawing); + setWithdrawing(null); + onClose(); + await onChanged(); + toast.success(t("skills.submissions.withdrawn")); + } catch (err) { + setError(err instanceof Error ? err.message : t("skills.submissions.actionFailed")); + setWithdrawing(null); + await onChanged(); + } finally { + setBusy(false); + } + }; const download = async () => { if (!submission || downloading) return; setDownloading(true); @@ -185,73 +240,111 @@ function SubmissionReview({ id, accessToken, queryScope, canReview, onClose, onC } }; return ( - !open && !busy && onClose()}> - - - {t("skills.submissions.reviewTitle", { name: submission?.skill_name ?? "" })} - - {detail.isPending &&

{t("skills.submissions.loading")}

} - {detail.isError && ( -
- {t("skills.submissions.loadFailed")} - -
- )} - {submission && ( - <> - - {!current &&

{t("skills.submissions.noCurrent")}

} - - {submission.has_archive && submission.proposal.agent_skill && ( - - )} - {canReview && ( - + )} + {submission && ( + <> + + - )} - {canDecide && ( - <> - -