From 103efe969bd8ce62c10f2eacc6baf4e75321ec99 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 29 Oct 2026 03:34:41 -0700 Subject: [PATCH] fix(router): count a Responses WebSocket turn's tokens before the next turn is admitted --- litellm/constants.py | 1 + litellm/llms/custom_httpx/llm_http_handler.py | 4 +- litellm/proxy/litellm_pre_call_utils.py | 2 + litellm/responses/main.py | 4 +- litellm/responses/streaming_iterator.py | 22 +- litellm/router.py | 144 +++++++++++-- litellm/router_strategy/lowest_tpm_rpm_v2.py | 60 +++--- .../track_deployment_metrics.py | 10 +- .../unit/proxy/test_litellm_pre_call_utils.py | 3 + .../test_responses_websocket_all_providers.py | 63 ++++++ .../test_router/test_router_helper_utils.py | 192 +++++++++++++++++- 11 files changed, 447 insertions(+), 58 deletions(-) diff --git a/litellm/constants.py b/litellm/constants.py index 42ad624026c..236ad61feeb 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -1666,6 +1666,7 @@ ROUTING_REQUEST_TAGS_METADATA_KEY: Final = "_routing_request_tags" BAGGAGE_MAPPED_TAGS_METADATA_KEY: Final = "_baggage_mapped_tags" VECTOR_STORE_CALLER_METADATA_KWARG: Final = "litellm_vector_store_caller_metadata" ROUTER_USAGE_COUNTED_TOKENS_METADATA_KEY: Final = "_litellm_router_usage_counted_tokens" +USAGE_BASED_ROUTING_COUNTED_TOKENS_METADATA_KEY: Final = "_litellm_usage_based_routing_counted_tokens" INTERNAL_CALL_ORIGIN_METADATA_KEY: Final = "internal_call_origin" BILLED_RETRY_ATTEMPT_METADATA_KEY: Final = "billed_retry_attempt" MANAGED_BACKGROUND_RESPONSE_METADATA_KEY: Final = "_managed_background_response" diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index e7da6586b97..8ff1e2d83df 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -244,7 +244,7 @@ if TYPE_CHECKING: FakeAnthropicMessagesStreamIterator, ) from litellm.llms.base_llm.passthrough.transformation import BasePassthroughConfig - from litellm.responses.streaming_iterator import ResponsesWsFrameGuard + from litellm.responses.streaming_iterator import ResponsesWsFrameGuard, ResponsesWsTurnUsageRecorder from litellm.types.google_genai.main import GenerateContentResponse from litellm.types.llms.openai_evals import ( CancelEvalResponse, @@ -7143,6 +7143,7 @@ class BaseLLMHTTPHandler: first_message: str | None = None, request_defaults: ResponsesWebSocketRequestDefaults | None = None, turn_admission: Callable[[], Awaitable[None]] | None = None, + turn_usage_recorder: "ResponsesWsTurnUsageRecorder | None" = None, frame_guard: "ResponsesWsFrameGuard | None" = None, **kwargs: object, ) -> Exception | None: @@ -7303,6 +7304,7 @@ class BaseLLMHTTPHandler: custom_llm_provider=custom_llm_provider, request_defaults=request_defaults, turn_admission=turn_admission, + turn_usage_recorder=turn_usage_recorder, frame_guard=frame_guard, ) return await streaming.bidirectional_forward() diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index 14d941ff62c..81b06ab80c2 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -45,6 +45,7 @@ from litellm.constants import ( SESSION_ID_GENERATED_METADATA_KEY, SESSION_ID_OMITTED_METADATA_KEY, TYPESAFE_DANGER_ZONE_HEADER_PREFIX, + USAGE_BASED_ROUTING_COUNTED_TOKENS_METADATA_KEY, VECTOR_STORE_CALLER_METADATA_KWARG, X_LITELLM_DISABLE_CALLBACKS, ) @@ -518,6 +519,7 @@ _ROUTER_RESERVED_METADATA_FIELDS: Final = frozenset( "request_retry_count", CLIENT_OUTPUT_CEILING_METADATA_KEY, ROUTER_USAGE_COUNTED_TOKENS_METADATA_KEY, + USAGE_BASED_ROUTING_COUNTED_TOKENS_METADATA_KEY, } ) _ALLOW_CLIENT_PRICING_OVERRIDE_METADATA_KEY: Final = "allow_client_pricing_override" diff --git a/litellm/responses/main.py b/litellm/responses/main.py index d474ed3991f..ce0cbf89280 100644 --- a/litellm/responses/main.py +++ b/litellm/responses/main.py @@ -78,7 +78,7 @@ from litellm.types.llms.openai import ( # Handle ResponseText import with fallback if TYPE_CHECKING: - from litellm.responses.streaming_iterator import ResponsesWsFrameGuard + from litellm.responses.streaming_iterator import ResponsesWsFrameGuard, ResponsesWsTurnUsageRecorder from litellm.types.llms.openai import ResponseText else: ResponseText = str # Fallback for ResponseText import @@ -2781,6 +2781,7 @@ async def _aresponses_websocket( api_key: str | None = None, timeout: float | None = None, turn_admission: Callable[[], Awaitable[None]] | None = None, + turn_usage_recorder: "ResponsesWsTurnUsageRecorder | None" = None, responses_ws_frame_guard: "ResponsesWsFrameGuard | None" = None, **kwargs, ) -> Exception | None: @@ -2875,6 +2876,7 @@ async def _aresponses_websocket( custom_llm_provider=_custom_llm_provider, request_defaults=_build_responses_websocket_request_defaults(deployment_kwargs), turn_admission=turn_admission, + turn_usage_recorder=turn_usage_recorder, frame_guard=responses_ws_frame_guard, **remaining_kwargs, ) diff --git a/litellm/responses/streaming_iterator.py b/litellm/responses/streaming_iterator.py index 6c57bfd2a3f..3232237b0f4 100644 --- a/litellm/responses/streaming_iterator.py +++ b/litellm/responses/streaming_iterator.py @@ -2245,6 +2245,7 @@ class GuardedResponsesWsFrame: ResponsesWsFrameGuard: TypeAlias = Callable[[str], Awaitable[GuardedResponsesWsFrame]] +ResponsesWsTurnUsageRecorder: TypeAlias = Callable[[Mapping[str, object], int], Awaitable[None]] async def _reject_guarded_turn( @@ -2444,6 +2445,7 @@ class ResponsesWebSocketStreaming: custom_llm_provider: str | None = None, request_defaults: ResponsesWebSocketRequestDefaults | None = None, turn_admission: Callable[[], Awaitable[None]] | None = None, + turn_usage_recorder: ResponsesWsTurnUsageRecorder | None = None, frame_guard: ResponsesWsFrameGuard | None = None, ): self.websocket = websocket @@ -2466,6 +2468,7 @@ class ResponsesWebSocketStreaming: self.authorized_model: str | None = authorized_model self.request_defaults: ResponsesWebSocketRequestDefaults | None = request_defaults self.turn_admission: Callable[[], Awaitable[None]] | None = turn_admission + self.turn_usage_recorder: ResponsesWsTurnUsageRecorder | None = turn_usage_recorder def _start_turn( self, client_frame: str | None, guardrail_information: tuple[Mapping[str, object], ...] = () @@ -2537,7 +2540,7 @@ class ResponsesWebSocketStreaming: "pii_tokens": {**(known if _is_str_mapping(known) else {}), **pii_tokens}, } - def _record_backend_event(self, event: str) -> None: + async def _record_backend_event(self, event: str) -> None: event_obj: Final = _json_object_or_none(event) if event_obj is None or event_obj.get("type") not in RESPONSES_WS_LOGGED_EVENT_TYPES: return @@ -2546,10 +2549,25 @@ class ResponsesWebSocketStreaming: turn: Final = self._open_turns[0].with_event(event_obj) if event_obj.get("type") in _RESPONSES_WS_TERMINAL_EVENT_TYPES: self._open_turns = self._open_turns[1:] + if event_obj.get("type") in _HISTORY_TERMINAL_EVENT_TYPES: + await self._record_turn_usage(turn) self._log_turn(turn) return self._open_turns = (turn, *self._open_turns[1:]) + async def _record_turn_usage(self, turn: _ResponsesWsTurn) -> None: + from litellm.cost_calculator import ResponsesWebSocketTokenUsageProcessor + + turn_litellm_params: Final = _json_object_or_none( + turn.logging_obj.litellm_params # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # Logging.litellm_params is untyped + ) + if self.turn_usage_recorder is None or turn_litellm_params is None: + return + usage: Final = ResponsesWebSocketTokenUsageProcessor.collect_and_combine_usage_from_responses_ws_results( + turn.events + ) + await self.turn_usage_recorder(turn_litellm_params, usage.total_tokens) + def _log_open_turns(self) -> None: open_turns: Final = self._open_turns self._open_turns = () @@ -2634,7 +2652,7 @@ class ResponsesWebSocketStreaming: # Log the output-masked form so PII redacted by apply_to_output # guardrails does not appear in success logs. - self._record_backend_event(wrapped_str) + await self._record_backend_event(wrapped_str) await self.websocket.send_text(wrapped_str) diff --git a/litellm/router.py b/litellm/router.py index e01f6474ded..7c3cbeb393a 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -102,6 +102,7 @@ from litellm.constants import ( ROUTING_REQUEST_TAGS_METADATA_KEY, RUNTIME_UPDATABLE_ROUTER_SETTINGS, SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY, + USAGE_BASED_ROUTING_COUNTED_TOKENS_METADATA_KEY, ) from litellm.integrations.custom_guardrail import CustomGuardrail, is_guardrail_intervention from litellm.integrations.custom_logger import CustomLogger @@ -869,6 +870,30 @@ def _request_model_group(kwargs: Mapping[str, object]) -> str | None: return model_group if isinstance(model_group, str) else None +def _routed_deployment_id(deployment_metadata: Mapping[str, object]) -> str | None: + model_info: Final = deployment_metadata.get("model_info") + deployment_id: Final = ( + _MODEL_INFO_ADAPTER.validate_python(model_info).get("id") if isinstance(model_info, dict) else None + ) + return None if deployment_id is None else str(deployment_id) + + +async def _count_before_callbacks( + deployment_metadata: dict[str, object], # mutable-ok: the success callbacks read the marker from it + marker: str, + tokens: int, + count: Callable[[], Awaitable[object]] | None, +) -> None: + if count is None: + return + deployment_metadata[marker] = tokens # rebind-ok: the success callbacks read the marker off this call's metadata + try: + await count() + except Exception: + deployment_metadata.pop(marker, None) + raise + + def _mid_stream_error_before_output(error: "MidStreamFallbackError") -> bool: return not error.generated_content or error.is_pre_first_chunk @@ -6423,7 +6448,10 @@ class Router: "model": model_name, **_with_router_resolved_session_model(kwargs.get("session"), model_name), **( - {"turn_admission": partial(self.async_admit_deployment_turn, model, deployment, kwargs)} + { + "turn_admission": partial(self.async_admit_deployment_turn, model, deployment, kwargs), + "turn_usage_recorder": partial(self.async_record_deployment_turn_usage, model, kwargs), + } if generic_function_name == "_aresponses_websocket" else {} ), @@ -9565,30 +9593,114 @@ class Router: model_group: Final = request_kwargs.get("model") if deployment_metadata is None or not isinstance(model_group, str): return - model_info: Final = deployment_metadata["model_info"] - deployment_id: Final = model_info.get("id") if isinstance(model_info, dict) else None - if deployment_id is None: - return total_tokens: Final = response_total_token_count(response) - deployment_name: Final = deployment_metadata.get("deployment") - deployment_metadata[ROUTER_USAGE_COUNTED_TOKENS_METADATA_KEY] = total_tokens - try: - await self._increment_deployment_usage( - deployment_id=str(deployment_id), - deployment_name=deployment_name if isinstance(deployment_name, str) else None, + await _count_before_callbacks( + deployment_metadata, + ROUTER_USAGE_COUNTED_TOKENS_METADATA_KEY, + total_tokens, + self._deployment_usage_count( + deployment_metadata=deployment_metadata, model_group=model_group, total_tokens=total_tokens, - rpm_increment=1, parent_otel_span=get_parent_otel_span_from_kwargs(request_kwargs), - ) - except Exception: - deployment_metadata.pop(ROUTER_USAGE_COUNTED_TOKENS_METADATA_KEY, None) - raise + ), + ) except Exception as e: verbose_router_logger.debug( "litellm.router.Router::increment_deployment_usage_for_response(): Exception occured - %s", e ) + async def async_record_deployment_turn_usage( + self, + model_group: str, + request_kwargs: Mapping[str, object], + turn_litellm_params: Mapping[str, object], + total_tokens: int, + ) -> None: + try: + deployment_metadata: Final = find_deployment_metadata(turn_litellm_params) + if deployment_metadata is None: + return + parent_otel_span: Final = get_parent_otel_span_from_kwargs(request_kwargs) + await _count_before_callbacks( + deployment_metadata, + ROUTER_USAGE_COUNTED_TOKENS_METADATA_KEY, + total_tokens, + self._deployment_usage_count( + deployment_metadata=deployment_metadata, + model_group=model_group, + total_tokens=total_tokens, + parent_otel_span=parent_otel_span, + ), + ) + await _count_before_callbacks( + deployment_metadata, + USAGE_BASED_ROUTING_COUNTED_TOKENS_METADATA_KEY, + total_tokens, + self._routing_strategy_usage_count( + deployment_metadata=deployment_metadata, + model_group=model_group, + request_kwargs=request_kwargs, + total_tokens=total_tokens, + parent_otel_span=parent_otel_span, + ), + ) + except Exception as e: # noqa: BLE001 # a failed early count must not break the turn; its logging counts it + verbose_router_logger.debug( + "litellm.router.Router::async_record_deployment_turn_usage(): Exception occured - %s", e + ) + + def _deployment_usage_count( + self, + *, + deployment_metadata: Mapping[str, object], + model_group: str, + total_tokens: int, + parent_otel_span: Span | None, + ) -> Callable[[], Awaitable[object]] | None: + deployment_id: Final = _routed_deployment_id(deployment_metadata) + if deployment_id is None: + return None + deployment_name: Final = deployment_metadata.get("deployment") + return partial( + self._increment_deployment_usage, + deployment_id=deployment_id, + deployment_name=deployment_name if isinstance(deployment_name, str) else None, + model_group=model_group, + total_tokens=total_tokens, + rpm_increment=1, + parent_otel_span=parent_otel_span, + ) + + def _routing_strategy_usage_count( + self, + *, + deployment_metadata: Mapping[str, object], + model_group: str, + request_kwargs: Mapping[str, object], + total_tokens: int, + parent_otel_span: Span | None, + ) -> Callable[[], Awaitable[object]] | None: + routing_kwargs: Final = {key: value for key, value in request_kwargs.items() if key != "litellm_logging_obj"} + _, selector = self._get_routing_context( # pyright: ignore[reportUnknownMemberType] # legacy signature takes an untyped request dict + model=model_group, request_kwargs=routing_kwargs + ) + deployment_id: Final = _routed_deployment_id(deployment_metadata) + deployment_name: Final = deployment_metadata.get("deployment") + if ( + not isinstance(selector, LowestTPMLoggingHandler_v2) + or deployment_id is None + or not isinstance(deployment_name, str) + ): + return None + return partial( + selector.async_increment_tpm, + deployment_id=deployment_id, + deployment_name=deployment_name, + tokens=total_tokens, + parent_otel_span=parent_otel_span, + ) + async def _increment_deployment_usage( self, *, diff --git a/litellm/router_strategy/lowest_tpm_rpm_v2.py b/litellm/router_strategy/lowest_tpm_rpm_v2.py index 0719b30d36d..5999e3ace96 100644 --- a/litellm/router_strategy/lowest_tpm_rpm_v2.py +++ b/litellm/router_strategy/lowest_tpm_rpm_v2.py @@ -8,15 +8,19 @@ from dataclasses import dataclass from typing import TYPE_CHECKING, Any, Final import httpx +from pydantic import TypeAdapter +from typing_extensions import ReadOnly, TypedDict import litellm from litellm import token_counter from litellm._internal_context import with_service_target from litellm._logging import verbose_logger, verbose_router_logger from litellm.caching.caching import DualCache +from litellm.constants import USAGE_BASED_ROUTING_COUNTED_TOKENS_METADATA_KEY from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.batch_utils import is_batch_retrieve_call_type from litellm.litellm_core_utils.core_helpers import get_parent_otel_span_from_kwargs +from litellm.router_utils.router_callbacks.track_deployment_metrics import get_counted_usage_tokens from litellm.types.router import RouterErrors from litellm.types.utils import LiteLLMPydanticObjectBase, StandardLoggingPayload from litellm.utils import get_utc_datetime, print_verbose @@ -35,6 +39,13 @@ class RoutingArgs(LiteLLMPydanticObjectBase): ttl: int = 1 * 60 # 1min (RPM/TPM expire key) +class _SuccessKwargs(TypedDict, total=False): + litellm_params: ReadOnly[Mapping[str, object] | None] + + +_SUCCESS_KWARGS_ADAPTER: Final = TypeAdapter(_SuccessKwargs) + + _active_prefetched_usage: Final[ContextVar["PrefetchedUsage | None"]] = ContextVar("prefetched_usage", default=None) @@ -296,7 +307,7 @@ class LowestTPMLoggingHandler_v2(BaseRoutingStrategy, CustomLogger): ) @with_service_target("router_usage") - async def async_log_success_event(self, kwargs, response_obj, start_time, end_time): + async def async_log_success_event(self, kwargs, response_obj, start_time, end_time) -> None: if is_batch_retrieve_call_type(kwargs.get("call_type")): return try: @@ -308,30 +319,19 @@ class LowestTPMLoggingHandler_v2(BaseRoutingStrategy, CustomLogger): raise ValueError("standard_logging_object not passed in.") model_group: Final = standard_logging_object.get("model_group") model: Final = standard_logging_object["hidden_params"]["litellm_model_name"] - id = standard_logging_object.get("model_id") - if model_group is None or id is None: - return - elif isinstance(id, int): - id = str(id) + model_id: Final = standard_logging_object.get("model_id") total_tokens: Final = standard_logging_object.get("total_tokens") - # ------------ - # Setup values - # ------------ - dt: Final = get_utc_datetime() - current_minute: Final = dt.strftime("%H-%M") # use the same timezone regardless of system clock - - tpm_key: Final = f"{id}:{model}:tpm:{current_minute}" - # ------------ - # Update usage - # ------------ - # update cache - parent_otel_span: Final = get_parent_otel_span_from_kwargs(kwargs) - ## TPM - await self.router_cache.async_increment_cache_post_call( - key=tpm_key, - value=total_tokens, - ttl=self.routing_args.ttl, - parent_otel_span=parent_otel_span, + if model_group is None or not isinstance(model_id, (str, int)) or not isinstance(total_tokens, int): + return + counted_tokens: Final = get_counted_usage_tokens( + _SUCCESS_KWARGS_ADAPTER.validate_python(kwargs).get("litellm_params") or {}, + USAGE_BASED_ROUTING_COUNTED_TOKENS_METADATA_KEY, + ) + await self.async_increment_tpm( + deployment_id=str(model_id), + deployment_name=model if isinstance(model, str) else None, + tokens=total_tokens if counted_tokens is None else max(0, total_tokens - counted_tokens), + parent_otel_span=get_parent_otel_span_from_kwargs(kwargs), ) ### TESTING ### @@ -342,6 +342,18 @@ class LowestTPMLoggingHandler_v2(BaseRoutingStrategy, CustomLogger): "litellm.proxy.hooks.lowest_tpm_rpm_v2.py::async_log_success_event(): Exception occured - %s", e ) + @with_service_target("router_usage") + async def async_increment_tpm( + self, *, deployment_id: str, deployment_name: str | None, tokens: int, parent_otel_span: Span | None + ) -> None: + current_minute: Final = get_utc_datetime().strftime("%H-%M") + await self.router_cache.async_increment_cache_post_call( + key=f"{deployment_id}:{deployment_name}:tpm:{current_minute}", + value=tokens, + ttl=self.routing_args.ttl, + parent_otel_span=parent_otel_span, + ) + def _return_potential_deployments( self, healthy_deployments: list[dict], diff --git a/litellm/router_utils/router_callbacks/track_deployment_metrics.py b/litellm/router_utils/router_callbacks/track_deployment_metrics.py index 6b422e98ec8..2866cdbc0ca 100644 --- a/litellm/router_utils/router_callbacks/track_deployment_metrics.py +++ b/litellm/router_utils/router_callbacks/track_deployment_metrics.py @@ -29,14 +29,12 @@ def find_deployment_metadata(kwargs: Mapping[str, object]) -> dict[str, object] return next((bucket for bucket in buckets if isinstance(bucket, dict) and "model_info" in bucket), None) -def get_counted_usage_tokens(litellm_params: Mapping[str, object]) -> int | None: +def get_counted_usage_tokens( + litellm_params: Mapping[str, object], marker: str = ROUTER_USAGE_COUNTED_TOKENS_METADATA_KEY +) -> int | None: buckets: Final = (litellm_params.get(channel) for channel in _METADATA_CHANNELS) counted: Final = next( - ( - bucket[ROUTER_USAGE_COUNTED_TOKENS_METADATA_KEY] - for bucket in buckets - if isinstance(bucket, dict) and ROUTER_USAGE_COUNTED_TOKENS_METADATA_KEY in bucket - ), + (bucket[marker] for bucket in buckets if isinstance(bucket, dict) and marker in bucket), None, ) return counted if isinstance(counted, int) and not isinstance(counted, bool) else None diff --git a/tests/unit/proxy/test_litellm_pre_call_utils.py b/tests/unit/proxy/test_litellm_pre_call_utils.py index 81a196af676..d777953b501 100644 --- a/tests/unit/proxy/test_litellm_pre_call_utils.py +++ b/tests/unit/proxy/test_litellm_pre_call_utils.py @@ -25,6 +25,7 @@ from litellm.constants import ( SERVER_STREAMING_CLASSIFICATION_MARKER, SESSION_ID_GENERATED_METADATA_KEY, SESSION_ID_OMITTED_METADATA_KEY, + USAGE_BASED_ROUTING_COUNTED_TOKENS_METADATA_KEY, ) from litellm.litellm_core_utils.core_helpers import get_litellm_metadata_from_kwargs from litellm.litellm_core_utils.get_provider_specific_headers import ( @@ -8759,6 +8760,7 @@ _PLANTED_STAMPS = { "previous_models": [{"exception_type": "Timeout", "deployment_id": "spoofed", "attempted_retries": 101}], "_client_output_ceiling": {"api_base": "https://attacker.example"}, ROUTER_USAGE_COUNTED_TOKENS_METADATA_KEY: 10**9, + USAGE_BASED_ROUTING_COUNTED_TOKENS_METADATA_KEY: 10**9, "client_key": "client_value", } @@ -8793,6 +8795,7 @@ async def test_add_litellm_data_to_request_strips_router_reserved_stamps_from_bo assert "request_retry_count" not in updated["metadata"] assert "previous_models" not in updated["metadata"] assert ROUTER_USAGE_COUNTED_TOKENS_METADATA_KEY not in updated["metadata"] + assert USAGE_BASED_ROUTING_COUNTED_TOKENS_METADATA_KEY not in updated["metadata"] assert updated["metadata"]["client_key"] == "client_value" diff --git a/tests/unit/responses/test_responses_websocket_all_providers.py b/tests/unit/responses/test_responses_websocket_all_providers.py index ab52e6d29c3..2e4ff06646e 100644 --- a/tests/unit/responses/test_responses_websocket_all_providers.py +++ b/tests/unit/responses/test_responses_websocket_all_providers.py @@ -1582,6 +1582,9 @@ class TestWebSocketProjectQuotaEnforcement: quota_callback.enforce_project_io_token_quota_for_frame.assert_awaited_once() +_TURN_USAGE: Final = {"input_tokens": 20, "output_tokens": 15, "total_tokens": 35} + + def _turn_frames() -> tuple[str, str, str]: return tuple(json.dumps({"type": "response.create", "input": word}) for word in ("one", "two", "three")) @@ -1662,6 +1665,66 @@ def _errors(websocket: _ClientSocket) -> list[tuple[object, object]]: class TestWebSocketTurnAdmission: """A turn after the connection's first is admitted by the deployment again before it reaches the provider.""" + @pytest.mark.asyncio + @pytest.mark.parametrize( + ("terminal_event", "counted_before_it_reaches_the_client"), + [ + ({"type": "response.completed", "response": {"usage": _TURN_USAGE}}, [("ws-dep", 35)]), + ({"type": "response.incomplete", "response": {"usage": _TURN_USAGE}}, [("ws-dep", 35)]), + ({"type": "response.failed", "response": {"usage": _TURN_USAGE, "error": {"code": "server_error"}}}, []), + ], + ) + async def test_native_relay_counts_a_finished_turn_before_forwarding_its_terminal_event( + self, terminal_event: dict[str, object], counted_before_it_reaches_the_client: list[tuple[object, int]] + ): + from unittest.mock import AsyncMock + + import websockets.exceptions + + counted: Final[list[tuple[object, int]]] = [] + + async def record_turn_usage(turn_litellm_params: Mapping[str, object], total_tokens: int) -> None: + await asyncio.sleep(0) + metadata: Final = turn_litellm_params["litellm_metadata"] + assert isinstance(metadata, dict) + counted.append((metadata["model_info"]["id"], total_tokens)) + + class Backend: + def __init__(self, *events: dict[str, object]) -> None: + self.events: Final = list(events) + + async def recv(self, decode: bool = False) -> str: + if not self.events: + raise websockets.exceptions.ConnectionClosed(None, None) + return json.dumps(self.events.pop(0)) + + class Client: + def __init__(self) -> None: + self.seen: Final[list[tuple[object, list[tuple[object, int]]]]] = [] + + async def send_text(self, data: str) -> None: + self.seen.append((json.loads(data)["type"], list(counted))) + + logging_obj: Final = MagicMock() + logging_obj.litellm_params = {"litellm_metadata": {"model_info": {"id": "ws-dep"}}} + logging_obj.copy_for_new_call.return_value = logging_obj + logging_obj.dispatch_success_handlers = AsyncMock() + logging_obj.dispatch_failure_handlers = AsyncMock() + client: Final = Client() + handler: Final = _make_streaming( + websocket=client, + backend_ws=Backend({"type": "response.created", "response": {}}, terminal_event), + logging_obj=logging_obj, + turn_usage_recorder=record_turn_usage, + ) + + await handler.backend_to_client() + + assert client.seen == [ + ("response.created", []), + (terminal_event["type"], counted_before_it_reaches_the_client), + ] + @pytest.mark.asyncio async def test_native_relay_drops_a_later_turn_the_deployment_refuses(self): from unittest.mock import AsyncMock diff --git a/tests/unit/test_router/test_router_helper_utils.py b/tests/unit/test_router/test_router_helper_utils.py index 513136ae150..032085026dc 100644 --- a/tests/unit/test_router/test_router_helper_utils.py +++ b/tests/unit/test_router/test_router_helper_utils.py @@ -4,7 +4,7 @@ import gc import json import os import weakref -from collections.abc import Iterator, Mapping +from collections.abc import Awaitable, Callable, Iterator, Mapping from datetime import datetime, timezone from types import MethodType from typing import Final @@ -19,8 +19,11 @@ from litellm.caching.in_memory_cache import InMemoryCache from litellm.constants import DEFAULT_AUTO_ROUTER_MAX_INPUT_CHARS, ROUTER_USAGE_COUNTED_TOKENS_METADATA_KEY from litellm.integrations.custom_logger import CustomLogger from litellm.integrations.SlackAlerting.slack_alerting import SlackAlerting +from litellm.litellm_core_utils.get_litellm_params import get_litellm_params +from litellm.litellm_core_utils.litellm_logging import Logging from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper from litellm.router_strategy.adaptive_router.hooks import AdaptiveRouterPostCallHook +from litellm.types.caching import RedisPipelineIncrementOperation from litellm.types.litellm_params import RoutingStrategyName from litellm.types.router import AlertingConfig, Deployment, DeploymentTypedDict, LiteLLM_Params, ModelInfo from litellm.types.utils import ( @@ -545,31 +548,203 @@ async def _kwargs_routed_to(router: Router, model: str, function_name: str) -> d routed.update(kwargs) call.__name__ = function_name - await router._ageneric_api_call_with_fallbacks_helper(model=model, original_generic_function=call) + await router._ageneric_api_call_with_fallbacks_helper( + model=model, original_generic_function=call, litellm_metadata={"model_group": model} + ) return routed -async def _finish_35_token_websocket_turn(routed: Mapping[str, object], call_id: str) -> None: +def _open_websocket_turn(routed: Mapping[str, object], call_id: str) -> Logging: import time - from litellm.litellm_core_utils.litellm_logging import Logging - - turn: Final = Logging( + connection: Final = Logging( model="gpt-5-mini", messages=[], stream=False, call_type=CallTypes.aresponses_websocket.value, start_time=time.time(), - litellm_call_id=call_id, + litellm_call_id=f"{call_id}-connection", function_id=call_id, ) - turn.update_environment_variables(litellm_params={"litellm_metadata": routed["litellm_metadata"]}, optional_params={}) + connection.update_environment_variables(litellm_params=get_litellm_params(**routed), optional_params={}) + return connection.copy_for_new_call(litellm_call_id=call_id, start_time=datetime.now()) + + +async def _complete_35_token_turn(turn: Logging) -> None: await turn.dispatch_success_handlers( [{"type": "response.completed", "response": {"usage": {"input_tokens": 20, "output_tokens": 15, "total_tokens": 35}}}], prefer_async_handlers=True, ) +async def _finish_35_token_websocket_turn(routed: Mapping[str, object], call_id: str) -> None: + await _complete_35_token_turn(_open_websocket_turn(routed, call_id)) + + +def _with_success_callbacks_of(monkeypatch: pytest.MonkeyPatch, build_router: Callable[[], Router]) -> Router: + monkeypatch.setattr(litellm, "callbacks", []) + monkeypatch.setattr(litellm, "success_callback", []) + monkeypatch.setattr(litellm, "_async_success_callback", []) + router: Final = build_router() + monkeypatch.setattr(litellm, "_async_success_callback", [*litellm._async_success_callback, *litellm.callbacks]) + return router + + +def _usage_based_routing_router(monkeypatch: pytest.MonkeyPatch, model_name: str, tpm: int) -> Router: + return _with_success_callbacks_of( + monkeypatch, + lambda: Router( + model_list=[ + { + "model_name": model_name, + "litellm_params": {"model": "openai/gpt-5-mini", "api_key": "sk-fake", "tpm": tpm}, + "model_info": {"id": f"{model_name}-dep"}, + } + ], + routing_strategy="usage-based-routing-v2", + ), + ) + + +_TurnAdmission = Callable[[], Awaitable[None]] +_TurnUsageRecorder = Callable[[Mapping[str, object], int], Awaitable[None]] + + +async def _websocket_turn_hooks( + router: Router, model_name: str +) -> tuple[dict[str, object], _TurnAdmission, _TurnUsageRecorder]: + routed: Final = await _kwargs_routed_to(router, model_name, "_aresponses_websocket") + admit_turn: Final = routed["turn_admission"] + record_turn: Final = routed["turn_usage_recorder"] + assert callable(admit_turn) + assert callable(record_turn) + return routed, admit_turn, record_turn + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("router_minute_pinned") +async def test_usage_based_routing_refuses_the_next_websocket_turn_before_the_finished_turn_s_logging_runs( + monkeypatch: pytest.MonkeyPatch, +): + router: Final = _usage_based_routing_router(monkeypatch, "ws-tpm-v2-early", tpm=30) + routed, admit_turn, record_turn = await _websocket_turn_hooks(router, "ws-tpm-v2-early") + await admit_turn() + + await record_turn(_open_websocket_turn(routed, "ws-tpm-v2-early-turn").litellm_params, 35) + + with pytest.raises(litellm.RateLimitError, match="ws-tpm-v2-early"): + await admit_turn() + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("router_minute_pinned") +async def test_websocket_turn_counted_before_its_logging_is_not_counted_again_when_the_logging_runs( + monkeypatch: pytest.MonkeyPatch, +): + router: Final = _usage_based_routing_router(monkeypatch, "ws-tpm-v2-once", tpm=69) + routed, admit_turn, record_turn = await _websocket_turn_hooks(router, "ws-tpm-v2-once") + await admit_turn() + first_turn: Final = _open_websocket_turn(routed, "ws-tpm-v2-once-first") + + await record_turn(first_turn.litellm_params, 35) + assert await router.get_model_group_usage("ws-tpm-v2-once") == (35, 1) + await _complete_35_token_turn(first_turn) + + assert await router.get_model_group_usage("ws-tpm-v2-once") == (35, 1) + await admit_turn() + await record_turn(_open_websocket_turn(routed, "ws-tpm-v2-once-second").litellm_params, 35) + with pytest.raises(litellm.RateLimitError, match="ws-tpm-v2-once"): + await admit_turn() + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("router_minute_pinned") +async def test_usage_based_routing_counts_an_http_call_once(monkeypatch: pytest.MonkeyPatch): + from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER + + router: Final = _usage_based_routing_router(monkeypatch, "http-tpm-v2", tpm=1000) + + response: Final = await router.acompletion( + model="http-tpm-v2", messages=[{"role": "user", "content": "hi"}], mock_response="pong" + ) + await asyncio.sleep(0) + await GLOBAL_LOGGING_WORKER.flush() + + tpm_keys, _ = router.lowesttpm_logger_v2.usage_counter_keys(router.get_model_list(model_name="http-tpm-v2")) + assert await router.cache.async_get_cache(key=tpm_keys[0]) == response.usage.total_tokens + + +class _FirstIncrementFailingCache(DualCache): + def __init__(self) -> None: + super().__init__(in_memory_cache=InMemoryCache()) + self.failed: bool = False + + async def async_increment_cache_pipeline( + self, increment_list: list[RedisPipelineIncrementOperation], **kwargs: object + ) -> list[float] | None: + if self.failed: + return await super().async_increment_cache_pipeline(increment_list, **kwargs) + self.failed = True + raise RuntimeError("cache unavailable") + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("router_minute_pinned") +async def test_websocket_turn_whose_early_count_fails_is_counted_in_full_by_its_logging( + monkeypatch: pytest.MonkeyPatch, +): + router: Final = _with_success_callbacks_of(monkeypatch, lambda: _rpm_tpm_router("ws-early-count-fails")) + router.cache = _FirstIncrementFailingCache() + routed, _, record_turn = await _websocket_turn_hooks(router, "gpt-5-mini") + turn: Final = _open_websocket_turn(routed, "ws-early-count-fails-turn") + + await record_turn(turn.litellm_params, 35) + await _complete_35_token_turn(turn) + + assert await router.get_model_group_usage("gpt-5-mini") == (35, 1) + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("router_minute_pinned") +async def test_websocket_turn_on_a_router_without_usage_based_routing_is_counted_once( + monkeypatch: pytest.MonkeyPatch, +): + router: Final = _with_success_callbacks_of(monkeypatch, lambda: _rpm_tpm_router("ws-default-routing")) + routed, _, record_turn = await _websocket_turn_hooks(router, "gpt-5-mini") + turn: Final = _open_websocket_turn(routed, "ws-default-routing-turn") + + await record_turn(turn.litellm_params, 35) + assert await router.get_model_group_usage("gpt-5-mini") == (35, 1) + await _complete_35_token_turn(turn) + + assert await router.get_model_group_usage("gpt-5-mini") == (35, 1) + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("router_minute_pinned") +@pytest.mark.parametrize( + "deployment_metadata", + [{}, {"model_info": {"id": None}}], + ids=["no_deployment", "deployment_without_id"], +) +async def test_websocket_turn_not_attributed_to_a_deployment_does_not_throttle_the_next_turn( + monkeypatch: pytest.MonkeyPatch, deployment_metadata: dict[str, object] +): + router: Final = _usage_based_routing_router(monkeypatch, "ws-tpm-v2-unrouted", tpm=30) + routed, admit_turn, record_turn = await _websocket_turn_hooks(router, "ws-tpm-v2-unrouted") + await admit_turn() + turn: Final = _open_websocket_turn( + {**routed, "litellm_metadata": {"model_group": "ws-tpm-v2-unrouted", **deployment_metadata}}, + "ws-tpm-v2-unrouted-turn", + ) + + await record_turn(turn.litellm_params, 35) + await _complete_35_token_turn(turn) + + await admit_turn() + assert await router.get_model_group_usage("ws-tpm-v2-unrouted") == (None, None) + + @pytest.mark.asyncio async def test_responses_websocket_turn_is_refused_once_its_deployment_crosses_budget(): router = Router( @@ -1198,6 +1373,7 @@ def _rpm_tpm_router(model_id: str) -> Router: def router_minute_pinned(monkeypatch: pytest.MonkeyPatch) -> datetime: pinned: Final = datetime(2026, 1, 1, 12, 0, 30, tzinfo=timezone.utc) monkeypatch.setattr("litellm.router.get_utc_datetime", lambda: pinned) + monkeypatch.setattr("litellm.router_strategy.lowest_tpm_rpm_v2.get_utc_datetime", lambda: pinned) return pinned