fix(router): count a Responses WebSocket turn's tokens before the next turn is admitted

This commit is contained in:
mateo-berri 2026-10-29 03:34:41 -07:00
parent 5eb1042bbf
commit 103efe969b
11 changed files with 447 additions and 58 deletions

View file

@ -1666,6 +1666,7 @@ ROUTING_REQUEST_TAGS_METADATA_KEY: Final = "_routing_request_tags"
BAGGAGE_MAPPED_TAGS_METADATA_KEY: Final = "_baggage_mapped_tags"
VECTOR_STORE_CALLER_METADATA_KWARG: Final = "litellm_vector_store_caller_metadata"
ROUTER_USAGE_COUNTED_TOKENS_METADATA_KEY: Final = "_litellm_router_usage_counted_tokens"
USAGE_BASED_ROUTING_COUNTED_TOKENS_METADATA_KEY: Final = "_litellm_usage_based_routing_counted_tokens"
INTERNAL_CALL_ORIGIN_METADATA_KEY: Final = "internal_call_origin"
BILLED_RETRY_ATTEMPT_METADATA_KEY: Final = "billed_retry_attempt"
MANAGED_BACKGROUND_RESPONSE_METADATA_KEY: Final = "_managed_background_response"

View file

@ -244,7 +244,7 @@ if TYPE_CHECKING:
FakeAnthropicMessagesStreamIterator,
)
from litellm.llms.base_llm.passthrough.transformation import BasePassthroughConfig
from litellm.responses.streaming_iterator import ResponsesWsFrameGuard
from litellm.responses.streaming_iterator import ResponsesWsFrameGuard, ResponsesWsTurnUsageRecorder
from litellm.types.google_genai.main import GenerateContentResponse
from litellm.types.llms.openai_evals import (
CancelEvalResponse,
@ -7143,6 +7143,7 @@ class BaseLLMHTTPHandler:
first_message: str | None = None,
request_defaults: ResponsesWebSocketRequestDefaults | None = None,
turn_admission: Callable[[], Awaitable[None]] | None = None,
turn_usage_recorder: "ResponsesWsTurnUsageRecorder | None" = None,
frame_guard: "ResponsesWsFrameGuard | None" = None,
**kwargs: object,
) -> Exception | None:
@ -7303,6 +7304,7 @@ class BaseLLMHTTPHandler:
custom_llm_provider=custom_llm_provider,
request_defaults=request_defaults,
turn_admission=turn_admission,
turn_usage_recorder=turn_usage_recorder,
frame_guard=frame_guard,
)
return await streaming.bidirectional_forward()

View file

@ -45,6 +45,7 @@ from litellm.constants import (
SESSION_ID_GENERATED_METADATA_KEY,
SESSION_ID_OMITTED_METADATA_KEY,
TYPESAFE_DANGER_ZONE_HEADER_PREFIX,
USAGE_BASED_ROUTING_COUNTED_TOKENS_METADATA_KEY,
VECTOR_STORE_CALLER_METADATA_KWARG,
X_LITELLM_DISABLE_CALLBACKS,
)
@ -518,6 +519,7 @@ _ROUTER_RESERVED_METADATA_FIELDS: Final = frozenset(
"request_retry_count",
CLIENT_OUTPUT_CEILING_METADATA_KEY,
ROUTER_USAGE_COUNTED_TOKENS_METADATA_KEY,
USAGE_BASED_ROUTING_COUNTED_TOKENS_METADATA_KEY,
}
)
_ALLOW_CLIENT_PRICING_OVERRIDE_METADATA_KEY: Final = "allow_client_pricing_override"

View file

@ -78,7 +78,7 @@ from litellm.types.llms.openai import (
# Handle ResponseText import with fallback
if TYPE_CHECKING:
from litellm.responses.streaming_iterator import ResponsesWsFrameGuard
from litellm.responses.streaming_iterator import ResponsesWsFrameGuard, ResponsesWsTurnUsageRecorder
from litellm.types.llms.openai import ResponseText
else:
ResponseText = str # Fallback for ResponseText import
@ -2781,6 +2781,7 @@ async def _aresponses_websocket(
api_key: str | None = None,
timeout: float | None = None,
turn_admission: Callable[[], Awaitable[None]] | None = None,
turn_usage_recorder: "ResponsesWsTurnUsageRecorder | None" = None,
responses_ws_frame_guard: "ResponsesWsFrameGuard | None" = None,
**kwargs,
) -> Exception | None:
@ -2875,6 +2876,7 @@ async def _aresponses_websocket(
custom_llm_provider=_custom_llm_provider,
request_defaults=_build_responses_websocket_request_defaults(deployment_kwargs),
turn_admission=turn_admission,
turn_usage_recorder=turn_usage_recorder,
frame_guard=responses_ws_frame_guard,
**remaining_kwargs,
)

View file

@ -2245,6 +2245,7 @@ class GuardedResponsesWsFrame:
ResponsesWsFrameGuard: TypeAlias = Callable[[str], Awaitable[GuardedResponsesWsFrame]]
ResponsesWsTurnUsageRecorder: TypeAlias = Callable[[Mapping[str, object], int], Awaitable[None]]
async def _reject_guarded_turn(
@ -2444,6 +2445,7 @@ class ResponsesWebSocketStreaming:
custom_llm_provider: str | None = None,
request_defaults: ResponsesWebSocketRequestDefaults | None = None,
turn_admission: Callable[[], Awaitable[None]] | None = None,
turn_usage_recorder: ResponsesWsTurnUsageRecorder | None = None,
frame_guard: ResponsesWsFrameGuard | None = None,
):
self.websocket = websocket
@ -2466,6 +2468,7 @@ class ResponsesWebSocketStreaming:
self.authorized_model: str | None = authorized_model
self.request_defaults: ResponsesWebSocketRequestDefaults | None = request_defaults
self.turn_admission: Callable[[], Awaitable[None]] | None = turn_admission
self.turn_usage_recorder: ResponsesWsTurnUsageRecorder | None = turn_usage_recorder
def _start_turn(
self, client_frame: str | None, guardrail_information: tuple[Mapping[str, object], ...] = ()
@ -2537,7 +2540,7 @@ class ResponsesWebSocketStreaming:
"pii_tokens": {**(known if _is_str_mapping(known) else {}), **pii_tokens},
}
def _record_backend_event(self, event: str) -> None:
async def _record_backend_event(self, event: str) -> None:
event_obj: Final = _json_object_or_none(event)
if event_obj is None or event_obj.get("type") not in RESPONSES_WS_LOGGED_EVENT_TYPES:
return
@ -2546,10 +2549,25 @@ class ResponsesWebSocketStreaming:
turn: Final = self._open_turns[0].with_event(event_obj)
if event_obj.get("type") in _RESPONSES_WS_TERMINAL_EVENT_TYPES:
self._open_turns = self._open_turns[1:]
if event_obj.get("type") in _HISTORY_TERMINAL_EVENT_TYPES:
await self._record_turn_usage(turn)
self._log_turn(turn)
return
self._open_turns = (turn, *self._open_turns[1:])
async def _record_turn_usage(self, turn: _ResponsesWsTurn) -> None:
from litellm.cost_calculator import ResponsesWebSocketTokenUsageProcessor
turn_litellm_params: Final = _json_object_or_none(
turn.logging_obj.litellm_params # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # Logging.litellm_params is untyped
)
if self.turn_usage_recorder is None or turn_litellm_params is None:
return
usage: Final = ResponsesWebSocketTokenUsageProcessor.collect_and_combine_usage_from_responses_ws_results(
turn.events
)
await self.turn_usage_recorder(turn_litellm_params, usage.total_tokens)
def _log_open_turns(self) -> None:
open_turns: Final = self._open_turns
self._open_turns = ()
@ -2634,7 +2652,7 @@ class ResponsesWebSocketStreaming:
# Log the output-masked form so PII redacted by apply_to_output
# guardrails does not appear in success logs.
self._record_backend_event(wrapped_str)
await self._record_backend_event(wrapped_str)
await self.websocket.send_text(wrapped_str)

View file

@ -102,6 +102,7 @@ from litellm.constants import (
ROUTING_REQUEST_TAGS_METADATA_KEY,
RUNTIME_UPDATABLE_ROUTER_SETTINGS,
SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY,
USAGE_BASED_ROUTING_COUNTED_TOKENS_METADATA_KEY,
)
from litellm.integrations.custom_guardrail import CustomGuardrail, is_guardrail_intervention
from litellm.integrations.custom_logger import CustomLogger
@ -869,6 +870,30 @@ def _request_model_group(kwargs: Mapping[str, object]) -> str | None:
return model_group if isinstance(model_group, str) else None
def _routed_deployment_id(deployment_metadata: Mapping[str, object]) -> str | None:
model_info: Final = deployment_metadata.get("model_info")
deployment_id: Final = (
_MODEL_INFO_ADAPTER.validate_python(model_info).get("id") if isinstance(model_info, dict) else None
)
return None if deployment_id is None else str(deployment_id)
async def _count_before_callbacks(
deployment_metadata: dict[str, object], # mutable-ok: the success callbacks read the marker from it
marker: str,
tokens: int,
count: Callable[[], Awaitable[object]] | None,
) -> None:
if count is None:
return
deployment_metadata[marker] = tokens # rebind-ok: the success callbacks read the marker off this call's metadata
try:
await count()
except Exception:
deployment_metadata.pop(marker, None)
raise
def _mid_stream_error_before_output(error: "MidStreamFallbackError") -> bool:
return not error.generated_content or error.is_pre_first_chunk
@ -6423,7 +6448,10 @@ class Router:
"model": model_name,
**_with_router_resolved_session_model(kwargs.get("session"), model_name),
**(
{"turn_admission": partial(self.async_admit_deployment_turn, model, deployment, kwargs)}
{
"turn_admission": partial(self.async_admit_deployment_turn, model, deployment, kwargs),
"turn_usage_recorder": partial(self.async_record_deployment_turn_usage, model, kwargs),
}
if generic_function_name == "_aresponses_websocket"
else {}
),
@ -9565,30 +9593,114 @@ class Router:
model_group: Final = request_kwargs.get("model")
if deployment_metadata is None or not isinstance(model_group, str):
return
model_info: Final = deployment_metadata["model_info"]
deployment_id: Final = model_info.get("id") if isinstance(model_info, dict) else None
if deployment_id is None:
return
total_tokens: Final = response_total_token_count(response)
deployment_name: Final = deployment_metadata.get("deployment")
deployment_metadata[ROUTER_USAGE_COUNTED_TOKENS_METADATA_KEY] = total_tokens
try:
await self._increment_deployment_usage(
deployment_id=str(deployment_id),
deployment_name=deployment_name if isinstance(deployment_name, str) else None,
await _count_before_callbacks(
deployment_metadata,
ROUTER_USAGE_COUNTED_TOKENS_METADATA_KEY,
total_tokens,
self._deployment_usage_count(
deployment_metadata=deployment_metadata,
model_group=model_group,
total_tokens=total_tokens,
rpm_increment=1,
parent_otel_span=get_parent_otel_span_from_kwargs(request_kwargs),
)
except Exception:
deployment_metadata.pop(ROUTER_USAGE_COUNTED_TOKENS_METADATA_KEY, None)
raise
),
)
except Exception as e:
verbose_router_logger.debug(
"litellm.router.Router::increment_deployment_usage_for_response(): Exception occured - %s", e
)
async def async_record_deployment_turn_usage(
self,
model_group: str,
request_kwargs: Mapping[str, object],
turn_litellm_params: Mapping[str, object],
total_tokens: int,
) -> None:
try:
deployment_metadata: Final = find_deployment_metadata(turn_litellm_params)
if deployment_metadata is None:
return
parent_otel_span: Final = get_parent_otel_span_from_kwargs(request_kwargs)
await _count_before_callbacks(
deployment_metadata,
ROUTER_USAGE_COUNTED_TOKENS_METADATA_KEY,
total_tokens,
self._deployment_usage_count(
deployment_metadata=deployment_metadata,
model_group=model_group,
total_tokens=total_tokens,
parent_otel_span=parent_otel_span,
),
)
await _count_before_callbacks(
deployment_metadata,
USAGE_BASED_ROUTING_COUNTED_TOKENS_METADATA_KEY,
total_tokens,
self._routing_strategy_usage_count(
deployment_metadata=deployment_metadata,
model_group=model_group,
request_kwargs=request_kwargs,
total_tokens=total_tokens,
parent_otel_span=parent_otel_span,
),
)
except Exception as e: # noqa: BLE001 # a failed early count must not break the turn; its logging counts it
verbose_router_logger.debug(
"litellm.router.Router::async_record_deployment_turn_usage(): Exception occured - %s", e
)
def _deployment_usage_count(
self,
*,
deployment_metadata: Mapping[str, object],
model_group: str,
total_tokens: int,
parent_otel_span: Span | None,
) -> Callable[[], Awaitable[object]] | None:
deployment_id: Final = _routed_deployment_id(deployment_metadata)
if deployment_id is None:
return None
deployment_name: Final = deployment_metadata.get("deployment")
return partial(
self._increment_deployment_usage,
deployment_id=deployment_id,
deployment_name=deployment_name if isinstance(deployment_name, str) else None,
model_group=model_group,
total_tokens=total_tokens,
rpm_increment=1,
parent_otel_span=parent_otel_span,
)
def _routing_strategy_usage_count(
self,
*,
deployment_metadata: Mapping[str, object],
model_group: str,
request_kwargs: Mapping[str, object],
total_tokens: int,
parent_otel_span: Span | None,
) -> Callable[[], Awaitable[object]] | None:
routing_kwargs: Final = {key: value for key, value in request_kwargs.items() if key != "litellm_logging_obj"}
_, selector = self._get_routing_context( # pyright: ignore[reportUnknownMemberType] # legacy signature takes an untyped request dict
model=model_group, request_kwargs=routing_kwargs
)
deployment_id: Final = _routed_deployment_id(deployment_metadata)
deployment_name: Final = deployment_metadata.get("deployment")
if (
not isinstance(selector, LowestTPMLoggingHandler_v2)
or deployment_id is None
or not isinstance(deployment_name, str)
):
return None
return partial(
selector.async_increment_tpm,
deployment_id=deployment_id,
deployment_name=deployment_name,
tokens=total_tokens,
parent_otel_span=parent_otel_span,
)
async def _increment_deployment_usage(
self,
*,

View file

@ -8,15 +8,19 @@ from dataclasses import dataclass
from typing import TYPE_CHECKING, Any, Final
import httpx
from pydantic import TypeAdapter
from typing_extensions import ReadOnly, TypedDict
import litellm
from litellm import token_counter
from litellm._internal_context import with_service_target
from litellm._logging import verbose_logger, verbose_router_logger
from litellm.caching.caching import DualCache
from litellm.constants import USAGE_BASED_ROUTING_COUNTED_TOKENS_METADATA_KEY
from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.batch_utils import is_batch_retrieve_call_type
from litellm.litellm_core_utils.core_helpers import get_parent_otel_span_from_kwargs
from litellm.router_utils.router_callbacks.track_deployment_metrics import get_counted_usage_tokens
from litellm.types.router import RouterErrors
from litellm.types.utils import LiteLLMPydanticObjectBase, StandardLoggingPayload
from litellm.utils import get_utc_datetime, print_verbose
@ -35,6 +39,13 @@ class RoutingArgs(LiteLLMPydanticObjectBase):
ttl: int = 1 * 60 # 1min (RPM/TPM expire key)
class _SuccessKwargs(TypedDict, total=False):
litellm_params: ReadOnly[Mapping[str, object] | None]
_SUCCESS_KWARGS_ADAPTER: Final = TypeAdapter(_SuccessKwargs)
_active_prefetched_usage: Final[ContextVar["PrefetchedUsage | None"]] = ContextVar("prefetched_usage", default=None)
@ -296,7 +307,7 @@ class LowestTPMLoggingHandler_v2(BaseRoutingStrategy, CustomLogger):
)
@with_service_target("router_usage")
async def async_log_success_event(self, kwargs, response_obj, start_time, end_time):
async def async_log_success_event(self, kwargs, response_obj, start_time, end_time) -> None:
if is_batch_retrieve_call_type(kwargs.get("call_type")):
return
try:
@ -308,30 +319,19 @@ class LowestTPMLoggingHandler_v2(BaseRoutingStrategy, CustomLogger):
raise ValueError("standard_logging_object not passed in.")
model_group: Final = standard_logging_object.get("model_group")
model: Final = standard_logging_object["hidden_params"]["litellm_model_name"]
id = standard_logging_object.get("model_id")
if model_group is None or id is None:
return
elif isinstance(id, int):
id = str(id)
model_id: Final = standard_logging_object.get("model_id")
total_tokens: Final = standard_logging_object.get("total_tokens")
# ------------
# Setup values
# ------------
dt: Final = get_utc_datetime()
current_minute: Final = dt.strftime("%H-%M") # use the same timezone regardless of system clock
tpm_key: Final = f"{id}:{model}:tpm:{current_minute}"
# ------------
# Update usage
# ------------
# update cache
parent_otel_span: Final = get_parent_otel_span_from_kwargs(kwargs)
## TPM
await self.router_cache.async_increment_cache_post_call(
key=tpm_key,
value=total_tokens,
ttl=self.routing_args.ttl,
parent_otel_span=parent_otel_span,
if model_group is None or not isinstance(model_id, (str, int)) or not isinstance(total_tokens, int):
return
counted_tokens: Final = get_counted_usage_tokens(
_SUCCESS_KWARGS_ADAPTER.validate_python(kwargs).get("litellm_params") or {},
USAGE_BASED_ROUTING_COUNTED_TOKENS_METADATA_KEY,
)
await self.async_increment_tpm(
deployment_id=str(model_id),
deployment_name=model if isinstance(model, str) else None,
tokens=total_tokens if counted_tokens is None else max(0, total_tokens - counted_tokens),
parent_otel_span=get_parent_otel_span_from_kwargs(kwargs),
)
### TESTING ###
@ -342,6 +342,18 @@ class LowestTPMLoggingHandler_v2(BaseRoutingStrategy, CustomLogger):
"litellm.proxy.hooks.lowest_tpm_rpm_v2.py::async_log_success_event(): Exception occured - %s", e
)
@with_service_target("router_usage")
async def async_increment_tpm(
self, *, deployment_id: str, deployment_name: str | None, tokens: int, parent_otel_span: Span | None
) -> None:
current_minute: Final = get_utc_datetime().strftime("%H-%M")
await self.router_cache.async_increment_cache_post_call(
key=f"{deployment_id}:{deployment_name}:tpm:{current_minute}",
value=tokens,
ttl=self.routing_args.ttl,
parent_otel_span=parent_otel_span,
)
def _return_potential_deployments(
self,
healthy_deployments: list[dict],

View file

@ -29,14 +29,12 @@ def find_deployment_metadata(kwargs: Mapping[str, object]) -> dict[str, object]
return next((bucket for bucket in buckets if isinstance(bucket, dict) and "model_info" in bucket), None)
def get_counted_usage_tokens(litellm_params: Mapping[str, object]) -> int | None:
def get_counted_usage_tokens(
litellm_params: Mapping[str, object], marker: str = ROUTER_USAGE_COUNTED_TOKENS_METADATA_KEY
) -> int | None:
buckets: Final = (litellm_params.get(channel) for channel in _METADATA_CHANNELS)
counted: Final = next(
(
bucket[ROUTER_USAGE_COUNTED_TOKENS_METADATA_KEY]
for bucket in buckets
if isinstance(bucket, dict) and ROUTER_USAGE_COUNTED_TOKENS_METADATA_KEY in bucket
),
(bucket[marker] for bucket in buckets if isinstance(bucket, dict) and marker in bucket),
None,
)
return counted if isinstance(counted, int) and not isinstance(counted, bool) else None

View file

@ -25,6 +25,7 @@ from litellm.constants import (
SERVER_STREAMING_CLASSIFICATION_MARKER,
SESSION_ID_GENERATED_METADATA_KEY,
SESSION_ID_OMITTED_METADATA_KEY,
USAGE_BASED_ROUTING_COUNTED_TOKENS_METADATA_KEY,
)
from litellm.litellm_core_utils.core_helpers import get_litellm_metadata_from_kwargs
from litellm.litellm_core_utils.get_provider_specific_headers import (
@ -8759,6 +8760,7 @@ _PLANTED_STAMPS = {
"previous_models": [{"exception_type": "Timeout", "deployment_id": "spoofed", "attempted_retries": 101}],
"_client_output_ceiling": {"api_base": "https://attacker.example"},
ROUTER_USAGE_COUNTED_TOKENS_METADATA_KEY: 10**9,
USAGE_BASED_ROUTING_COUNTED_TOKENS_METADATA_KEY: 10**9,
"client_key": "client_value",
}
@ -8793,6 +8795,7 @@ async def test_add_litellm_data_to_request_strips_router_reserved_stamps_from_bo
assert "request_retry_count" not in updated["metadata"]
assert "previous_models" not in updated["metadata"]
assert ROUTER_USAGE_COUNTED_TOKENS_METADATA_KEY not in updated["metadata"]
assert USAGE_BASED_ROUTING_COUNTED_TOKENS_METADATA_KEY not in updated["metadata"]
assert updated["metadata"]["client_key"] == "client_value"

View file

@ -1582,6 +1582,9 @@ class TestWebSocketProjectQuotaEnforcement:
quota_callback.enforce_project_io_token_quota_for_frame.assert_awaited_once()
_TURN_USAGE: Final = {"input_tokens": 20, "output_tokens": 15, "total_tokens": 35}
def _turn_frames() -> tuple[str, str, str]:
return tuple(json.dumps({"type": "response.create", "input": word}) for word in ("one", "two", "three"))
@ -1662,6 +1665,66 @@ def _errors(websocket: _ClientSocket) -> list[tuple[object, object]]:
class TestWebSocketTurnAdmission:
"""A turn after the connection's first is admitted by the deployment again before it reaches the provider."""
@pytest.mark.asyncio
@pytest.mark.parametrize(
("terminal_event", "counted_before_it_reaches_the_client"),
[
({"type": "response.completed", "response": {"usage": _TURN_USAGE}}, [("ws-dep", 35)]),
({"type": "response.incomplete", "response": {"usage": _TURN_USAGE}}, [("ws-dep", 35)]),
({"type": "response.failed", "response": {"usage": _TURN_USAGE, "error": {"code": "server_error"}}}, []),
],
)
async def test_native_relay_counts_a_finished_turn_before_forwarding_its_terminal_event(
self, terminal_event: dict[str, object], counted_before_it_reaches_the_client: list[tuple[object, int]]
):
from unittest.mock import AsyncMock
import websockets.exceptions
counted: Final[list[tuple[object, int]]] = []
async def record_turn_usage(turn_litellm_params: Mapping[str, object], total_tokens: int) -> None:
await asyncio.sleep(0)
metadata: Final = turn_litellm_params["litellm_metadata"]
assert isinstance(metadata, dict)
counted.append((metadata["model_info"]["id"], total_tokens))
class Backend:
def __init__(self, *events: dict[str, object]) -> None:
self.events: Final = list(events)
async def recv(self, decode: bool = False) -> str:
if not self.events:
raise websockets.exceptions.ConnectionClosed(None, None)
return json.dumps(self.events.pop(0))
class Client:
def __init__(self) -> None:
self.seen: Final[list[tuple[object, list[tuple[object, int]]]]] = []
async def send_text(self, data: str) -> None:
self.seen.append((json.loads(data)["type"], list(counted)))
logging_obj: Final = MagicMock()
logging_obj.litellm_params = {"litellm_metadata": {"model_info": {"id": "ws-dep"}}}
logging_obj.copy_for_new_call.return_value = logging_obj
logging_obj.dispatch_success_handlers = AsyncMock()
logging_obj.dispatch_failure_handlers = AsyncMock()
client: Final = Client()
handler: Final = _make_streaming(
websocket=client,
backend_ws=Backend({"type": "response.created", "response": {}}, terminal_event),
logging_obj=logging_obj,
turn_usage_recorder=record_turn_usage,
)
await handler.backend_to_client()
assert client.seen == [
("response.created", []),
(terminal_event["type"], counted_before_it_reaches_the_client),
]
@pytest.mark.asyncio
async def test_native_relay_drops_a_later_turn_the_deployment_refuses(self):
from unittest.mock import AsyncMock

View file

@ -4,7 +4,7 @@ import gc
import json
import os
import weakref
from collections.abc import Iterator, Mapping
from collections.abc import Awaitable, Callable, Iterator, Mapping
from datetime import datetime, timezone
from types import MethodType
from typing import Final
@ -19,8 +19,11 @@ from litellm.caching.in_memory_cache import InMemoryCache
from litellm.constants import DEFAULT_AUTO_ROUTER_MAX_INPUT_CHARS, ROUTER_USAGE_COUNTED_TOKENS_METADATA_KEY
from litellm.integrations.custom_logger import CustomLogger
from litellm.integrations.SlackAlerting.slack_alerting import SlackAlerting
from litellm.litellm_core_utils.get_litellm_params import get_litellm_params
from litellm.litellm_core_utils.litellm_logging import Logging
from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper
from litellm.router_strategy.adaptive_router.hooks import AdaptiveRouterPostCallHook
from litellm.types.caching import RedisPipelineIncrementOperation
from litellm.types.litellm_params import RoutingStrategyName
from litellm.types.router import AlertingConfig, Deployment, DeploymentTypedDict, LiteLLM_Params, ModelInfo
from litellm.types.utils import (
@ -545,31 +548,203 @@ async def _kwargs_routed_to(router: Router, model: str, function_name: str) -> d
routed.update(kwargs)
call.__name__ = function_name
await router._ageneric_api_call_with_fallbacks_helper(model=model, original_generic_function=call)
await router._ageneric_api_call_with_fallbacks_helper(
model=model, original_generic_function=call, litellm_metadata={"model_group": model}
)
return routed
async def _finish_35_token_websocket_turn(routed: Mapping[str, object], call_id: str) -> None:
def _open_websocket_turn(routed: Mapping[str, object], call_id: str) -> Logging:
import time
from litellm.litellm_core_utils.litellm_logging import Logging
turn: Final = Logging(
connection: Final = Logging(
model="gpt-5-mini",
messages=[],
stream=False,
call_type=CallTypes.aresponses_websocket.value,
start_time=time.time(),
litellm_call_id=call_id,
litellm_call_id=f"{call_id}-connection",
function_id=call_id,
)
turn.update_environment_variables(litellm_params={"litellm_metadata": routed["litellm_metadata"]}, optional_params={})
connection.update_environment_variables(litellm_params=get_litellm_params(**routed), optional_params={})
return connection.copy_for_new_call(litellm_call_id=call_id, start_time=datetime.now())
async def _complete_35_token_turn(turn: Logging) -> None:
await turn.dispatch_success_handlers(
[{"type": "response.completed", "response": {"usage": {"input_tokens": 20, "output_tokens": 15, "total_tokens": 35}}}],
prefer_async_handlers=True,
)
async def _finish_35_token_websocket_turn(routed: Mapping[str, object], call_id: str) -> None:
await _complete_35_token_turn(_open_websocket_turn(routed, call_id))
def _with_success_callbacks_of(monkeypatch: pytest.MonkeyPatch, build_router: Callable[[], Router]) -> Router:
monkeypatch.setattr(litellm, "callbacks", [])
monkeypatch.setattr(litellm, "success_callback", [])
monkeypatch.setattr(litellm, "_async_success_callback", [])
router: Final = build_router()
monkeypatch.setattr(litellm, "_async_success_callback", [*litellm._async_success_callback, *litellm.callbacks])
return router
def _usage_based_routing_router(monkeypatch: pytest.MonkeyPatch, model_name: str, tpm: int) -> Router:
return _with_success_callbacks_of(
monkeypatch,
lambda: Router(
model_list=[
{
"model_name": model_name,
"litellm_params": {"model": "openai/gpt-5-mini", "api_key": "sk-fake", "tpm": tpm},
"model_info": {"id": f"{model_name}-dep"},
}
],
routing_strategy="usage-based-routing-v2",
),
)
_TurnAdmission = Callable[[], Awaitable[None]]
_TurnUsageRecorder = Callable[[Mapping[str, object], int], Awaitable[None]]
async def _websocket_turn_hooks(
router: Router, model_name: str
) -> tuple[dict[str, object], _TurnAdmission, _TurnUsageRecorder]:
routed: Final = await _kwargs_routed_to(router, model_name, "_aresponses_websocket")
admit_turn: Final = routed["turn_admission"]
record_turn: Final = routed["turn_usage_recorder"]
assert callable(admit_turn)
assert callable(record_turn)
return routed, admit_turn, record_turn
@pytest.mark.asyncio
@pytest.mark.usefixtures("router_minute_pinned")
async def test_usage_based_routing_refuses_the_next_websocket_turn_before_the_finished_turn_s_logging_runs(
monkeypatch: pytest.MonkeyPatch,
):
router: Final = _usage_based_routing_router(monkeypatch, "ws-tpm-v2-early", tpm=30)
routed, admit_turn, record_turn = await _websocket_turn_hooks(router, "ws-tpm-v2-early")
await admit_turn()
await record_turn(_open_websocket_turn(routed, "ws-tpm-v2-early-turn").litellm_params, 35)
with pytest.raises(litellm.RateLimitError, match="ws-tpm-v2-early"):
await admit_turn()
@pytest.mark.asyncio
@pytest.mark.usefixtures("router_minute_pinned")
async def test_websocket_turn_counted_before_its_logging_is_not_counted_again_when_the_logging_runs(
monkeypatch: pytest.MonkeyPatch,
):
router: Final = _usage_based_routing_router(monkeypatch, "ws-tpm-v2-once", tpm=69)
routed, admit_turn, record_turn = await _websocket_turn_hooks(router, "ws-tpm-v2-once")
await admit_turn()
first_turn: Final = _open_websocket_turn(routed, "ws-tpm-v2-once-first")
await record_turn(first_turn.litellm_params, 35)
assert await router.get_model_group_usage("ws-tpm-v2-once") == (35, 1)
await _complete_35_token_turn(first_turn)
assert await router.get_model_group_usage("ws-tpm-v2-once") == (35, 1)
await admit_turn()
await record_turn(_open_websocket_turn(routed, "ws-tpm-v2-once-second").litellm_params, 35)
with pytest.raises(litellm.RateLimitError, match="ws-tpm-v2-once"):
await admit_turn()
@pytest.mark.asyncio
@pytest.mark.usefixtures("router_minute_pinned")
async def test_usage_based_routing_counts_an_http_call_once(monkeypatch: pytest.MonkeyPatch):
from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
router: Final = _usage_based_routing_router(monkeypatch, "http-tpm-v2", tpm=1000)
response: Final = await router.acompletion(
model="http-tpm-v2", messages=[{"role": "user", "content": "hi"}], mock_response="pong"
)
await asyncio.sleep(0)
await GLOBAL_LOGGING_WORKER.flush()
tpm_keys, _ = router.lowesttpm_logger_v2.usage_counter_keys(router.get_model_list(model_name="http-tpm-v2"))
assert await router.cache.async_get_cache(key=tpm_keys[0]) == response.usage.total_tokens
class _FirstIncrementFailingCache(DualCache):
def __init__(self) -> None:
super().__init__(in_memory_cache=InMemoryCache())
self.failed: bool = False
async def async_increment_cache_pipeline(
self, increment_list: list[RedisPipelineIncrementOperation], **kwargs: object
) -> list[float] | None:
if self.failed:
return await super().async_increment_cache_pipeline(increment_list, **kwargs)
self.failed = True
raise RuntimeError("cache unavailable")
@pytest.mark.asyncio
@pytest.mark.usefixtures("router_minute_pinned")
async def test_websocket_turn_whose_early_count_fails_is_counted_in_full_by_its_logging(
monkeypatch: pytest.MonkeyPatch,
):
router: Final = _with_success_callbacks_of(monkeypatch, lambda: _rpm_tpm_router("ws-early-count-fails"))
router.cache = _FirstIncrementFailingCache()
routed, _, record_turn = await _websocket_turn_hooks(router, "gpt-5-mini")
turn: Final = _open_websocket_turn(routed, "ws-early-count-fails-turn")
await record_turn(turn.litellm_params, 35)
await _complete_35_token_turn(turn)
assert await router.get_model_group_usage("gpt-5-mini") == (35, 1)
@pytest.mark.asyncio
@pytest.mark.usefixtures("router_minute_pinned")
async def test_websocket_turn_on_a_router_without_usage_based_routing_is_counted_once(
monkeypatch: pytest.MonkeyPatch,
):
router: Final = _with_success_callbacks_of(monkeypatch, lambda: _rpm_tpm_router("ws-default-routing"))
routed, _, record_turn = await _websocket_turn_hooks(router, "gpt-5-mini")
turn: Final = _open_websocket_turn(routed, "ws-default-routing-turn")
await record_turn(turn.litellm_params, 35)
assert await router.get_model_group_usage("gpt-5-mini") == (35, 1)
await _complete_35_token_turn(turn)
assert await router.get_model_group_usage("gpt-5-mini") == (35, 1)
@pytest.mark.asyncio
@pytest.mark.usefixtures("router_minute_pinned")
@pytest.mark.parametrize(
"deployment_metadata",
[{}, {"model_info": {"id": None}}],
ids=["no_deployment", "deployment_without_id"],
)
async def test_websocket_turn_not_attributed_to_a_deployment_does_not_throttle_the_next_turn(
monkeypatch: pytest.MonkeyPatch, deployment_metadata: dict[str, object]
):
router: Final = _usage_based_routing_router(monkeypatch, "ws-tpm-v2-unrouted", tpm=30)
routed, admit_turn, record_turn = await _websocket_turn_hooks(router, "ws-tpm-v2-unrouted")
await admit_turn()
turn: Final = _open_websocket_turn(
{**routed, "litellm_metadata": {"model_group": "ws-tpm-v2-unrouted", **deployment_metadata}},
"ws-tpm-v2-unrouted-turn",
)
await record_turn(turn.litellm_params, 35)
await _complete_35_token_turn(turn)
await admit_turn()
assert await router.get_model_group_usage("ws-tpm-v2-unrouted") == (None, None)
@pytest.mark.asyncio
async def test_responses_websocket_turn_is_refused_once_its_deployment_crosses_budget():
router = Router(
@ -1198,6 +1373,7 @@ def _rpm_tpm_router(model_id: str) -> Router:
def router_minute_pinned(monkeypatch: pytest.MonkeyPatch) -> datetime:
pinned: Final = datetime(2026, 1, 1, 12, 0, 30, tzinfo=timezone.utc)
monkeypatch.setattr("litellm.router.get_utc_datetime", lambda: pinned)
monkeypatch.setattr("litellm.router_strategy.lowest_tpm_rpm_v2.get_utc_datetime", lambda: pinned)
return pinned