fix(router): check a responses socket's open without reserving and read shared token counters first

A Responses WebSocket open now runs the same itpm/otpm check a turn runs and
reserves nothing. It used to reserve an estimate that falls back to the
model's max output, so a socket could not open under any otpm below that.
The open still records a zero reservation for its own minute, so the first
frame, which the relay forwards without a turn check, counts its actual
tokens like every later turn.

This drops the plumbing that only carried the open's hold into the first
turn. The turn admission callable takes no arguments again, and the native
relay detaches only the budget reservation, as before this change.

The itpm/otpm turn check reads Redis first and falls back to this
instance's in-memory count only without Redis or when the circuit breaker
is open. These counters go down when a call settles, so an in-memory copy
at the limit could refuse turns after the shared count dropped. The tpm
check keeps its in-memory shortcut, since its counter only grows within a
minute.

Responses usage that reports input_tokens_details.cached_tokens now leaves
cached input out of itpm, as chat usage already did.
This commit is contained in:
mateo-berri 2026-10-29 14:23:09 -07:00
parent 42249f81f5
commit be078aa315
9 changed files with 292 additions and 329 deletions

View file

@ -174,14 +174,6 @@ def without_budget_reservation(
}
def without_io_token_reservation(metadata: Mapping[str, object]) -> Mapping[str, object]:
return {key: value for key, value in metadata.items() if key not in IO_TOKEN_RESERVATION_KEYS}
def io_token_reservation(metadata: Mapping[str, object]) -> Mapping[str, object]:
return MappingProxyType({key: value for key, value in metadata.items() if key in IO_TOKEN_RESERVATION_KEYS})
def forwarded_internal_call_metadata(
parent_metadata: Mapping[str, object] | None,
call_origin: InternalCallOrigin,

View file

@ -46,11 +46,9 @@ from litellm.litellm_core_utils.core_helpers import (
)
from litellm.litellm_core_utils.exception_mapping_utils import with_deployment_api_base
from litellm.litellm_core_utils.internal_call_metadata import (
io_token_reservation,
is_background_response,
requests_background,
without_budget_reservation,
without_io_token_reservation,
)
from litellm.litellm_core_utils.litellm_logging import BilledRoundsSpend
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
@ -2156,17 +2154,17 @@ class _RefusedTurn:
rejection: Exception
ResponsesWsTurnAdmission: TypeAlias = Callable[[Mapping[str, object]], Awaitable[Mapping[str, object]]]
ResponsesWsTurnAdmission: TypeAlias = Callable[[], Awaitable[Mapping[str, object]]]
async def _admit_frame_turn(
turn_admission: ResponsesWsTurnAdmission | None, raw_message: str, held_reservation: Mapping[str, object]
turn_admission: ResponsesWsTurnAdmission | None, raw_message: str
) -> _AdmittedTurn | _RefusedTurn:
frame: Final = _json_object_or_none(raw_message)
if turn_admission is None or frame is None or frame.get("type") != "response.create":
return _AdmittedTurn()
try:
return _AdmittedTurn(await turn_admission(held_reservation))
return _AdmittedTurn(await turn_admission())
except RateLimitError as refusal:
return _RefusedTurn(refusal)
except Exception as e:
@ -2519,32 +2517,22 @@ class ResponsesWebSocketStreaming:
turn_logging_obj: Final = self.logging_obj.copy_for_new_call(
litellm_call_id=str(uuid.uuid4()), start_time=start_time
)
self._detach_admission_reservations()
self._detach_budget_reservation()
_set_turn_guardrail_information(turn_logging_obj, guardrail_information)
_set_turn_io_token_reservation(turn_logging_obj, io_token_reservation)
turn_logging_obj.model_call_details["messages"] = list(_logged_user_messages(client_frame))
turn_logging_obj.pre_call(input=client_frame, api_key="")
self._open_turns = (*self._open_turns, _ResponsesWsTurn(logging_obj=turn_logging_obj))
def _held_admission_reservation(self) -> Mapping[str, object]:
litellm_params: Final = _json_object_or_none(
self.logging_obj.litellm_params # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # Logging.litellm_params is untyped
)
buckets: Final = (
() if litellm_params is None else (litellm_params.get(key) for key in ("metadata", "litellm_metadata"))
)
held: Final = (io_token_reservation(bucket) for bucket in buckets if _is_json_object(bucket))
return next((reservation for reservation in held if reservation), _NO_IO_TOKEN_RESERVATION)
def _detach_admission_reservations(self) -> None:
"""Only the first turn settles the connection's admission reservations; later turns charge their own usage."""
def _detach_budget_reservation(self) -> None:
"""Only the first turn settles the connection's admission reservation; later turns charge their cost directly."""
litellm_params: Final = _json_object_or_none(
self.logging_obj.litellm_params # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # Logging.litellm_params is untyped
)
if litellm_params is None:
return
self.logging_obj.litellm_params = {
key: without_io_token_reservation(without_budget_reservation(value))
key: without_budget_reservation(value)
if key in _PER_TURN_METADATA_KEYS and _is_json_object(value)
else value
for key, value in litellm_params.items()
@ -2990,7 +2978,7 @@ class ResponsesWebSocketStreaming:
async def _admit_or_reject_turn(
self, guarded: GuardedResponsesWsFrame, message: str, turn_admission: ResponsesWsTurnAdmission | None
) -> _AdmittedTurn | None:
verdict: Final = await _admit_frame_turn(turn_admission, message, self._held_admission_reservation())
verdict: Final = await _admit_frame_turn(turn_admission, message)
match verdict:
case _AdmittedTurn():
return verdict
@ -3419,21 +3407,6 @@ class ManagedResponsesWebSocketHandler:
if self.litellm_metadata:
call_kwargs["litellm_metadata"] = dict(self.litellm_metadata)
def _held_io_token_reservation(self) -> Mapping[str, object]:
connection_metadata: Final = self.extra_kwargs.get("metadata")
held_on_connection: Final = (
io_token_reservation(connection_metadata)
if _is_json_object(connection_metadata)
else _NO_IO_TOKEN_RESERVATION
)
return held_on_connection or io_token_reservation(self.litellm_metadata)
def _detach_io_token_reservation(self) -> None:
connection_metadata: Final = self.extra_kwargs.get("metadata")
self.litellm_metadata = without_io_token_reservation(self.litellm_metadata)
if _is_json_object(connection_metadata):
self.extra_kwargs = {**self.extra_kwargs, "metadata": without_io_token_reservation(connection_metadata)}
async def _guard_frame(self, raw_message: str) -> GuardedResponsesWsFrame | None:
"""Same per-frame pre_call guardrail walk as the native socket: a rejected frame errors and is not sent."""
if self.frame_guard is None:
@ -3584,7 +3557,7 @@ class ManagedResponsesWebSocketHandler:
guarded: Final = await self._guard_frame(raw_message)
if guarded is None:
return
verdict: Final = await _admit_frame_turn(turn_admission, raw_message, self._held_io_token_reservation())
verdict: Final = await _admit_frame_turn(turn_admission, raw_message)
if isinstance(verdict, _RefusedTurn):
if self._failure is None:
self._failure = verdict.rejection
@ -3617,15 +3590,15 @@ class ManagedResponsesWebSocketHandler:
call_kwargs["litellm_metadata"] = self._metadata_with_turn_guardrails(
call_kwargs.get("litellm_metadata"), guarded.guardrail_information
)
if verdict.reservation:
existing_metadata: Final = call_kwargs.get("litellm_metadata")
call_kwargs["litellm_metadata"] = {
**(existing_metadata if _is_json_object(existing_metadata) else {}),
**verdict.reservation,
}
self._update_proxy_request(call_kwargs, requested_model or self.model_group or model)
call_kwargs.update(self.extra_kwargs)
self._detach_io_token_reservation()
call_kwargs.update(
{
channel: {**bucket, **verdict.reservation}
for channel, bucket in call_kwargs.items()
if channel in _PER_TURN_METADATA_KEYS and _is_json_object(bucket)
}
)
try:
terminal_event: Final = await self._stream_and_forward(model, call_kwargs)

View file

@ -10452,7 +10452,6 @@ class Router(DeprecatedRouterMembers):
model_group: str,
opened_on: Mapping[str, object],
request_kwargs: Mapping[str, object],
held_reservation: Mapping[str, object] = MappingProxyType({}),
) -> Mapping[str, object]:
deployment: Final = self._deployment_as_configured_now(opened_on)
rate_limit_check: Final = next(
@ -10461,9 +10460,7 @@ class Router(DeprecatedRouterMembers):
io_token_admission: Final = (
MappingProxyType({})
if rate_limit_check is None
else await async_admit_io_token_turn(
rate_limit_check.dual_cache, deployment, held_reservation, get_utc_datetime()
)
else await async_admit_io_token_turn(rate_limit_check.dual_cache, deployment, get_utc_datetime())
)
budget_limiter: Final = self._core_get_router_deployment_budget_limiter()
if budget_limiter is not None:

View file

@ -185,27 +185,32 @@ def _resolve_max_tokens(request_kwargs: dict[str, Any] | None, deployment: dict)
return 4096
def _usage_field(container: object, name: str) -> object:
if is_str_keyed_mapping(container):
return container.get(name)
return getattr(container, name, None)
def _cached_prompt_tokens(usage: object) -> int:
reported: Final = (
_usage_field(_usage_field(usage, "prompt_tokens_details"), "cached_tokens"),
_usage_field(_usage_field(usage, "input_tokens_details"), "cached_tokens"),
_usage_field(usage, "cache_read_input_tokens"),
)
return next((int(tokens) for tokens in reported if isinstance(tokens, int | float) and tokens), 0)
def _get_usage_tokens(usage: object) -> tuple[int, int, int]:
if usage is None:
return 0, 0, 0
if hasattr(usage, "prompt_tokens") or hasattr(usage, "input_tokens"):
prompt = int(getattr(usage, "prompt_tokens", None) or getattr(usage, "input_tokens", 0) or 0)
completion = int(getattr(usage, "completion_tokens", None) or getattr(usage, "output_tokens", 0) or 0)
cached = 0
details: object = getattr(usage, "prompt_tokens_details", None)
if details is not None:
cached = int(getattr(details, "cached_tokens", 0) or 0)
if not cached:
cached = int(getattr(usage, "cache_read_input_tokens", 0) or 0)
return prompt, completion, cached
return prompt, completion, _cached_prompt_tokens(usage)
if isinstance(usage, dict):
prompt = int(usage.get("prompt_tokens") or usage.get("input_tokens") or 0)
completion = int(usage.get("completion_tokens") or usage.get("output_tokens") or 0)
details = usage.get("prompt_tokens_details") or {}
cached = int(details.get("cached_tokens", 0) or 0) if isinstance(details, dict) else 0
if not cached:
cached = int(usage.get("cache_read_input_tokens") or 0)
return prompt, completion, cached
return prompt, completion, _cached_prompt_tokens(usage)
return 0, 0, 0
@ -266,22 +271,22 @@ def _resolve_reconcile_usage_tokens(
return 0, 0, False
def _stash_reservation_in_metadata(
request_kwargs: dict[str, object] | None,
*,
itpm_reserved: int,
otpm_reserved: int,
itpm_cache_key: str | None,
otpm_cache_key: str | None,
) -> None:
def _reservation(
itpm_reserved: int, otpm_reserved: int, itpm_cache_key: str | None, otpm_cache_key: str | None
) -> Mapping[str, object]:
return MappingProxyType(
{
ITPM_RESERVED_KEY: itpm_reserved,
OTPM_RESERVED_KEY: otpm_reserved,
ITPM_CACHE_KEY: itpm_cache_key,
OTPM_CACHE_KEY: otpm_cache_key,
}
)
def _stash_reservation_in_metadata(request_kwargs: dict[str, object] | None, reservation: Mapping[str, object]) -> None:
if not request_kwargs:
return
reservation: Final = {
ITPM_RESERVED_KEY: itpm_reserved,
OTPM_RESERVED_KEY: otpm_reserved,
ITPM_CACHE_KEY: itpm_cache_key,
OTPM_CACHE_KEY: otpm_cache_key,
}
for channel in ("metadata", "litellm_metadata"):
existing = request_kwargs.get(channel)
if isinstance(existing, dict):
@ -483,10 +488,12 @@ def io_token_pre_call_check(
_stash_reservation_in_metadata(
request_kwargs,
itpm_reserved=itpm_reserved,
otpm_reserved=otpm_reserved,
itpm_cache_key=itpm_key if itpm_limit is not None else None,
otpm_cache_key=otpm_key if otpm_limit is not None else None,
_reservation(
itpm_reserved,
otpm_reserved,
itpm_key if itpm_limit is not None else None,
otpm_key if otpm_limit is not None else None,
),
)
return deployment
@ -502,6 +509,12 @@ async def async_io_token_pre_call_check(
return deployment
request_kwargs: Final = get_io_token_rate_limit_request_kwargs()
if request_kwargs is not None and request_kwargs.get("websocket") is not None:
_stash_reservation_in_metadata(
request_kwargs,
await async_admit_io_token_turn(dual_cache, deployment, get_utc_datetime(), parent_otel_span),
)
return deployment
_model: Final = _tokenizer_model(deployment)
estimated_input: Final = await offload_token_count(_estimate_input_tokens)(request_kwargs, model=_model)
max_tokens: Final = _resolve_max_tokens(request_kwargs, deployment)
@ -553,50 +566,39 @@ async def async_io_token_pre_call_check(
_stash_reservation_in_metadata(
request_kwargs,
itpm_reserved=itpm_reserved,
otpm_reserved=otpm_reserved,
itpm_cache_key=itpm_key if itpm_limit is not None else None,
otpm_cache_key=otpm_key if otpm_limit is not None else None,
_reservation(
itpm_reserved,
otpm_reserved,
itpm_key if itpm_limit is not None else None,
otpm_key if otpm_limit is not None else None,
),
)
return deployment
@with_service_target(ROUTER_USAGE_TARGET)
async def async_current_usage(
dual_cache: DualCache, key: str, limit: int, parent_otel_span: Span | None = None
) -> int | None:
local_usage: Final = await dual_cache.async_get_cache(key=key, local_only=True)
async def _async_shared_usage(dual_cache: DualCache, key: str, parent_otel_span: Span | None) -> int | None:
redis_cache: Final = dual_cache.redis_cache
if redis_cache is None or (local_usage is not None and local_usage >= limit):
return local_usage
try:
return await redis_cache.async_get_cache(key=key, parent_otel_span=parent_otel_span)
except RedisCircuitBreakerOpenError:
return local_usage
def _held_on(held_reservation: Mapping[str, object], reserved_key: str, cache_key_key: str, cache_key: str) -> int:
held: Final = held_reservation.get(reserved_key)
if held_reservation.get(cache_key_key) != cache_key or not isinstance(held, int):
return 0
return held
if redis_cache is not None:
with contextlib.suppress(RedisCircuitBreakerOpenError):
return await redis_cache.async_get_cache(key=key, parent_otel_span=parent_otel_span)
return await dual_cache.async_get_cache(key=key, local_only=True)
async def _raise_if_at_limit(
dual_cache: DualCache, key: str, limit: int | None, held: int, limit_label: str, parent_otel_span: Span | None
dual_cache: DualCache, key: str, limit: int | None, limit_label: str, parent_otel_span: Span | None
) -> None:
if limit is None:
return
usage: Final = await async_current_usage(dual_cache, key, limit + held, parent_otel_span)
if usage is not None and usage - held >= limit:
raise _rate_limit_error(limit_label, limit, usage - held)
usage: Final = await _async_shared_usage(dual_cache, key, parent_otel_span)
if usage is not None and usage >= limit:
raise _rate_limit_error(limit_label, limit, usage)
@with_service_target(ROUTER_USAGE_TARGET)
async def async_admit_io_token_turn(
dual_cache: DualCache,
deployment: Mapping[str, object],
held_reservation: Mapping[str, object],
now: datetime,
parent_otel_span: Span | None = None,
) -> Mapping[str, object]:
@ -608,19 +610,10 @@ async def async_admit_io_token_turn(
if cache_keys is None:
return MappingProxyType({})
itpm_key, otpm_key = cache_keys
itpm_held: Final = _held_on(held_reservation, ITPM_RESERVED_KEY, ITPM_CACHE_KEY, itpm_key)
otpm_held: Final = _held_on(held_reservation, OTPM_RESERVED_KEY, OTPM_CACHE_KEY, otpm_key)
await _raise_if_at_limit(dual_cache, itpm_key, itpm_limit, itpm_held, "ITPM", parent_otel_span)
await _raise_if_at_limit(dual_cache, otpm_key, otpm_limit, otpm_held, "OTPM", parent_otel_span)
if held_reservation:
return MappingProxyType({})
return MappingProxyType(
{
ITPM_RESERVED_KEY: 0,
OTPM_RESERVED_KEY: 0,
ITPM_CACHE_KEY: itpm_key if itpm_limit is not None else None,
OTPM_CACHE_KEY: otpm_key if otpm_limit is not None else None,
}
await _raise_if_at_limit(dual_cache, itpm_key, itpm_limit, "ITPM", parent_otel_span)
await _raise_if_at_limit(dual_cache, otpm_key, otpm_limit, "OTPM", parent_otel_span)
return _reservation(
0, 0, itpm_key if itpm_limit is not None else None, otpm_key if otpm_limit is not None else None
)

View file

@ -26,7 +26,6 @@ from litellm.caching.dual_cache import DualCache
from litellm.caching.redis_cache import RedisCircuitBreakerOpenError
from litellm.integrations.custom_logger import CustomLogger
from litellm.router_utils.pre_call_checks.io_token_rate_limit_check import (
async_current_usage,
async_io_token_pre_call_check,
async_io_token_reconcile_success,
async_io_token_refund_failure,
@ -290,7 +289,14 @@ class ModelRateLimitingCheck(CustomLogger):
@with_service_target(ROUTER_USAGE_TARGET)
async def _async_get_current_tpm(self, tpm_key: str, tpm_limit: int, parent_otel_span: Span | None) -> int | None:
return await async_current_usage(self.dual_cache, tpm_key, tpm_limit, parent_otel_span)
local_tpm: Final = await self.dual_cache.async_get_cache(key=tpm_key, local_only=True)
redis_cache: Final = self.dual_cache.redis_cache
if redis_cache is None or (local_tpm is not None and local_tpm >= tpm_limit):
return local_tpm
try:
return await redis_cache.async_get_cache(key=tpm_key, parent_otel_span=parent_otel_span)
except RedisCircuitBreakerOpenError:
return local_tpm
@with_service_target(ROUTER_USAGE_TARGET)
def pre_call_check(self, deployment: dict) -> dict | None:

View file

@ -518,9 +518,7 @@ def test_a_tpm_limit_lowered_while_the_socket_is_open_applies_to_its_next_turn(
assert deployment.forwarded_turns() == 1
def test_a_socket_on_an_otpm_limited_deployment_reserves_once_and_serves_every_turn(
limited: Gateway, cert: tuple[Path, Path]
) -> None:
def test_a_socket_on_an_otpm_limited_deployment_serves_every_turn(limited: Gateway, cert: tuple[Path, Path]) -> None:
with responses_peer(cert) as peer, limited.scenario() as scenario:
deployment: Final = _native(scenario, peer, otpm=10_000)
key: Final = scenario.key(models=[deployment.name])

View file

@ -10,10 +10,10 @@ Tests that:
import asyncio
import itertools
import json
from collections.abc import AsyncIterator, Callable, Mapping
from collections.abc import AsyncIterator, Mapping
from datetime import datetime, timezone
from functools import partial
from typing import TYPE_CHECKING, Final, TypeAlias
from typing import TYPE_CHECKING, Final
from unittest.mock import AsyncMock, MagicMock
import pytest
@ -49,6 +49,7 @@ from litellm.llms.xai.responses.transformation import XAIResponsesAPIConfig
from litellm.types.utils import GenericStreamingChunk
if TYPE_CHECKING:
from litellm.caching.dual_cache import DualCache
from litellm.litellm_core_utils.litellm_logging import Logging
from litellm.responses.streaming_iterator import ResponsesWsTurnAdmission
from litellm.router_utils.pre_call_checks.model_rate_limit_check import ModelRateLimitingCheck
@ -1667,19 +1668,13 @@ def _guard_recording(bill_refusal: MagicMock):
_OPENED_ON: Final = "global_router:ws-dep:openai/gpt-5-mini:otpm:12-34"
_TURNED_ON: Final = "global_router:ws-dep:openai/gpt-5-mini:otpm:12-35"
def _otpm_reservation(reserved: int, cache_key: str) -> Mapping[str, object]:
from litellm.router_utils.pre_call_checks.io_token_rate_limit_check import (
ITPM_CACHE_KEY,
ITPM_RESERVED_KEY,
OTPM_CACHE_KEY,
OTPM_RESERVED_KEY,
)
return {ITPM_RESERVED_KEY: 0, OTPM_RESERVED_KEY: reserved, ITPM_CACHE_KEY: None, OTPM_CACHE_KEY: cache_key}
_OPENED_AT: Final = datetime(2026, 1, 1, 12, 34, 56, tzinfo=timezone.utc)
_TURNED_AT: Final = datetime(2026, 1, 1, 12, 35, 5, tzinfo=timezone.utc)
_WS_DEPLOYMENT: Final = {
"litellm_params": {"model": "openai/gpt-5-mini", "otpm": 4096},
"model_info": {"id": "ws-dep"},
}
_OTHER_USAGE: Final = 10
_OUTPUT_TOKENS_PER_TURN: Final = 2
_UPSTREAM_ERROR: Final = json.dumps({"type": "error", "error": {"type": "server_error", "message": "upstream failed"}})
@ -1698,25 +1693,23 @@ def _turn_completed() -> str:
)
def _admitting(reservation: Mapping[str, object]) -> AsyncMock:
return AsyncMock(return_value=reservation)
async def _socket_opened_on(dual_cache: "DualCache") -> Mapping[str, object]:
from litellm.router_utils.pre_call_checks.io_token_rate_limit_check import async_admit_io_token_turn
await dual_cache.async_increment_cache(key=_OPENED_ON, value=_OTHER_USAGE, ttl=60)
await dual_cache.async_increment_cache(key=_TURNED_ON, value=_OTHER_USAGE, ttl=60)
return await async_admit_io_token_turn(dual_cache, _WS_DEPLOYMENT, _OPENED_AT)
_FIRST_TURN_ENDS: Final = (
pytest.param(_turn_completed(), _OUTPUT_TOKENS_PER_TURN, id="first_turn_completed"),
pytest.param(_UPSTREAM_ERROR, 0, id="first_turn_failed"),
)
_LaterTurnAdmission: TypeAlias = Callable[[], AsyncMock | None]
_LATER_TURN_ADMISSIONS: Final = (
pytest.param(
lambda: _admitting(_otpm_reservation(0, _TURNED_ON)), 2 * _OUTPUT_TOKENS_PER_TURN, id="admitted_this_minute"
),
pytest.param(lambda: _admitting({}), None, id="limits_removed_mid_socket"),
pytest.param(lambda: None, None, id="no_turn_admission"),
)
def _later_turns_admitted_through(dual_cache: "DualCache") -> "ResponsesWsTurnAdmission":
from litellm.router_utils.pre_call_checks.io_token_rate_limit_check import async_admit_io_token_turn
return partial(async_admit_io_token_turn, dual_cache, _WS_DEPLOYMENT, _TURNED_AT)
def _connection_settling_turns_through(rate_limits: "ModelRateLimitingCheck") -> "Logging":
def _connection_settling_turns_through(
rate_limits: "ModelRateLimitingCheck", opened_with: Mapping[str, object]
) -> "Logging":
import time
from litellm.cost_calculator import ResponsesWebSocketTokenUsageProcessor
@ -1746,20 +1739,10 @@ def _connection_settling_turns_through(rate_limits: "ModelRateLimitingCheck") ->
litellm_call_id="connection-call",
function_id="connection-call",
)
connection.litellm_params = {
"metadata": dict(_otpm_reservation(4096, _OPENED_ON)),
"litellm_metadata": dict(_otpm_reservation(4096, _OPENED_ON)),
}
connection.litellm_params = {"metadata": dict(opened_with), "litellm_metadata": dict(opened_with)}
return connection
_OPENED_AT: Final = datetime(2026, 1, 1, 12, 34, 56, tzinfo=timezone.utc)
_WS_DEPLOYMENT_AT_ITS_OTPM: Final = {
"litellm_params": {"model": "openai/gpt-5-mini", "otpm": 4096},
"model_info": {"id": "ws-dep"},
}
class _TwoOutputTokenTurns(CustomLLM):
def __init__(self, failing_turns: frozenset[int] = frozenset()) -> None:
super().__init__()
@ -1785,6 +1768,7 @@ async def _run_managed_relay(
rate_limits: "ModelRateLimitingCheck",
frames: tuple[str, ...],
turn_admission: "ResponsesWsTurnAdmission | None",
opened_with: Mapping[str, object],
) -> _ClientSocket:
import litellm
from litellm.litellm_core_utils.litellm_logging import Logging
@ -1807,10 +1791,10 @@ async def _run_managed_relay(
litellm_call_id="test-id",
function_id="test-func",
),
litellm_metadata={"user_api_key_hash": "k1", **_otpm_reservation(4096, _OPENED_ON)},
litellm_metadata={"user_api_key_hash": "k1", **opened_with},
first_message=first,
turn_admission=turn_admission,
metadata=dict(_otpm_reservation(4096, _OPENED_ON)),
metadata=dict(opened_with),
)
await handler.run()
@ -1888,14 +1872,15 @@ class TestWebSocketTurnAdmission:
assert "crossed budget" in str(failure)
@pytest.mark.asyncio
@pytest.mark.parametrize(("first_turn_end", "opened_on_tokens"), _FIRST_TURN_ENDS)
@pytest.mark.parametrize(("turn_admission", "turned_on_tokens"), _LATER_TURN_ADMISSIONS)
async def test_native_relay_settles_the_open_reservation_on_the_first_turn_only(
self,
first_turn_end: str,
opened_on_tokens: int,
turn_admission: _LaterTurnAdmission,
turned_on_tokens: int | None,
@pytest.mark.parametrize(
("first_turn_end", "first_turn_tokens"),
[
pytest.param(_turn_completed(), _OUTPUT_TOKENS_PER_TURN, id="first_turn_completed"),
pytest.param(_UPSTREAM_ERROR, 0, id="first_turn_failed"),
],
)
async def test_native_relay_counts_the_first_turn_on_the_minute_it_opened_and_each_later_turn_on_its_own(
self, first_turn_end: str, first_turn_tokens: int
) -> None:
import websockets.exceptions # noqa: F401 (lazy submodule must be importable)
@ -1904,7 +1889,7 @@ class TestWebSocketTurnAdmission:
from litellm.router_utils.pre_call_checks.model_rate_limit_check import ModelRateLimitingCheck
dual_cache: Final = DualCache()
await dual_cache.async_increment_cache(key=_OPENED_ON, value=4096, ttl=60)
opened_with: Final = await _socket_opened_on(dual_cache)
first, second, third = _turn_frames()
backend_ws: Final = MagicMock()
backend_ws.send = AsyncMock()
@ -1915,141 +1900,46 @@ class TestWebSocketTurnAdmission:
websocket=_ClientSocket(second, third),
backend_ws=backend_ws,
request_data={},
logging_obj=_connection_settling_turns_through(ModelRateLimitingCheck(dual_cache)),
logging_obj=_connection_settling_turns_through(ModelRateLimitingCheck(dual_cache), opened_with),
first_message=first,
turn_admission=turn_admission(),
turn_admission=_later_turns_admitted_through(dual_cache),
)
await handler.client_to_backend()
await handler.backend_to_client()
await drain_logging_tasks()
assert await dual_cache.async_get_cache(key=_OPENED_ON) == opened_on_tokens
assert await dual_cache.async_get_cache(key=_TURNED_ON) == turned_on_tokens
@pytest.mark.asyncio
async def test_native_relay_does_not_settle_the_open_reservation_again_for_an_event_no_turn_asked_for(
self,
) -> None:
import websockets.exceptions # noqa: F401 (lazy submodule must be importable)
from litellm.caching.dual_cache import DualCache
from litellm.responses.streaming_iterator import drain_logging_tasks
from litellm.router_utils.pre_call_checks.model_rate_limit_check import ModelRateLimitingCheck
dual_cache: Final = DualCache()
await dual_cache.async_increment_cache(key=_OPENED_ON, value=4096, ttl=60)
first, _, _ = _turn_frames()
backend_ws: Final = MagicMock()
backend_ws.send = AsyncMock()
backend_ws.recv = AsyncMock(side_effect=[_turn_completed(), _UPSTREAM_ERROR, Exception("stop")])
handler: Final = _make_streaming(
websocket=_ClientSocket(),
backend_ws=backend_ws,
request_data={},
logging_obj=_connection_settling_turns_through(ModelRateLimitingCheck(dual_cache)),
first_message=first,
turn_admission=_admitting(_otpm_reservation(0, _TURNED_ON)),
)
await handler.client_to_backend()
await handler.backend_to_client()
await drain_logging_tasks()
assert await dual_cache.async_get_cache(key=_OPENED_ON) == _OUTPUT_TOKENS_PER_TURN
assert await dual_cache.async_get_cache(key=_OPENED_ON) == _OTHER_USAGE + first_turn_tokens
assert await dual_cache.async_get_cache(key=_TURNED_ON) == _OTHER_USAGE + 2 * _OUTPUT_TOKENS_PER_TURN
@pytest.mark.asyncio
@pytest.mark.parametrize(
("failing_turns", "opened_on_tokens"),
("failing_turns", "first_turn_tokens"),
[
pytest.param(frozenset(), _OUTPUT_TOKENS_PER_TURN, id="first_turn_completed"),
pytest.param(frozenset({1}), 0, id="first_turn_failed"),
],
)
@pytest.mark.parametrize(("turn_admission", "turned_on_tokens"), _LATER_TURN_ADMISSIONS)
async def test_managed_relay_settles_the_open_reservation_on_the_first_turn_only(
self,
monkeypatch: pytest.MonkeyPatch,
failing_turns: frozenset[int],
opened_on_tokens: int,
turn_admission: _LaterTurnAdmission,
turned_on_tokens: int | None,
async def test_managed_relay_counts_the_first_turn_on_the_minute_it_opened_and_each_later_turn_on_its_own(
self, monkeypatch: pytest.MonkeyPatch, failing_turns: frozenset[int], first_turn_tokens: int
) -> None:
from litellm.caching.dual_cache import DualCache
from litellm.router_utils.pre_call_checks.model_rate_limit_check import ModelRateLimitingCheck
dual_cache: Final = DualCache()
await dual_cache.async_increment_cache(key=_OPENED_ON, value=4096, ttl=60)
opened_with: Final = await _socket_opened_on(dual_cache)
await _run_managed_relay(
monkeypatch,
_TwoOutputTokenTurns(failing_turns),
ModelRateLimitingCheck(dual_cache),
_turn_frames(),
turn_admission(),
_later_turns_admitted_through(dual_cache),
opened_with,
)
assert await dual_cache.async_get_cache(key=_OPENED_ON) == opened_on_tokens
assert await dual_cache.async_get_cache(key=_TURNED_ON) == turned_on_tokens
@pytest.mark.asyncio
async def test_managed_relay_answers_and_settles_the_first_prompt_after_a_warmup_when_the_open_hold_fills_the_minute(
self, monkeypatch: pytest.MonkeyPatch
) -> None:
from litellm.caching.dual_cache import DualCache
from litellm.router_utils.pre_call_checks.io_token_rate_limit_check import async_admit_io_token_turn
from litellm.router_utils.pre_call_checks.model_rate_limit_check import ModelRateLimitingCheck
dual_cache: Final = DualCache()
await dual_cache.async_increment_cache(key=_OPENED_ON, value=4096, ttl=60)
warmup: Final = json.dumps({"type": "response.create", "generate": False, "input": []})
first, _, _ = _turn_frames()
websocket: Final = await _run_managed_relay(
monkeypatch,
_TwoOutputTokenTurns(),
ModelRateLimitingCheck(dual_cache),
(warmup, first),
partial(async_admit_io_token_turn, dual_cache, _WS_DEPLOYMENT_AT_ITS_OTPM, now=_OPENED_AT),
)
assert _errors(websocket) == []
assert await dual_cache.async_get_cache(key=_OPENED_ON) == _OUTPUT_TOKENS_PER_TURN
@pytest.mark.asyncio
async def test_native_relay_answers_and_settles_the_first_prompt_after_a_refused_first_frame_when_the_open_hold_fills_the_minute(
self,
) -> None:
import websockets.exceptions # noqa: F401 (lazy submodule must be importable)
from litellm.caching.dual_cache import DualCache
from litellm.responses.streaming_iterator import drain_logging_tasks
from litellm.router_utils.pre_call_checks.io_token_rate_limit_check import async_admit_io_token_turn
from litellm.router_utils.pre_call_checks.model_rate_limit_check import ModelRateLimitingCheck
dual_cache: Final = DualCache()
await dual_cache.async_increment_cache(key=_OPENED_ON, value=4096, ttl=60)
first, second, _ = _turn_frames()
backend_ws: Final = MagicMock()
backend_ws.send = AsyncMock()
backend_ws.recv = AsyncMock(side_effect=[_turn_completed(), Exception("stop")])
websocket: Final = _ClientSocket(second)
handler: Final = _make_streaming(
websocket=websocket,
backend_ws=backend_ws,
request_data={},
logging_obj=_connection_settling_turns_through(ModelRateLimitingCheck(dual_cache)),
first_message=first,
frame_guard=_guard_blocking("one"),
turn_admission=partial(async_admit_io_token_turn, dual_cache, _WS_DEPLOYMENT_AT_ITS_OTPM, now=_OPENED_AT),
)
await handler.client_to_backend()
await handler.backend_to_client()
await drain_logging_tasks()
assert [(status, error["type"]) for status, error in _errors(websocket)] == [(None, "invalid_request_error")]
assert await dual_cache.async_get_cache(key=_OPENED_ON) == _OUTPUT_TOKENS_PER_TURN
assert await dual_cache.async_get_cache(key=_OPENED_ON) == _OTHER_USAGE + first_turn_tokens
assert await dual_cache.async_get_cache(key=_TURNED_ON) == _OTHER_USAGE + 2 * _OUTPUT_TOKENS_PER_TURN
@pytest.mark.asyncio
async def test_managed_relay_skips_a_later_turn_the_deployment_refuses(self, monkeypatch: pytest.MonkeyPatch):

View file

@ -3,12 +3,14 @@ Tests for separate ITPM/OTPM deployment rate limits (enforce_model_rate_limits).
"""
import asyncio
from collections.abc import Mapping
import threading
from collections.abc import Iterator, Mapping
from datetime import datetime, timezone
from typing import Final
from unittest.mock import AsyncMock, MagicMock
import pytest
from fakeredis import TcpFakeServer
import litellm
from litellm import Router
@ -36,6 +38,7 @@ from litellm.router_utils.pre_call_checks.io_token_rate_limit_check import (
from litellm.router_utils.pre_call_checks.model_rate_limit_check import (
ModelRateLimitingCheck,
)
from litellm.types.llms.openai import InputTokensDetails, ResponseAPIUsage, ResponsesAPIResponse
from litellm.types.utils import ModelResponse, Usage
@ -1146,15 +1149,6 @@ def _turn_key(kind: str, minute: str = "12-34") -> str:
return f"global_router:ws-turn-{kind}:openai/gpt-5-mini:{kind}:{minute}"
def _socket_hold(kind: str, held: int, cache_key: str) -> Mapping[str, object]:
return {
ITPM_RESERVED_KEY: held if kind == "itpm" else 0,
OTPM_RESERVED_KEY: held if kind == "otpm" else 0,
ITPM_CACHE_KEY: cache_key if kind == "itpm" else None,
OTPM_CACHE_KEY: cache_key if kind == "otpm" else None,
}
@pytest.mark.asyncio
@pytest.mark.parametrize("kind", ["itpm", "otpm"])
@pytest.mark.parametrize("used", [100, 101, 250])
@ -1163,7 +1157,7 @@ async def test_turn_admission_refuses_once_the_minutes_tokens_reach_the_limit(ki
await dual_cache.async_increment_cache(key=_turn_key(kind), value=used, ttl=60)
with pytest.raises(litellm.RateLimitError, match=f"{kind.upper()} limit=100, current usage={used}"):
await async_admit_io_token_turn(dual_cache, _turn_deployment(kind), {}, _TURN_AT)
await async_admit_io_token_turn(dual_cache, _turn_deployment(kind), _TURN_AT)
@pytest.mark.asyncio
@ -1174,7 +1168,7 @@ async def test_an_admitted_turn_reserves_nothing_and_counts_its_actual_tokens(ki
await dual_cache.async_increment_cache(key=_turn_key(kind), value=used, ttl=60)
actual: Final = _ACTUAL_TOKENS
reservation: Final = await async_admit_io_token_turn(dual_cache, _turn_deployment(kind), {}, _TURN_AT)
reservation: Final = await async_admit_io_token_turn(dual_cache, _turn_deployment(kind), _TURN_AT)
counter_key: Final = reservation[ITPM_CACHE_KEY if kind == "itpm" else OTPM_CACHE_KEY]
assert isinstance(counter_key, str)
assert await dual_cache.async_get_cache(key=counter_key) == used
@ -1196,44 +1190,99 @@ async def test_an_admitted_turn_reserves_nothing_and_counts_its_actual_tokens(ki
async def test_a_turn_without_io_token_counters_is_admitted_with_nothing_to_settle(
deployment: dict[str, object],
) -> None:
assert await async_admit_io_token_turn(DualCache(), deployment, {}, _TURN_AT) == {}
assert await async_admit_io_token_turn(DualCache(), deployment, _TURN_AT) == {}
@pytest.mark.asyncio
@pytest.mark.parametrize("kind", ["itpm", "otpm"])
@pytest.mark.parametrize("used_by_others", [0, 1, 99])
async def test_a_sockets_first_turn_is_not_refused_by_its_own_opening_hold_and_settles_it(
kind: str, used_by_others: int
async def test_a_socket_open_reserves_nothing_and_its_first_turn_counts_its_actual_tokens_on_its_own_minute(
kind: str, used_by_others: int, monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.setattr(
"litellm.router_utils.pre_call_checks.io_token_rate_limit_check.get_utc_datetime", lambda: _TURN_AT
)
dual_cache: Final = DualCache()
held: Final = _socket_hold(kind, 100, _turn_key(kind))
await dual_cache.async_increment_cache(key=_turn_key(kind), value=used_by_others + 100, ttl=60)
await dual_cache.async_increment_cache(key=_turn_key(kind), value=used_by_others, ttl=60)
socket_open: Final[dict[str, object]] = {"websocket": object(), "metadata": {}}
set_io_token_rate_limit_request_kwargs(socket_open)
admitted: Final = await async_admit_io_token_turn(dual_cache, _turn_deployment(kind), held, _TURN_AT)
assert await async_io_token_pre_call_check(dual_cache, _turn_deployment(kind)) == _turn_deployment(kind)
assert await dual_cache.async_get_cache(key=_turn_key(kind)) == used_by_others
opened_with: Final = socket_open["metadata"]
assert isinstance(opened_with, dict)
await async_io_token_reconcile_success(
dual_cache,
_proxy_success_kwargs({**held, **admitted}),
_reply_using(_ACTUAL_TOKENS["itpm"], _ACTUAL_TOKENS["otpm"]),
dual_cache, _proxy_success_kwargs(opened_with), _reply_using(_ACTUAL_TOKENS["itpm"], _ACTUAL_TOKENS["otpm"])
)
assert await dual_cache.async_get_cache(key=_turn_key(kind)) == used_by_others + _ACTUAL_TOKENS[kind]
@pytest.fixture
def shared_redis() -> Iterator[RedisCache]:
server: Final = TcpFakeServer(("127.0.0.1", 0), server_type="redis")
threading.Thread(target=server.serve_forever, daemon=True).start()
try:
yield RedisCache(host="127.0.0.1", port=server.server_address[1])
finally:
server.shutdown()
server.server_close()
@pytest.mark.asyncio
@pytest.mark.parametrize("kind", ["itpm", "otpm"])
@pytest.mark.parametrize(
("held_on_minute", "counted_this_minute"),
[pytest.param("12-34", 200, id="others_reach_the_limit"), pytest.param("12-33", 100, id="hold_from_last_minute")],
)
async def test_a_sockets_first_turn_is_refused_once_other_usage_reaches_the_limit(
kind: str, held_on_minute: str, counted_this_minute: int
async def test_a_turn_is_admitted_once_the_shared_counter_drops_below_the_limit_this_instance_last_read(
kind: str, shared_redis: RedisCache
) -> None:
dual_cache: Final = DualCache()
await dual_cache.async_increment_cache(key=_turn_key(kind), value=counted_this_minute, ttl=60)
held: Final = _socket_hold(kind, 100, _turn_key(kind, held_on_minute))
dual_cache: Final = DualCache(in_memory_cache=InMemoryCache(), redis_cache=shared_redis)
await dual_cache.in_memory_cache.async_increment(key=_turn_key(kind), value=100)
await shared_redis.async_increment(key=_turn_key(kind), value=99)
with pytest.raises(litellm.RateLimitError, match=f"{kind.upper()} limit=100, current usage=100"):
await async_admit_io_token_turn(dual_cache, _turn_deployment(kind), held, _TURN_AT)
admitted: Final = await async_admit_io_token_turn(dual_cache, _turn_deployment(kind), _TURN_AT)
assert admitted[ITPM_CACHE_KEY if kind == "itpm" else OTPM_CACHE_KEY] == _turn_key(kind)
@pytest.mark.asyncio
@pytest.mark.parametrize(
"responses_reply",
[
pytest.param(
ResponsesAPIResponse(
id="resp_cached",
created_at=0,
output=[],
usage=ResponseAPIUsage(
input_tokens=50,
output_tokens=5,
total_tokens=55,
input_tokens_details=InputTokensDetails(cached_tokens=30),
),
),
id="usage_object",
),
pytest.param(
{
"usage": {
"input_tokens": 50,
"output_tokens": 5,
"total_tokens": 55,
"input_tokens_details": {"cached_tokens": 30},
}
},
id="usage_dict",
),
],
)
async def test_a_turn_counts_only_its_uncached_responses_input_toward_itpm(responses_reply: object) -> None:
dual_cache: Final = DualCache()
await dual_cache.async_increment_cache(key=_turn_key("itpm"), value=10, ttl=60)
reservation: Final = await async_admit_io_token_turn(dual_cache, _turn_deployment("itpm"), _TURN_AT)
await async_io_token_reconcile_success(dual_cache, _proxy_success_kwargs(reservation), responses_reply)
assert await dual_cache.async_get_cache(key=_turn_key("itpm")) == 10 + 50 - 30
def _shared_counter_answering(answer: int | Exception) -> DualCache:
@ -1246,7 +1295,7 @@ def _shared_counter_answering(answer: int | Exception) -> DualCache:
@pytest.mark.parametrize("kind", ["itpm", "otpm"])
async def test_a_later_turn_is_refused_when_another_instance_filled_the_shared_minute(kind: str) -> None:
with pytest.raises(litellm.RateLimitError, match=f"{kind.upper()} limit=100, current usage=100"):
await async_admit_io_token_turn(_shared_counter_answering(100), _turn_deployment(kind), {}, _TURN_AT)
await async_admit_io_token_turn(_shared_counter_answering(100), _turn_deployment(kind), _TURN_AT)
@pytest.mark.asyncio
@ -1257,6 +1306,6 @@ async def test_a_later_turn_is_admitted_on_this_instance_s_count_when_the_shared
dual_cache: Final = _shared_counter_answering(RedisCircuitBreakerOpenError("circuit open"))
await dual_cache.in_memory_cache.async_increment(key=_turn_key(kind), value=99)
admitted: Final = await async_admit_io_token_turn(dual_cache, _turn_deployment(kind), {}, _TURN_AT)
admitted: Final = await async_admit_io_token_turn(dual_cache, _turn_deployment(kind), _TURN_AT)
assert admitted[ITPM_CACHE_KEY if kind == "itpm" else OTPM_CACHE_KEY] == _turn_key(kind)

View file

@ -538,14 +538,14 @@ async def test_deployment_callback_on_success_counts_a_responses_websocket_turn(
assert await router.cache.async_get_cache(key=tpm_key) == 17
async def _kwargs_routed_to(router: Router, model: str, function_name: str) -> dict[str, object]:
async def _kwargs_routed_to(router: Router, model: str, function_name: str, **request: object) -> dict[str, object]:
routed: Final[dict[str, object]] = {}
async def call(**kwargs: object) -> None:
routed.update(kwargs)
call.__name__ = function_name
await router._ageneric_api_call_with_fallbacks_helper(model=model, original_generic_function=call)
await router._ageneric_api_call_with_fallbacks_helper(model=model, original_generic_function=call, **request)
return routed
@ -733,13 +733,75 @@ def _io_limited_router(kind: str, **litellm_params: object) -> Router:
)
async def _count_minutes_tokens(router: Router, kind: str, tokens: int, minute: datetime) -> None:
await router.cache.async_increment_cache(
key=f"global_router:ws-io-dep:openai/ws-io-unpriced:{kind}:{minute.strftime('%H-%M')}",
value=tokens,
ttl=120,
def _minutes_counter_key(kind: str, minute: datetime, model: str = "openai/ws-io-unpriced") -> str:
return f"global_router:ws-io-dep:{model}:{kind}:{minute.strftime('%H-%M')}"
async def _count_minutes_tokens(
router: Router, kind: str, tokens: int, minute: datetime, model: str = "openai/ws-io-unpriced"
) -> None:
await router.cache.async_increment_cache(key=_minutes_counter_key(kind, minute, model), value=tokens, ttl=120)
_LONG_OUTPUT_MODEL: Final = "openai/gpt-5.4-mini"
def _long_output_router() -> tuple[Router, Mapping[str, int]]:
max_output: Final = litellm.get_model_info(_LONG_OUTPUT_MODEL)["max_output_tokens"]
assert isinstance(max_output, int)
limits: Final = {"itpm": 10_000, "otpm": max_output // 2}
return _io_limited_router("itpm", model=_LONG_OUTPUT_MODEL, otpm=limits["otpm"]), limits
@pytest.mark.asyncio
@pytest.mark.parametrize("kind", ["itpm", "otpm"])
async def test_responses_websocket_opens_without_counting_tokens_until_the_minute_reaches_deployment_io_limit(
kind: str, router_minute_pinned: datetime
):
router, limits = _long_output_router()
await _kwargs_routed_to(router, "ws-io", "_aresponses_websocket", websocket=object())
assert [
await router.cache.async_get_cache(key=_minutes_counter_key(counted, router_minute_pinned, _LONG_OUTPUT_MODEL))
for counted in ("itpm", "otpm")
] == [None, None]
await _count_minutes_tokens(router, kind, limits[kind], router_minute_pinned, _LONG_OUTPUT_MODEL)
with pytest.raises(litellm.RateLimitError, match=f"{kind.upper()} limit={limits[kind]}, current usage="):
await _kwargs_routed_to(router, "ws-io", "_aresponses_websocket", websocket=object())
@pytest.mark.asyncio
async def test_responses_websocket_first_turn_counts_its_actual_tokens_on_the_minute_it_opened(
router_minute_pinned: datetime,
):
from litellm.cost_calculator import ResponsesWebSocketTokenUsageProcessor
from litellm.litellm_core_utils.get_litellm_params import get_litellm_params
from litellm.router_utils.pre_call_checks.model_rate_limit_check import ModelRateLimitingCheck
router, _ = _long_output_router()
socket_metadata: Final = (await _kwargs_routed_to(router, "ws-io", "_aresponses_websocket", websocket=object()))[
"litellm_metadata"
]
assert isinstance(socket_metadata, dict)
rate_limits: Final = next(
check for check in router.optional_callbacks or () if isinstance(check, ModelRateLimitingCheck)
)
await rate_limits.async_log_success_event(
{"litellm_params": get_litellm_params(litellm_metadata=socket_metadata)},
ResponsesWebSocketTokenUsageProcessor.create_logging_object(
[{"type": "response.completed", "response": {"usage": {"input_tokens": 20, "output_tokens": 15}}}]
),
None,
None,
)
assert [
await router.cache.async_get_cache(key=_minutes_counter_key(counted, router_minute_pinned, _LONG_OUTPUT_MODEL))
for counted in ("itpm", "otpm")
] == [20, 15]
@pytest.mark.asyncio
@pytest.mark.parametrize("kind", ["itpm", "otpm"])
@ -1257,6 +1319,9 @@ def _rpm_tpm_router(model_id: str) -> Router:
def router_minute_pinned(monkeypatch: pytest.MonkeyPatch) -> datetime:
pinned: Final = datetime(2026, 1, 1, 12, 0, 30, tzinfo=timezone.utc)
monkeypatch.setattr("litellm.router.get_utc_datetime", lambda: pinned)
monkeypatch.setattr(
"litellm.router_utils.pre_call_checks.io_token_rate_limit_check.get_utc_datetime", lambda: pinned
)
return pinned