fix(router): mark only responses socket opens as check-only and count a turn on the minute it finishes

The check-only branch for a socket open keyed on a websocket value in the request kwargs, which any HTTP body can carry and which the realtime route also sets. The router now passes an opens_responses_socket flag into set_io_token_rate_limit_request_kwargs only for its _aresponses_websocket call, and the flag lives in the same context value as the request kwargs, so no request body can set it and realtime opens reserve and get refused as before

A zero reservation holds nothing to refund, so a socket turn now counts its real tokens on the minute the call finishes, read from the clock at settle time. A non-zero reservation still settles on the keys it reserved on so its refund hits the same counter

When the shared Redis read returns nothing, the check now falls back to this instance's in-memory count instead of admitting the turn with no count at all
This commit is contained in:
mateo-berri 2026-10-29 17:19:50 -07:00
parent 4e2342d475
commit 7d72b09652
6 changed files with 236 additions and 57 deletions

View file

@ -5111,6 +5111,7 @@ class Router(DeprecatedRouterMembers):
kwargs: dict,
function_name: str | None = None,
splits_stream_timeout: bool = False,
opens_responses_socket: bool = False,
) -> None:
"""
5 jobs:
@ -5200,7 +5201,11 @@ class Router(DeprecatedRouterMembers):
# here before it's wiped below, instead of relying on that attempt's
# (possibly still-pending) failure event to do it.
refund_stale_reservation_before_retry(self._core_cache, kwargs)
set_io_token_rate_limit_request_kwargs(kwargs, store_in_context=deployment_has_io_token_limits(deployment))
set_io_token_rate_limit_request_kwargs(
kwargs,
store_in_context=deployment_has_io_token_limits(deployment),
opens_responses_socket=opens_responses_socket,
)
kwargs[metadata_variable_name].setdefault(
ROUTING_REQUEST_TAGS_METADATA_KEY,
@ -6607,6 +6612,7 @@ class Router(DeprecatedRouterMembers):
kwargs=kwargs,
function_name=function_name,
splits_stream_timeout=handler_name in _STREAM_TIMEOUT_AWARE_GENERIC_HANDLERS,
opens_responses_socket=handler_name == "_aresponses_websocket",
)
data: Final = deployment["litellm_params"].copy()

View file

@ -13,6 +13,7 @@ from __future__ import annotations
import contextlib
import contextvars
from collections.abc import Mapping, MutableMapping
from dataclasses import dataclass
from datetime import datetime
from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Final
@ -24,7 +25,7 @@ from litellm import token_counter
from litellm._internal_context import with_service_target
from litellm._logging import verbose_router_logger
from litellm.caching.dual_cache import DualCache
from litellm.caching.redis_cache import RedisCircuitBreakerOpenError
from litellm.caching.redis_cache import RedisCache, RedisCircuitBreakerOpenError
from litellm.litellm_core_utils.internal_call_metadata import (
IO_TOKEN_RESERVATION_KEYS,
ITPM_CACHE_KEY,
@ -47,13 +48,22 @@ else:
RoutingArgsTTL: Final = 60
_io_token_rate_limit_request_kwargs: Final[contextvars.ContextVar[dict[str, object] | None]] = contextvars.ContextVar(
"io_token_rate_limit_request_kwargs",
@dataclass(frozen=True, slots=True)
class _IoTokenRateLimitRequest:
kwargs: dict[str, object] | None
opens_responses_socket: bool
_io_token_rate_limit_request: Final[contextvars.ContextVar[_IoTokenRateLimitRequest | None]] = contextvars.ContextVar(
"io_token_rate_limit_request",
default=None,
)
def set_io_token_rate_limit_request_kwargs(kwargs: dict[str, object] | None, store_in_context: bool = True) -> None:
def set_io_token_rate_limit_request_kwargs(
kwargs: dict[str, object] | None, store_in_context: bool = True, *, opens_responses_socket: bool = False
) -> None:
# The reservation sentinels are server-only, but `metadata` is caller
# controlled on proxy requests. Strip any client-supplied copies here (this
# runs before the router stashes its own reservation) so a forged
@ -67,11 +77,14 @@ def set_io_token_rate_limit_request_kwargs(kwargs: dict[str, object] | None, sto
# every other deployment overwrite the slot with None instead of the
# kwargs; overwriting (rather than skipping) also releases a previous
# request's kwargs when a context is reused.
_io_token_rate_limit_request_kwargs.set(kwargs if store_in_context else None)
_io_token_rate_limit_request.set(
_IoTokenRateLimitRequest(kwargs, opens_responses_socket) if store_in_context else None
)
def get_io_token_rate_limit_request_kwargs() -> dict[str, object] | None:
return _io_token_rate_limit_request_kwargs.get()
request: Final = _io_token_rate_limit_request.get()
return None if request is None else request.kwargs
def seconds_until_minute_reset() -> int:
@ -506,8 +519,9 @@ async def async_io_token_pre_call_check(
if itpm_limit is None and otpm_limit is None:
return deployment
request_kwargs: Final = get_io_token_rate_limit_request_kwargs()
if request_kwargs is not None and request_kwargs.get("websocket") is not None:
request: Final = _io_token_rate_limit_request.get()
request_kwargs: Final = None if request is None else request.kwargs
if request is not None and request.opens_responses_socket:
_stash_reservation_in_metadata(
request_kwargs,
await async_admit_io_token_turn(dual_cache, deployment, get_utc_datetime(), parent_otel_span),
@ -574,12 +588,19 @@ async def async_io_token_pre_call_check(
return deployment
async def _async_redis_usage(redis_cache: RedisCache | None, key: str, parent_otel_span: Span | None) -> int | None:
if redis_cache is None:
return None
with contextlib.suppress(RedisCircuitBreakerOpenError):
return await redis_cache.async_get_cache(key=key, parent_otel_span=parent_otel_span)
return None
@with_service_target(ROUTER_USAGE_TARGET)
async def _async_shared_usage(dual_cache: DualCache, key: str, parent_otel_span: Span | None) -> int | None:
redis_cache: Final = dual_cache.redis_cache
if redis_cache is not None:
with contextlib.suppress(RedisCircuitBreakerOpenError):
return await redis_cache.async_get_cache(key=key, parent_otel_span=parent_otel_span)
shared: Final = await _async_redis_usage(dual_cache.redis_cache, key, parent_otel_span)
if shared is not None:
return shared
return await dual_cache.async_get_cache(key=key, local_only=True)
@ -615,6 +636,12 @@ async def async_admit_io_token_turn(
)
def _settle_key(reserved_on: str | None, reserved: int, settled_at: datetime) -> str | None:
if reserved_on is None or reserved > 0:
return reserved_on
return f"{reserved_on.rpartition(':')[0]}:{settled_at.strftime('%H-%M')}"
@with_service_target(ROUTER_USAGE_TARGET)
def io_token_reconcile_success(
dual_cache: DualCache,
@ -623,9 +650,12 @@ def io_token_reconcile_success(
) -> None:
request_kwargs: Final[Mapping[str, object] | None] = kwargs
response: Final[object] = response_obj
itpm_reserved, otpm_reserved, itpm_key, otpm_key = _read_reservation_from_kwargs(request_kwargs)
if itpm_key is None and otpm_key is None:
itpm_reserved, otpm_reserved, itpm_reserved_on, otpm_reserved_on = _read_reservation_from_kwargs(request_kwargs)
if itpm_reserved_on is None and otpm_reserved_on is None:
return
settled_at: Final = get_utc_datetime()
itpm_key: Final = _settle_key(itpm_reserved_on, itpm_reserved, settled_at)
otpm_key: Final = _settle_key(otpm_reserved_on, otpm_reserved, settled_at)
billable_input, completion_tokens, usage_resolved = _resolve_reconcile_usage_tokens(request_kwargs, response)
@ -677,18 +707,20 @@ async def async_io_token_reconcile_success(
) -> None:
request_kwargs: Final[Mapping[str, object] | None] = kwargs
response: Final[object] = response_obj
itpm_reserved, otpm_reserved, itpm_key, otpm_key = _read_reservation_from_kwargs(request_kwargs)
if itpm_key is None and otpm_key is None:
itpm_reserved, otpm_reserved, itpm_reserved_on, otpm_reserved_on = _read_reservation_from_kwargs(request_kwargs)
if itpm_reserved_on is None and otpm_reserved_on is None:
return
# A held reservation settles on the exact key that holds it (which encodes
# the reservation's minute), so its refund lands on the counter it raised.
# A zero reservation holds nothing, so its actual tokens count on the minute
# the call finishes. Always clear the stash afterwards (even if an increment
# throws) so a retry or a duplicate success event can't re-process it.
settled_at: Final = get_utc_datetime()
itpm_key: Final = _settle_key(itpm_reserved_on, itpm_reserved, settled_at)
otpm_key: Final = _settle_key(otpm_reserved_on, otpm_reserved, settled_at)
billable_input, completion_tokens, usage_resolved = _resolve_reconcile_usage_tokens(request_kwargs, response)
# Reconcile against the exact key that held the reservation (which encodes
# the reservation's minute), not a key recomputed at response time. This
# tracks actual usage even when the pre-call estimate was 0, and avoids a
# minute-boundary mismatch for calls that span into the next minute. Always
# clear the stash afterwards (even if an increment throws) so a retry or a
# duplicate success event can't re-process it.
try:
if usage_resolved:
if itpm_key is not None:

View file

@ -1701,6 +1701,12 @@ async def _socket_opened_on(dual_cache: "DualCache") -> Mapping[str, object]:
return await async_admit_io_token_turn(dual_cache, _WS_DEPLOYMENT, _OPENED_AT)
def _turns_finish_at(monkeypatch: pytest.MonkeyPatch, finished_at: datetime) -> None:
monkeypatch.setattr(
"litellm.router_utils.pre_call_checks.io_token_rate_limit_check.get_utc_datetime", lambda: finished_at
)
def _later_turns_admitted_through(dual_cache: "DualCache") -> "ResponsesWsTurnAdmission":
from litellm.router_utils.pre_call_checks.io_token_rate_limit_check import async_admit_io_token_turn
@ -1929,8 +1935,8 @@ class TestWebSocketTurnAdmission:
pytest.param(_UPSTREAM_ERROR, 0, id="first_turn_failed"),
],
)
async def test_native_relay_counts_the_first_turn_on_the_minute_it_opened_and_each_later_turn_on_its_own(
self, first_turn_end: str, first_turn_tokens: int
async def test_native_relay_counts_every_turn_on_the_minute_it_finishes(
self, first_turn_end: str, first_turn_tokens: int, monkeypatch: pytest.MonkeyPatch
) -> None:
import websockets.exceptions # noqa: F401 (lazy submodule must be importable)
@ -1938,6 +1944,7 @@ class TestWebSocketTurnAdmission:
from litellm.responses.streaming_iterator import drain_logging_tasks
from litellm.router_utils.pre_call_checks.model_rate_limit_check import ModelRateLimitingCheck
_turns_finish_at(monkeypatch, _TURNED_AT)
dual_cache: Final = DualCache()
opened_with: Final = await _socket_opened_on(dual_cache)
first, second, third = _turn_frames()
@ -1959,8 +1966,11 @@ class TestWebSocketTurnAdmission:
await handler.backend_to_client()
await drain_logging_tasks()
assert await dual_cache.async_get_cache(key=_OPENED_ON) == _OTHER_USAGE + first_turn_tokens
assert await dual_cache.async_get_cache(key=_TURNED_ON) == _OTHER_USAGE + 2 * _OUTPUT_TOKENS_PER_TURN
assert await dual_cache.async_get_cache(key=_OPENED_ON) == _OTHER_USAGE
assert (
await dual_cache.async_get_cache(key=_TURNED_ON)
== _OTHER_USAGE + first_turn_tokens + 2 * _OUTPUT_TOKENS_PER_TURN
)
@pytest.mark.asyncio
@pytest.mark.parametrize(
@ -1971,7 +1981,7 @@ class TestWebSocketTurnAdmission:
],
)
async def test_native_relay_leaves_a_turn_s_cached_input_out_of_the_deployment_s_itpm(
self, cached_tokens: int, counted_input_tokens: int
self, cached_tokens: int, counted_input_tokens: int, monkeypatch: pytest.MonkeyPatch
) -> None:
import websockets.exceptions # noqa: F401 (lazy submodule must be importable)
@ -1980,6 +1990,7 @@ class TestWebSocketTurnAdmission:
from litellm.router_utils.pre_call_checks.io_token_rate_limit_check import async_admit_io_token_turn
from litellm.router_utils.pre_call_checks.model_rate_limit_check import ModelRateLimitingCheck
_turns_finish_at(monkeypatch, _OPENED_AT)
dual_cache: Final = DualCache()
opened_with: Final = await async_admit_io_token_turn(dual_cache, _ITPM_DEPLOYMENT, _OPENED_AT)
first, _, _ = _turn_frames()
@ -2008,12 +2019,13 @@ class TestWebSocketTurnAdmission:
pytest.param(frozenset({1}), 0, id="first_turn_failed"),
],
)
async def test_managed_relay_counts_the_first_turn_on_the_minute_it_opened_and_each_later_turn_on_its_own(
async def test_managed_relay_counts_every_turn_on_the_minute_it_finishes(
self, monkeypatch: pytest.MonkeyPatch, failing_turns: frozenset[int], first_turn_tokens: int
) -> None:
from litellm.caching.dual_cache import DualCache
from litellm.router_utils.pre_call_checks.model_rate_limit_check import ModelRateLimitingCheck
_turns_finish_at(monkeypatch, _TURNED_AT)
dual_cache: Final = DualCache()
opened_with: Final = await _socket_opened_on(dual_cache)
@ -2026,8 +2038,11 @@ class TestWebSocketTurnAdmission:
opened_with,
)
assert await dual_cache.async_get_cache(key=_OPENED_ON) == _OTHER_USAGE + first_turn_tokens
assert await dual_cache.async_get_cache(key=_TURNED_ON) == _OTHER_USAGE + 2 * _OUTPUT_TOKENS_PER_TURN
assert await dual_cache.async_get_cache(key=_OPENED_ON) == _OTHER_USAGE
assert (
await dual_cache.async_get_cache(key=_TURNED_ON)
== _OTHER_USAGE + first_turn_tokens + 2 * _OUTPUT_TOKENS_PER_TURN
)
@pytest.mark.asyncio
async def test_managed_relay_skips_a_later_turn_the_deployment_refuses(self, monkeypatch: pytest.MonkeyPatch):

View file

@ -1149,6 +1149,13 @@ def _turn_key(kind: str, minute: str = "12-34") -> str:
return f"global_router:ws-turn-{kind}:openai/gpt-5-mini:{kind}:{minute}"
@pytest.fixture
def settled_on_the_turn_s_minute(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setattr(
"litellm.router_utils.pre_call_checks.io_token_rate_limit_check.get_utc_datetime", lambda: _TURN_AT
)
@pytest.mark.asyncio
@pytest.mark.parametrize("kind", ["itpm", "otpm"])
@pytest.mark.parametrize("used", [100, 101, 250])
@ -1161,6 +1168,7 @@ async def test_turn_admission_refuses_once_the_minutes_tokens_reach_the_limit(ki
@pytest.mark.asyncio
@pytest.mark.usefixtures("settled_on_the_turn_s_minute")
@pytest.mark.parametrize("kind", ["itpm", "otpm"])
@pytest.mark.parametrize("used", [0, 1, 99])
async def test_an_admitted_turn_reserves_nothing_and_counts_its_actual_tokens(kind: str, used: int) -> None:
@ -1194,18 +1202,16 @@ async def test_a_turn_without_io_token_counters_is_admitted_with_nothing_to_sett
@pytest.mark.asyncio
@pytest.mark.usefixtures("settled_on_the_turn_s_minute")
@pytest.mark.parametrize("kind", ["itpm", "otpm"])
@pytest.mark.parametrize("used_by_others", [0, 1, 99])
async def test_a_socket_open_reserves_nothing_and_its_first_turn_counts_its_actual_tokens_on_its_own_minute(
kind: str, used_by_others: int, monkeypatch: pytest.MonkeyPatch
kind: str, used_by_others: int
) -> None:
monkeypatch.setattr(
"litellm.router_utils.pre_call_checks.io_token_rate_limit_check.get_utc_datetime", lambda: _TURN_AT
)
dual_cache: Final = DualCache()
await dual_cache.async_increment_cache(key=_turn_key(kind), value=used_by_others, ttl=60)
socket_open: Final[dict[str, object]] = {"websocket": object(), "metadata": {}}
set_io_token_rate_limit_request_kwargs(socket_open)
socket_open: Final[dict[str, object]] = {"metadata": {}}
set_io_token_rate_limit_request_kwargs(socket_open, opens_responses_socket=True)
assert await async_io_token_pre_call_check(dual_cache, _turn_deployment(kind)) == _turn_deployment(kind)
assert await dual_cache.async_get_cache(key=_turn_key(kind)) == used_by_others
@ -1230,6 +1236,51 @@ def shared_redis() -> Iterator[RedisCache]:
server.server_close()
@pytest.fixture
def unreachable_redis() -> RedisCache:
server: Final = TcpFakeServer(("127.0.0.1", 0), server_type="redis")
server.server_close()
return RedisCache(host="127.0.0.1", port=server.server_address[1])
@pytest.mark.asyncio
@pytest.mark.parametrize("kind", ["itpm", "otpm"])
async def test_a_turn_is_refused_on_this_instance_s_count_when_reading_the_shared_counter_fails(
kind: str, unreachable_redis: RedisCache
) -> None:
dual_cache: Final = DualCache(in_memory_cache=InMemoryCache(), redis_cache=unreachable_redis)
await dual_cache.in_memory_cache.async_increment(key=_turn_key(kind), value=100)
with pytest.raises(litellm.RateLimitError, match=f"{kind.upper()} limit=100, current usage=100"):
await async_admit_io_token_turn(dual_cache, _turn_deployment(kind), _TURN_AT)
@pytest.mark.asyncio
@pytest.mark.parametrize("kind", ["itpm", "otpm"])
@pytest.mark.parametrize("delivery", ["async", "sync"])
async def test_a_turn_finishing_in_the_next_minute_counts_its_actual_tokens_on_that_minute(
kind: str, delivery: str, monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.setattr(
"litellm.router_utils.pre_call_checks.io_token_rate_limit_check.get_utc_datetime",
lambda: datetime(2026, 1, 1, 12, 35, 10, tzinfo=timezone.utc),
)
dual_cache: Final = DualCache()
reservation: Final = await async_admit_io_token_turn(dual_cache, _turn_deployment(kind), _TURN_AT)
await _deliver_success(
ModelRateLimitingCheck(dual_cache=dual_cache),
delivery,
_proxy_success_kwargs(reservation),
_reply_using(_ACTUAL_TOKENS["itpm"], _ACTUAL_TOKENS["otpm"]),
)
assert [await dual_cache.async_get_cache(key=_turn_key(kind, minute)) for minute in ("12-34", "12-35")] == [
None,
_ACTUAL_TOKENS[kind],
]
@pytest.mark.asyncio
@pytest.mark.parametrize("kind", ["itpm", "otpm"])
async def test_a_turn_is_admitted_once_the_shared_counter_drops_below_the_limit_this_instance_last_read(
@ -1275,6 +1326,7 @@ async def test_a_turn_is_admitted_once_the_shared_counter_drops_below_the_limit_
),
],
)
@pytest.mark.usefixtures("settled_on_the_turn_s_minute")
async def test_a_turn_counts_only_its_uncached_responses_input_toward_itpm(responses_reply: object) -> None:
dual_cache: Final = DualCache()
await dual_cache.async_increment_cache(key=_turn_key("itpm"), value=10, ttl=60)

View file

@ -2199,7 +2199,11 @@ async def test_ageneric_api_call_deployment_model_overrides_alias():
)
def inject_alias_into_kwargs(
deployment: dict, kwargs: dict, function_name: str | None = None, splits_stream_timeout: bool = False
deployment: dict,
kwargs: dict,
function_name: str | None = None,
splits_stream_timeout: bool = False,
opens_responses_socket: bool = False,
) -> None:
# Simulate the alias leaking into kwargs (as happens when
# _ageneric_api_call_with_fallbacks sets kwargs["model"] = alias before

View file

@ -743,6 +743,31 @@ async def _count_minutes_tokens(
await router.cache.async_increment_cache(key=_minutes_counter_key(kind, minute, model), value=tokens, ttl=120)
async def _settle_turn(
router: Router, reservation: Mapping[str, object], input_tokens: int, output_tokens: int
) -> None:
from litellm.cost_calculator import ResponsesWebSocketTokenUsageProcessor
from litellm.litellm_core_utils.get_litellm_params import get_litellm_params
from litellm.router_utils.pre_call_checks.model_rate_limit_check import ModelRateLimitingCheck
rate_limits: Final = next(
check for check in router.optional_callbacks or () if isinstance(check, ModelRateLimitingCheck)
)
await rate_limits.async_log_success_event(
{"litellm_params": get_litellm_params(litellm_metadata=dict(reservation))},
ResponsesWebSocketTokenUsageProcessor.create_logging_object(
[
{
"type": "response.completed",
"response": {"usage": {"input_tokens": input_tokens, "output_tokens": output_tokens}},
}
]
),
None,
None,
)
_LONG_OUTPUT_MODEL: Final = "openai/gpt-5.4-mini"
@ -772,30 +797,16 @@ async def test_responses_websocket_opens_without_counting_tokens_until_the_minut
@pytest.mark.asyncio
async def test_responses_websocket_first_turn_counts_its_actual_tokens_on_the_minute_it_opened(
async def test_responses_websocket_first_turn_counts_its_actual_tokens_toward_deployment_itpm_and_otpm(
router_minute_pinned: datetime,
):
from litellm.cost_calculator import ResponsesWebSocketTokenUsageProcessor
from litellm.litellm_core_utils.get_litellm_params import get_litellm_params
from litellm.router_utils.pre_call_checks.model_rate_limit_check import ModelRateLimitingCheck
router, _ = _long_output_router()
socket_metadata: Final = (await _kwargs_routed_to(router, "ws-io", "_aresponses_websocket", websocket=object()))[
"litellm_metadata"
]
assert isinstance(socket_metadata, dict)
rate_limits: Final = next(
check for check in router.optional_callbacks or () if isinstance(check, ModelRateLimitingCheck)
)
await rate_limits.async_log_success_event(
{"litellm_params": get_litellm_params(litellm_metadata=socket_metadata)},
ResponsesWebSocketTokenUsageProcessor.create_logging_object(
[{"type": "response.completed", "response": {"usage": {"input_tokens": 20, "output_tokens": 15}}}]
),
None,
None,
)
await _settle_turn(router, socket_metadata, input_tokens=20, output_tokens=15)
assert [
await router.cache.async_get_cache(key=_minutes_counter_key(counted, router_minute_pinned, _LONG_OUTPUT_MODEL))
@ -803,6 +814,61 @@ async def test_responses_websocket_first_turn_counts_its_actual_tokens_on_the_mi
] == [20, 15]
_ADMITTED_LATE_IN_THE_MINUTE: Final = datetime(2026, 1, 1, 12, 0, 50, tzinfo=timezone.utc)
_FINISHED_IN_THE_NEXT_MINUTE: Final = datetime(2026, 1, 1, 12, 1, 15, tzinfo=timezone.utc)
@pytest.mark.asyncio
@pytest.mark.parametrize("kind", ["itpm", "otpm"])
@pytest.mark.parametrize("turn", ["first", "later"])
async def test_responses_websocket_turn_finishing_in_the_next_minute_counts_its_tokens_toward_that_minute(
kind: str, turn: str, monkeypatch: pytest.MonkeyPatch
):
_pin_router_clock(monkeypatch, _ADMITTED_LATE_IN_THE_MINUTE)
router: Final = _io_limited_router(kind)
socket: Final = await _kwargs_routed_to(router, "ws-io", "_aresponses_websocket", websocket=object())
admit_turn: Final = socket["turn_admission"]
assert callable(admit_turn)
reservation: Final = socket["litellm_metadata"] if turn == "first" else await admit_turn()
assert isinstance(reservation, Mapping)
_pin_router_clock(monkeypatch, _FINISHED_IN_THE_NEXT_MINUTE)
await _settle_turn(router, reservation, input_tokens=10_000, output_tokens=10_000)
with pytest.raises(litellm.RateLimitError, match=f"{kind.upper()} limit=10000, current usage=10000"):
await admit_turn()
@pytest.mark.asyncio
@pytest.mark.parametrize("body", [{}, {"websocket": True}], ids=["plain", "websocket_key"])
async def test_messages_call_reserves_max_tokens_against_deployment_otpm_whatever_websocket_key_its_body_carries(
body: Mapping[str, object], router_minute_pinned: datetime
):
router: Final = _io_limited_router("otpm")
for _ in range(3):
await _kwargs_routed_to(router, "ws-io", "anthropic_messages", max_tokens=3_000, **body)
assert await router.cache.async_get_cache(key=_minutes_counter_key("otpm", router_minute_pinned)) == 9_000
with pytest.raises(litellm.RateLimitError, match="OTPM limit=10000, current usage=12000"):
await _kwargs_routed_to(router, "ws-io", "anthropic_messages", max_tokens=3_000, **body)
async def _realtime_open_refusal(**socket: object) -> str:
with pytest.raises(litellm.RateLimitError) as refused:
await _kwargs_routed_to(_io_limited_router("otpm", otpm=50), "ws-io", "_arealtime", **socket)
return str(refused.value)
@pytest.mark.asyncio
@pytest.mark.usefixtures("router_minute_pinned")
async def test_realtime_open_reserves_and_is_refused_at_deployment_otpm_like_an_open_without_a_socket():
with_socket: Final = await _realtime_open_refusal(websocket=object())
assert "OTPM limit=50, current usage=" in with_socket
assert with_socket == await _realtime_open_refusal()
@pytest.mark.asyncio
@pytest.mark.parametrize("kind", ["itpm", "otpm"])
async def test_responses_websocket_turn_is_refused_once_the_minutes_tokens_reach_deployment_io_limit(
@ -1315,13 +1381,17 @@ def _rpm_tpm_router(model_id: str) -> Router:
)
@pytest.fixture
def router_minute_pinned(monkeypatch: pytest.MonkeyPatch) -> datetime:
pinned: Final = datetime(2026, 1, 1, 12, 0, 30, tzinfo=timezone.utc)
def _pin_router_clock(monkeypatch: pytest.MonkeyPatch, pinned: datetime) -> None:
monkeypatch.setattr("litellm.router.get_utc_datetime", lambda: pinned)
monkeypatch.setattr(
"litellm.router_utils.pre_call_checks.io_token_rate_limit_check.get_utc_datetime", lambda: pinned
)
@pytest.fixture
def router_minute_pinned(monkeypatch: pytest.MonkeyPatch) -> datetime:
pinned: Final = datetime(2026, 1, 1, 12, 0, 30, tzinfo=timezone.utc)
_pin_router_clock(monkeypatch, pinned)
return pinned