From 7d72b09652fa4d145f46941828ab2bb28b3d61e4 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 29 Oct 2026 17:19:50 -0700 Subject: [PATCH] fix(router): mark only responses socket opens as check-only and count a turn on the minute it finishes The check-only branch for a socket open keyed on a websocket value in the request kwargs, which any HTTP body can carry and which the realtime route also sets. The router now passes an opens_responses_socket flag into set_io_token_rate_limit_request_kwargs only for its _aresponses_websocket call, and the flag lives in the same context value as the request kwargs, so no request body can set it and realtime opens reserve and get refused as before A zero reservation holds nothing to refund, so a socket turn now counts its real tokens on the minute the call finishes, read from the clock at settle time. A non-zero reservation still settles on the keys it reserved on so its refund hits the same counter When the shared Redis read returns nothing, the check now falls back to this instance's in-memory count instead of admitting the turn with no count at all --- litellm/router.py | 8 +- .../io_token_rate_limit_check.py | 76 ++++++++---- .../test_responses_websocket_all_providers.py | 31 +++-- .../test_router/test_io_token_rate_limits.py | 64 ++++++++++- tests/unit/test_router/test_router.py | 6 +- .../test_router/test_router_helper_utils.py | 108 +++++++++++++++--- 6 files changed, 236 insertions(+), 57 deletions(-) diff --git a/litellm/router.py b/litellm/router.py index b087df7f1a1..1f310a2e57f 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -5111,6 +5111,7 @@ class Router(DeprecatedRouterMembers): kwargs: dict, function_name: str | None = None, splits_stream_timeout: bool = False, + opens_responses_socket: bool = False, ) -> None: """ 5 jobs: @@ -5200,7 +5201,11 @@ class Router(DeprecatedRouterMembers): # here before it's wiped below, instead of relying on that attempt's # (possibly still-pending) failure event to do it. refund_stale_reservation_before_retry(self._core_cache, kwargs) - set_io_token_rate_limit_request_kwargs(kwargs, store_in_context=deployment_has_io_token_limits(deployment)) + set_io_token_rate_limit_request_kwargs( + kwargs, + store_in_context=deployment_has_io_token_limits(deployment), + opens_responses_socket=opens_responses_socket, + ) kwargs[metadata_variable_name].setdefault( ROUTING_REQUEST_TAGS_METADATA_KEY, @@ -6607,6 +6612,7 @@ class Router(DeprecatedRouterMembers): kwargs=kwargs, function_name=function_name, splits_stream_timeout=handler_name in _STREAM_TIMEOUT_AWARE_GENERIC_HANDLERS, + opens_responses_socket=handler_name == "_aresponses_websocket", ) data: Final = deployment["litellm_params"].copy() diff --git a/litellm/router_utils/pre_call_checks/io_token_rate_limit_check.py b/litellm/router_utils/pre_call_checks/io_token_rate_limit_check.py index 89acad0a4cb..0afe8d182af 100644 --- a/litellm/router_utils/pre_call_checks/io_token_rate_limit_check.py +++ b/litellm/router_utils/pre_call_checks/io_token_rate_limit_check.py @@ -13,6 +13,7 @@ from __future__ import annotations import contextlib import contextvars from collections.abc import Mapping, MutableMapping +from dataclasses import dataclass from datetime import datetime from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final @@ -24,7 +25,7 @@ from litellm import token_counter from litellm._internal_context import with_service_target from litellm._logging import verbose_router_logger from litellm.caching.dual_cache import DualCache -from litellm.caching.redis_cache import RedisCircuitBreakerOpenError +from litellm.caching.redis_cache import RedisCache, RedisCircuitBreakerOpenError from litellm.litellm_core_utils.internal_call_metadata import ( IO_TOKEN_RESERVATION_KEYS, ITPM_CACHE_KEY, @@ -47,13 +48,22 @@ else: RoutingArgsTTL: Final = 60 -_io_token_rate_limit_request_kwargs: Final[contextvars.ContextVar[dict[str, object] | None]] = contextvars.ContextVar( - "io_token_rate_limit_request_kwargs", + +@dataclass(frozen=True, slots=True) +class _IoTokenRateLimitRequest: + kwargs: dict[str, object] | None + opens_responses_socket: bool + + +_io_token_rate_limit_request: Final[contextvars.ContextVar[_IoTokenRateLimitRequest | None]] = contextvars.ContextVar( + "io_token_rate_limit_request", default=None, ) -def set_io_token_rate_limit_request_kwargs(kwargs: dict[str, object] | None, store_in_context: bool = True) -> None: +def set_io_token_rate_limit_request_kwargs( + kwargs: dict[str, object] | None, store_in_context: bool = True, *, opens_responses_socket: bool = False +) -> None: # The reservation sentinels are server-only, but `metadata` is caller # controlled on proxy requests. Strip any client-supplied copies here (this # runs before the router stashes its own reservation) so a forged @@ -67,11 +77,14 @@ def set_io_token_rate_limit_request_kwargs(kwargs: dict[str, object] | None, sto # every other deployment overwrite the slot with None instead of the # kwargs; overwriting (rather than skipping) also releases a previous # request's kwargs when a context is reused. - _io_token_rate_limit_request_kwargs.set(kwargs if store_in_context else None) + _io_token_rate_limit_request.set( + _IoTokenRateLimitRequest(kwargs, opens_responses_socket) if store_in_context else None + ) def get_io_token_rate_limit_request_kwargs() -> dict[str, object] | None: - return _io_token_rate_limit_request_kwargs.get() + request: Final = _io_token_rate_limit_request.get() + return None if request is None else request.kwargs def seconds_until_minute_reset() -> int: @@ -506,8 +519,9 @@ async def async_io_token_pre_call_check( if itpm_limit is None and otpm_limit is None: return deployment - request_kwargs: Final = get_io_token_rate_limit_request_kwargs() - if request_kwargs is not None and request_kwargs.get("websocket") is not None: + request: Final = _io_token_rate_limit_request.get() + request_kwargs: Final = None if request is None else request.kwargs + if request is not None and request.opens_responses_socket: _stash_reservation_in_metadata( request_kwargs, await async_admit_io_token_turn(dual_cache, deployment, get_utc_datetime(), parent_otel_span), @@ -574,12 +588,19 @@ async def async_io_token_pre_call_check( return deployment +async def _async_redis_usage(redis_cache: RedisCache | None, key: str, parent_otel_span: Span | None) -> int | None: + if redis_cache is None: + return None + with contextlib.suppress(RedisCircuitBreakerOpenError): + return await redis_cache.async_get_cache(key=key, parent_otel_span=parent_otel_span) + return None + + @with_service_target(ROUTER_USAGE_TARGET) async def _async_shared_usage(dual_cache: DualCache, key: str, parent_otel_span: Span | None) -> int | None: - redis_cache: Final = dual_cache.redis_cache - if redis_cache is not None: - with contextlib.suppress(RedisCircuitBreakerOpenError): - return await redis_cache.async_get_cache(key=key, parent_otel_span=parent_otel_span) + shared: Final = await _async_redis_usage(dual_cache.redis_cache, key, parent_otel_span) + if shared is not None: + return shared return await dual_cache.async_get_cache(key=key, local_only=True) @@ -615,6 +636,12 @@ async def async_admit_io_token_turn( ) +def _settle_key(reserved_on: str | None, reserved: int, settled_at: datetime) -> str | None: + if reserved_on is None or reserved > 0: + return reserved_on + return f"{reserved_on.rpartition(':')[0]}:{settled_at.strftime('%H-%M')}" + + @with_service_target(ROUTER_USAGE_TARGET) def io_token_reconcile_success( dual_cache: DualCache, @@ -623,9 +650,12 @@ def io_token_reconcile_success( ) -> None: request_kwargs: Final[Mapping[str, object] | None] = kwargs response: Final[object] = response_obj - itpm_reserved, otpm_reserved, itpm_key, otpm_key = _read_reservation_from_kwargs(request_kwargs) - if itpm_key is None and otpm_key is None: + itpm_reserved, otpm_reserved, itpm_reserved_on, otpm_reserved_on = _read_reservation_from_kwargs(request_kwargs) + if itpm_reserved_on is None and otpm_reserved_on is None: return + settled_at: Final = get_utc_datetime() + itpm_key: Final = _settle_key(itpm_reserved_on, itpm_reserved, settled_at) + otpm_key: Final = _settle_key(otpm_reserved_on, otpm_reserved, settled_at) billable_input, completion_tokens, usage_resolved = _resolve_reconcile_usage_tokens(request_kwargs, response) @@ -677,18 +707,20 @@ async def async_io_token_reconcile_success( ) -> None: request_kwargs: Final[Mapping[str, object] | None] = kwargs response: Final[object] = response_obj - itpm_reserved, otpm_reserved, itpm_key, otpm_key = _read_reservation_from_kwargs(request_kwargs) - if itpm_key is None and otpm_key is None: + itpm_reserved, otpm_reserved, itpm_reserved_on, otpm_reserved_on = _read_reservation_from_kwargs(request_kwargs) + if itpm_reserved_on is None and otpm_reserved_on is None: return + # A held reservation settles on the exact key that holds it (which encodes + # the reservation's minute), so its refund lands on the counter it raised. + # A zero reservation holds nothing, so its actual tokens count on the minute + # the call finishes. Always clear the stash afterwards (even if an increment + # throws) so a retry or a duplicate success event can't re-process it. + settled_at: Final = get_utc_datetime() + itpm_key: Final = _settle_key(itpm_reserved_on, itpm_reserved, settled_at) + otpm_key: Final = _settle_key(otpm_reserved_on, otpm_reserved, settled_at) billable_input, completion_tokens, usage_resolved = _resolve_reconcile_usage_tokens(request_kwargs, response) - # Reconcile against the exact key that held the reservation (which encodes - # the reservation's minute), not a key recomputed at response time. This - # tracks actual usage even when the pre-call estimate was 0, and avoids a - # minute-boundary mismatch for calls that span into the next minute. Always - # clear the stash afterwards (even if an increment throws) so a retry or a - # duplicate success event can't re-process it. try: if usage_resolved: if itpm_key is not None: diff --git a/tests/unit/responses/test_responses_websocket_all_providers.py b/tests/unit/responses/test_responses_websocket_all_providers.py index 5009e1e58ac..c84c42277f2 100644 --- a/tests/unit/responses/test_responses_websocket_all_providers.py +++ b/tests/unit/responses/test_responses_websocket_all_providers.py @@ -1701,6 +1701,12 @@ async def _socket_opened_on(dual_cache: "DualCache") -> Mapping[str, object]: return await async_admit_io_token_turn(dual_cache, _WS_DEPLOYMENT, _OPENED_AT) +def _turns_finish_at(monkeypatch: pytest.MonkeyPatch, finished_at: datetime) -> None: + monkeypatch.setattr( + "litellm.router_utils.pre_call_checks.io_token_rate_limit_check.get_utc_datetime", lambda: finished_at + ) + + def _later_turns_admitted_through(dual_cache: "DualCache") -> "ResponsesWsTurnAdmission": from litellm.router_utils.pre_call_checks.io_token_rate_limit_check import async_admit_io_token_turn @@ -1929,8 +1935,8 @@ class TestWebSocketTurnAdmission: pytest.param(_UPSTREAM_ERROR, 0, id="first_turn_failed"), ], ) - async def test_native_relay_counts_the_first_turn_on_the_minute_it_opened_and_each_later_turn_on_its_own( - self, first_turn_end: str, first_turn_tokens: int + async def test_native_relay_counts_every_turn_on_the_minute_it_finishes( + self, first_turn_end: str, first_turn_tokens: int, monkeypatch: pytest.MonkeyPatch ) -> None: import websockets.exceptions # noqa: F401 (lazy submodule must be importable) @@ -1938,6 +1944,7 @@ class TestWebSocketTurnAdmission: from litellm.responses.streaming_iterator import drain_logging_tasks from litellm.router_utils.pre_call_checks.model_rate_limit_check import ModelRateLimitingCheck + _turns_finish_at(monkeypatch, _TURNED_AT) dual_cache: Final = DualCache() opened_with: Final = await _socket_opened_on(dual_cache) first, second, third = _turn_frames() @@ -1959,8 +1966,11 @@ class TestWebSocketTurnAdmission: await handler.backend_to_client() await drain_logging_tasks() - assert await dual_cache.async_get_cache(key=_OPENED_ON) == _OTHER_USAGE + first_turn_tokens - assert await dual_cache.async_get_cache(key=_TURNED_ON) == _OTHER_USAGE + 2 * _OUTPUT_TOKENS_PER_TURN + assert await dual_cache.async_get_cache(key=_OPENED_ON) == _OTHER_USAGE + assert ( + await dual_cache.async_get_cache(key=_TURNED_ON) + == _OTHER_USAGE + first_turn_tokens + 2 * _OUTPUT_TOKENS_PER_TURN + ) @pytest.mark.asyncio @pytest.mark.parametrize( @@ -1971,7 +1981,7 @@ class TestWebSocketTurnAdmission: ], ) async def test_native_relay_leaves_a_turn_s_cached_input_out_of_the_deployment_s_itpm( - self, cached_tokens: int, counted_input_tokens: int + self, cached_tokens: int, counted_input_tokens: int, monkeypatch: pytest.MonkeyPatch ) -> None: import websockets.exceptions # noqa: F401 (lazy submodule must be importable) @@ -1980,6 +1990,7 @@ class TestWebSocketTurnAdmission: from litellm.router_utils.pre_call_checks.io_token_rate_limit_check import async_admit_io_token_turn from litellm.router_utils.pre_call_checks.model_rate_limit_check import ModelRateLimitingCheck + _turns_finish_at(monkeypatch, _OPENED_AT) dual_cache: Final = DualCache() opened_with: Final = await async_admit_io_token_turn(dual_cache, _ITPM_DEPLOYMENT, _OPENED_AT) first, _, _ = _turn_frames() @@ -2008,12 +2019,13 @@ class TestWebSocketTurnAdmission: pytest.param(frozenset({1}), 0, id="first_turn_failed"), ], ) - async def test_managed_relay_counts_the_first_turn_on_the_minute_it_opened_and_each_later_turn_on_its_own( + async def test_managed_relay_counts_every_turn_on_the_minute_it_finishes( self, monkeypatch: pytest.MonkeyPatch, failing_turns: frozenset[int], first_turn_tokens: int ) -> None: from litellm.caching.dual_cache import DualCache from litellm.router_utils.pre_call_checks.model_rate_limit_check import ModelRateLimitingCheck + _turns_finish_at(monkeypatch, _TURNED_AT) dual_cache: Final = DualCache() opened_with: Final = await _socket_opened_on(dual_cache) @@ -2026,8 +2038,11 @@ class TestWebSocketTurnAdmission: opened_with, ) - assert await dual_cache.async_get_cache(key=_OPENED_ON) == _OTHER_USAGE + first_turn_tokens - assert await dual_cache.async_get_cache(key=_TURNED_ON) == _OTHER_USAGE + 2 * _OUTPUT_TOKENS_PER_TURN + assert await dual_cache.async_get_cache(key=_OPENED_ON) == _OTHER_USAGE + assert ( + await dual_cache.async_get_cache(key=_TURNED_ON) + == _OTHER_USAGE + first_turn_tokens + 2 * _OUTPUT_TOKENS_PER_TURN + ) @pytest.mark.asyncio async def test_managed_relay_skips_a_later_turn_the_deployment_refuses(self, monkeypatch: pytest.MonkeyPatch): diff --git a/tests/unit/test_router/test_io_token_rate_limits.py b/tests/unit/test_router/test_io_token_rate_limits.py index 99789076c91..b914d1286d7 100644 --- a/tests/unit/test_router/test_io_token_rate_limits.py +++ b/tests/unit/test_router/test_io_token_rate_limits.py @@ -1149,6 +1149,13 @@ def _turn_key(kind: str, minute: str = "12-34") -> str: return f"global_router:ws-turn-{kind}:openai/gpt-5-mini:{kind}:{minute}" +@pytest.fixture +def settled_on_the_turn_s_minute(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr( + "litellm.router_utils.pre_call_checks.io_token_rate_limit_check.get_utc_datetime", lambda: _TURN_AT + ) + + @pytest.mark.asyncio @pytest.mark.parametrize("kind", ["itpm", "otpm"]) @pytest.mark.parametrize("used", [100, 101, 250]) @@ -1161,6 +1168,7 @@ async def test_turn_admission_refuses_once_the_minutes_tokens_reach_the_limit(ki @pytest.mark.asyncio +@pytest.mark.usefixtures("settled_on_the_turn_s_minute") @pytest.mark.parametrize("kind", ["itpm", "otpm"]) @pytest.mark.parametrize("used", [0, 1, 99]) async def test_an_admitted_turn_reserves_nothing_and_counts_its_actual_tokens(kind: str, used: int) -> None: @@ -1194,18 +1202,16 @@ async def test_a_turn_without_io_token_counters_is_admitted_with_nothing_to_sett @pytest.mark.asyncio +@pytest.mark.usefixtures("settled_on_the_turn_s_minute") @pytest.mark.parametrize("kind", ["itpm", "otpm"]) @pytest.mark.parametrize("used_by_others", [0, 1, 99]) async def test_a_socket_open_reserves_nothing_and_its_first_turn_counts_its_actual_tokens_on_its_own_minute( - kind: str, used_by_others: int, monkeypatch: pytest.MonkeyPatch + kind: str, used_by_others: int ) -> None: - monkeypatch.setattr( - "litellm.router_utils.pre_call_checks.io_token_rate_limit_check.get_utc_datetime", lambda: _TURN_AT - ) dual_cache: Final = DualCache() await dual_cache.async_increment_cache(key=_turn_key(kind), value=used_by_others, ttl=60) - socket_open: Final[dict[str, object]] = {"websocket": object(), "metadata": {}} - set_io_token_rate_limit_request_kwargs(socket_open) + socket_open: Final[dict[str, object]] = {"metadata": {}} + set_io_token_rate_limit_request_kwargs(socket_open, opens_responses_socket=True) assert await async_io_token_pre_call_check(dual_cache, _turn_deployment(kind)) == _turn_deployment(kind) assert await dual_cache.async_get_cache(key=_turn_key(kind)) == used_by_others @@ -1230,6 +1236,51 @@ def shared_redis() -> Iterator[RedisCache]: server.server_close() +@pytest.fixture +def unreachable_redis() -> RedisCache: + server: Final = TcpFakeServer(("127.0.0.1", 0), server_type="redis") + server.server_close() + return RedisCache(host="127.0.0.1", port=server.server_address[1]) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("kind", ["itpm", "otpm"]) +async def test_a_turn_is_refused_on_this_instance_s_count_when_reading_the_shared_counter_fails( + kind: str, unreachable_redis: RedisCache +) -> None: + dual_cache: Final = DualCache(in_memory_cache=InMemoryCache(), redis_cache=unreachable_redis) + await dual_cache.in_memory_cache.async_increment(key=_turn_key(kind), value=100) + + with pytest.raises(litellm.RateLimitError, match=f"{kind.upper()} limit=100, current usage=100"): + await async_admit_io_token_turn(dual_cache, _turn_deployment(kind), _TURN_AT) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("kind", ["itpm", "otpm"]) +@pytest.mark.parametrize("delivery", ["async", "sync"]) +async def test_a_turn_finishing_in_the_next_minute_counts_its_actual_tokens_on_that_minute( + kind: str, delivery: str, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr( + "litellm.router_utils.pre_call_checks.io_token_rate_limit_check.get_utc_datetime", + lambda: datetime(2026, 1, 1, 12, 35, 10, tzinfo=timezone.utc), + ) + dual_cache: Final = DualCache() + reservation: Final = await async_admit_io_token_turn(dual_cache, _turn_deployment(kind), _TURN_AT) + + await _deliver_success( + ModelRateLimitingCheck(dual_cache=dual_cache), + delivery, + _proxy_success_kwargs(reservation), + _reply_using(_ACTUAL_TOKENS["itpm"], _ACTUAL_TOKENS["otpm"]), + ) + + assert [await dual_cache.async_get_cache(key=_turn_key(kind, minute)) for minute in ("12-34", "12-35")] == [ + None, + _ACTUAL_TOKENS[kind], + ] + + @pytest.mark.asyncio @pytest.mark.parametrize("kind", ["itpm", "otpm"]) async def test_a_turn_is_admitted_once_the_shared_counter_drops_below_the_limit_this_instance_last_read( @@ -1275,6 +1326,7 @@ async def test_a_turn_is_admitted_once_the_shared_counter_drops_below_the_limit_ ), ], ) +@pytest.mark.usefixtures("settled_on_the_turn_s_minute") async def test_a_turn_counts_only_its_uncached_responses_input_toward_itpm(responses_reply: object) -> None: dual_cache: Final = DualCache() await dual_cache.async_increment_cache(key=_turn_key("itpm"), value=10, ttl=60) diff --git a/tests/unit/test_router/test_router.py b/tests/unit/test_router/test_router.py index a0a7dd78193..fa17a5a745e 100644 --- a/tests/unit/test_router/test_router.py +++ b/tests/unit/test_router/test_router.py @@ -2199,7 +2199,11 @@ async def test_ageneric_api_call_deployment_model_overrides_alias(): ) def inject_alias_into_kwargs( - deployment: dict, kwargs: dict, function_name: str | None = None, splits_stream_timeout: bool = False + deployment: dict, + kwargs: dict, + function_name: str | None = None, + splits_stream_timeout: bool = False, + opens_responses_socket: bool = False, ) -> None: # Simulate the alias leaking into kwargs (as happens when # _ageneric_api_call_with_fallbacks sets kwargs["model"] = alias before diff --git a/tests/unit/test_router/test_router_helper_utils.py b/tests/unit/test_router/test_router_helper_utils.py index be99c6f9350..2a82825a33c 100644 --- a/tests/unit/test_router/test_router_helper_utils.py +++ b/tests/unit/test_router/test_router_helper_utils.py @@ -743,6 +743,31 @@ async def _count_minutes_tokens( await router.cache.async_increment_cache(key=_minutes_counter_key(kind, minute, model), value=tokens, ttl=120) +async def _settle_turn( + router: Router, reservation: Mapping[str, object], input_tokens: int, output_tokens: int +) -> None: + from litellm.cost_calculator import ResponsesWebSocketTokenUsageProcessor + from litellm.litellm_core_utils.get_litellm_params import get_litellm_params + from litellm.router_utils.pre_call_checks.model_rate_limit_check import ModelRateLimitingCheck + + rate_limits: Final = next( + check for check in router.optional_callbacks or () if isinstance(check, ModelRateLimitingCheck) + ) + await rate_limits.async_log_success_event( + {"litellm_params": get_litellm_params(litellm_metadata=dict(reservation))}, + ResponsesWebSocketTokenUsageProcessor.create_logging_object( + [ + { + "type": "response.completed", + "response": {"usage": {"input_tokens": input_tokens, "output_tokens": output_tokens}}, + } + ] + ), + None, + None, + ) + + _LONG_OUTPUT_MODEL: Final = "openai/gpt-5.4-mini" @@ -772,30 +797,16 @@ async def test_responses_websocket_opens_without_counting_tokens_until_the_minut @pytest.mark.asyncio -async def test_responses_websocket_first_turn_counts_its_actual_tokens_on_the_minute_it_opened( +async def test_responses_websocket_first_turn_counts_its_actual_tokens_toward_deployment_itpm_and_otpm( router_minute_pinned: datetime, ): - from litellm.cost_calculator import ResponsesWebSocketTokenUsageProcessor - from litellm.litellm_core_utils.get_litellm_params import get_litellm_params - from litellm.router_utils.pre_call_checks.model_rate_limit_check import ModelRateLimitingCheck - router, _ = _long_output_router() socket_metadata: Final = (await _kwargs_routed_to(router, "ws-io", "_aresponses_websocket", websocket=object()))[ "litellm_metadata" ] assert isinstance(socket_metadata, dict) - rate_limits: Final = next( - check for check in router.optional_callbacks or () if isinstance(check, ModelRateLimitingCheck) - ) - await rate_limits.async_log_success_event( - {"litellm_params": get_litellm_params(litellm_metadata=socket_metadata)}, - ResponsesWebSocketTokenUsageProcessor.create_logging_object( - [{"type": "response.completed", "response": {"usage": {"input_tokens": 20, "output_tokens": 15}}}] - ), - None, - None, - ) + await _settle_turn(router, socket_metadata, input_tokens=20, output_tokens=15) assert [ await router.cache.async_get_cache(key=_minutes_counter_key(counted, router_minute_pinned, _LONG_OUTPUT_MODEL)) @@ -803,6 +814,61 @@ async def test_responses_websocket_first_turn_counts_its_actual_tokens_on_the_mi ] == [20, 15] +_ADMITTED_LATE_IN_THE_MINUTE: Final = datetime(2026, 1, 1, 12, 0, 50, tzinfo=timezone.utc) +_FINISHED_IN_THE_NEXT_MINUTE: Final = datetime(2026, 1, 1, 12, 1, 15, tzinfo=timezone.utc) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("kind", ["itpm", "otpm"]) +@pytest.mark.parametrize("turn", ["first", "later"]) +async def test_responses_websocket_turn_finishing_in_the_next_minute_counts_its_tokens_toward_that_minute( + kind: str, turn: str, monkeypatch: pytest.MonkeyPatch +): + _pin_router_clock(monkeypatch, _ADMITTED_LATE_IN_THE_MINUTE) + router: Final = _io_limited_router(kind) + socket: Final = await _kwargs_routed_to(router, "ws-io", "_aresponses_websocket", websocket=object()) + admit_turn: Final = socket["turn_admission"] + assert callable(admit_turn) + reservation: Final = socket["litellm_metadata"] if turn == "first" else await admit_turn() + assert isinstance(reservation, Mapping) + _pin_router_clock(monkeypatch, _FINISHED_IN_THE_NEXT_MINUTE) + + await _settle_turn(router, reservation, input_tokens=10_000, output_tokens=10_000) + + with pytest.raises(litellm.RateLimitError, match=f"{kind.upper()} limit=10000, current usage=10000"): + await admit_turn() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("body", [{}, {"websocket": True}], ids=["plain", "websocket_key"]) +async def test_messages_call_reserves_max_tokens_against_deployment_otpm_whatever_websocket_key_its_body_carries( + body: Mapping[str, object], router_minute_pinned: datetime +): + router: Final = _io_limited_router("otpm") + + for _ in range(3): + await _kwargs_routed_to(router, "ws-io", "anthropic_messages", max_tokens=3_000, **body) + + assert await router.cache.async_get_cache(key=_minutes_counter_key("otpm", router_minute_pinned)) == 9_000 + with pytest.raises(litellm.RateLimitError, match="OTPM limit=10000, current usage=12000"): + await _kwargs_routed_to(router, "ws-io", "anthropic_messages", max_tokens=3_000, **body) + + +async def _realtime_open_refusal(**socket: object) -> str: + with pytest.raises(litellm.RateLimitError) as refused: + await _kwargs_routed_to(_io_limited_router("otpm", otpm=50), "ws-io", "_arealtime", **socket) + return str(refused.value) + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("router_minute_pinned") +async def test_realtime_open_reserves_and_is_refused_at_deployment_otpm_like_an_open_without_a_socket(): + with_socket: Final = await _realtime_open_refusal(websocket=object()) + + assert "OTPM limit=50, current usage=" in with_socket + assert with_socket == await _realtime_open_refusal() + + @pytest.mark.asyncio @pytest.mark.parametrize("kind", ["itpm", "otpm"]) async def test_responses_websocket_turn_is_refused_once_the_minutes_tokens_reach_deployment_io_limit( @@ -1315,13 +1381,17 @@ def _rpm_tpm_router(model_id: str) -> Router: ) -@pytest.fixture -def router_minute_pinned(monkeypatch: pytest.MonkeyPatch) -> datetime: - pinned: Final = datetime(2026, 1, 1, 12, 0, 30, tzinfo=timezone.utc) +def _pin_router_clock(monkeypatch: pytest.MonkeyPatch, pinned: datetime) -> None: monkeypatch.setattr("litellm.router.get_utc_datetime", lambda: pinned) monkeypatch.setattr( "litellm.router_utils.pre_call_checks.io_token_rate_limit_check.get_utc_datetime", lambda: pinned ) + + +@pytest.fixture +def router_minute_pinned(monkeypatch: pytest.MonkeyPatch) -> datetime: + pinned: Final = datetime(2026, 1, 1, 12, 0, 30, tzinfo=timezone.utc) + _pin_router_clock(monkeypatch, pinned) return pinned