diff --git a/litellm/router.py b/litellm/router.py index b087df7f1a1..1f310a2e57f 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -5111,6 +5111,7 @@ class Router(DeprecatedRouterMembers): kwargs: dict, function_name: str | None = None, splits_stream_timeout: bool = False, + opens_responses_socket: bool = False, ) -> None: """ 5 jobs: @@ -5200,7 +5201,11 @@ class Router(DeprecatedRouterMembers): # here before it's wiped below, instead of relying on that attempt's # (possibly still-pending) failure event to do it. refund_stale_reservation_before_retry(self._core_cache, kwargs) - set_io_token_rate_limit_request_kwargs(kwargs, store_in_context=deployment_has_io_token_limits(deployment)) + set_io_token_rate_limit_request_kwargs( + kwargs, + store_in_context=deployment_has_io_token_limits(deployment), + opens_responses_socket=opens_responses_socket, + ) kwargs[metadata_variable_name].setdefault( ROUTING_REQUEST_TAGS_METADATA_KEY, @@ -6607,6 +6612,7 @@ class Router(DeprecatedRouterMembers): kwargs=kwargs, function_name=function_name, splits_stream_timeout=handler_name in _STREAM_TIMEOUT_AWARE_GENERIC_HANDLERS, + opens_responses_socket=handler_name == "_aresponses_websocket", ) data: Final = deployment["litellm_params"].copy() diff --git a/litellm/router_utils/pre_call_checks/io_token_rate_limit_check.py b/litellm/router_utils/pre_call_checks/io_token_rate_limit_check.py index 89acad0a4cb..0afe8d182af 100644 --- a/litellm/router_utils/pre_call_checks/io_token_rate_limit_check.py +++ b/litellm/router_utils/pre_call_checks/io_token_rate_limit_check.py @@ -13,6 +13,7 @@ from __future__ import annotations import contextlib import contextvars from collections.abc import Mapping, MutableMapping +from dataclasses import dataclass from datetime import datetime from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final @@ -24,7 +25,7 @@ from litellm import token_counter from litellm._internal_context import with_service_target from litellm._logging import verbose_router_logger from litellm.caching.dual_cache import DualCache -from litellm.caching.redis_cache import RedisCircuitBreakerOpenError +from litellm.caching.redis_cache import RedisCache, RedisCircuitBreakerOpenError from litellm.litellm_core_utils.internal_call_metadata import ( IO_TOKEN_RESERVATION_KEYS, ITPM_CACHE_KEY, @@ -47,13 +48,22 @@ else: RoutingArgsTTL: Final = 60 -_io_token_rate_limit_request_kwargs: Final[contextvars.ContextVar[dict[str, object] | None]] = contextvars.ContextVar( - "io_token_rate_limit_request_kwargs", + +@dataclass(frozen=True, slots=True) +class _IoTokenRateLimitRequest: + kwargs: dict[str, object] | None + opens_responses_socket: bool + + +_io_token_rate_limit_request: Final[contextvars.ContextVar[_IoTokenRateLimitRequest | None]] = contextvars.ContextVar( + "io_token_rate_limit_request", default=None, ) -def set_io_token_rate_limit_request_kwargs(kwargs: dict[str, object] | None, store_in_context: bool = True) -> None: +def set_io_token_rate_limit_request_kwargs( + kwargs: dict[str, object] | None, store_in_context: bool = True, *, opens_responses_socket: bool = False +) -> None: # The reservation sentinels are server-only, but `metadata` is caller # controlled on proxy requests. Strip any client-supplied copies here (this # runs before the router stashes its own reservation) so a forged @@ -67,11 +77,14 @@ def set_io_token_rate_limit_request_kwargs(kwargs: dict[str, object] | None, sto # every other deployment overwrite the slot with None instead of the # kwargs; overwriting (rather than skipping) also releases a previous # request's kwargs when a context is reused. - _io_token_rate_limit_request_kwargs.set(kwargs if store_in_context else None) + _io_token_rate_limit_request.set( + _IoTokenRateLimitRequest(kwargs, opens_responses_socket) if store_in_context else None + ) def get_io_token_rate_limit_request_kwargs() -> dict[str, object] | None: - return _io_token_rate_limit_request_kwargs.get() + request: Final = _io_token_rate_limit_request.get() + return None if request is None else request.kwargs def seconds_until_minute_reset() -> int: @@ -506,8 +519,9 @@ async def async_io_token_pre_call_check( if itpm_limit is None and otpm_limit is None: return deployment - request_kwargs: Final = get_io_token_rate_limit_request_kwargs() - if request_kwargs is not None and request_kwargs.get("websocket") is not None: + request: Final = _io_token_rate_limit_request.get() + request_kwargs: Final = None if request is None else request.kwargs + if request is not None and request.opens_responses_socket: _stash_reservation_in_metadata( request_kwargs, await async_admit_io_token_turn(dual_cache, deployment, get_utc_datetime(), parent_otel_span), @@ -574,12 +588,19 @@ async def async_io_token_pre_call_check( return deployment +async def _async_redis_usage(redis_cache: RedisCache | None, key: str, parent_otel_span: Span | None) -> int | None: + if redis_cache is None: + return None + with contextlib.suppress(RedisCircuitBreakerOpenError): + return await redis_cache.async_get_cache(key=key, parent_otel_span=parent_otel_span) + return None + + @with_service_target(ROUTER_USAGE_TARGET) async def _async_shared_usage(dual_cache: DualCache, key: str, parent_otel_span: Span | None) -> int | None: - redis_cache: Final = dual_cache.redis_cache - if redis_cache is not None: - with contextlib.suppress(RedisCircuitBreakerOpenError): - return await redis_cache.async_get_cache(key=key, parent_otel_span=parent_otel_span) + shared: Final = await _async_redis_usage(dual_cache.redis_cache, key, parent_otel_span) + if shared is not None: + return shared return await dual_cache.async_get_cache(key=key, local_only=True) @@ -615,6 +636,12 @@ async def async_admit_io_token_turn( ) +def _settle_key(reserved_on: str | None, reserved: int, settled_at: datetime) -> str | None: + if reserved_on is None or reserved > 0: + return reserved_on + return f"{reserved_on.rpartition(':')[0]}:{settled_at.strftime('%H-%M')}" + + @with_service_target(ROUTER_USAGE_TARGET) def io_token_reconcile_success( dual_cache: DualCache, @@ -623,9 +650,12 @@ def io_token_reconcile_success( ) -> None: request_kwargs: Final[Mapping[str, object] | None] = kwargs response: Final[object] = response_obj - itpm_reserved, otpm_reserved, itpm_key, otpm_key = _read_reservation_from_kwargs(request_kwargs) - if itpm_key is None and otpm_key is None: + itpm_reserved, otpm_reserved, itpm_reserved_on, otpm_reserved_on = _read_reservation_from_kwargs(request_kwargs) + if itpm_reserved_on is None and otpm_reserved_on is None: return + settled_at: Final = get_utc_datetime() + itpm_key: Final = _settle_key(itpm_reserved_on, itpm_reserved, settled_at) + otpm_key: Final = _settle_key(otpm_reserved_on, otpm_reserved, settled_at) billable_input, completion_tokens, usage_resolved = _resolve_reconcile_usage_tokens(request_kwargs, response) @@ -677,18 +707,20 @@ async def async_io_token_reconcile_success( ) -> None: request_kwargs: Final[Mapping[str, object] | None] = kwargs response: Final[object] = response_obj - itpm_reserved, otpm_reserved, itpm_key, otpm_key = _read_reservation_from_kwargs(request_kwargs) - if itpm_key is None and otpm_key is None: + itpm_reserved, otpm_reserved, itpm_reserved_on, otpm_reserved_on = _read_reservation_from_kwargs(request_kwargs) + if itpm_reserved_on is None and otpm_reserved_on is None: return + # A held reservation settles on the exact key that holds it (which encodes + # the reservation's minute), so its refund lands on the counter it raised. + # A zero reservation holds nothing, so its actual tokens count on the minute + # the call finishes. Always clear the stash afterwards (even if an increment + # throws) so a retry or a duplicate success event can't re-process it. + settled_at: Final = get_utc_datetime() + itpm_key: Final = _settle_key(itpm_reserved_on, itpm_reserved, settled_at) + otpm_key: Final = _settle_key(otpm_reserved_on, otpm_reserved, settled_at) billable_input, completion_tokens, usage_resolved = _resolve_reconcile_usage_tokens(request_kwargs, response) - # Reconcile against the exact key that held the reservation (which encodes - # the reservation's minute), not a key recomputed at response time. This - # tracks actual usage even when the pre-call estimate was 0, and avoids a - # minute-boundary mismatch for calls that span into the next minute. Always - # clear the stash afterwards (even if an increment throws) so a retry or a - # duplicate success event can't re-process it. try: if usage_resolved: if itpm_key is not None: diff --git a/tests/unit/responses/test_responses_websocket_all_providers.py b/tests/unit/responses/test_responses_websocket_all_providers.py index 5009e1e58ac..c84c42277f2 100644 --- a/tests/unit/responses/test_responses_websocket_all_providers.py +++ b/tests/unit/responses/test_responses_websocket_all_providers.py @@ -1701,6 +1701,12 @@ async def _socket_opened_on(dual_cache: "DualCache") -> Mapping[str, object]: return await async_admit_io_token_turn(dual_cache, _WS_DEPLOYMENT, _OPENED_AT) +def _turns_finish_at(monkeypatch: pytest.MonkeyPatch, finished_at: datetime) -> None: + monkeypatch.setattr( + "litellm.router_utils.pre_call_checks.io_token_rate_limit_check.get_utc_datetime", lambda: finished_at + ) + + def _later_turns_admitted_through(dual_cache: "DualCache") -> "ResponsesWsTurnAdmission": from litellm.router_utils.pre_call_checks.io_token_rate_limit_check import async_admit_io_token_turn @@ -1929,8 +1935,8 @@ class TestWebSocketTurnAdmission: pytest.param(_UPSTREAM_ERROR, 0, id="first_turn_failed"), ], ) - async def test_native_relay_counts_the_first_turn_on_the_minute_it_opened_and_each_later_turn_on_its_own( - self, first_turn_end: str, first_turn_tokens: int + async def test_native_relay_counts_every_turn_on_the_minute_it_finishes( + self, first_turn_end: str, first_turn_tokens: int, monkeypatch: pytest.MonkeyPatch ) -> None: import websockets.exceptions # noqa: F401 (lazy submodule must be importable) @@ -1938,6 +1944,7 @@ class TestWebSocketTurnAdmission: from litellm.responses.streaming_iterator import drain_logging_tasks from litellm.router_utils.pre_call_checks.model_rate_limit_check import ModelRateLimitingCheck + _turns_finish_at(monkeypatch, _TURNED_AT) dual_cache: Final = DualCache() opened_with: Final = await _socket_opened_on(dual_cache) first, second, third = _turn_frames() @@ -1959,8 +1966,11 @@ class TestWebSocketTurnAdmission: await handler.backend_to_client() await drain_logging_tasks() - assert await dual_cache.async_get_cache(key=_OPENED_ON) == _OTHER_USAGE + first_turn_tokens - assert await dual_cache.async_get_cache(key=_TURNED_ON) == _OTHER_USAGE + 2 * _OUTPUT_TOKENS_PER_TURN + assert await dual_cache.async_get_cache(key=_OPENED_ON) == _OTHER_USAGE + assert ( + await dual_cache.async_get_cache(key=_TURNED_ON) + == _OTHER_USAGE + first_turn_tokens + 2 * _OUTPUT_TOKENS_PER_TURN + ) @pytest.mark.asyncio @pytest.mark.parametrize( @@ -1971,7 +1981,7 @@ class TestWebSocketTurnAdmission: ], ) async def test_native_relay_leaves_a_turn_s_cached_input_out_of_the_deployment_s_itpm( - self, cached_tokens: int, counted_input_tokens: int + self, cached_tokens: int, counted_input_tokens: int, monkeypatch: pytest.MonkeyPatch ) -> None: import websockets.exceptions # noqa: F401 (lazy submodule must be importable) @@ -1980,6 +1990,7 @@ class TestWebSocketTurnAdmission: from litellm.router_utils.pre_call_checks.io_token_rate_limit_check import async_admit_io_token_turn from litellm.router_utils.pre_call_checks.model_rate_limit_check import ModelRateLimitingCheck + _turns_finish_at(monkeypatch, _OPENED_AT) dual_cache: Final = DualCache() opened_with: Final = await async_admit_io_token_turn(dual_cache, _ITPM_DEPLOYMENT, _OPENED_AT) first, _, _ = _turn_frames() @@ -2008,12 +2019,13 @@ class TestWebSocketTurnAdmission: pytest.param(frozenset({1}), 0, id="first_turn_failed"), ], ) - async def test_managed_relay_counts_the_first_turn_on_the_minute_it_opened_and_each_later_turn_on_its_own( + async def test_managed_relay_counts_every_turn_on_the_minute_it_finishes( self, monkeypatch: pytest.MonkeyPatch, failing_turns: frozenset[int], first_turn_tokens: int ) -> None: from litellm.caching.dual_cache import DualCache from litellm.router_utils.pre_call_checks.model_rate_limit_check import ModelRateLimitingCheck + _turns_finish_at(monkeypatch, _TURNED_AT) dual_cache: Final = DualCache() opened_with: Final = await _socket_opened_on(dual_cache) @@ -2026,8 +2038,11 @@ class TestWebSocketTurnAdmission: opened_with, ) - assert await dual_cache.async_get_cache(key=_OPENED_ON) == _OTHER_USAGE + first_turn_tokens - assert await dual_cache.async_get_cache(key=_TURNED_ON) == _OTHER_USAGE + 2 * _OUTPUT_TOKENS_PER_TURN + assert await dual_cache.async_get_cache(key=_OPENED_ON) == _OTHER_USAGE + assert ( + await dual_cache.async_get_cache(key=_TURNED_ON) + == _OTHER_USAGE + first_turn_tokens + 2 * _OUTPUT_TOKENS_PER_TURN + ) @pytest.mark.asyncio async def test_managed_relay_skips_a_later_turn_the_deployment_refuses(self, monkeypatch: pytest.MonkeyPatch): diff --git a/tests/unit/test_router/test_io_token_rate_limits.py b/tests/unit/test_router/test_io_token_rate_limits.py index 99789076c91..b914d1286d7 100644 --- a/tests/unit/test_router/test_io_token_rate_limits.py +++ b/tests/unit/test_router/test_io_token_rate_limits.py @@ -1149,6 +1149,13 @@ def _turn_key(kind: str, minute: str = "12-34") -> str: return f"global_router:ws-turn-{kind}:openai/gpt-5-mini:{kind}:{minute}" +@pytest.fixture +def settled_on_the_turn_s_minute(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr( + "litellm.router_utils.pre_call_checks.io_token_rate_limit_check.get_utc_datetime", lambda: _TURN_AT + ) + + @pytest.mark.asyncio @pytest.mark.parametrize("kind", ["itpm", "otpm"]) @pytest.mark.parametrize("used", [100, 101, 250]) @@ -1161,6 +1168,7 @@ async def test_turn_admission_refuses_once_the_minutes_tokens_reach_the_limit(ki @pytest.mark.asyncio +@pytest.mark.usefixtures("settled_on_the_turn_s_minute") @pytest.mark.parametrize("kind", ["itpm", "otpm"]) @pytest.mark.parametrize("used", [0, 1, 99]) async def test_an_admitted_turn_reserves_nothing_and_counts_its_actual_tokens(kind: str, used: int) -> None: @@ -1194,18 +1202,16 @@ async def test_a_turn_without_io_token_counters_is_admitted_with_nothing_to_sett @pytest.mark.asyncio +@pytest.mark.usefixtures("settled_on_the_turn_s_minute") @pytest.mark.parametrize("kind", ["itpm", "otpm"]) @pytest.mark.parametrize("used_by_others", [0, 1, 99]) async def test_a_socket_open_reserves_nothing_and_its_first_turn_counts_its_actual_tokens_on_its_own_minute( - kind: str, used_by_others: int, monkeypatch: pytest.MonkeyPatch + kind: str, used_by_others: int ) -> None: - monkeypatch.setattr( - "litellm.router_utils.pre_call_checks.io_token_rate_limit_check.get_utc_datetime", lambda: _TURN_AT - ) dual_cache: Final = DualCache() await dual_cache.async_increment_cache(key=_turn_key(kind), value=used_by_others, ttl=60) - socket_open: Final[dict[str, object]] = {"websocket": object(), "metadata": {}} - set_io_token_rate_limit_request_kwargs(socket_open) + socket_open: Final[dict[str, object]] = {"metadata": {}} + set_io_token_rate_limit_request_kwargs(socket_open, opens_responses_socket=True) assert await async_io_token_pre_call_check(dual_cache, _turn_deployment(kind)) == _turn_deployment(kind) assert await dual_cache.async_get_cache(key=_turn_key(kind)) == used_by_others @@ -1230,6 +1236,51 @@ def shared_redis() -> Iterator[RedisCache]: server.server_close() +@pytest.fixture +def unreachable_redis() -> RedisCache: + server: Final = TcpFakeServer(("127.0.0.1", 0), server_type="redis") + server.server_close() + return RedisCache(host="127.0.0.1", port=server.server_address[1]) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("kind", ["itpm", "otpm"]) +async def test_a_turn_is_refused_on_this_instance_s_count_when_reading_the_shared_counter_fails( + kind: str, unreachable_redis: RedisCache +) -> None: + dual_cache: Final = DualCache(in_memory_cache=InMemoryCache(), redis_cache=unreachable_redis) + await dual_cache.in_memory_cache.async_increment(key=_turn_key(kind), value=100) + + with pytest.raises(litellm.RateLimitError, match=f"{kind.upper()} limit=100, current usage=100"): + await async_admit_io_token_turn(dual_cache, _turn_deployment(kind), _TURN_AT) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("kind", ["itpm", "otpm"]) +@pytest.mark.parametrize("delivery", ["async", "sync"]) +async def test_a_turn_finishing_in_the_next_minute_counts_its_actual_tokens_on_that_minute( + kind: str, delivery: str, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr( + "litellm.router_utils.pre_call_checks.io_token_rate_limit_check.get_utc_datetime", + lambda: datetime(2026, 1, 1, 12, 35, 10, tzinfo=timezone.utc), + ) + dual_cache: Final = DualCache() + reservation: Final = await async_admit_io_token_turn(dual_cache, _turn_deployment(kind), _TURN_AT) + + await _deliver_success( + ModelRateLimitingCheck(dual_cache=dual_cache), + delivery, + _proxy_success_kwargs(reservation), + _reply_using(_ACTUAL_TOKENS["itpm"], _ACTUAL_TOKENS["otpm"]), + ) + + assert [await dual_cache.async_get_cache(key=_turn_key(kind, minute)) for minute in ("12-34", "12-35")] == [ + None, + _ACTUAL_TOKENS[kind], + ] + + @pytest.mark.asyncio @pytest.mark.parametrize("kind", ["itpm", "otpm"]) async def test_a_turn_is_admitted_once_the_shared_counter_drops_below_the_limit_this_instance_last_read( @@ -1275,6 +1326,7 @@ async def test_a_turn_is_admitted_once_the_shared_counter_drops_below_the_limit_ ), ], ) +@pytest.mark.usefixtures("settled_on_the_turn_s_minute") async def test_a_turn_counts_only_its_uncached_responses_input_toward_itpm(responses_reply: object) -> None: dual_cache: Final = DualCache() await dual_cache.async_increment_cache(key=_turn_key("itpm"), value=10, ttl=60) diff --git a/tests/unit/test_router/test_router.py b/tests/unit/test_router/test_router.py index a0a7dd78193..fa17a5a745e 100644 --- a/tests/unit/test_router/test_router.py +++ b/tests/unit/test_router/test_router.py @@ -2199,7 +2199,11 @@ async def test_ageneric_api_call_deployment_model_overrides_alias(): ) def inject_alias_into_kwargs( - deployment: dict, kwargs: dict, function_name: str | None = None, splits_stream_timeout: bool = False + deployment: dict, + kwargs: dict, + function_name: str | None = None, + splits_stream_timeout: bool = False, + opens_responses_socket: bool = False, ) -> None: # Simulate the alias leaking into kwargs (as happens when # _ageneric_api_call_with_fallbacks sets kwargs["model"] = alias before diff --git a/tests/unit/test_router/test_router_helper_utils.py b/tests/unit/test_router/test_router_helper_utils.py index be99c6f9350..2a82825a33c 100644 --- a/tests/unit/test_router/test_router_helper_utils.py +++ b/tests/unit/test_router/test_router_helper_utils.py @@ -743,6 +743,31 @@ async def _count_minutes_tokens( await router.cache.async_increment_cache(key=_minutes_counter_key(kind, minute, model), value=tokens, ttl=120) +async def _settle_turn( + router: Router, reservation: Mapping[str, object], input_tokens: int, output_tokens: int +) -> None: + from litellm.cost_calculator import ResponsesWebSocketTokenUsageProcessor + from litellm.litellm_core_utils.get_litellm_params import get_litellm_params + from litellm.router_utils.pre_call_checks.model_rate_limit_check import ModelRateLimitingCheck + + rate_limits: Final = next( + check for check in router.optional_callbacks or () if isinstance(check, ModelRateLimitingCheck) + ) + await rate_limits.async_log_success_event( + {"litellm_params": get_litellm_params(litellm_metadata=dict(reservation))}, + ResponsesWebSocketTokenUsageProcessor.create_logging_object( + [ + { + "type": "response.completed", + "response": {"usage": {"input_tokens": input_tokens, "output_tokens": output_tokens}}, + } + ] + ), + None, + None, + ) + + _LONG_OUTPUT_MODEL: Final = "openai/gpt-5.4-mini" @@ -772,30 +797,16 @@ async def test_responses_websocket_opens_without_counting_tokens_until_the_minut @pytest.mark.asyncio -async def test_responses_websocket_first_turn_counts_its_actual_tokens_on_the_minute_it_opened( +async def test_responses_websocket_first_turn_counts_its_actual_tokens_toward_deployment_itpm_and_otpm( router_minute_pinned: datetime, ): - from litellm.cost_calculator import ResponsesWebSocketTokenUsageProcessor - from litellm.litellm_core_utils.get_litellm_params import get_litellm_params - from litellm.router_utils.pre_call_checks.model_rate_limit_check import ModelRateLimitingCheck - router, _ = _long_output_router() socket_metadata: Final = (await _kwargs_routed_to(router, "ws-io", "_aresponses_websocket", websocket=object()))[ "litellm_metadata" ] assert isinstance(socket_metadata, dict) - rate_limits: Final = next( - check for check in router.optional_callbacks or () if isinstance(check, ModelRateLimitingCheck) - ) - await rate_limits.async_log_success_event( - {"litellm_params": get_litellm_params(litellm_metadata=socket_metadata)}, - ResponsesWebSocketTokenUsageProcessor.create_logging_object( - [{"type": "response.completed", "response": {"usage": {"input_tokens": 20, "output_tokens": 15}}}] - ), - None, - None, - ) + await _settle_turn(router, socket_metadata, input_tokens=20, output_tokens=15) assert [ await router.cache.async_get_cache(key=_minutes_counter_key(counted, router_minute_pinned, _LONG_OUTPUT_MODEL)) @@ -803,6 +814,61 @@ async def test_responses_websocket_first_turn_counts_its_actual_tokens_on_the_mi ] == [20, 15] +_ADMITTED_LATE_IN_THE_MINUTE: Final = datetime(2026, 1, 1, 12, 0, 50, tzinfo=timezone.utc) +_FINISHED_IN_THE_NEXT_MINUTE: Final = datetime(2026, 1, 1, 12, 1, 15, tzinfo=timezone.utc) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("kind", ["itpm", "otpm"]) +@pytest.mark.parametrize("turn", ["first", "later"]) +async def test_responses_websocket_turn_finishing_in_the_next_minute_counts_its_tokens_toward_that_minute( + kind: str, turn: str, monkeypatch: pytest.MonkeyPatch +): + _pin_router_clock(monkeypatch, _ADMITTED_LATE_IN_THE_MINUTE) + router: Final = _io_limited_router(kind) + socket: Final = await _kwargs_routed_to(router, "ws-io", "_aresponses_websocket", websocket=object()) + admit_turn: Final = socket["turn_admission"] + assert callable(admit_turn) + reservation: Final = socket["litellm_metadata"] if turn == "first" else await admit_turn() + assert isinstance(reservation, Mapping) + _pin_router_clock(monkeypatch, _FINISHED_IN_THE_NEXT_MINUTE) + + await _settle_turn(router, reservation, input_tokens=10_000, output_tokens=10_000) + + with pytest.raises(litellm.RateLimitError, match=f"{kind.upper()} limit=10000, current usage=10000"): + await admit_turn() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("body", [{}, {"websocket": True}], ids=["plain", "websocket_key"]) +async def test_messages_call_reserves_max_tokens_against_deployment_otpm_whatever_websocket_key_its_body_carries( + body: Mapping[str, object], router_minute_pinned: datetime +): + router: Final = _io_limited_router("otpm") + + for _ in range(3): + await _kwargs_routed_to(router, "ws-io", "anthropic_messages", max_tokens=3_000, **body) + + assert await router.cache.async_get_cache(key=_minutes_counter_key("otpm", router_minute_pinned)) == 9_000 + with pytest.raises(litellm.RateLimitError, match="OTPM limit=10000, current usage=12000"): + await _kwargs_routed_to(router, "ws-io", "anthropic_messages", max_tokens=3_000, **body) + + +async def _realtime_open_refusal(**socket: object) -> str: + with pytest.raises(litellm.RateLimitError) as refused: + await _kwargs_routed_to(_io_limited_router("otpm", otpm=50), "ws-io", "_arealtime", **socket) + return str(refused.value) + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("router_minute_pinned") +async def test_realtime_open_reserves_and_is_refused_at_deployment_otpm_like_an_open_without_a_socket(): + with_socket: Final = await _realtime_open_refusal(websocket=object()) + + assert "OTPM limit=50, current usage=" in with_socket + assert with_socket == await _realtime_open_refusal() + + @pytest.mark.asyncio @pytest.mark.parametrize("kind", ["itpm", "otpm"]) async def test_responses_websocket_turn_is_refused_once_the_minutes_tokens_reach_deployment_io_limit( @@ -1315,13 +1381,17 @@ def _rpm_tpm_router(model_id: str) -> Router: ) -@pytest.fixture -def router_minute_pinned(monkeypatch: pytest.MonkeyPatch) -> datetime: - pinned: Final = datetime(2026, 1, 1, 12, 0, 30, tzinfo=timezone.utc) +def _pin_router_clock(monkeypatch: pytest.MonkeyPatch, pinned: datetime) -> None: monkeypatch.setattr("litellm.router.get_utc_datetime", lambda: pinned) monkeypatch.setattr( "litellm.router_utils.pre_call_checks.io_token_rate_limit_check.get_utc_datetime", lambda: pinned ) + + +@pytest.fixture +def router_minute_pinned(monkeypatch: pytest.MonkeyPatch) -> datetime: + pinned: Final = datetime(2026, 1, 1, 12, 0, 30, tzinfo=timezone.utc) + _pin_router_clock(monkeypatch, pinned) return pinned