diff --git a/litellm/litellm_core_utils/internal_call_metadata.py b/litellm/litellm_core_utils/internal_call_metadata.py index ded7825206f..453d4358b35 100644 --- a/litellm/litellm_core_utils/internal_call_metadata.py +++ b/litellm/litellm_core_utils/internal_call_metadata.py @@ -174,14 +174,6 @@ def without_budget_reservation( } -def without_io_token_reservation(metadata: Mapping[str, object]) -> Mapping[str, object]: - return {key: value for key, value in metadata.items() if key not in IO_TOKEN_RESERVATION_KEYS} - - -def io_token_reservation(metadata: Mapping[str, object]) -> Mapping[str, object]: - return MappingProxyType({key: value for key, value in metadata.items() if key in IO_TOKEN_RESERVATION_KEYS}) - - def forwarded_internal_call_metadata( parent_metadata: Mapping[str, object] | None, call_origin: InternalCallOrigin, diff --git a/litellm/responses/streaming_iterator.py b/litellm/responses/streaming_iterator.py index 07ff38536df..77980b603ec 100644 --- a/litellm/responses/streaming_iterator.py +++ b/litellm/responses/streaming_iterator.py @@ -46,11 +46,9 @@ from litellm.litellm_core_utils.core_helpers import ( ) from litellm.litellm_core_utils.exception_mapping_utils import with_deployment_api_base from litellm.litellm_core_utils.internal_call_metadata import ( - io_token_reservation, is_background_response, requests_background, without_budget_reservation, - without_io_token_reservation, ) from litellm.litellm_core_utils.litellm_logging import BilledRoundsSpend from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj @@ -2156,17 +2154,17 @@ class _RefusedTurn: rejection: Exception -ResponsesWsTurnAdmission: TypeAlias = Callable[[Mapping[str, object]], Awaitable[Mapping[str, object]]] +ResponsesWsTurnAdmission: TypeAlias = Callable[[], Awaitable[Mapping[str, object]]] async def _admit_frame_turn( - turn_admission: ResponsesWsTurnAdmission | None, raw_message: str, held_reservation: Mapping[str, object] + turn_admission: ResponsesWsTurnAdmission | None, raw_message: str ) -> _AdmittedTurn | _RefusedTurn: frame: Final = _json_object_or_none(raw_message) if turn_admission is None or frame is None or frame.get("type") != "response.create": return _AdmittedTurn() try: - return _AdmittedTurn(await turn_admission(held_reservation)) + return _AdmittedTurn(await turn_admission()) except RateLimitError as refusal: return _RefusedTurn(refusal) except Exception as e: @@ -2519,32 +2517,22 @@ class ResponsesWebSocketStreaming: turn_logging_obj: Final = self.logging_obj.copy_for_new_call( litellm_call_id=str(uuid.uuid4()), start_time=start_time ) - self._detach_admission_reservations() + self._detach_budget_reservation() _set_turn_guardrail_information(turn_logging_obj, guardrail_information) _set_turn_io_token_reservation(turn_logging_obj, io_token_reservation) turn_logging_obj.model_call_details["messages"] = list(_logged_user_messages(client_frame)) turn_logging_obj.pre_call(input=client_frame, api_key="") self._open_turns = (*self._open_turns, _ResponsesWsTurn(logging_obj=turn_logging_obj)) - def _held_admission_reservation(self) -> Mapping[str, object]: - litellm_params: Final = _json_object_or_none( - self.logging_obj.litellm_params # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # Logging.litellm_params is untyped - ) - buckets: Final = ( - () if litellm_params is None else (litellm_params.get(key) for key in ("metadata", "litellm_metadata")) - ) - held: Final = (io_token_reservation(bucket) for bucket in buckets if _is_json_object(bucket)) - return next((reservation for reservation in held if reservation), _NO_IO_TOKEN_RESERVATION) - - def _detach_admission_reservations(self) -> None: - """Only the first turn settles the connection's admission reservations; later turns charge their own usage.""" + def _detach_budget_reservation(self) -> None: + """Only the first turn settles the connection's admission reservation; later turns charge their cost directly.""" litellm_params: Final = _json_object_or_none( self.logging_obj.litellm_params # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # Logging.litellm_params is untyped ) if litellm_params is None: return self.logging_obj.litellm_params = { - key: without_io_token_reservation(without_budget_reservation(value)) + key: without_budget_reservation(value) if key in _PER_TURN_METADATA_KEYS and _is_json_object(value) else value for key, value in litellm_params.items() @@ -2990,7 +2978,7 @@ class ResponsesWebSocketStreaming: async def _admit_or_reject_turn( self, guarded: GuardedResponsesWsFrame, message: str, turn_admission: ResponsesWsTurnAdmission | None ) -> _AdmittedTurn | None: - verdict: Final = await _admit_frame_turn(turn_admission, message, self._held_admission_reservation()) + verdict: Final = await _admit_frame_turn(turn_admission, message) match verdict: case _AdmittedTurn(): return verdict @@ -3419,21 +3407,6 @@ class ManagedResponsesWebSocketHandler: if self.litellm_metadata: call_kwargs["litellm_metadata"] = dict(self.litellm_metadata) - def _held_io_token_reservation(self) -> Mapping[str, object]: - connection_metadata: Final = self.extra_kwargs.get("metadata") - held_on_connection: Final = ( - io_token_reservation(connection_metadata) - if _is_json_object(connection_metadata) - else _NO_IO_TOKEN_RESERVATION - ) - return held_on_connection or io_token_reservation(self.litellm_metadata) - - def _detach_io_token_reservation(self) -> None: - connection_metadata: Final = self.extra_kwargs.get("metadata") - self.litellm_metadata = without_io_token_reservation(self.litellm_metadata) - if _is_json_object(connection_metadata): - self.extra_kwargs = {**self.extra_kwargs, "metadata": without_io_token_reservation(connection_metadata)} - async def _guard_frame(self, raw_message: str) -> GuardedResponsesWsFrame | None: """Same per-frame pre_call guardrail walk as the native socket: a rejected frame errors and is not sent.""" if self.frame_guard is None: @@ -3584,7 +3557,7 @@ class ManagedResponsesWebSocketHandler: guarded: Final = await self._guard_frame(raw_message) if guarded is None: return - verdict: Final = await _admit_frame_turn(turn_admission, raw_message, self._held_io_token_reservation()) + verdict: Final = await _admit_frame_turn(turn_admission, raw_message) if isinstance(verdict, _RefusedTurn): if self._failure is None: self._failure = verdict.rejection @@ -3617,15 +3590,15 @@ class ManagedResponsesWebSocketHandler: call_kwargs["litellm_metadata"] = self._metadata_with_turn_guardrails( call_kwargs.get("litellm_metadata"), guarded.guardrail_information ) - if verdict.reservation: - existing_metadata: Final = call_kwargs.get("litellm_metadata") - call_kwargs["litellm_metadata"] = { - **(existing_metadata if _is_json_object(existing_metadata) else {}), - **verdict.reservation, - } self._update_proxy_request(call_kwargs, requested_model or self.model_group or model) call_kwargs.update(self.extra_kwargs) - self._detach_io_token_reservation() + call_kwargs.update( + { + channel: {**bucket, **verdict.reservation} + for channel, bucket in call_kwargs.items() + if channel in _PER_TURN_METADATA_KEYS and _is_json_object(bucket) + } + ) try: terminal_event: Final = await self._stream_and_forward(model, call_kwargs) diff --git a/litellm/router.py b/litellm/router.py index e08e7451807..b087df7f1a1 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -10452,7 +10452,6 @@ class Router(DeprecatedRouterMembers): model_group: str, opened_on: Mapping[str, object], request_kwargs: Mapping[str, object], - held_reservation: Mapping[str, object] = MappingProxyType({}), ) -> Mapping[str, object]: deployment: Final = self._deployment_as_configured_now(opened_on) rate_limit_check: Final = next( @@ -10461,9 +10460,7 @@ class Router(DeprecatedRouterMembers): io_token_admission: Final = ( MappingProxyType({}) if rate_limit_check is None - else await async_admit_io_token_turn( - rate_limit_check.dual_cache, deployment, held_reservation, get_utc_datetime() - ) + else await async_admit_io_token_turn(rate_limit_check.dual_cache, deployment, get_utc_datetime()) ) budget_limiter: Final = self._core_get_router_deployment_budget_limiter() if budget_limiter is not None: diff --git a/litellm/router_utils/pre_call_checks/io_token_rate_limit_check.py b/litellm/router_utils/pre_call_checks/io_token_rate_limit_check.py index fd0edc290c5..fec0307f073 100644 --- a/litellm/router_utils/pre_call_checks/io_token_rate_limit_check.py +++ b/litellm/router_utils/pre_call_checks/io_token_rate_limit_check.py @@ -185,27 +185,32 @@ def _resolve_max_tokens(request_kwargs: dict[str, Any] | None, deployment: dict) return 4096 +def _usage_field(container: object, name: str) -> object: + if is_str_keyed_mapping(container): + return container.get(name) + return getattr(container, name, None) + + +def _cached_prompt_tokens(usage: object) -> int: + reported: Final = ( + _usage_field(_usage_field(usage, "prompt_tokens_details"), "cached_tokens"), + _usage_field(_usage_field(usage, "input_tokens_details"), "cached_tokens"), + _usage_field(usage, "cache_read_input_tokens"), + ) + return next((int(tokens) for tokens in reported if isinstance(tokens, int | float) and tokens), 0) + + def _get_usage_tokens(usage: object) -> tuple[int, int, int]: if usage is None: return 0, 0, 0 if hasattr(usage, "prompt_tokens") or hasattr(usage, "input_tokens"): prompt = int(getattr(usage, "prompt_tokens", None) or getattr(usage, "input_tokens", 0) or 0) completion = int(getattr(usage, "completion_tokens", None) or getattr(usage, "output_tokens", 0) or 0) - cached = 0 - details: object = getattr(usage, "prompt_tokens_details", None) - if details is not None: - cached = int(getattr(details, "cached_tokens", 0) or 0) - if not cached: - cached = int(getattr(usage, "cache_read_input_tokens", 0) or 0) - return prompt, completion, cached + return prompt, completion, _cached_prompt_tokens(usage) if isinstance(usage, dict): prompt = int(usage.get("prompt_tokens") or usage.get("input_tokens") or 0) completion = int(usage.get("completion_tokens") or usage.get("output_tokens") or 0) - details = usage.get("prompt_tokens_details") or {} - cached = int(details.get("cached_tokens", 0) or 0) if isinstance(details, dict) else 0 - if not cached: - cached = int(usage.get("cache_read_input_tokens") or 0) - return prompt, completion, cached + return prompt, completion, _cached_prompt_tokens(usage) return 0, 0, 0 @@ -266,22 +271,22 @@ def _resolve_reconcile_usage_tokens( return 0, 0, False -def _stash_reservation_in_metadata( - request_kwargs: dict[str, object] | None, - *, - itpm_reserved: int, - otpm_reserved: int, - itpm_cache_key: str | None, - otpm_cache_key: str | None, -) -> None: +def _reservation( + itpm_reserved: int, otpm_reserved: int, itpm_cache_key: str | None, otpm_cache_key: str | None +) -> Mapping[str, object]: + return MappingProxyType( + { + ITPM_RESERVED_KEY: itpm_reserved, + OTPM_RESERVED_KEY: otpm_reserved, + ITPM_CACHE_KEY: itpm_cache_key, + OTPM_CACHE_KEY: otpm_cache_key, + } + ) + + +def _stash_reservation_in_metadata(request_kwargs: dict[str, object] | None, reservation: Mapping[str, object]) -> None: if not request_kwargs: return - reservation: Final = { - ITPM_RESERVED_KEY: itpm_reserved, - OTPM_RESERVED_KEY: otpm_reserved, - ITPM_CACHE_KEY: itpm_cache_key, - OTPM_CACHE_KEY: otpm_cache_key, - } for channel in ("metadata", "litellm_metadata"): existing = request_kwargs.get(channel) if isinstance(existing, dict): @@ -483,10 +488,12 @@ def io_token_pre_call_check( _stash_reservation_in_metadata( request_kwargs, - itpm_reserved=itpm_reserved, - otpm_reserved=otpm_reserved, - itpm_cache_key=itpm_key if itpm_limit is not None else None, - otpm_cache_key=otpm_key if otpm_limit is not None else None, + _reservation( + itpm_reserved, + otpm_reserved, + itpm_key if itpm_limit is not None else None, + otpm_key if otpm_limit is not None else None, + ), ) return deployment @@ -502,6 +509,12 @@ async def async_io_token_pre_call_check( return deployment request_kwargs: Final = get_io_token_rate_limit_request_kwargs() + if request_kwargs is not None and request_kwargs.get("websocket") is not None: + _stash_reservation_in_metadata( + request_kwargs, + await async_admit_io_token_turn(dual_cache, deployment, get_utc_datetime(), parent_otel_span), + ) + return deployment _model: Final = _tokenizer_model(deployment) estimated_input: Final = await offload_token_count(_estimate_input_tokens)(request_kwargs, model=_model) max_tokens: Final = _resolve_max_tokens(request_kwargs, deployment) @@ -553,50 +566,39 @@ async def async_io_token_pre_call_check( _stash_reservation_in_metadata( request_kwargs, - itpm_reserved=itpm_reserved, - otpm_reserved=otpm_reserved, - itpm_cache_key=itpm_key if itpm_limit is not None else None, - otpm_cache_key=otpm_key if otpm_limit is not None else None, + _reservation( + itpm_reserved, + otpm_reserved, + itpm_key if itpm_limit is not None else None, + otpm_key if otpm_limit is not None else None, + ), ) return deployment @with_service_target(ROUTER_USAGE_TARGET) -async def async_current_usage( - dual_cache: DualCache, key: str, limit: int, parent_otel_span: Span | None = None -) -> int | None: - local_usage: Final = await dual_cache.async_get_cache(key=key, local_only=True) +async def _async_shared_usage(dual_cache: DualCache, key: str, parent_otel_span: Span | None) -> int | None: redis_cache: Final = dual_cache.redis_cache - if redis_cache is None or (local_usage is not None and local_usage >= limit): - return local_usage - try: - return await redis_cache.async_get_cache(key=key, parent_otel_span=parent_otel_span) - except RedisCircuitBreakerOpenError: - return local_usage - - -def _held_on(held_reservation: Mapping[str, object], reserved_key: str, cache_key_key: str, cache_key: str) -> int: - held: Final = held_reservation.get(reserved_key) - if held_reservation.get(cache_key_key) != cache_key or not isinstance(held, int): - return 0 - return held + if redis_cache is not None: + with contextlib.suppress(RedisCircuitBreakerOpenError): + return await redis_cache.async_get_cache(key=key, parent_otel_span=parent_otel_span) + return await dual_cache.async_get_cache(key=key, local_only=True) async def _raise_if_at_limit( - dual_cache: DualCache, key: str, limit: int | None, held: int, limit_label: str, parent_otel_span: Span | None + dual_cache: DualCache, key: str, limit: int | None, limit_label: str, parent_otel_span: Span | None ) -> None: if limit is None: return - usage: Final = await async_current_usage(dual_cache, key, limit + held, parent_otel_span) - if usage is not None and usage - held >= limit: - raise _rate_limit_error(limit_label, limit, usage - held) + usage: Final = await _async_shared_usage(dual_cache, key, parent_otel_span) + if usage is not None and usage >= limit: + raise _rate_limit_error(limit_label, limit, usage) @with_service_target(ROUTER_USAGE_TARGET) async def async_admit_io_token_turn( dual_cache: DualCache, deployment: Mapping[str, object], - held_reservation: Mapping[str, object], now: datetime, parent_otel_span: Span | None = None, ) -> Mapping[str, object]: @@ -608,19 +610,10 @@ async def async_admit_io_token_turn( if cache_keys is None: return MappingProxyType({}) itpm_key, otpm_key = cache_keys - itpm_held: Final = _held_on(held_reservation, ITPM_RESERVED_KEY, ITPM_CACHE_KEY, itpm_key) - otpm_held: Final = _held_on(held_reservation, OTPM_RESERVED_KEY, OTPM_CACHE_KEY, otpm_key) - await _raise_if_at_limit(dual_cache, itpm_key, itpm_limit, itpm_held, "ITPM", parent_otel_span) - await _raise_if_at_limit(dual_cache, otpm_key, otpm_limit, otpm_held, "OTPM", parent_otel_span) - if held_reservation: - return MappingProxyType({}) - return MappingProxyType( - { - ITPM_RESERVED_KEY: 0, - OTPM_RESERVED_KEY: 0, - ITPM_CACHE_KEY: itpm_key if itpm_limit is not None else None, - OTPM_CACHE_KEY: otpm_key if otpm_limit is not None else None, - } + await _raise_if_at_limit(dual_cache, itpm_key, itpm_limit, "ITPM", parent_otel_span) + await _raise_if_at_limit(dual_cache, otpm_key, otpm_limit, "OTPM", parent_otel_span) + return _reservation( + 0, 0, itpm_key if itpm_limit is not None else None, otpm_key if otpm_limit is not None else None ) diff --git a/litellm/router_utils/pre_call_checks/model_rate_limit_check.py b/litellm/router_utils/pre_call_checks/model_rate_limit_check.py index 140fa342577..0f8acf8782a 100644 --- a/litellm/router_utils/pre_call_checks/model_rate_limit_check.py +++ b/litellm/router_utils/pre_call_checks/model_rate_limit_check.py @@ -26,7 +26,6 @@ from litellm.caching.dual_cache import DualCache from litellm.caching.redis_cache import RedisCircuitBreakerOpenError from litellm.integrations.custom_logger import CustomLogger from litellm.router_utils.pre_call_checks.io_token_rate_limit_check import ( - async_current_usage, async_io_token_pre_call_check, async_io_token_reconcile_success, async_io_token_refund_failure, @@ -290,7 +289,14 @@ class ModelRateLimitingCheck(CustomLogger): @with_service_target(ROUTER_USAGE_TARGET) async def _async_get_current_tpm(self, tpm_key: str, tpm_limit: int, parent_otel_span: Span | None) -> int | None: - return await async_current_usage(self.dual_cache, tpm_key, tpm_limit, parent_otel_span) + local_tpm: Final = await self.dual_cache.async_get_cache(key=tpm_key, local_only=True) + redis_cache: Final = self.dual_cache.redis_cache + if redis_cache is None or (local_tpm is not None and local_tpm >= tpm_limit): + return local_tpm + try: + return await redis_cache.async_get_cache(key=tpm_key, parent_otel_span=parent_otel_span) + except RedisCircuitBreakerOpenError: + return local_tpm @with_service_target(ROUTER_USAGE_TARGET) def pre_call_check(self, deployment: dict) -> dict | None: diff --git a/tests/integration/routing/test_responses_websocket_turn_admission.py b/tests/integration/routing/test_responses_websocket_turn_admission.py index 4d5dc0dda52..7b0e14d044f 100644 --- a/tests/integration/routing/test_responses_websocket_turn_admission.py +++ b/tests/integration/routing/test_responses_websocket_turn_admission.py @@ -518,9 +518,7 @@ def test_a_tpm_limit_lowered_while_the_socket_is_open_applies_to_its_next_turn( assert deployment.forwarded_turns() == 1 -def test_a_socket_on_an_otpm_limited_deployment_reserves_once_and_serves_every_turn( - limited: Gateway, cert: tuple[Path, Path] -) -> None: +def test_a_socket_on_an_otpm_limited_deployment_serves_every_turn(limited: Gateway, cert: tuple[Path, Path]) -> None: with responses_peer(cert) as peer, limited.scenario() as scenario: deployment: Final = _native(scenario, peer, otpm=10_000) key: Final = scenario.key(models=[deployment.name]) diff --git a/tests/unit/responses/test_responses_websocket_all_providers.py b/tests/unit/responses/test_responses_websocket_all_providers.py index d8e260e2cbe..823ff087bb9 100644 --- a/tests/unit/responses/test_responses_websocket_all_providers.py +++ b/tests/unit/responses/test_responses_websocket_all_providers.py @@ -10,10 +10,10 @@ Tests that: import asyncio import itertools import json -from collections.abc import AsyncIterator, Callable, Mapping +from collections.abc import AsyncIterator, Mapping from datetime import datetime, timezone from functools import partial -from typing import TYPE_CHECKING, Final, TypeAlias +from typing import TYPE_CHECKING, Final from unittest.mock import AsyncMock, MagicMock import pytest @@ -49,6 +49,7 @@ from litellm.llms.xai.responses.transformation import XAIResponsesAPIConfig from litellm.types.utils import GenericStreamingChunk if TYPE_CHECKING: + from litellm.caching.dual_cache import DualCache from litellm.litellm_core_utils.litellm_logging import Logging from litellm.responses.streaming_iterator import ResponsesWsTurnAdmission from litellm.router_utils.pre_call_checks.model_rate_limit_check import ModelRateLimitingCheck @@ -1667,19 +1668,13 @@ def _guard_recording(bill_refusal: MagicMock): _OPENED_ON: Final = "global_router:ws-dep:openai/gpt-5-mini:otpm:12-34" _TURNED_ON: Final = "global_router:ws-dep:openai/gpt-5-mini:otpm:12-35" - - -def _otpm_reservation(reserved: int, cache_key: str) -> Mapping[str, object]: - from litellm.router_utils.pre_call_checks.io_token_rate_limit_check import ( - ITPM_CACHE_KEY, - ITPM_RESERVED_KEY, - OTPM_CACHE_KEY, - OTPM_RESERVED_KEY, - ) - - return {ITPM_RESERVED_KEY: 0, OTPM_RESERVED_KEY: reserved, ITPM_CACHE_KEY: None, OTPM_CACHE_KEY: cache_key} - - +_OPENED_AT: Final = datetime(2026, 1, 1, 12, 34, 56, tzinfo=timezone.utc) +_TURNED_AT: Final = datetime(2026, 1, 1, 12, 35, 5, tzinfo=timezone.utc) +_WS_DEPLOYMENT: Final = { + "litellm_params": {"model": "openai/gpt-5-mini", "otpm": 4096}, + "model_info": {"id": "ws-dep"}, +} +_OTHER_USAGE: Final = 10 _OUTPUT_TOKENS_PER_TURN: Final = 2 _UPSTREAM_ERROR: Final = json.dumps({"type": "error", "error": {"type": "server_error", "message": "upstream failed"}}) @@ -1698,25 +1693,23 @@ def _turn_completed() -> str: ) -def _admitting(reservation: Mapping[str, object]) -> AsyncMock: - return AsyncMock(return_value=reservation) +async def _socket_opened_on(dual_cache: "DualCache") -> Mapping[str, object]: + from litellm.router_utils.pre_call_checks.io_token_rate_limit_check import async_admit_io_token_turn + + await dual_cache.async_increment_cache(key=_OPENED_ON, value=_OTHER_USAGE, ttl=60) + await dual_cache.async_increment_cache(key=_TURNED_ON, value=_OTHER_USAGE, ttl=60) + return await async_admit_io_token_turn(dual_cache, _WS_DEPLOYMENT, _OPENED_AT) -_FIRST_TURN_ENDS: Final = ( - pytest.param(_turn_completed(), _OUTPUT_TOKENS_PER_TURN, id="first_turn_completed"), - pytest.param(_UPSTREAM_ERROR, 0, id="first_turn_failed"), -) -_LaterTurnAdmission: TypeAlias = Callable[[], AsyncMock | None] -_LATER_TURN_ADMISSIONS: Final = ( - pytest.param( - lambda: _admitting(_otpm_reservation(0, _TURNED_ON)), 2 * _OUTPUT_TOKENS_PER_TURN, id="admitted_this_minute" - ), - pytest.param(lambda: _admitting({}), None, id="limits_removed_mid_socket"), - pytest.param(lambda: None, None, id="no_turn_admission"), -) +def _later_turns_admitted_through(dual_cache: "DualCache") -> "ResponsesWsTurnAdmission": + from litellm.router_utils.pre_call_checks.io_token_rate_limit_check import async_admit_io_token_turn + + return partial(async_admit_io_token_turn, dual_cache, _WS_DEPLOYMENT, _TURNED_AT) -def _connection_settling_turns_through(rate_limits: "ModelRateLimitingCheck") -> "Logging": +def _connection_settling_turns_through( + rate_limits: "ModelRateLimitingCheck", opened_with: Mapping[str, object] +) -> "Logging": import time from litellm.cost_calculator import ResponsesWebSocketTokenUsageProcessor @@ -1746,20 +1739,10 @@ def _connection_settling_turns_through(rate_limits: "ModelRateLimitingCheck") -> litellm_call_id="connection-call", function_id="connection-call", ) - connection.litellm_params = { - "metadata": dict(_otpm_reservation(4096, _OPENED_ON)), - "litellm_metadata": dict(_otpm_reservation(4096, _OPENED_ON)), - } + connection.litellm_params = {"metadata": dict(opened_with), "litellm_metadata": dict(opened_with)} return connection -_OPENED_AT: Final = datetime(2026, 1, 1, 12, 34, 56, tzinfo=timezone.utc) -_WS_DEPLOYMENT_AT_ITS_OTPM: Final = { - "litellm_params": {"model": "openai/gpt-5-mini", "otpm": 4096}, - "model_info": {"id": "ws-dep"}, -} - - class _TwoOutputTokenTurns(CustomLLM): def __init__(self, failing_turns: frozenset[int] = frozenset()) -> None: super().__init__() @@ -1785,6 +1768,7 @@ async def _run_managed_relay( rate_limits: "ModelRateLimitingCheck", frames: tuple[str, ...], turn_admission: "ResponsesWsTurnAdmission | None", + opened_with: Mapping[str, object], ) -> _ClientSocket: import litellm from litellm.litellm_core_utils.litellm_logging import Logging @@ -1807,10 +1791,10 @@ async def _run_managed_relay( litellm_call_id="test-id", function_id="test-func", ), - litellm_metadata={"user_api_key_hash": "k1", **_otpm_reservation(4096, _OPENED_ON)}, + litellm_metadata={"user_api_key_hash": "k1", **opened_with}, first_message=first, turn_admission=turn_admission, - metadata=dict(_otpm_reservation(4096, _OPENED_ON)), + metadata=dict(opened_with), ) await handler.run() @@ -1888,14 +1872,15 @@ class TestWebSocketTurnAdmission: assert "crossed budget" in str(failure) @pytest.mark.asyncio - @pytest.mark.parametrize(("first_turn_end", "opened_on_tokens"), _FIRST_TURN_ENDS) - @pytest.mark.parametrize(("turn_admission", "turned_on_tokens"), _LATER_TURN_ADMISSIONS) - async def test_native_relay_settles_the_open_reservation_on_the_first_turn_only( - self, - first_turn_end: str, - opened_on_tokens: int, - turn_admission: _LaterTurnAdmission, - turned_on_tokens: int | None, + @pytest.mark.parametrize( + ("first_turn_end", "first_turn_tokens"), + [ + pytest.param(_turn_completed(), _OUTPUT_TOKENS_PER_TURN, id="first_turn_completed"), + pytest.param(_UPSTREAM_ERROR, 0, id="first_turn_failed"), + ], + ) + async def test_native_relay_counts_the_first_turn_on_the_minute_it_opened_and_each_later_turn_on_its_own( + self, first_turn_end: str, first_turn_tokens: int ) -> None: import websockets.exceptions # noqa: F401 (lazy submodule must be importable) @@ -1904,7 +1889,7 @@ class TestWebSocketTurnAdmission: from litellm.router_utils.pre_call_checks.model_rate_limit_check import ModelRateLimitingCheck dual_cache: Final = DualCache() - await dual_cache.async_increment_cache(key=_OPENED_ON, value=4096, ttl=60) + opened_with: Final = await _socket_opened_on(dual_cache) first, second, third = _turn_frames() backend_ws: Final = MagicMock() backend_ws.send = AsyncMock() @@ -1915,141 +1900,46 @@ class TestWebSocketTurnAdmission: websocket=_ClientSocket(second, third), backend_ws=backend_ws, request_data={}, - logging_obj=_connection_settling_turns_through(ModelRateLimitingCheck(dual_cache)), + logging_obj=_connection_settling_turns_through(ModelRateLimitingCheck(dual_cache), opened_with), first_message=first, - turn_admission=turn_admission(), + turn_admission=_later_turns_admitted_through(dual_cache), ) await handler.client_to_backend() await handler.backend_to_client() await drain_logging_tasks() - assert await dual_cache.async_get_cache(key=_OPENED_ON) == opened_on_tokens - assert await dual_cache.async_get_cache(key=_TURNED_ON) == turned_on_tokens - - @pytest.mark.asyncio - async def test_native_relay_does_not_settle_the_open_reservation_again_for_an_event_no_turn_asked_for( - self, - ) -> None: - import websockets.exceptions # noqa: F401 (lazy submodule must be importable) - - from litellm.caching.dual_cache import DualCache - from litellm.responses.streaming_iterator import drain_logging_tasks - from litellm.router_utils.pre_call_checks.model_rate_limit_check import ModelRateLimitingCheck - - dual_cache: Final = DualCache() - await dual_cache.async_increment_cache(key=_OPENED_ON, value=4096, ttl=60) - first, _, _ = _turn_frames() - backend_ws: Final = MagicMock() - backend_ws.send = AsyncMock() - backend_ws.recv = AsyncMock(side_effect=[_turn_completed(), _UPSTREAM_ERROR, Exception("stop")]) - handler: Final = _make_streaming( - websocket=_ClientSocket(), - backend_ws=backend_ws, - request_data={}, - logging_obj=_connection_settling_turns_through(ModelRateLimitingCheck(dual_cache)), - first_message=first, - turn_admission=_admitting(_otpm_reservation(0, _TURNED_ON)), - ) - - await handler.client_to_backend() - await handler.backend_to_client() - await drain_logging_tasks() - - assert await dual_cache.async_get_cache(key=_OPENED_ON) == _OUTPUT_TOKENS_PER_TURN + assert await dual_cache.async_get_cache(key=_OPENED_ON) == _OTHER_USAGE + first_turn_tokens + assert await dual_cache.async_get_cache(key=_TURNED_ON) == _OTHER_USAGE + 2 * _OUTPUT_TOKENS_PER_TURN @pytest.mark.asyncio @pytest.mark.parametrize( - ("failing_turns", "opened_on_tokens"), + ("failing_turns", "first_turn_tokens"), [ pytest.param(frozenset(), _OUTPUT_TOKENS_PER_TURN, id="first_turn_completed"), pytest.param(frozenset({1}), 0, id="first_turn_failed"), ], ) - @pytest.mark.parametrize(("turn_admission", "turned_on_tokens"), _LATER_TURN_ADMISSIONS) - async def test_managed_relay_settles_the_open_reservation_on_the_first_turn_only( - self, - monkeypatch: pytest.MonkeyPatch, - failing_turns: frozenset[int], - opened_on_tokens: int, - turn_admission: _LaterTurnAdmission, - turned_on_tokens: int | None, + async def test_managed_relay_counts_the_first_turn_on_the_minute_it_opened_and_each_later_turn_on_its_own( + self, monkeypatch: pytest.MonkeyPatch, failing_turns: frozenset[int], first_turn_tokens: int ) -> None: from litellm.caching.dual_cache import DualCache from litellm.router_utils.pre_call_checks.model_rate_limit_check import ModelRateLimitingCheck dual_cache: Final = DualCache() - await dual_cache.async_increment_cache(key=_OPENED_ON, value=4096, ttl=60) + opened_with: Final = await _socket_opened_on(dual_cache) await _run_managed_relay( monkeypatch, _TwoOutputTokenTurns(failing_turns), ModelRateLimitingCheck(dual_cache), _turn_frames(), - turn_admission(), + _later_turns_admitted_through(dual_cache), + opened_with, ) - assert await dual_cache.async_get_cache(key=_OPENED_ON) == opened_on_tokens - assert await dual_cache.async_get_cache(key=_TURNED_ON) == turned_on_tokens - - @pytest.mark.asyncio - async def test_managed_relay_answers_and_settles_the_first_prompt_after_a_warmup_when_the_open_hold_fills_the_minute( - self, monkeypatch: pytest.MonkeyPatch - ) -> None: - from litellm.caching.dual_cache import DualCache - from litellm.router_utils.pre_call_checks.io_token_rate_limit_check import async_admit_io_token_turn - from litellm.router_utils.pre_call_checks.model_rate_limit_check import ModelRateLimitingCheck - - dual_cache: Final = DualCache() - await dual_cache.async_increment_cache(key=_OPENED_ON, value=4096, ttl=60) - warmup: Final = json.dumps({"type": "response.create", "generate": False, "input": []}) - first, _, _ = _turn_frames() - - websocket: Final = await _run_managed_relay( - monkeypatch, - _TwoOutputTokenTurns(), - ModelRateLimitingCheck(dual_cache), - (warmup, first), - partial(async_admit_io_token_turn, dual_cache, _WS_DEPLOYMENT_AT_ITS_OTPM, now=_OPENED_AT), - ) - - assert _errors(websocket) == [] - assert await dual_cache.async_get_cache(key=_OPENED_ON) == _OUTPUT_TOKENS_PER_TURN - - @pytest.mark.asyncio - async def test_native_relay_answers_and_settles_the_first_prompt_after_a_refused_first_frame_when_the_open_hold_fills_the_minute( - self, - ) -> None: - import websockets.exceptions # noqa: F401 (lazy submodule must be importable) - - from litellm.caching.dual_cache import DualCache - from litellm.responses.streaming_iterator import drain_logging_tasks - from litellm.router_utils.pre_call_checks.io_token_rate_limit_check import async_admit_io_token_turn - from litellm.router_utils.pre_call_checks.model_rate_limit_check import ModelRateLimitingCheck - - dual_cache: Final = DualCache() - await dual_cache.async_increment_cache(key=_OPENED_ON, value=4096, ttl=60) - first, second, _ = _turn_frames() - backend_ws: Final = MagicMock() - backend_ws.send = AsyncMock() - backend_ws.recv = AsyncMock(side_effect=[_turn_completed(), Exception("stop")]) - websocket: Final = _ClientSocket(second) - handler: Final = _make_streaming( - websocket=websocket, - backend_ws=backend_ws, - request_data={}, - logging_obj=_connection_settling_turns_through(ModelRateLimitingCheck(dual_cache)), - first_message=first, - frame_guard=_guard_blocking("one"), - turn_admission=partial(async_admit_io_token_turn, dual_cache, _WS_DEPLOYMENT_AT_ITS_OTPM, now=_OPENED_AT), - ) - - await handler.client_to_backend() - await handler.backend_to_client() - await drain_logging_tasks() - - assert [(status, error["type"]) for status, error in _errors(websocket)] == [(None, "invalid_request_error")] - assert await dual_cache.async_get_cache(key=_OPENED_ON) == _OUTPUT_TOKENS_PER_TURN + assert await dual_cache.async_get_cache(key=_OPENED_ON) == _OTHER_USAGE + first_turn_tokens + assert await dual_cache.async_get_cache(key=_TURNED_ON) == _OTHER_USAGE + 2 * _OUTPUT_TOKENS_PER_TURN @pytest.mark.asyncio async def test_managed_relay_skips_a_later_turn_the_deployment_refuses(self, monkeypatch: pytest.MonkeyPatch): diff --git a/tests/unit/test_router/test_io_token_rate_limits.py b/tests/unit/test_router/test_io_token_rate_limits.py index af4a5a4217f..99789076c91 100644 --- a/tests/unit/test_router/test_io_token_rate_limits.py +++ b/tests/unit/test_router/test_io_token_rate_limits.py @@ -3,12 +3,14 @@ Tests for separate ITPM/OTPM deployment rate limits (enforce_model_rate_limits). """ import asyncio -from collections.abc import Mapping +import threading +from collections.abc import Iterator, Mapping from datetime import datetime, timezone from typing import Final from unittest.mock import AsyncMock, MagicMock import pytest +from fakeredis import TcpFakeServer import litellm from litellm import Router @@ -36,6 +38,7 @@ from litellm.router_utils.pre_call_checks.io_token_rate_limit_check import ( from litellm.router_utils.pre_call_checks.model_rate_limit_check import ( ModelRateLimitingCheck, ) +from litellm.types.llms.openai import InputTokensDetails, ResponseAPIUsage, ResponsesAPIResponse from litellm.types.utils import ModelResponse, Usage @@ -1146,15 +1149,6 @@ def _turn_key(kind: str, minute: str = "12-34") -> str: return f"global_router:ws-turn-{kind}:openai/gpt-5-mini:{kind}:{minute}" -def _socket_hold(kind: str, held: int, cache_key: str) -> Mapping[str, object]: - return { - ITPM_RESERVED_KEY: held if kind == "itpm" else 0, - OTPM_RESERVED_KEY: held if kind == "otpm" else 0, - ITPM_CACHE_KEY: cache_key if kind == "itpm" else None, - OTPM_CACHE_KEY: cache_key if kind == "otpm" else None, - } - - @pytest.mark.asyncio @pytest.mark.parametrize("kind", ["itpm", "otpm"]) @pytest.mark.parametrize("used", [100, 101, 250]) @@ -1163,7 +1157,7 @@ async def test_turn_admission_refuses_once_the_minutes_tokens_reach_the_limit(ki await dual_cache.async_increment_cache(key=_turn_key(kind), value=used, ttl=60) with pytest.raises(litellm.RateLimitError, match=f"{kind.upper()} limit=100, current usage={used}"): - await async_admit_io_token_turn(dual_cache, _turn_deployment(kind), {}, _TURN_AT) + await async_admit_io_token_turn(dual_cache, _turn_deployment(kind), _TURN_AT) @pytest.mark.asyncio @@ -1174,7 +1168,7 @@ async def test_an_admitted_turn_reserves_nothing_and_counts_its_actual_tokens(ki await dual_cache.async_increment_cache(key=_turn_key(kind), value=used, ttl=60) actual: Final = _ACTUAL_TOKENS - reservation: Final = await async_admit_io_token_turn(dual_cache, _turn_deployment(kind), {}, _TURN_AT) + reservation: Final = await async_admit_io_token_turn(dual_cache, _turn_deployment(kind), _TURN_AT) counter_key: Final = reservation[ITPM_CACHE_KEY if kind == "itpm" else OTPM_CACHE_KEY] assert isinstance(counter_key, str) assert await dual_cache.async_get_cache(key=counter_key) == used @@ -1196,44 +1190,99 @@ async def test_an_admitted_turn_reserves_nothing_and_counts_its_actual_tokens(ki async def test_a_turn_without_io_token_counters_is_admitted_with_nothing_to_settle( deployment: dict[str, object], ) -> None: - assert await async_admit_io_token_turn(DualCache(), deployment, {}, _TURN_AT) == {} + assert await async_admit_io_token_turn(DualCache(), deployment, _TURN_AT) == {} @pytest.mark.asyncio @pytest.mark.parametrize("kind", ["itpm", "otpm"]) @pytest.mark.parametrize("used_by_others", [0, 1, 99]) -async def test_a_sockets_first_turn_is_not_refused_by_its_own_opening_hold_and_settles_it( - kind: str, used_by_others: int +async def test_a_socket_open_reserves_nothing_and_its_first_turn_counts_its_actual_tokens_on_its_own_minute( + kind: str, used_by_others: int, monkeypatch: pytest.MonkeyPatch ) -> None: + monkeypatch.setattr( + "litellm.router_utils.pre_call_checks.io_token_rate_limit_check.get_utc_datetime", lambda: _TURN_AT + ) dual_cache: Final = DualCache() - held: Final = _socket_hold(kind, 100, _turn_key(kind)) - await dual_cache.async_increment_cache(key=_turn_key(kind), value=used_by_others + 100, ttl=60) + await dual_cache.async_increment_cache(key=_turn_key(kind), value=used_by_others, ttl=60) + socket_open: Final[dict[str, object]] = {"websocket": object(), "metadata": {}} + set_io_token_rate_limit_request_kwargs(socket_open) - admitted: Final = await async_admit_io_token_turn(dual_cache, _turn_deployment(kind), held, _TURN_AT) + assert await async_io_token_pre_call_check(dual_cache, _turn_deployment(kind)) == _turn_deployment(kind) + assert await dual_cache.async_get_cache(key=_turn_key(kind)) == used_by_others + + opened_with: Final = socket_open["metadata"] + assert isinstance(opened_with, dict) await async_io_token_reconcile_success( - dual_cache, - _proxy_success_kwargs({**held, **admitted}), - _reply_using(_ACTUAL_TOKENS["itpm"], _ACTUAL_TOKENS["otpm"]), + dual_cache, _proxy_success_kwargs(opened_with), _reply_using(_ACTUAL_TOKENS["itpm"], _ACTUAL_TOKENS["otpm"]) ) assert await dual_cache.async_get_cache(key=_turn_key(kind)) == used_by_others + _ACTUAL_TOKENS[kind] +@pytest.fixture +def shared_redis() -> Iterator[RedisCache]: + server: Final = TcpFakeServer(("127.0.0.1", 0), server_type="redis") + threading.Thread(target=server.serve_forever, daemon=True).start() + try: + yield RedisCache(host="127.0.0.1", port=server.server_address[1]) + finally: + server.shutdown() + server.server_close() + + @pytest.mark.asyncio @pytest.mark.parametrize("kind", ["itpm", "otpm"]) -@pytest.mark.parametrize( - ("held_on_minute", "counted_this_minute"), - [pytest.param("12-34", 200, id="others_reach_the_limit"), pytest.param("12-33", 100, id="hold_from_last_minute")], -) -async def test_a_sockets_first_turn_is_refused_once_other_usage_reaches_the_limit( - kind: str, held_on_minute: str, counted_this_minute: int +async def test_a_turn_is_admitted_once_the_shared_counter_drops_below_the_limit_this_instance_last_read( + kind: str, shared_redis: RedisCache ) -> None: - dual_cache: Final = DualCache() - await dual_cache.async_increment_cache(key=_turn_key(kind), value=counted_this_minute, ttl=60) - held: Final = _socket_hold(kind, 100, _turn_key(kind, held_on_minute)) + dual_cache: Final = DualCache(in_memory_cache=InMemoryCache(), redis_cache=shared_redis) + await dual_cache.in_memory_cache.async_increment(key=_turn_key(kind), value=100) + await shared_redis.async_increment(key=_turn_key(kind), value=99) - with pytest.raises(litellm.RateLimitError, match=f"{kind.upper()} limit=100, current usage=100"): - await async_admit_io_token_turn(dual_cache, _turn_deployment(kind), held, _TURN_AT) + admitted: Final = await async_admit_io_token_turn(dual_cache, _turn_deployment(kind), _TURN_AT) + + assert admitted[ITPM_CACHE_KEY if kind == "itpm" else OTPM_CACHE_KEY] == _turn_key(kind) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "responses_reply", + [ + pytest.param( + ResponsesAPIResponse( + id="resp_cached", + created_at=0, + output=[], + usage=ResponseAPIUsage( + input_tokens=50, + output_tokens=5, + total_tokens=55, + input_tokens_details=InputTokensDetails(cached_tokens=30), + ), + ), + id="usage_object", + ), + pytest.param( + { + "usage": { + "input_tokens": 50, + "output_tokens": 5, + "total_tokens": 55, + "input_tokens_details": {"cached_tokens": 30}, + } + }, + id="usage_dict", + ), + ], +) +async def test_a_turn_counts_only_its_uncached_responses_input_toward_itpm(responses_reply: object) -> None: + dual_cache: Final = DualCache() + await dual_cache.async_increment_cache(key=_turn_key("itpm"), value=10, ttl=60) + + reservation: Final = await async_admit_io_token_turn(dual_cache, _turn_deployment("itpm"), _TURN_AT) + await async_io_token_reconcile_success(dual_cache, _proxy_success_kwargs(reservation), responses_reply) + + assert await dual_cache.async_get_cache(key=_turn_key("itpm")) == 10 + 50 - 30 def _shared_counter_answering(answer: int | Exception) -> DualCache: @@ -1246,7 +1295,7 @@ def _shared_counter_answering(answer: int | Exception) -> DualCache: @pytest.mark.parametrize("kind", ["itpm", "otpm"]) async def test_a_later_turn_is_refused_when_another_instance_filled_the_shared_minute(kind: str) -> None: with pytest.raises(litellm.RateLimitError, match=f"{kind.upper()} limit=100, current usage=100"): - await async_admit_io_token_turn(_shared_counter_answering(100), _turn_deployment(kind), {}, _TURN_AT) + await async_admit_io_token_turn(_shared_counter_answering(100), _turn_deployment(kind), _TURN_AT) @pytest.mark.asyncio @@ -1257,6 +1306,6 @@ async def test_a_later_turn_is_admitted_on_this_instance_s_count_when_the_shared dual_cache: Final = _shared_counter_answering(RedisCircuitBreakerOpenError("circuit open")) await dual_cache.in_memory_cache.async_increment(key=_turn_key(kind), value=99) - admitted: Final = await async_admit_io_token_turn(dual_cache, _turn_deployment(kind), {}, _TURN_AT) + admitted: Final = await async_admit_io_token_turn(dual_cache, _turn_deployment(kind), _TURN_AT) assert admitted[ITPM_CACHE_KEY if kind == "itpm" else OTPM_CACHE_KEY] == _turn_key(kind) diff --git a/tests/unit/test_router/test_router_helper_utils.py b/tests/unit/test_router/test_router_helper_utils.py index 4c34894e094..be99c6f9350 100644 --- a/tests/unit/test_router/test_router_helper_utils.py +++ b/tests/unit/test_router/test_router_helper_utils.py @@ -538,14 +538,14 @@ async def test_deployment_callback_on_success_counts_a_responses_websocket_turn( assert await router.cache.async_get_cache(key=tpm_key) == 17 -async def _kwargs_routed_to(router: Router, model: str, function_name: str) -> dict[str, object]: +async def _kwargs_routed_to(router: Router, model: str, function_name: str, **request: object) -> dict[str, object]: routed: Final[dict[str, object]] = {} async def call(**kwargs: object) -> None: routed.update(kwargs) call.__name__ = function_name - await router._ageneric_api_call_with_fallbacks_helper(model=model, original_generic_function=call) + await router._ageneric_api_call_with_fallbacks_helper(model=model, original_generic_function=call, **request) return routed @@ -733,13 +733,75 @@ def _io_limited_router(kind: str, **litellm_params: object) -> Router: ) -async def _count_minutes_tokens(router: Router, kind: str, tokens: int, minute: datetime) -> None: - await router.cache.async_increment_cache( - key=f"global_router:ws-io-dep:openai/ws-io-unpriced:{kind}:{minute.strftime('%H-%M')}", - value=tokens, - ttl=120, +def _minutes_counter_key(kind: str, minute: datetime, model: str = "openai/ws-io-unpriced") -> str: + return f"global_router:ws-io-dep:{model}:{kind}:{minute.strftime('%H-%M')}" + + +async def _count_minutes_tokens( + router: Router, kind: str, tokens: int, minute: datetime, model: str = "openai/ws-io-unpriced" +) -> None: + await router.cache.async_increment_cache(key=_minutes_counter_key(kind, minute, model), value=tokens, ttl=120) + + +_LONG_OUTPUT_MODEL: Final = "openai/gpt-5.4-mini" + + +def _long_output_router() -> tuple[Router, Mapping[str, int]]: + max_output: Final = litellm.get_model_info(_LONG_OUTPUT_MODEL)["max_output_tokens"] + assert isinstance(max_output, int) + limits: Final = {"itpm": 10_000, "otpm": max_output // 2} + return _io_limited_router("itpm", model=_LONG_OUTPUT_MODEL, otpm=limits["otpm"]), limits + + +@pytest.mark.asyncio +@pytest.mark.parametrize("kind", ["itpm", "otpm"]) +async def test_responses_websocket_opens_without_counting_tokens_until_the_minute_reaches_deployment_io_limit( + kind: str, router_minute_pinned: datetime +): + router, limits = _long_output_router() + + await _kwargs_routed_to(router, "ws-io", "_aresponses_websocket", websocket=object()) + + assert [ + await router.cache.async_get_cache(key=_minutes_counter_key(counted, router_minute_pinned, _LONG_OUTPUT_MODEL)) + for counted in ("itpm", "otpm") + ] == [None, None] + await _count_minutes_tokens(router, kind, limits[kind], router_minute_pinned, _LONG_OUTPUT_MODEL) + with pytest.raises(litellm.RateLimitError, match=f"{kind.upper()} limit={limits[kind]}, current usage="): + await _kwargs_routed_to(router, "ws-io", "_aresponses_websocket", websocket=object()) + + +@pytest.mark.asyncio +async def test_responses_websocket_first_turn_counts_its_actual_tokens_on_the_minute_it_opened( + router_minute_pinned: datetime, +): + from litellm.cost_calculator import ResponsesWebSocketTokenUsageProcessor + from litellm.litellm_core_utils.get_litellm_params import get_litellm_params + from litellm.router_utils.pre_call_checks.model_rate_limit_check import ModelRateLimitingCheck + + router, _ = _long_output_router() + socket_metadata: Final = (await _kwargs_routed_to(router, "ws-io", "_aresponses_websocket", websocket=object()))[ + "litellm_metadata" + ] + assert isinstance(socket_metadata, dict) + rate_limits: Final = next( + check for check in router.optional_callbacks or () if isinstance(check, ModelRateLimitingCheck) ) + await rate_limits.async_log_success_event( + {"litellm_params": get_litellm_params(litellm_metadata=socket_metadata)}, + ResponsesWebSocketTokenUsageProcessor.create_logging_object( + [{"type": "response.completed", "response": {"usage": {"input_tokens": 20, "output_tokens": 15}}}] + ), + None, + None, + ) + + assert [ + await router.cache.async_get_cache(key=_minutes_counter_key(counted, router_minute_pinned, _LONG_OUTPUT_MODEL)) + for counted in ("itpm", "otpm") + ] == [20, 15] + @pytest.mark.asyncio @pytest.mark.parametrize("kind", ["itpm", "otpm"]) @@ -1257,6 +1319,9 @@ def _rpm_tpm_router(model_id: str) -> Router: def router_minute_pinned(monkeypatch: pytest.MonkeyPatch) -> datetime: pinned: Final = datetime(2026, 1, 1, 12, 0, 30, tzinfo=timezone.utc) monkeypatch.setattr("litellm.router.get_utc_datetime", lambda: pinned) + monkeypatch.setattr( + "litellm.router_utils.pre_call_checks.io_token_rate_limit_check.get_utc_datetime", lambda: pinned + ) return pinned