mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
fix(router): check a responses socket's open without reserving and read shared token counters first
A Responses WebSocket open now runs the same itpm/otpm check a turn runs and reserves nothing. It used to reserve an estimate that falls back to the model's max output, so a socket could not open under any otpm below that. The open still records a zero reservation for its own minute, so the first frame, which the relay forwards without a turn check, counts its actual tokens like every later turn. This drops the plumbing that only carried the open's hold into the first turn. The turn admission callable takes no arguments again, and the native relay detaches only the budget reservation, as before this change. The itpm/otpm turn check reads Redis first and falls back to this instance's in-memory count only without Redis or when the circuit breaker is open. These counters go down when a call settles, so an in-memory copy at the limit could refuse turns after the shared count dropped. The tpm check keeps its in-memory shortcut, since its counter only grows within a minute. Responses usage that reports input_tokens_details.cached_tokens now leaves cached input out of itpm, as chat usage already did.
This commit is contained in:
parent
42249f81f5
commit
be078aa315
9 changed files with 292 additions and 329 deletions
|
|
@ -174,14 +174,6 @@ def without_budget_reservation(
|
|||
}
|
||||
|
||||
|
||||
def without_io_token_reservation(metadata: Mapping[str, object]) -> Mapping[str, object]:
|
||||
return {key: value for key, value in metadata.items() if key not in IO_TOKEN_RESERVATION_KEYS}
|
||||
|
||||
|
||||
def io_token_reservation(metadata: Mapping[str, object]) -> Mapping[str, object]:
|
||||
return MappingProxyType({key: value for key, value in metadata.items() if key in IO_TOKEN_RESERVATION_KEYS})
|
||||
|
||||
|
||||
def forwarded_internal_call_metadata(
|
||||
parent_metadata: Mapping[str, object] | None,
|
||||
call_origin: InternalCallOrigin,
|
||||
|
|
|
|||
|
|
@ -46,11 +46,9 @@ from litellm.litellm_core_utils.core_helpers import (
|
|||
)
|
||||
from litellm.litellm_core_utils.exception_mapping_utils import with_deployment_api_base
|
||||
from litellm.litellm_core_utils.internal_call_metadata import (
|
||||
io_token_reservation,
|
||||
is_background_response,
|
||||
requests_background,
|
||||
without_budget_reservation,
|
||||
without_io_token_reservation,
|
||||
)
|
||||
from litellm.litellm_core_utils.litellm_logging import BilledRoundsSpend
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
|
|
@ -2156,17 +2154,17 @@ class _RefusedTurn:
|
|||
rejection: Exception
|
||||
|
||||
|
||||
ResponsesWsTurnAdmission: TypeAlias = Callable[[Mapping[str, object]], Awaitable[Mapping[str, object]]]
|
||||
ResponsesWsTurnAdmission: TypeAlias = Callable[[], Awaitable[Mapping[str, object]]]
|
||||
|
||||
|
||||
async def _admit_frame_turn(
|
||||
turn_admission: ResponsesWsTurnAdmission | None, raw_message: str, held_reservation: Mapping[str, object]
|
||||
turn_admission: ResponsesWsTurnAdmission | None, raw_message: str
|
||||
) -> _AdmittedTurn | _RefusedTurn:
|
||||
frame: Final = _json_object_or_none(raw_message)
|
||||
if turn_admission is None or frame is None or frame.get("type") != "response.create":
|
||||
return _AdmittedTurn()
|
||||
try:
|
||||
return _AdmittedTurn(await turn_admission(held_reservation))
|
||||
return _AdmittedTurn(await turn_admission())
|
||||
except RateLimitError as refusal:
|
||||
return _RefusedTurn(refusal)
|
||||
except Exception as e:
|
||||
|
|
@ -2519,32 +2517,22 @@ class ResponsesWebSocketStreaming:
|
|||
turn_logging_obj: Final = self.logging_obj.copy_for_new_call(
|
||||
litellm_call_id=str(uuid.uuid4()), start_time=start_time
|
||||
)
|
||||
self._detach_admission_reservations()
|
||||
self._detach_budget_reservation()
|
||||
_set_turn_guardrail_information(turn_logging_obj, guardrail_information)
|
||||
_set_turn_io_token_reservation(turn_logging_obj, io_token_reservation)
|
||||
turn_logging_obj.model_call_details["messages"] = list(_logged_user_messages(client_frame))
|
||||
turn_logging_obj.pre_call(input=client_frame, api_key="")
|
||||
self._open_turns = (*self._open_turns, _ResponsesWsTurn(logging_obj=turn_logging_obj))
|
||||
|
||||
def _held_admission_reservation(self) -> Mapping[str, object]:
|
||||
litellm_params: Final = _json_object_or_none(
|
||||
self.logging_obj.litellm_params # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # Logging.litellm_params is untyped
|
||||
)
|
||||
buckets: Final = (
|
||||
() if litellm_params is None else (litellm_params.get(key) for key in ("metadata", "litellm_metadata"))
|
||||
)
|
||||
held: Final = (io_token_reservation(bucket) for bucket in buckets if _is_json_object(bucket))
|
||||
return next((reservation for reservation in held if reservation), _NO_IO_TOKEN_RESERVATION)
|
||||
|
||||
def _detach_admission_reservations(self) -> None:
|
||||
"""Only the first turn settles the connection's admission reservations; later turns charge their own usage."""
|
||||
def _detach_budget_reservation(self) -> None:
|
||||
"""Only the first turn settles the connection's admission reservation; later turns charge their cost directly."""
|
||||
litellm_params: Final = _json_object_or_none(
|
||||
self.logging_obj.litellm_params # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # Logging.litellm_params is untyped
|
||||
)
|
||||
if litellm_params is None:
|
||||
return
|
||||
self.logging_obj.litellm_params = {
|
||||
key: without_io_token_reservation(without_budget_reservation(value))
|
||||
key: without_budget_reservation(value)
|
||||
if key in _PER_TURN_METADATA_KEYS and _is_json_object(value)
|
||||
else value
|
||||
for key, value in litellm_params.items()
|
||||
|
|
@ -2990,7 +2978,7 @@ class ResponsesWebSocketStreaming:
|
|||
async def _admit_or_reject_turn(
|
||||
self, guarded: GuardedResponsesWsFrame, message: str, turn_admission: ResponsesWsTurnAdmission | None
|
||||
) -> _AdmittedTurn | None:
|
||||
verdict: Final = await _admit_frame_turn(turn_admission, message, self._held_admission_reservation())
|
||||
verdict: Final = await _admit_frame_turn(turn_admission, message)
|
||||
match verdict:
|
||||
case _AdmittedTurn():
|
||||
return verdict
|
||||
|
|
@ -3419,21 +3407,6 @@ class ManagedResponsesWebSocketHandler:
|
|||
if self.litellm_metadata:
|
||||
call_kwargs["litellm_metadata"] = dict(self.litellm_metadata)
|
||||
|
||||
def _held_io_token_reservation(self) -> Mapping[str, object]:
|
||||
connection_metadata: Final = self.extra_kwargs.get("metadata")
|
||||
held_on_connection: Final = (
|
||||
io_token_reservation(connection_metadata)
|
||||
if _is_json_object(connection_metadata)
|
||||
else _NO_IO_TOKEN_RESERVATION
|
||||
)
|
||||
return held_on_connection or io_token_reservation(self.litellm_metadata)
|
||||
|
||||
def _detach_io_token_reservation(self) -> None:
|
||||
connection_metadata: Final = self.extra_kwargs.get("metadata")
|
||||
self.litellm_metadata = without_io_token_reservation(self.litellm_metadata)
|
||||
if _is_json_object(connection_metadata):
|
||||
self.extra_kwargs = {**self.extra_kwargs, "metadata": without_io_token_reservation(connection_metadata)}
|
||||
|
||||
async def _guard_frame(self, raw_message: str) -> GuardedResponsesWsFrame | None:
|
||||
"""Same per-frame pre_call guardrail walk as the native socket: a rejected frame errors and is not sent."""
|
||||
if self.frame_guard is None:
|
||||
|
|
@ -3584,7 +3557,7 @@ class ManagedResponsesWebSocketHandler:
|
|||
guarded: Final = await self._guard_frame(raw_message)
|
||||
if guarded is None:
|
||||
return
|
||||
verdict: Final = await _admit_frame_turn(turn_admission, raw_message, self._held_io_token_reservation())
|
||||
verdict: Final = await _admit_frame_turn(turn_admission, raw_message)
|
||||
if isinstance(verdict, _RefusedTurn):
|
||||
if self._failure is None:
|
||||
self._failure = verdict.rejection
|
||||
|
|
@ -3617,15 +3590,15 @@ class ManagedResponsesWebSocketHandler:
|
|||
call_kwargs["litellm_metadata"] = self._metadata_with_turn_guardrails(
|
||||
call_kwargs.get("litellm_metadata"), guarded.guardrail_information
|
||||
)
|
||||
if verdict.reservation:
|
||||
existing_metadata: Final = call_kwargs.get("litellm_metadata")
|
||||
call_kwargs["litellm_metadata"] = {
|
||||
**(existing_metadata if _is_json_object(existing_metadata) else {}),
|
||||
**verdict.reservation,
|
||||
}
|
||||
self._update_proxy_request(call_kwargs, requested_model or self.model_group or model)
|
||||
call_kwargs.update(self.extra_kwargs)
|
||||
self._detach_io_token_reservation()
|
||||
call_kwargs.update(
|
||||
{
|
||||
channel: {**bucket, **verdict.reservation}
|
||||
for channel, bucket in call_kwargs.items()
|
||||
if channel in _PER_TURN_METADATA_KEYS and _is_json_object(bucket)
|
||||
}
|
||||
)
|
||||
|
||||
try:
|
||||
terminal_event: Final = await self._stream_and_forward(model, call_kwargs)
|
||||
|
|
|
|||
|
|
@ -10452,7 +10452,6 @@ class Router(DeprecatedRouterMembers):
|
|||
model_group: str,
|
||||
opened_on: Mapping[str, object],
|
||||
request_kwargs: Mapping[str, object],
|
||||
held_reservation: Mapping[str, object] = MappingProxyType({}),
|
||||
) -> Mapping[str, object]:
|
||||
deployment: Final = self._deployment_as_configured_now(opened_on)
|
||||
rate_limit_check: Final = next(
|
||||
|
|
@ -10461,9 +10460,7 @@ class Router(DeprecatedRouterMembers):
|
|||
io_token_admission: Final = (
|
||||
MappingProxyType({})
|
||||
if rate_limit_check is None
|
||||
else await async_admit_io_token_turn(
|
||||
rate_limit_check.dual_cache, deployment, held_reservation, get_utc_datetime()
|
||||
)
|
||||
else await async_admit_io_token_turn(rate_limit_check.dual_cache, deployment, get_utc_datetime())
|
||||
)
|
||||
budget_limiter: Final = self._core_get_router_deployment_budget_limiter()
|
||||
if budget_limiter is not None:
|
||||
|
|
|
|||
|
|
@ -185,27 +185,32 @@ def _resolve_max_tokens(request_kwargs: dict[str, Any] | None, deployment: dict)
|
|||
return 4096
|
||||
|
||||
|
||||
def _usage_field(container: object, name: str) -> object:
|
||||
if is_str_keyed_mapping(container):
|
||||
return container.get(name)
|
||||
return getattr(container, name, None)
|
||||
|
||||
|
||||
def _cached_prompt_tokens(usage: object) -> int:
|
||||
reported: Final = (
|
||||
_usage_field(_usage_field(usage, "prompt_tokens_details"), "cached_tokens"),
|
||||
_usage_field(_usage_field(usage, "input_tokens_details"), "cached_tokens"),
|
||||
_usage_field(usage, "cache_read_input_tokens"),
|
||||
)
|
||||
return next((int(tokens) for tokens in reported if isinstance(tokens, int | float) and tokens), 0)
|
||||
|
||||
|
||||
def _get_usage_tokens(usage: object) -> tuple[int, int, int]:
|
||||
if usage is None:
|
||||
return 0, 0, 0
|
||||
if hasattr(usage, "prompt_tokens") or hasattr(usage, "input_tokens"):
|
||||
prompt = int(getattr(usage, "prompt_tokens", None) or getattr(usage, "input_tokens", 0) or 0)
|
||||
completion = int(getattr(usage, "completion_tokens", None) or getattr(usage, "output_tokens", 0) or 0)
|
||||
cached = 0
|
||||
details: object = getattr(usage, "prompt_tokens_details", None)
|
||||
if details is not None:
|
||||
cached = int(getattr(details, "cached_tokens", 0) or 0)
|
||||
if not cached:
|
||||
cached = int(getattr(usage, "cache_read_input_tokens", 0) or 0)
|
||||
return prompt, completion, cached
|
||||
return prompt, completion, _cached_prompt_tokens(usage)
|
||||
if isinstance(usage, dict):
|
||||
prompt = int(usage.get("prompt_tokens") or usage.get("input_tokens") or 0)
|
||||
completion = int(usage.get("completion_tokens") or usage.get("output_tokens") or 0)
|
||||
details = usage.get("prompt_tokens_details") or {}
|
||||
cached = int(details.get("cached_tokens", 0) or 0) if isinstance(details, dict) else 0
|
||||
if not cached:
|
||||
cached = int(usage.get("cache_read_input_tokens") or 0)
|
||||
return prompt, completion, cached
|
||||
return prompt, completion, _cached_prompt_tokens(usage)
|
||||
return 0, 0, 0
|
||||
|
||||
|
||||
|
|
@ -266,22 +271,22 @@ def _resolve_reconcile_usage_tokens(
|
|||
return 0, 0, False
|
||||
|
||||
|
||||
def _stash_reservation_in_metadata(
|
||||
request_kwargs: dict[str, object] | None,
|
||||
*,
|
||||
itpm_reserved: int,
|
||||
otpm_reserved: int,
|
||||
itpm_cache_key: str | None,
|
||||
otpm_cache_key: str | None,
|
||||
) -> None:
|
||||
def _reservation(
|
||||
itpm_reserved: int, otpm_reserved: int, itpm_cache_key: str | None, otpm_cache_key: str | None
|
||||
) -> Mapping[str, object]:
|
||||
return MappingProxyType(
|
||||
{
|
||||
ITPM_RESERVED_KEY: itpm_reserved,
|
||||
OTPM_RESERVED_KEY: otpm_reserved,
|
||||
ITPM_CACHE_KEY: itpm_cache_key,
|
||||
OTPM_CACHE_KEY: otpm_cache_key,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _stash_reservation_in_metadata(request_kwargs: dict[str, object] | None, reservation: Mapping[str, object]) -> None:
|
||||
if not request_kwargs:
|
||||
return
|
||||
reservation: Final = {
|
||||
ITPM_RESERVED_KEY: itpm_reserved,
|
||||
OTPM_RESERVED_KEY: otpm_reserved,
|
||||
ITPM_CACHE_KEY: itpm_cache_key,
|
||||
OTPM_CACHE_KEY: otpm_cache_key,
|
||||
}
|
||||
for channel in ("metadata", "litellm_metadata"):
|
||||
existing = request_kwargs.get(channel)
|
||||
if isinstance(existing, dict):
|
||||
|
|
@ -483,10 +488,12 @@ def io_token_pre_call_check(
|
|||
|
||||
_stash_reservation_in_metadata(
|
||||
request_kwargs,
|
||||
itpm_reserved=itpm_reserved,
|
||||
otpm_reserved=otpm_reserved,
|
||||
itpm_cache_key=itpm_key if itpm_limit is not None else None,
|
||||
otpm_cache_key=otpm_key if otpm_limit is not None else None,
|
||||
_reservation(
|
||||
itpm_reserved,
|
||||
otpm_reserved,
|
||||
itpm_key if itpm_limit is not None else None,
|
||||
otpm_key if otpm_limit is not None else None,
|
||||
),
|
||||
)
|
||||
return deployment
|
||||
|
||||
|
|
@ -502,6 +509,12 @@ async def async_io_token_pre_call_check(
|
|||
return deployment
|
||||
|
||||
request_kwargs: Final = get_io_token_rate_limit_request_kwargs()
|
||||
if request_kwargs is not None and request_kwargs.get("websocket") is not None:
|
||||
_stash_reservation_in_metadata(
|
||||
request_kwargs,
|
||||
await async_admit_io_token_turn(dual_cache, deployment, get_utc_datetime(), parent_otel_span),
|
||||
)
|
||||
return deployment
|
||||
_model: Final = _tokenizer_model(deployment)
|
||||
estimated_input: Final = await offload_token_count(_estimate_input_tokens)(request_kwargs, model=_model)
|
||||
max_tokens: Final = _resolve_max_tokens(request_kwargs, deployment)
|
||||
|
|
@ -553,50 +566,39 @@ async def async_io_token_pre_call_check(
|
|||
|
||||
_stash_reservation_in_metadata(
|
||||
request_kwargs,
|
||||
itpm_reserved=itpm_reserved,
|
||||
otpm_reserved=otpm_reserved,
|
||||
itpm_cache_key=itpm_key if itpm_limit is not None else None,
|
||||
otpm_cache_key=otpm_key if otpm_limit is not None else None,
|
||||
_reservation(
|
||||
itpm_reserved,
|
||||
otpm_reserved,
|
||||
itpm_key if itpm_limit is not None else None,
|
||||
otpm_key if otpm_limit is not None else None,
|
||||
),
|
||||
)
|
||||
return deployment
|
||||
|
||||
|
||||
@with_service_target(ROUTER_USAGE_TARGET)
|
||||
async def async_current_usage(
|
||||
dual_cache: DualCache, key: str, limit: int, parent_otel_span: Span | None = None
|
||||
) -> int | None:
|
||||
local_usage: Final = await dual_cache.async_get_cache(key=key, local_only=True)
|
||||
async def _async_shared_usage(dual_cache: DualCache, key: str, parent_otel_span: Span | None) -> int | None:
|
||||
redis_cache: Final = dual_cache.redis_cache
|
||||
if redis_cache is None or (local_usage is not None and local_usage >= limit):
|
||||
return local_usage
|
||||
try:
|
||||
return await redis_cache.async_get_cache(key=key, parent_otel_span=parent_otel_span)
|
||||
except RedisCircuitBreakerOpenError:
|
||||
return local_usage
|
||||
|
||||
|
||||
def _held_on(held_reservation: Mapping[str, object], reserved_key: str, cache_key_key: str, cache_key: str) -> int:
|
||||
held: Final = held_reservation.get(reserved_key)
|
||||
if held_reservation.get(cache_key_key) != cache_key or not isinstance(held, int):
|
||||
return 0
|
||||
return held
|
||||
if redis_cache is not None:
|
||||
with contextlib.suppress(RedisCircuitBreakerOpenError):
|
||||
return await redis_cache.async_get_cache(key=key, parent_otel_span=parent_otel_span)
|
||||
return await dual_cache.async_get_cache(key=key, local_only=True)
|
||||
|
||||
|
||||
async def _raise_if_at_limit(
|
||||
dual_cache: DualCache, key: str, limit: int | None, held: int, limit_label: str, parent_otel_span: Span | None
|
||||
dual_cache: DualCache, key: str, limit: int | None, limit_label: str, parent_otel_span: Span | None
|
||||
) -> None:
|
||||
if limit is None:
|
||||
return
|
||||
usage: Final = await async_current_usage(dual_cache, key, limit + held, parent_otel_span)
|
||||
if usage is not None and usage - held >= limit:
|
||||
raise _rate_limit_error(limit_label, limit, usage - held)
|
||||
usage: Final = await _async_shared_usage(dual_cache, key, parent_otel_span)
|
||||
if usage is not None and usage >= limit:
|
||||
raise _rate_limit_error(limit_label, limit, usage)
|
||||
|
||||
|
||||
@with_service_target(ROUTER_USAGE_TARGET)
|
||||
async def async_admit_io_token_turn(
|
||||
dual_cache: DualCache,
|
||||
deployment: Mapping[str, object],
|
||||
held_reservation: Mapping[str, object],
|
||||
now: datetime,
|
||||
parent_otel_span: Span | None = None,
|
||||
) -> Mapping[str, object]:
|
||||
|
|
@ -608,19 +610,10 @@ async def async_admit_io_token_turn(
|
|||
if cache_keys is None:
|
||||
return MappingProxyType({})
|
||||
itpm_key, otpm_key = cache_keys
|
||||
itpm_held: Final = _held_on(held_reservation, ITPM_RESERVED_KEY, ITPM_CACHE_KEY, itpm_key)
|
||||
otpm_held: Final = _held_on(held_reservation, OTPM_RESERVED_KEY, OTPM_CACHE_KEY, otpm_key)
|
||||
await _raise_if_at_limit(dual_cache, itpm_key, itpm_limit, itpm_held, "ITPM", parent_otel_span)
|
||||
await _raise_if_at_limit(dual_cache, otpm_key, otpm_limit, otpm_held, "OTPM", parent_otel_span)
|
||||
if held_reservation:
|
||||
return MappingProxyType({})
|
||||
return MappingProxyType(
|
||||
{
|
||||
ITPM_RESERVED_KEY: 0,
|
||||
OTPM_RESERVED_KEY: 0,
|
||||
ITPM_CACHE_KEY: itpm_key if itpm_limit is not None else None,
|
||||
OTPM_CACHE_KEY: otpm_key if otpm_limit is not None else None,
|
||||
}
|
||||
await _raise_if_at_limit(dual_cache, itpm_key, itpm_limit, "ITPM", parent_otel_span)
|
||||
await _raise_if_at_limit(dual_cache, otpm_key, otpm_limit, "OTPM", parent_otel_span)
|
||||
return _reservation(
|
||||
0, 0, itpm_key if itpm_limit is not None else None, otpm_key if otpm_limit is not None else None
|
||||
)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -26,7 +26,6 @@ from litellm.caching.dual_cache import DualCache
|
|||
from litellm.caching.redis_cache import RedisCircuitBreakerOpenError
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.router_utils.pre_call_checks.io_token_rate_limit_check import (
|
||||
async_current_usage,
|
||||
async_io_token_pre_call_check,
|
||||
async_io_token_reconcile_success,
|
||||
async_io_token_refund_failure,
|
||||
|
|
@ -290,7 +289,14 @@ class ModelRateLimitingCheck(CustomLogger):
|
|||
|
||||
@with_service_target(ROUTER_USAGE_TARGET)
|
||||
async def _async_get_current_tpm(self, tpm_key: str, tpm_limit: int, parent_otel_span: Span | None) -> int | None:
|
||||
return await async_current_usage(self.dual_cache, tpm_key, tpm_limit, parent_otel_span)
|
||||
local_tpm: Final = await self.dual_cache.async_get_cache(key=tpm_key, local_only=True)
|
||||
redis_cache: Final = self.dual_cache.redis_cache
|
||||
if redis_cache is None or (local_tpm is not None and local_tpm >= tpm_limit):
|
||||
return local_tpm
|
||||
try:
|
||||
return await redis_cache.async_get_cache(key=tpm_key, parent_otel_span=parent_otel_span)
|
||||
except RedisCircuitBreakerOpenError:
|
||||
return local_tpm
|
||||
|
||||
@with_service_target(ROUTER_USAGE_TARGET)
|
||||
def pre_call_check(self, deployment: dict) -> dict | None:
|
||||
|
|
|
|||
|
|
@ -518,9 +518,7 @@ def test_a_tpm_limit_lowered_while_the_socket_is_open_applies_to_its_next_turn(
|
|||
assert deployment.forwarded_turns() == 1
|
||||
|
||||
|
||||
def test_a_socket_on_an_otpm_limited_deployment_reserves_once_and_serves_every_turn(
|
||||
limited: Gateway, cert: tuple[Path, Path]
|
||||
) -> None:
|
||||
def test_a_socket_on_an_otpm_limited_deployment_serves_every_turn(limited: Gateway, cert: tuple[Path, Path]) -> None:
|
||||
with responses_peer(cert) as peer, limited.scenario() as scenario:
|
||||
deployment: Final = _native(scenario, peer, otpm=10_000)
|
||||
key: Final = scenario.key(models=[deployment.name])
|
||||
|
|
|
|||
|
|
@ -10,10 +10,10 @@ Tests that:
|
|||
import asyncio
|
||||
import itertools
|
||||
import json
|
||||
from collections.abc import AsyncIterator, Callable, Mapping
|
||||
from collections.abc import AsyncIterator, Mapping
|
||||
from datetime import datetime, timezone
|
||||
from functools import partial
|
||||
from typing import TYPE_CHECKING, Final, TypeAlias
|
||||
from typing import TYPE_CHECKING, Final
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
|
||||
import pytest
|
||||
|
|
@ -49,6 +49,7 @@ from litellm.llms.xai.responses.transformation import XAIResponsesAPIConfig
|
|||
from litellm.types.utils import GenericStreamingChunk
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.caching.dual_cache import DualCache
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging
|
||||
from litellm.responses.streaming_iterator import ResponsesWsTurnAdmission
|
||||
from litellm.router_utils.pre_call_checks.model_rate_limit_check import ModelRateLimitingCheck
|
||||
|
|
@ -1667,19 +1668,13 @@ def _guard_recording(bill_refusal: MagicMock):
|
|||
|
||||
_OPENED_ON: Final = "global_router:ws-dep:openai/gpt-5-mini:otpm:12-34"
|
||||
_TURNED_ON: Final = "global_router:ws-dep:openai/gpt-5-mini:otpm:12-35"
|
||||
|
||||
|
||||
def _otpm_reservation(reserved: int, cache_key: str) -> Mapping[str, object]:
|
||||
from litellm.router_utils.pre_call_checks.io_token_rate_limit_check import (
|
||||
ITPM_CACHE_KEY,
|
||||
ITPM_RESERVED_KEY,
|
||||
OTPM_CACHE_KEY,
|
||||
OTPM_RESERVED_KEY,
|
||||
)
|
||||
|
||||
return {ITPM_RESERVED_KEY: 0, OTPM_RESERVED_KEY: reserved, ITPM_CACHE_KEY: None, OTPM_CACHE_KEY: cache_key}
|
||||
|
||||
|
||||
_OPENED_AT: Final = datetime(2026, 1, 1, 12, 34, 56, tzinfo=timezone.utc)
|
||||
_TURNED_AT: Final = datetime(2026, 1, 1, 12, 35, 5, tzinfo=timezone.utc)
|
||||
_WS_DEPLOYMENT: Final = {
|
||||
"litellm_params": {"model": "openai/gpt-5-mini", "otpm": 4096},
|
||||
"model_info": {"id": "ws-dep"},
|
||||
}
|
||||
_OTHER_USAGE: Final = 10
|
||||
_OUTPUT_TOKENS_PER_TURN: Final = 2
|
||||
_UPSTREAM_ERROR: Final = json.dumps({"type": "error", "error": {"type": "server_error", "message": "upstream failed"}})
|
||||
|
||||
|
|
@ -1698,25 +1693,23 @@ def _turn_completed() -> str:
|
|||
)
|
||||
|
||||
|
||||
def _admitting(reservation: Mapping[str, object]) -> AsyncMock:
|
||||
return AsyncMock(return_value=reservation)
|
||||
async def _socket_opened_on(dual_cache: "DualCache") -> Mapping[str, object]:
|
||||
from litellm.router_utils.pre_call_checks.io_token_rate_limit_check import async_admit_io_token_turn
|
||||
|
||||
await dual_cache.async_increment_cache(key=_OPENED_ON, value=_OTHER_USAGE, ttl=60)
|
||||
await dual_cache.async_increment_cache(key=_TURNED_ON, value=_OTHER_USAGE, ttl=60)
|
||||
return await async_admit_io_token_turn(dual_cache, _WS_DEPLOYMENT, _OPENED_AT)
|
||||
|
||||
|
||||
_FIRST_TURN_ENDS: Final = (
|
||||
pytest.param(_turn_completed(), _OUTPUT_TOKENS_PER_TURN, id="first_turn_completed"),
|
||||
pytest.param(_UPSTREAM_ERROR, 0, id="first_turn_failed"),
|
||||
)
|
||||
_LaterTurnAdmission: TypeAlias = Callable[[], AsyncMock | None]
|
||||
_LATER_TURN_ADMISSIONS: Final = (
|
||||
pytest.param(
|
||||
lambda: _admitting(_otpm_reservation(0, _TURNED_ON)), 2 * _OUTPUT_TOKENS_PER_TURN, id="admitted_this_minute"
|
||||
),
|
||||
pytest.param(lambda: _admitting({}), None, id="limits_removed_mid_socket"),
|
||||
pytest.param(lambda: None, None, id="no_turn_admission"),
|
||||
)
|
||||
def _later_turns_admitted_through(dual_cache: "DualCache") -> "ResponsesWsTurnAdmission":
|
||||
from litellm.router_utils.pre_call_checks.io_token_rate_limit_check import async_admit_io_token_turn
|
||||
|
||||
return partial(async_admit_io_token_turn, dual_cache, _WS_DEPLOYMENT, _TURNED_AT)
|
||||
|
||||
|
||||
def _connection_settling_turns_through(rate_limits: "ModelRateLimitingCheck") -> "Logging":
|
||||
def _connection_settling_turns_through(
|
||||
rate_limits: "ModelRateLimitingCheck", opened_with: Mapping[str, object]
|
||||
) -> "Logging":
|
||||
import time
|
||||
|
||||
from litellm.cost_calculator import ResponsesWebSocketTokenUsageProcessor
|
||||
|
|
@ -1746,20 +1739,10 @@ def _connection_settling_turns_through(rate_limits: "ModelRateLimitingCheck") ->
|
|||
litellm_call_id="connection-call",
|
||||
function_id="connection-call",
|
||||
)
|
||||
connection.litellm_params = {
|
||||
"metadata": dict(_otpm_reservation(4096, _OPENED_ON)),
|
||||
"litellm_metadata": dict(_otpm_reservation(4096, _OPENED_ON)),
|
||||
}
|
||||
connection.litellm_params = {"metadata": dict(opened_with), "litellm_metadata": dict(opened_with)}
|
||||
return connection
|
||||
|
||||
|
||||
_OPENED_AT: Final = datetime(2026, 1, 1, 12, 34, 56, tzinfo=timezone.utc)
|
||||
_WS_DEPLOYMENT_AT_ITS_OTPM: Final = {
|
||||
"litellm_params": {"model": "openai/gpt-5-mini", "otpm": 4096},
|
||||
"model_info": {"id": "ws-dep"},
|
||||
}
|
||||
|
||||
|
||||
class _TwoOutputTokenTurns(CustomLLM):
|
||||
def __init__(self, failing_turns: frozenset[int] = frozenset()) -> None:
|
||||
super().__init__()
|
||||
|
|
@ -1785,6 +1768,7 @@ async def _run_managed_relay(
|
|||
rate_limits: "ModelRateLimitingCheck",
|
||||
frames: tuple[str, ...],
|
||||
turn_admission: "ResponsesWsTurnAdmission | None",
|
||||
opened_with: Mapping[str, object],
|
||||
) -> _ClientSocket:
|
||||
import litellm
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging
|
||||
|
|
@ -1807,10 +1791,10 @@ async def _run_managed_relay(
|
|||
litellm_call_id="test-id",
|
||||
function_id="test-func",
|
||||
),
|
||||
litellm_metadata={"user_api_key_hash": "k1", **_otpm_reservation(4096, _OPENED_ON)},
|
||||
litellm_metadata={"user_api_key_hash": "k1", **opened_with},
|
||||
first_message=first,
|
||||
turn_admission=turn_admission,
|
||||
metadata=dict(_otpm_reservation(4096, _OPENED_ON)),
|
||||
metadata=dict(opened_with),
|
||||
)
|
||||
|
||||
await handler.run()
|
||||
|
|
@ -1888,14 +1872,15 @@ class TestWebSocketTurnAdmission:
|
|||
assert "crossed budget" in str(failure)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(("first_turn_end", "opened_on_tokens"), _FIRST_TURN_ENDS)
|
||||
@pytest.mark.parametrize(("turn_admission", "turned_on_tokens"), _LATER_TURN_ADMISSIONS)
|
||||
async def test_native_relay_settles_the_open_reservation_on_the_first_turn_only(
|
||||
self,
|
||||
first_turn_end: str,
|
||||
opened_on_tokens: int,
|
||||
turn_admission: _LaterTurnAdmission,
|
||||
turned_on_tokens: int | None,
|
||||
@pytest.mark.parametrize(
|
||||
("first_turn_end", "first_turn_tokens"),
|
||||
[
|
||||
pytest.param(_turn_completed(), _OUTPUT_TOKENS_PER_TURN, id="first_turn_completed"),
|
||||
pytest.param(_UPSTREAM_ERROR, 0, id="first_turn_failed"),
|
||||
],
|
||||
)
|
||||
async def test_native_relay_counts_the_first_turn_on_the_minute_it_opened_and_each_later_turn_on_its_own(
|
||||
self, first_turn_end: str, first_turn_tokens: int
|
||||
) -> None:
|
||||
import websockets.exceptions # noqa: F401 (lazy submodule must be importable)
|
||||
|
||||
|
|
@ -1904,7 +1889,7 @@ class TestWebSocketTurnAdmission:
|
|||
from litellm.router_utils.pre_call_checks.model_rate_limit_check import ModelRateLimitingCheck
|
||||
|
||||
dual_cache: Final = DualCache()
|
||||
await dual_cache.async_increment_cache(key=_OPENED_ON, value=4096, ttl=60)
|
||||
opened_with: Final = await _socket_opened_on(dual_cache)
|
||||
first, second, third = _turn_frames()
|
||||
backend_ws: Final = MagicMock()
|
||||
backend_ws.send = AsyncMock()
|
||||
|
|
@ -1915,141 +1900,46 @@ class TestWebSocketTurnAdmission:
|
|||
websocket=_ClientSocket(second, third),
|
||||
backend_ws=backend_ws,
|
||||
request_data={},
|
||||
logging_obj=_connection_settling_turns_through(ModelRateLimitingCheck(dual_cache)),
|
||||
logging_obj=_connection_settling_turns_through(ModelRateLimitingCheck(dual_cache), opened_with),
|
||||
first_message=first,
|
||||
turn_admission=turn_admission(),
|
||||
turn_admission=_later_turns_admitted_through(dual_cache),
|
||||
)
|
||||
|
||||
await handler.client_to_backend()
|
||||
await handler.backend_to_client()
|
||||
await drain_logging_tasks()
|
||||
|
||||
assert await dual_cache.async_get_cache(key=_OPENED_ON) == opened_on_tokens
|
||||
assert await dual_cache.async_get_cache(key=_TURNED_ON) == turned_on_tokens
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_native_relay_does_not_settle_the_open_reservation_again_for_an_event_no_turn_asked_for(
|
||||
self,
|
||||
) -> None:
|
||||
import websockets.exceptions # noqa: F401 (lazy submodule must be importable)
|
||||
|
||||
from litellm.caching.dual_cache import DualCache
|
||||
from litellm.responses.streaming_iterator import drain_logging_tasks
|
||||
from litellm.router_utils.pre_call_checks.model_rate_limit_check import ModelRateLimitingCheck
|
||||
|
||||
dual_cache: Final = DualCache()
|
||||
await dual_cache.async_increment_cache(key=_OPENED_ON, value=4096, ttl=60)
|
||||
first, _, _ = _turn_frames()
|
||||
backend_ws: Final = MagicMock()
|
||||
backend_ws.send = AsyncMock()
|
||||
backend_ws.recv = AsyncMock(side_effect=[_turn_completed(), _UPSTREAM_ERROR, Exception("stop")])
|
||||
handler: Final = _make_streaming(
|
||||
websocket=_ClientSocket(),
|
||||
backend_ws=backend_ws,
|
||||
request_data={},
|
||||
logging_obj=_connection_settling_turns_through(ModelRateLimitingCheck(dual_cache)),
|
||||
first_message=first,
|
||||
turn_admission=_admitting(_otpm_reservation(0, _TURNED_ON)),
|
||||
)
|
||||
|
||||
await handler.client_to_backend()
|
||||
await handler.backend_to_client()
|
||||
await drain_logging_tasks()
|
||||
|
||||
assert await dual_cache.async_get_cache(key=_OPENED_ON) == _OUTPUT_TOKENS_PER_TURN
|
||||
assert await dual_cache.async_get_cache(key=_OPENED_ON) == _OTHER_USAGE + first_turn_tokens
|
||||
assert await dual_cache.async_get_cache(key=_TURNED_ON) == _OTHER_USAGE + 2 * _OUTPUT_TOKENS_PER_TURN
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("failing_turns", "opened_on_tokens"),
|
||||
("failing_turns", "first_turn_tokens"),
|
||||
[
|
||||
pytest.param(frozenset(), _OUTPUT_TOKENS_PER_TURN, id="first_turn_completed"),
|
||||
pytest.param(frozenset({1}), 0, id="first_turn_failed"),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize(("turn_admission", "turned_on_tokens"), _LATER_TURN_ADMISSIONS)
|
||||
async def test_managed_relay_settles_the_open_reservation_on_the_first_turn_only(
|
||||
self,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
failing_turns: frozenset[int],
|
||||
opened_on_tokens: int,
|
||||
turn_admission: _LaterTurnAdmission,
|
||||
turned_on_tokens: int | None,
|
||||
async def test_managed_relay_counts_the_first_turn_on_the_minute_it_opened_and_each_later_turn_on_its_own(
|
||||
self, monkeypatch: pytest.MonkeyPatch, failing_turns: frozenset[int], first_turn_tokens: int
|
||||
) -> None:
|
||||
from litellm.caching.dual_cache import DualCache
|
||||
from litellm.router_utils.pre_call_checks.model_rate_limit_check import ModelRateLimitingCheck
|
||||
|
||||
dual_cache: Final = DualCache()
|
||||
await dual_cache.async_increment_cache(key=_OPENED_ON, value=4096, ttl=60)
|
||||
opened_with: Final = await _socket_opened_on(dual_cache)
|
||||
|
||||
await _run_managed_relay(
|
||||
monkeypatch,
|
||||
_TwoOutputTokenTurns(failing_turns),
|
||||
ModelRateLimitingCheck(dual_cache),
|
||||
_turn_frames(),
|
||||
turn_admission(),
|
||||
_later_turns_admitted_through(dual_cache),
|
||||
opened_with,
|
||||
)
|
||||
|
||||
assert await dual_cache.async_get_cache(key=_OPENED_ON) == opened_on_tokens
|
||||
assert await dual_cache.async_get_cache(key=_TURNED_ON) == turned_on_tokens
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_managed_relay_answers_and_settles_the_first_prompt_after_a_warmup_when_the_open_hold_fills_the_minute(
|
||||
self, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
from litellm.caching.dual_cache import DualCache
|
||||
from litellm.router_utils.pre_call_checks.io_token_rate_limit_check import async_admit_io_token_turn
|
||||
from litellm.router_utils.pre_call_checks.model_rate_limit_check import ModelRateLimitingCheck
|
||||
|
||||
dual_cache: Final = DualCache()
|
||||
await dual_cache.async_increment_cache(key=_OPENED_ON, value=4096, ttl=60)
|
||||
warmup: Final = json.dumps({"type": "response.create", "generate": False, "input": []})
|
||||
first, _, _ = _turn_frames()
|
||||
|
||||
websocket: Final = await _run_managed_relay(
|
||||
monkeypatch,
|
||||
_TwoOutputTokenTurns(),
|
||||
ModelRateLimitingCheck(dual_cache),
|
||||
(warmup, first),
|
||||
partial(async_admit_io_token_turn, dual_cache, _WS_DEPLOYMENT_AT_ITS_OTPM, now=_OPENED_AT),
|
||||
)
|
||||
|
||||
assert _errors(websocket) == []
|
||||
assert await dual_cache.async_get_cache(key=_OPENED_ON) == _OUTPUT_TOKENS_PER_TURN
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_native_relay_answers_and_settles_the_first_prompt_after_a_refused_first_frame_when_the_open_hold_fills_the_minute(
|
||||
self,
|
||||
) -> None:
|
||||
import websockets.exceptions # noqa: F401 (lazy submodule must be importable)
|
||||
|
||||
from litellm.caching.dual_cache import DualCache
|
||||
from litellm.responses.streaming_iterator import drain_logging_tasks
|
||||
from litellm.router_utils.pre_call_checks.io_token_rate_limit_check import async_admit_io_token_turn
|
||||
from litellm.router_utils.pre_call_checks.model_rate_limit_check import ModelRateLimitingCheck
|
||||
|
||||
dual_cache: Final = DualCache()
|
||||
await dual_cache.async_increment_cache(key=_OPENED_ON, value=4096, ttl=60)
|
||||
first, second, _ = _turn_frames()
|
||||
backend_ws: Final = MagicMock()
|
||||
backend_ws.send = AsyncMock()
|
||||
backend_ws.recv = AsyncMock(side_effect=[_turn_completed(), Exception("stop")])
|
||||
websocket: Final = _ClientSocket(second)
|
||||
handler: Final = _make_streaming(
|
||||
websocket=websocket,
|
||||
backend_ws=backend_ws,
|
||||
request_data={},
|
||||
logging_obj=_connection_settling_turns_through(ModelRateLimitingCheck(dual_cache)),
|
||||
first_message=first,
|
||||
frame_guard=_guard_blocking("one"),
|
||||
turn_admission=partial(async_admit_io_token_turn, dual_cache, _WS_DEPLOYMENT_AT_ITS_OTPM, now=_OPENED_AT),
|
||||
)
|
||||
|
||||
await handler.client_to_backend()
|
||||
await handler.backend_to_client()
|
||||
await drain_logging_tasks()
|
||||
|
||||
assert [(status, error["type"]) for status, error in _errors(websocket)] == [(None, "invalid_request_error")]
|
||||
assert await dual_cache.async_get_cache(key=_OPENED_ON) == _OUTPUT_TOKENS_PER_TURN
|
||||
assert await dual_cache.async_get_cache(key=_OPENED_ON) == _OTHER_USAGE + first_turn_tokens
|
||||
assert await dual_cache.async_get_cache(key=_TURNED_ON) == _OTHER_USAGE + 2 * _OUTPUT_TOKENS_PER_TURN
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_managed_relay_skips_a_later_turn_the_deployment_refuses(self, monkeypatch: pytest.MonkeyPatch):
|
||||
|
|
|
|||
|
|
@ -3,12 +3,14 @@ Tests for separate ITPM/OTPM deployment rate limits (enforce_model_rate_limits).
|
|||
"""
|
||||
|
||||
import asyncio
|
||||
from collections.abc import Mapping
|
||||
import threading
|
||||
from collections.abc import Iterator, Mapping
|
||||
from datetime import datetime, timezone
|
||||
from typing import Final
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
|
||||
import pytest
|
||||
from fakeredis import TcpFakeServer
|
||||
|
||||
import litellm
|
||||
from litellm import Router
|
||||
|
|
@ -36,6 +38,7 @@ from litellm.router_utils.pre_call_checks.io_token_rate_limit_check import (
|
|||
from litellm.router_utils.pre_call_checks.model_rate_limit_check import (
|
||||
ModelRateLimitingCheck,
|
||||
)
|
||||
from litellm.types.llms.openai import InputTokensDetails, ResponseAPIUsage, ResponsesAPIResponse
|
||||
from litellm.types.utils import ModelResponse, Usage
|
||||
|
||||
|
||||
|
|
@ -1146,15 +1149,6 @@ def _turn_key(kind: str, minute: str = "12-34") -> str:
|
|||
return f"global_router:ws-turn-{kind}:openai/gpt-5-mini:{kind}:{minute}"
|
||||
|
||||
|
||||
def _socket_hold(kind: str, held: int, cache_key: str) -> Mapping[str, object]:
|
||||
return {
|
||||
ITPM_RESERVED_KEY: held if kind == "itpm" else 0,
|
||||
OTPM_RESERVED_KEY: held if kind == "otpm" else 0,
|
||||
ITPM_CACHE_KEY: cache_key if kind == "itpm" else None,
|
||||
OTPM_CACHE_KEY: cache_key if kind == "otpm" else None,
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("kind", ["itpm", "otpm"])
|
||||
@pytest.mark.parametrize("used", [100, 101, 250])
|
||||
|
|
@ -1163,7 +1157,7 @@ async def test_turn_admission_refuses_once_the_minutes_tokens_reach_the_limit(ki
|
|||
await dual_cache.async_increment_cache(key=_turn_key(kind), value=used, ttl=60)
|
||||
|
||||
with pytest.raises(litellm.RateLimitError, match=f"{kind.upper()} limit=100, current usage={used}"):
|
||||
await async_admit_io_token_turn(dual_cache, _turn_deployment(kind), {}, _TURN_AT)
|
||||
await async_admit_io_token_turn(dual_cache, _turn_deployment(kind), _TURN_AT)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
@ -1174,7 +1168,7 @@ async def test_an_admitted_turn_reserves_nothing_and_counts_its_actual_tokens(ki
|
|||
await dual_cache.async_increment_cache(key=_turn_key(kind), value=used, ttl=60)
|
||||
actual: Final = _ACTUAL_TOKENS
|
||||
|
||||
reservation: Final = await async_admit_io_token_turn(dual_cache, _turn_deployment(kind), {}, _TURN_AT)
|
||||
reservation: Final = await async_admit_io_token_turn(dual_cache, _turn_deployment(kind), _TURN_AT)
|
||||
counter_key: Final = reservation[ITPM_CACHE_KEY if kind == "itpm" else OTPM_CACHE_KEY]
|
||||
assert isinstance(counter_key, str)
|
||||
assert await dual_cache.async_get_cache(key=counter_key) == used
|
||||
|
|
@ -1196,44 +1190,99 @@ async def test_an_admitted_turn_reserves_nothing_and_counts_its_actual_tokens(ki
|
|||
async def test_a_turn_without_io_token_counters_is_admitted_with_nothing_to_settle(
|
||||
deployment: dict[str, object],
|
||||
) -> None:
|
||||
assert await async_admit_io_token_turn(DualCache(), deployment, {}, _TURN_AT) == {}
|
||||
assert await async_admit_io_token_turn(DualCache(), deployment, _TURN_AT) == {}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("kind", ["itpm", "otpm"])
|
||||
@pytest.mark.parametrize("used_by_others", [0, 1, 99])
|
||||
async def test_a_sockets_first_turn_is_not_refused_by_its_own_opening_hold_and_settles_it(
|
||||
kind: str, used_by_others: int
|
||||
async def test_a_socket_open_reserves_nothing_and_its_first_turn_counts_its_actual_tokens_on_its_own_minute(
|
||||
kind: str, used_by_others: int, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
monkeypatch.setattr(
|
||||
"litellm.router_utils.pre_call_checks.io_token_rate_limit_check.get_utc_datetime", lambda: _TURN_AT
|
||||
)
|
||||
dual_cache: Final = DualCache()
|
||||
held: Final = _socket_hold(kind, 100, _turn_key(kind))
|
||||
await dual_cache.async_increment_cache(key=_turn_key(kind), value=used_by_others + 100, ttl=60)
|
||||
await dual_cache.async_increment_cache(key=_turn_key(kind), value=used_by_others, ttl=60)
|
||||
socket_open: Final[dict[str, object]] = {"websocket": object(), "metadata": {}}
|
||||
set_io_token_rate_limit_request_kwargs(socket_open)
|
||||
|
||||
admitted: Final = await async_admit_io_token_turn(dual_cache, _turn_deployment(kind), held, _TURN_AT)
|
||||
assert await async_io_token_pre_call_check(dual_cache, _turn_deployment(kind)) == _turn_deployment(kind)
|
||||
assert await dual_cache.async_get_cache(key=_turn_key(kind)) == used_by_others
|
||||
|
||||
opened_with: Final = socket_open["metadata"]
|
||||
assert isinstance(opened_with, dict)
|
||||
await async_io_token_reconcile_success(
|
||||
dual_cache,
|
||||
_proxy_success_kwargs({**held, **admitted}),
|
||||
_reply_using(_ACTUAL_TOKENS["itpm"], _ACTUAL_TOKENS["otpm"]),
|
||||
dual_cache, _proxy_success_kwargs(opened_with), _reply_using(_ACTUAL_TOKENS["itpm"], _ACTUAL_TOKENS["otpm"])
|
||||
)
|
||||
|
||||
assert await dual_cache.async_get_cache(key=_turn_key(kind)) == used_by_others + _ACTUAL_TOKENS[kind]
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def shared_redis() -> Iterator[RedisCache]:
|
||||
server: Final = TcpFakeServer(("127.0.0.1", 0), server_type="redis")
|
||||
threading.Thread(target=server.serve_forever, daemon=True).start()
|
||||
try:
|
||||
yield RedisCache(host="127.0.0.1", port=server.server_address[1])
|
||||
finally:
|
||||
server.shutdown()
|
||||
server.server_close()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("kind", ["itpm", "otpm"])
|
||||
@pytest.mark.parametrize(
|
||||
("held_on_minute", "counted_this_minute"),
|
||||
[pytest.param("12-34", 200, id="others_reach_the_limit"), pytest.param("12-33", 100, id="hold_from_last_minute")],
|
||||
)
|
||||
async def test_a_sockets_first_turn_is_refused_once_other_usage_reaches_the_limit(
|
||||
kind: str, held_on_minute: str, counted_this_minute: int
|
||||
async def test_a_turn_is_admitted_once_the_shared_counter_drops_below_the_limit_this_instance_last_read(
|
||||
kind: str, shared_redis: RedisCache
|
||||
) -> None:
|
||||
dual_cache: Final = DualCache()
|
||||
await dual_cache.async_increment_cache(key=_turn_key(kind), value=counted_this_minute, ttl=60)
|
||||
held: Final = _socket_hold(kind, 100, _turn_key(kind, held_on_minute))
|
||||
dual_cache: Final = DualCache(in_memory_cache=InMemoryCache(), redis_cache=shared_redis)
|
||||
await dual_cache.in_memory_cache.async_increment(key=_turn_key(kind), value=100)
|
||||
await shared_redis.async_increment(key=_turn_key(kind), value=99)
|
||||
|
||||
with pytest.raises(litellm.RateLimitError, match=f"{kind.upper()} limit=100, current usage=100"):
|
||||
await async_admit_io_token_turn(dual_cache, _turn_deployment(kind), held, _TURN_AT)
|
||||
admitted: Final = await async_admit_io_token_turn(dual_cache, _turn_deployment(kind), _TURN_AT)
|
||||
|
||||
assert admitted[ITPM_CACHE_KEY if kind == "itpm" else OTPM_CACHE_KEY] == _turn_key(kind)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"responses_reply",
|
||||
[
|
||||
pytest.param(
|
||||
ResponsesAPIResponse(
|
||||
id="resp_cached",
|
||||
created_at=0,
|
||||
output=[],
|
||||
usage=ResponseAPIUsage(
|
||||
input_tokens=50,
|
||||
output_tokens=5,
|
||||
total_tokens=55,
|
||||
input_tokens_details=InputTokensDetails(cached_tokens=30),
|
||||
),
|
||||
),
|
||||
id="usage_object",
|
||||
),
|
||||
pytest.param(
|
||||
{
|
||||
"usage": {
|
||||
"input_tokens": 50,
|
||||
"output_tokens": 5,
|
||||
"total_tokens": 55,
|
||||
"input_tokens_details": {"cached_tokens": 30},
|
||||
}
|
||||
},
|
||||
id="usage_dict",
|
||||
),
|
||||
],
|
||||
)
|
||||
async def test_a_turn_counts_only_its_uncached_responses_input_toward_itpm(responses_reply: object) -> None:
|
||||
dual_cache: Final = DualCache()
|
||||
await dual_cache.async_increment_cache(key=_turn_key("itpm"), value=10, ttl=60)
|
||||
|
||||
reservation: Final = await async_admit_io_token_turn(dual_cache, _turn_deployment("itpm"), _TURN_AT)
|
||||
await async_io_token_reconcile_success(dual_cache, _proxy_success_kwargs(reservation), responses_reply)
|
||||
|
||||
assert await dual_cache.async_get_cache(key=_turn_key("itpm")) == 10 + 50 - 30
|
||||
|
||||
|
||||
def _shared_counter_answering(answer: int | Exception) -> DualCache:
|
||||
|
|
@ -1246,7 +1295,7 @@ def _shared_counter_answering(answer: int | Exception) -> DualCache:
|
|||
@pytest.mark.parametrize("kind", ["itpm", "otpm"])
|
||||
async def test_a_later_turn_is_refused_when_another_instance_filled_the_shared_minute(kind: str) -> None:
|
||||
with pytest.raises(litellm.RateLimitError, match=f"{kind.upper()} limit=100, current usage=100"):
|
||||
await async_admit_io_token_turn(_shared_counter_answering(100), _turn_deployment(kind), {}, _TURN_AT)
|
||||
await async_admit_io_token_turn(_shared_counter_answering(100), _turn_deployment(kind), _TURN_AT)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
@ -1257,6 +1306,6 @@ async def test_a_later_turn_is_admitted_on_this_instance_s_count_when_the_shared
|
|||
dual_cache: Final = _shared_counter_answering(RedisCircuitBreakerOpenError("circuit open"))
|
||||
await dual_cache.in_memory_cache.async_increment(key=_turn_key(kind), value=99)
|
||||
|
||||
admitted: Final = await async_admit_io_token_turn(dual_cache, _turn_deployment(kind), {}, _TURN_AT)
|
||||
admitted: Final = await async_admit_io_token_turn(dual_cache, _turn_deployment(kind), _TURN_AT)
|
||||
|
||||
assert admitted[ITPM_CACHE_KEY if kind == "itpm" else OTPM_CACHE_KEY] == _turn_key(kind)
|
||||
|
|
|
|||
|
|
@ -538,14 +538,14 @@ async def test_deployment_callback_on_success_counts_a_responses_websocket_turn(
|
|||
assert await router.cache.async_get_cache(key=tpm_key) == 17
|
||||
|
||||
|
||||
async def _kwargs_routed_to(router: Router, model: str, function_name: str) -> dict[str, object]:
|
||||
async def _kwargs_routed_to(router: Router, model: str, function_name: str, **request: object) -> dict[str, object]:
|
||||
routed: Final[dict[str, object]] = {}
|
||||
|
||||
async def call(**kwargs: object) -> None:
|
||||
routed.update(kwargs)
|
||||
|
||||
call.__name__ = function_name
|
||||
await router._ageneric_api_call_with_fallbacks_helper(model=model, original_generic_function=call)
|
||||
await router._ageneric_api_call_with_fallbacks_helper(model=model, original_generic_function=call, **request)
|
||||
return routed
|
||||
|
||||
|
||||
|
|
@ -733,13 +733,75 @@ def _io_limited_router(kind: str, **litellm_params: object) -> Router:
|
|||
)
|
||||
|
||||
|
||||
async def _count_minutes_tokens(router: Router, kind: str, tokens: int, minute: datetime) -> None:
|
||||
await router.cache.async_increment_cache(
|
||||
key=f"global_router:ws-io-dep:openai/ws-io-unpriced:{kind}:{minute.strftime('%H-%M')}",
|
||||
value=tokens,
|
||||
ttl=120,
|
||||
def _minutes_counter_key(kind: str, minute: datetime, model: str = "openai/ws-io-unpriced") -> str:
|
||||
return f"global_router:ws-io-dep:{model}:{kind}:{minute.strftime('%H-%M')}"
|
||||
|
||||
|
||||
async def _count_minutes_tokens(
|
||||
router: Router, kind: str, tokens: int, minute: datetime, model: str = "openai/ws-io-unpriced"
|
||||
) -> None:
|
||||
await router.cache.async_increment_cache(key=_minutes_counter_key(kind, minute, model), value=tokens, ttl=120)
|
||||
|
||||
|
||||
_LONG_OUTPUT_MODEL: Final = "openai/gpt-5.4-mini"
|
||||
|
||||
|
||||
def _long_output_router() -> tuple[Router, Mapping[str, int]]:
|
||||
max_output: Final = litellm.get_model_info(_LONG_OUTPUT_MODEL)["max_output_tokens"]
|
||||
assert isinstance(max_output, int)
|
||||
limits: Final = {"itpm": 10_000, "otpm": max_output // 2}
|
||||
return _io_limited_router("itpm", model=_LONG_OUTPUT_MODEL, otpm=limits["otpm"]), limits
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("kind", ["itpm", "otpm"])
|
||||
async def test_responses_websocket_opens_without_counting_tokens_until_the_minute_reaches_deployment_io_limit(
|
||||
kind: str, router_minute_pinned: datetime
|
||||
):
|
||||
router, limits = _long_output_router()
|
||||
|
||||
await _kwargs_routed_to(router, "ws-io", "_aresponses_websocket", websocket=object())
|
||||
|
||||
assert [
|
||||
await router.cache.async_get_cache(key=_minutes_counter_key(counted, router_minute_pinned, _LONG_OUTPUT_MODEL))
|
||||
for counted in ("itpm", "otpm")
|
||||
] == [None, None]
|
||||
await _count_minutes_tokens(router, kind, limits[kind], router_minute_pinned, _LONG_OUTPUT_MODEL)
|
||||
with pytest.raises(litellm.RateLimitError, match=f"{kind.upper()} limit={limits[kind]}, current usage="):
|
||||
await _kwargs_routed_to(router, "ws-io", "_aresponses_websocket", websocket=object())
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_responses_websocket_first_turn_counts_its_actual_tokens_on_the_minute_it_opened(
|
||||
router_minute_pinned: datetime,
|
||||
):
|
||||
from litellm.cost_calculator import ResponsesWebSocketTokenUsageProcessor
|
||||
from litellm.litellm_core_utils.get_litellm_params import get_litellm_params
|
||||
from litellm.router_utils.pre_call_checks.model_rate_limit_check import ModelRateLimitingCheck
|
||||
|
||||
router, _ = _long_output_router()
|
||||
socket_metadata: Final = (await _kwargs_routed_to(router, "ws-io", "_aresponses_websocket", websocket=object()))[
|
||||
"litellm_metadata"
|
||||
]
|
||||
assert isinstance(socket_metadata, dict)
|
||||
rate_limits: Final = next(
|
||||
check for check in router.optional_callbacks or () if isinstance(check, ModelRateLimitingCheck)
|
||||
)
|
||||
|
||||
await rate_limits.async_log_success_event(
|
||||
{"litellm_params": get_litellm_params(litellm_metadata=socket_metadata)},
|
||||
ResponsesWebSocketTokenUsageProcessor.create_logging_object(
|
||||
[{"type": "response.completed", "response": {"usage": {"input_tokens": 20, "output_tokens": 15}}}]
|
||||
),
|
||||
None,
|
||||
None,
|
||||
)
|
||||
|
||||
assert [
|
||||
await router.cache.async_get_cache(key=_minutes_counter_key(counted, router_minute_pinned, _LONG_OUTPUT_MODEL))
|
||||
for counted in ("itpm", "otpm")
|
||||
] == [20, 15]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("kind", ["itpm", "otpm"])
|
||||
|
|
@ -1257,6 +1319,9 @@ def _rpm_tpm_router(model_id: str) -> Router:
|
|||
def router_minute_pinned(monkeypatch: pytest.MonkeyPatch) -> datetime:
|
||||
pinned: Final = datetime(2026, 1, 1, 12, 0, 30, tzinfo=timezone.utc)
|
||||
monkeypatch.setattr("litellm.router.get_utc_datetime", lambda: pinned)
|
||||
monkeypatch.setattr(
|
||||
"litellm.router_utils.pre_call_checks.io_token_rate_limit_check.get_utc_datetime", lambda: pinned
|
||||
)
|
||||
return pinned
|
||||
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue