fix(router): keep io token reservation keys out of the metadata sent to providers

The itpm/otpm pre-call check wrote its reservation keys into both metadata and litellm_metadata. On the routes the router serves through its generic helper (Responses, Anthropic Messages, and the rest), metadata is the client's own field and goes to the provider as is. So _litellm_itpm_cache_key and _litellm_otpm_cache_key, which carry the deployment id and the model name, reached the provider, came back in the echoed response metadata, and counted toward the provider's 16-key cap. The managed Responses socket relay merged each turn's reservation into the frame's metadata the same way, and a native socket opened with metadata sent the keys on every turn through its request defaults

The router now passes the limiter the metadata field it uses for its own fields on that call (metadata for chat, litellm_metadata for the generic routes), and the reservation goes only there. metadata_channel is a required keyword, so a new caller has to name the field it writes. The managed relay puts each turn's reservation only on litellm_metadata

With the reservation only on litellm_metadata, a retried call on the generic routes refunded its first attempt twice and pushed the deployment's counters below zero. The failure hook read the reservation from the logging copy in litellm_params.metadata and cleared only that copy, so the router's litellm_metadata still held it and the retry refunded it again. The hooks now also read and clear litellm_params.litellm_metadata, which is the router's own dict

The proxy does not strip client keys that start with _litellm_ on the way in, so dropping that prefix at the provider boundary would also drop keys clients send today

The io token tests now run Redis in process with fakeredis instead of a TCP fake server
This commit is contained in:
mateo-berri 2026-10-29 19:19:55 -07:00
parent 7d72b09652
commit e1cbafce24
6 changed files with 378 additions and 75 deletions

View file

@ -3592,13 +3592,11 @@ class ManagedResponsesWebSocketHandler:
)
self._update_proxy_request(call_kwargs, requested_model or self.model_group or model)
call_kwargs.update(self.extra_kwargs)
call_kwargs.update(
{
channel: {**bucket, **verdict.reservation}
for channel, bucket in call_kwargs.items()
if channel in _PER_TURN_METADATA_KEYS and _is_json_object(bucket)
}
)
turn_litellm_metadata: Final = call_kwargs.get("litellm_metadata")
call_kwargs["litellm_metadata"] = {
**(turn_litellm_metadata if _is_json_object(turn_litellm_metadata) else {}),
**verdict.reservation,
}
try:
terminal_event: Final = await self._stream_and_forward(model, call_kwargs)

View file

@ -5205,6 +5205,7 @@ class Router(DeprecatedRouterMembers):
kwargs,
store_in_context=deployment_has_io_token_limits(deployment),
opens_responses_socket=opens_responses_socket,
metadata_channel=metadata_variable_name,
)
kwargs[metadata_variable_name].setdefault(

View file

@ -53,6 +53,7 @@ RoutingArgsTTL: Final = 60
class _IoTokenRateLimitRequest:
kwargs: dict[str, object] | None
opens_responses_socket: bool
metadata_channel: str
_io_token_rate_limit_request: Final[contextvars.ContextVar[_IoTokenRateLimitRequest | None]] = contextvars.ContextVar(
@ -62,7 +63,11 @@ _io_token_rate_limit_request: Final[contextvars.ContextVar[_IoTokenRateLimitRequ
def set_io_token_rate_limit_request_kwargs(
kwargs: dict[str, object] | None, store_in_context: bool = True, *, opens_responses_socket: bool = False
kwargs: dict[str, object] | None,
store_in_context: bool = True,
*,
opens_responses_socket: bool = False,
metadata_channel: str,
) -> None:
# The reservation sentinels are server-only, but `metadata` is caller
# controlled on proxy requests. Strip any client-supplied copies here (this
@ -78,7 +83,7 @@ def set_io_token_rate_limit_request_kwargs(
# kwargs; overwriting (rather than skipping) also releases a previous
# request's kwargs when a context is reused.
_io_token_rate_limit_request.set(
_IoTokenRateLimitRequest(kwargs, opens_responses_socket) if store_in_context else None
_IoTokenRateLimitRequest(kwargs, opens_responses_socket, metadata_channel) if store_in_context else None
)
@ -295,15 +300,10 @@ def _reservation(
)
def _stash_reservation_in_metadata(request_kwargs: dict[str, object] | None, reservation: Mapping[str, object]) -> None:
if not request_kwargs:
return
for channel in ("metadata", "litellm_metadata"):
existing = request_kwargs.get(channel)
if isinstance(existing, dict):
existing.update(reservation)
elif channel == "metadata":
request_kwargs[channel] = dict(reservation)
def _stash_reservation_in_metadata(request: _IoTokenRateLimitRequest | None, reservation: Mapping[str, object]) -> None:
metadata: Final = None if request is None or not request.kwargs else request.kwargs.get(request.metadata_channel)
if isinstance(metadata, dict):
metadata.update(reservation)
def _extract_reservation(reservation: Mapping[str, int | str | None]) -> tuple[int, int, str | None, str | None]:
@ -332,7 +332,9 @@ def _reservation_channels(kwargs: Mapping[str, object] | None) -> tuple[object,
return ()
top_level: Final = (kwargs.get("metadata"), kwargs.get("litellm_metadata"))
litellm_params: Final = _as_mutable_mapping(kwargs.get("litellm_params"))
from_params: Final = () if litellm_params is None else (litellm_params.get("metadata"),)
from_params: Final = (
() if litellm_params is None else (litellm_params.get("metadata"), litellm_params.get("litellm_metadata"))
)
standard_logging_object: Final = _as_mutable_mapping(kwargs.get("standard_logging_object"))
from_logging_object: Final = () if standard_logging_object is None else (standard_logging_object.get("metadata"),)
return top_level + from_params + from_logging_object
@ -453,7 +455,8 @@ def io_token_pre_call_check(
if itpm_limit is None and otpm_limit is None:
return deployment
request_kwargs: Final = get_io_token_rate_limit_request_kwargs()
request: Final = _io_token_rate_limit_request.get()
request_kwargs: Final = None if request is None else request.kwargs
_model: Final = _tokenizer_model(deployment)
estimated_input: Final = _estimate_input_tokens(request_kwargs, model=_model)
max_tokens: Final = _resolve_max_tokens(request_kwargs, deployment)
@ -498,7 +501,7 @@ def io_token_pre_call_check(
raise
_stash_reservation_in_metadata(
request_kwargs,
request,
_reservation(
itpm_reserved,
otpm_reserved,
@ -523,7 +526,7 @@ async def async_io_token_pre_call_check(
request_kwargs: Final = None if request is None else request.kwargs
if request is not None and request.opens_responses_socket:
_stash_reservation_in_metadata(
request_kwargs,
request,
await async_admit_io_token_turn(dual_cache, deployment, get_utc_datetime(), parent_otel_span),
)
return deployment
@ -577,7 +580,7 @@ async def async_io_token_pre_call_check(
raise
_stash_reservation_in_metadata(
request_kwargs,
request,
_reservation(
itpm_reserved,
otpm_reserved,

View file

@ -16,7 +16,9 @@ from functools import partial
from typing import TYPE_CHECKING, Final
from unittest.mock import AsyncMock, MagicMock
import httpx
import pytest
import respx
from litellm.llms.azure.responses.transformation import AzureOpenAIResponsesAPIConfig
from litellm.llms.chatgpt.responses.transformation import ChatGPTResponsesAPIConfig
@ -1799,6 +1801,56 @@ def _connection_logged_through(rate_limits: "ModelRateLimitingCheck", opened_wit
return connection
def _provider_turn_stream(_request: httpx.Request) -> httpx.Response:
completed: Final = {
"id": "resp_turn",
"object": "response",
"created_at": 1767271505,
"status": "completed",
"model": "gpt-5-mini",
"output": [],
"parallel_tool_calls": True,
"tool_choice": "auto",
"tools": [],
"usage": {"input_tokens": 5, "output_tokens": _OUTPUT_TOKENS_PER_TURN, "total_tokens": 7},
}
return httpx.Response(
200,
headers={"content-type": "text/event-stream"},
text="".join(
f"event: {kind}\ndata: {json.dumps({'type': kind, 'response': completed})}\n\n"
for kind in ("response.created", "response.completed")
),
)
class _ProviderResponsesSocket:
"""A provider's native Responses socket that answers each ``response.create`` with a completed turn."""
def __init__(self) -> None:
self.sent: Final[list[dict[str, object]]] = []
self._replies: Final[asyncio.Queue[str]] = asyncio.Queue()
def connect(self, _url: str, **_kwargs: object) -> "_ProviderResponsesSocket":
return self
async def __aenter__(self) -> "_ProviderResponsesSocket":
return self
async def __aexit__(self, *_exc_info: object) -> None:
return None
async def send(self, message: str) -> None:
self.sent.append(json.loads(message))
await self._replies.put(_turn_completed())
async def recv(self, decode: bool = False) -> str:
return await self._replies.get()
async def close(self) -> None:
return None
class _TwoOutputTokenTurns(CustomLLM):
def __init__(self, failing_turns: frozenset[int] = frozenset()) -> None:
super().__init__()
@ -1827,19 +1879,33 @@ async def _run_managed_relay(
opened_with: Mapping[str, object],
) -> _ClientSocket:
import litellm
monkeypatch.setattr(litellm, "custom_provider_map", [{"provider": "ws-turn-llm", "custom_handler": provider}])
return await _relay_turns_to(monkeypatch, "ws-turn-llm/turns", rate_limits, frames, turn_admission, opened_with)
async def _relay_turns_to(
monkeypatch: pytest.MonkeyPatch,
model: str,
rate_limits: "ModelRateLimitingCheck",
frames: tuple[str, ...],
turn_admission: "ResponsesWsTurnAdmission | None",
opened_with: Mapping[str, object],
**connection: object,
) -> _ClientSocket:
import litellm
from litellm.litellm_core_utils.litellm_logging import Logging
from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
from litellm.responses.streaming_iterator import ManagedResponsesWebSocketHandler, drain_logging_tasks
monkeypatch.setattr(litellm, "custom_provider_map", [{"provider": "ws-turn-llm", "custom_handler": provider}])
monkeypatch.setattr(litellm, "callbacks", [rate_limits])
first, *later = frames
websocket: Final = _ClientSocket(*later)
handler: Final = ManagedResponsesWebSocketHandler(
websocket=websocket,
model="ws-turn-llm/turns",
model=model,
logging_obj=Logging(
model="ws-turn-llm/turns",
model=model,
messages=[],
stream=True,
call_type="aresponses",
@ -1850,7 +1916,7 @@ async def _run_managed_relay(
litellm_metadata={"user_api_key_hash": "k1", **opened_with},
first_message=first,
turn_admission=turn_admission,
metadata=dict(opened_with),
**connection,
)
await handler.run()
@ -2044,6 +2110,78 @@ class TestWebSocketTurnAdmission:
== _OTHER_USAGE + first_turn_tokens + 2 * _OUTPUT_TOKENS_PER_TURN
)
@pytest.mark.asyncio
async def test_managed_relay_sends_the_provider_only_the_metadata_each_turn_carries(
self, monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter
) -> None:
import litellm
from litellm.caching.dual_cache import DualCache
from litellm.router_utils.pre_call_checks.model_rate_limit_check import ModelRateLimitingCheck
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
litellm.in_memory_llm_clients_cache.flush_cache()
_turns_finish_at(monkeypatch, _TURNED_AT)
provider: Final = respx_mock.post("https://api.openai.com/v1/responses").mock(side_effect=_provider_turn_stream)
client_metadata: Final = {"session": "abc", "team": "search"}
dual_cache: Final = DualCache()
await _relay_turns_to(
monkeypatch,
"openai/gpt-5-mini",
ModelRateLimitingCheck(dual_cache),
tuple(
json.dumps({"type": "response.create", "input": word, "metadata": client_metadata})
for word in ("one", "two")
),
_later_turns_admitted_through(dual_cache),
await _socket_opened_on(dual_cache),
api_key="sk-fake",
)
assert [json.loads(call.request.read()).get("metadata") for call in provider.calls] == [
client_metadata,
client_metadata,
]
assert await dual_cache.async_get_cache(key=_TURNED_ON) == _OTHER_USAGE + 2 * _OUTPUT_TOKENS_PER_TURN
@pytest.mark.asyncio
async def test_native_relay_sends_the_provider_only_the_metadata_the_socket_opened_with(
self, monkeypatch: pytest.MonkeyPatch
) -> None:
from unittest.mock import patch
from litellm import Router
monkeypatch.setenv("LITELLM_RUST", "false")
provider: Final = _ProviderResponsesSocket()
client_metadata: Final = {"session": "abc", "team": "search"}
first, second, _ = _turn_frames()
router: Final = Router(
model_list=[
{
"model_name": "ws-io",
"litellm_params": {
"model": "openai/gpt-5-mini",
"api_key": "sk-fake",
"itpm": 40_000,
"otpm": 4096,
},
"model_info": {"id": "ws-dep"},
}
],
optional_pre_call_checks=["enforce_model_rate_limits"],
)
with patch("websockets.connect", provider.connect):
await router._aresponses_websocket(
model="ws-io",
websocket=_ClientSocket(second),
first_message=first,
metadata=dict(client_metadata),
)
assert [frame.get("metadata") for frame in provider.sent] == [client_metadata, client_metadata]
@pytest.mark.asyncio
async def test_managed_relay_skips_a_later_turn_the_deployment_refuses(self, monkeypatch: pytest.MonkeyPatch):
from unittest.mock import AsyncMock

View file

@ -3,14 +3,14 @@ Tests for separate ITPM/OTPM deployment rate limits (enforce_model_rate_limits).
"""
import asyncio
import threading
from collections.abc import Iterator, Mapping
from collections.abc import Mapping
from datetime import datetime, timezone
from typing import Final
from unittest.mock import AsyncMock, MagicMock
import pytest
from fakeredis import TcpFakeServer
from fakeredis import FakeRedis, FakeServer
from fakeredis.aioredis import FakeRedis as FakeAsyncRedis
import litellm
from litellm import Router
@ -100,7 +100,7 @@ class TestModelRateLimitingCheckIOTokens:
"max_tokens": 10,
"metadata": {},
}
set_io_token_rate_limit_request_kwargs(request_kwargs)
set_io_token_rate_limit_request_kwargs(request_kwargs, metadata_channel="metadata")
await check.async_pre_call_check(deployment)
minute = get_utc_datetime().strftime("%H-%M")
@ -158,7 +158,8 @@ class TestModelRateLimitingCheckIOTokens:
],
"max_tokens": 10,
"metadata": {},
}
},
metadata_channel="metadata",
)
with pytest.raises(litellm.RateLimitError) as exc_info:
@ -188,7 +189,8 @@ class TestModelRateLimitingCheckIOTokens:
"messages": [{"role": "user", "content": "hi"}],
"max_tokens": max_tokens,
"metadata": {},
}
},
metadata_channel="metadata",
)
async def _attempt():
@ -240,7 +242,8 @@ class TestModelRateLimitingCheckIOTokens:
{
"max_tokens": 5,
"metadata": {},
}
},
metadata_channel="metadata",
)
async def _attempt():
@ -305,7 +308,7 @@ class TestModelRateLimitingCheckIOTokens:
}
request_kwargs = {"max_tokens": 5, "metadata": {}}
set_io_token_rate_limit_request_kwargs(request_kwargs)
set_io_token_rate_limit_request_kwargs(request_kwargs, metadata_channel="metadata")
await check.async_pre_call_check(deployment)
minute = get_utc_datetime().strftime("%H-%M")
@ -348,15 +351,15 @@ class TestModelRateLimitingCheckIOTokens:
"model_name": "opus",
}
request_kwargs = {"max_tokens": 5, "metadata": {}}
set_io_token_rate_limit_request_kwargs(request_kwargs)
set_io_token_rate_limit_request_kwargs(request_kwargs, metadata_channel="metadata")
await check.async_pre_call_check(deployment)
# Second zero-estimate call still fits within the itpm=2 limit.
set_io_token_rate_limit_request_kwargs({"max_tokens": 5, "metadata": {}})
set_io_token_rate_limit_request_kwargs({"max_tokens": 5, "metadata": {}}, metadata_channel="metadata")
await check.async_pre_call_check(deployment)
# A third exceeds the limit and is rejected.
set_io_token_rate_limit_request_kwargs({"max_tokens": 5, "metadata": {}})
set_io_token_rate_limit_request_kwargs({"max_tokens": 5, "metadata": {}}, metadata_channel="metadata")
with pytest.raises(litellm.RateLimitError):
await check.async_pre_call_check(deployment)
@ -377,7 +380,7 @@ class TestModelRateLimitingCheckIOTokens:
"max_tokens": 0,
"metadata": {},
}
set_io_token_rate_limit_request_kwargs(zero_output_kwargs)
set_io_token_rate_limit_request_kwargs(zero_output_kwargs, metadata_channel="metadata")
await check.async_pre_call_check(deployment)
zero_output_otpm_key = zero_output_kwargs["metadata"][OTPM_CACHE_KEY]
@ -389,7 +392,7 @@ class TestModelRateLimitingCheckIOTokens:
"max_tokens": 5,
"metadata": {},
}
set_io_token_rate_limit_request_kwargs(normal_output_kwargs)
set_io_token_rate_limit_request_kwargs(normal_output_kwargs, metadata_channel="metadata")
await check.async_pre_call_check(deployment)
normal_output_otpm_key = normal_output_kwargs["metadata"][OTPM_CACHE_KEY]
@ -414,7 +417,7 @@ class TestModelRateLimitingCheckIOTokens:
"max_tokens": 10,
"metadata": {},
}
set_io_token_rate_limit_request_kwargs(request_kwargs)
set_io_token_rate_limit_request_kwargs(request_kwargs, metadata_channel="metadata")
check.pre_call_check(deployment)
minute = get_utc_datetime().strftime("%H-%M")
@ -550,7 +553,7 @@ class TestModelRateLimitingCheckIOTokens:
# The retry proceeds to stash deployment B's own reservation on the
# same dict; it starts clean, unaffected by A's cleared sentinels.
set_io_token_rate_limit_request_kwargs(shared_kwargs)
set_io_token_rate_limit_request_kwargs(shared_kwargs, metadata_channel="metadata")
itpm_key_b = f"global_router:io-retry-b:bedrock_mantle/test-b:itpm:{minute}"
shared_kwargs["metadata"][ITPM_RESERVED_KEY] = 4
shared_kwargs["metadata"][ITPM_CACHE_KEY] = itpm_key_b
@ -566,7 +569,7 @@ class TestModelRateLimitingCheckIOTokens:
"litellm_metadata": {OTPM_RESERVED_KEY: 7},
"litellm_params": {"metadata": {OTPM_CACHE_KEY: "attacker:key:otpm:00-00"}},
}
set_io_token_rate_limit_request_kwargs(forged)
set_io_token_rate_limit_request_kwargs(forged, metadata_channel="metadata")
stored = get_io_token_rate_limit_request_kwargs()
assert ITPM_RESERVED_KEY not in stored["metadata"]
@ -592,7 +595,7 @@ class TestModelRateLimitingCheckIOTokens:
},
}
# The router sanitizes the request kwargs before the call runs.
set_io_token_rate_limit_request_kwargs(kwargs)
set_io_token_rate_limit_request_kwargs(kwargs, metadata_channel="metadata")
response = ModelResponse(
choices=[{"message": {"role": "assistant", "content": "ok"}, "index": 0, "finish_reason": "stop"}],
usage=Usage(prompt_tokens=1, completion_tokens=1, total_tokens=2),
@ -627,7 +630,8 @@ class TestModelRateLimitingCheckIOTokens:
"messages": [{"role": "user", "content": "hello world"}],
"max_tokens": 5,
"metadata": {},
}
},
metadata_channel="metadata",
)
with pytest.raises(RuntimeError):
@ -692,7 +696,7 @@ class TestModelRateLimitingCheckIOTokens:
"max_tokens": 5,
"metadata": {},
}
set_io_token_rate_limit_request_kwargs(request_kwargs)
set_io_token_rate_limit_request_kwargs(request_kwargs, metadata_channel="metadata")
result = await async_io_token_pre_call_check(dual_cache, deployment)
@ -863,7 +867,7 @@ class TestModelRateLimitingCheckIOTokens:
"max_tokens": 5,
"metadata": {},
}
set_io_token_rate_limit_request_kwargs(request_kwargs)
set_io_token_rate_limit_request_kwargs(request_kwargs, metadata_channel="metadata")
with caplog.at_level(logging.WARNING, logger="LiteLLM Router"):
with pytest.raises(litellm.RateLimitError):
@ -1004,15 +1008,15 @@ class TestContextSlotRetention:
"messages": [{"role": "user", "content": "x" * 1000}],
"metadata": {ITPM_RESERVED_KEY: 999, ITPM_CACHE_KEY: "forged"},
}
set_io_token_rate_limit_request_kwargs(kwargs, store_in_context=False)
set_io_token_rate_limit_request_kwargs(kwargs, store_in_context=False, metadata_channel="metadata")
assert get_io_token_rate_limit_request_kwargs() is None
assert ITPM_RESERVED_KEY not in kwargs["metadata"]
assert ITPM_CACHE_KEY not in kwargs["metadata"]
set_io_token_rate_limit_request_kwargs(kwargs, store_in_context=True)
set_io_token_rate_limit_request_kwargs(kwargs, store_in_context=True, metadata_channel="metadata")
assert get_io_token_rate_limit_request_kwargs() is kwargs
set_io_token_rate_limit_request_kwargs(kwargs, store_in_context=False)
set_io_token_rate_limit_request_kwargs(kwargs, store_in_context=False, metadata_channel="metadata")
assert get_io_token_rate_limit_request_kwargs() is None
@pytest.mark.asyncio
@ -1025,7 +1029,7 @@ class TestContextSlotRetention:
}
]
)
set_io_token_rate_limit_request_kwargs(None)
set_io_token_rate_limit_request_kwargs(None, metadata_channel="metadata")
kwargs = {"messages": [{"role": "user", "content": "hello"}], "metadata": {}}
deployment = router.get_deployment_by_model_group_name("plain")
assert deployment is not None
@ -1043,7 +1047,7 @@ class TestContextSlotRetention:
],
optional_pre_call_checks=["enforce_model_rate_limits"],
)
set_io_token_rate_limit_request_kwargs(None)
set_io_token_rate_limit_request_kwargs(None, metadata_channel="metadata")
kwargs = {"messages": [{"role": "user", "content": "hello"}], "metadata": {}}
deployment = router.get_deployment_by_model_group_name("limited")
assert deployment is not None
@ -1060,7 +1064,7 @@ async def test_custom_azure_deployment_name_reserves_itpm_on_its_base_models_tok
"max_tokens": 5,
"metadata": metadata,
}
set_io_token_rate_limit_request_kwargs(request_kwargs)
set_io_token_rate_limit_request_kwargs(request_kwargs, metadata_channel="metadata")
await async_io_token_pre_call_check(DualCache(), deployment)
return metadata[ITPM_RESERVED_KEY]
@ -1211,7 +1215,7 @@ async def test_a_socket_open_reserves_nothing_and_its_first_turn_counts_its_actu
dual_cache: Final = DualCache()
await dual_cache.async_increment_cache(key=_turn_key(kind), value=used_by_others, ttl=60)
socket_open: Final[dict[str, object]] = {"metadata": {}}
set_io_token_rate_limit_request_kwargs(socket_open, opens_responses_socket=True)
set_io_token_rate_limit_request_kwargs(socket_open, opens_responses_socket=True, metadata_channel="metadata")
assert await async_io_token_pre_call_check(dual_cache, _turn_deployment(kind)) == _turn_deployment(kind)
assert await dual_cache.async_get_cache(key=_turn_key(kind)) == used_by_others
@ -1225,30 +1229,23 @@ async def test_a_socket_open_reserves_nothing_and_its_first_turn_counts_its_actu
assert await dual_cache.async_get_cache(key=_turn_key(kind)) == used_by_others + _ACTUAL_TOKENS[kind]
@pytest.fixture
def shared_redis() -> Iterator[RedisCache]:
server: Final = TcpFakeServer(("127.0.0.1", 0), server_type="redis")
threading.Thread(target=server.serve_forever, daemon=True).start()
try:
yield RedisCache(host="127.0.0.1", port=server.server_address[1])
finally:
server.shutdown()
server.server_close()
class _RedisCacheOnFakeServer(RedisCache):
def __init__(self, server: FakeServer) -> None:
super().__init__(host="io-token-turn-redis", connection_pool=FakeRedis(server=server).connection_pool)
self._fake_async_client: Final = FakeAsyncRedis(server=server)
@pytest.fixture
def unreachable_redis() -> RedisCache:
server: Final = TcpFakeServer(("127.0.0.1", 0), server_type="redis")
server.server_close()
return RedisCache(host="127.0.0.1", port=server.server_address[1])
def init_async_client(self) -> FakeAsyncRedis:
return self._fake_async_client
@pytest.mark.asyncio
@pytest.mark.parametrize("kind", ["itpm", "otpm"])
async def test_a_turn_is_refused_on_this_instance_s_count_when_reading_the_shared_counter_fails(
kind: str, unreachable_redis: RedisCache
kind: str,
) -> None:
dual_cache: Final = DualCache(in_memory_cache=InMemoryCache(), redis_cache=unreachable_redis)
server: Final = FakeServer()
dual_cache: Final = DualCache(in_memory_cache=InMemoryCache(), redis_cache=_RedisCacheOnFakeServer(server))
server.connected = False
await dual_cache.in_memory_cache.async_increment(key=_turn_key(kind), value=100)
with pytest.raises(litellm.RateLimitError, match=f"{kind.upper()} limit=100, current usage=100"):
@ -1284,11 +1281,12 @@ async def test_a_turn_finishing_in_the_next_minute_counts_its_actual_tokens_on_t
@pytest.mark.asyncio
@pytest.mark.parametrize("kind", ["itpm", "otpm"])
async def test_a_turn_is_admitted_once_the_shared_counter_drops_below_the_limit_this_instance_last_read(
kind: str, shared_redis: RedisCache
kind: str,
) -> None:
dual_cache: Final = DualCache(in_memory_cache=InMemoryCache(), redis_cache=shared_redis)
redis_cache: Final = _RedisCacheOnFakeServer(FakeServer())
dual_cache: Final = DualCache(in_memory_cache=InMemoryCache(), redis_cache=redis_cache)
await dual_cache.in_memory_cache.async_increment(key=_turn_key(kind), value=100)
await shared_redis.async_increment(key=_turn_key(kind), value=99)
await redis_cache.async_increment(key=_turn_key(kind), value=99)
admitted: Final = await async_admit_io_token_turn(dual_cache, _turn_deployment(kind), _TURN_AT)

View file

@ -4,13 +4,15 @@ import gc
import json
import os
import weakref
from collections.abc import Iterator, Mapping
from collections.abc import Awaitable, Callable, Iterator, Mapping
from datetime import datetime, timezone
from types import MethodType
from typing import Final
from unittest.mock import AsyncMock, MagicMock, patch
import httpx
import pytest
import respx
import litellm
from litellm import Router
@ -902,6 +904,169 @@ async def test_responses_websocket_turn_refused_for_tokens_leaves_the_daily_requ
await admit_turn()
_PROVIDER_RESPONSE: Final = {
"id": "resp_io",
"object": "response",
"created_at": 1767268830,
"status": "completed",
"model": "ws-io-unpriced",
"output": [
{
"type": "message",
"id": "msg_io",
"status": "completed",
"role": "assistant",
"content": [{"type": "output_text", "text": "Done.", "annotations": []}],
}
],
"parallel_tool_calls": True,
"tool_choice": "auto",
"tools": [],
"usage": {"input_tokens": 10, "output_tokens": 5, "total_tokens": 15},
}
@pytest.mark.asyncio
@pytest.mark.parametrize(
"client_metadata", [None, {"session": "abc", "team": "search"}], ids=["without_metadata", "with_metadata"]
)
async def test_responses_call_sends_the_provider_only_the_client_metadata_and_settles_deployment_io_tokens(
client_metadata: Mapping[str, str] | None,
router_minute_pinned: datetime,
monkeypatch: pytest.MonkeyPatch,
respx_mock: respx.MockRouter,
):
from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
from litellm.responses.streaming_iterator import drain_logging_tasks
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
litellm.in_memory_llm_clients_cache.flush_cache()
provider: Final = respx_mock.post("https://api.openai.com/v1/responses").mock(
return_value=httpx.Response(200, json=_PROVIDER_RESPONSE)
)
router: Final = _io_limited_router("itpm", otpm=10_000)
await router.aresponses(
model="ws-io",
input="hi",
max_output_tokens=50,
**({} if client_metadata is None else {"metadata": dict(client_metadata)}),
)
await drain_logging_tasks()
await asyncio.wait_for(GLOBAL_LOGGING_WORKER.flush(), timeout=10.0)
assert [json.loads(call.request.read()).get("metadata") for call in provider.calls] == [client_metadata]
assert [
await router.cache.async_get_cache(key=_minutes_counter_key(kind, router_minute_pinned))
for kind in ("itpm", "otpm")
] == [10, 5]
@pytest.mark.parametrize(
"client_metadata", [None, {"session": "abc", "team": "search"}], ids=["without_metadata", "with_metadata"]
)
def test_sync_responses_call_sends_the_provider_only_the_client_metadata(
client_metadata: Mapping[str, str] | None, monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter
):
litellm.in_memory_llm_clients_cache.flush_cache()
provider: Final = respx_mock.post("https://api.openai.com/v1/responses").mock(
return_value=httpx.Response(200, json=_PROVIDER_RESPONSE)
)
_io_limited_router("itpm", otpm=10_000).responses(
model="ws-io",
input="hi",
max_output_tokens=50,
**({} if client_metadata is None else {"metadata": dict(client_metadata)}),
)
assert [json.loads(call.request.read()).get("metadata") for call in provider.calls] == [client_metadata]
_ANTHROPIC_PROVIDER_RESPONSE: Final = {
"id": "msg_io",
"type": "message",
"role": "assistant",
"model": "ws-io-unpriced",
"content": [{"type": "text", "text": "Done."}],
"stop_reason": "end_turn",
"stop_sequence": None,
"usage": {"input_tokens": 10, "output_tokens": 5},
}
async def _responses_call(router: Router) -> object:
return await router.aresponses(model="ws-io", input="hi", max_output_tokens=50)
async def _messages_call(router: Router) -> object:
return await router.aanthropic_messages(model="ws-io", messages=[{"role": "user", "content": "hi"}], max_tokens=50)
@pytest.mark.asyncio
@pytest.mark.parametrize(
("model", "provider_url", "provider_response", "call"),
[
pytest.param(
"openai/ws-io-unpriced",
"https://api.openai.com/v1/responses",
_PROVIDER_RESPONSE,
_responses_call,
id="responses",
),
pytest.param(
"anthropic/ws-io-unpriced",
"https://api.anthropic.com/v1/messages",
_ANTHROPIC_PROVIDER_RESPONSE,
_messages_call,
id="messages",
),
],
)
async def test_retried_call_settles_deployment_io_tokens_to_the_successful_attempt_s_tokens(
model: str,
provider_url: str,
provider_response: Mapping[str, object],
call: Callable[[Router], Awaitable[object]],
router_minute_pinned: datetime,
monkeypatch: pytest.MonkeyPatch,
respx_mock: respx.MockRouter,
):
from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
from litellm.responses.streaming_iterator import drain_logging_tasks
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
litellm.in_memory_llm_clients_cache.flush_cache()
provider: Final = respx_mock.post(provider_url).mock(
side_effect=[
httpx.Response(500, json={"error": {"message": "upstream failed", "type": "server_error"}}),
httpx.Response(200, json=provider_response),
]
)
router: Final = Router(
model_list=[
{
"model_name": "ws-io",
"litellm_params": {"model": model, "api_key": "sk-fake", "itpm": 10_000, "otpm": 10_000},
"model_info": {"id": "ws-io-dep"},
}
],
optional_pre_call_checks=["enforce_model_rate_limits"],
num_retries=1,
retry_after=0,
)
await call(router)
await drain_logging_tasks()
await asyncio.wait_for(GLOBAL_LOGGING_WORKER.flush(), timeout=10.0)
assert provider.call_count == 2
assert [
await router.cache.async_get_cache(key=_minutes_counter_key(kind, router_minute_pinned, model))
for kind in ("itpm", "otpm")
] == [10, 5]
@pytest.mark.asyncio
async def test_deployment_callback_on_success_tracks_tpm_for_io_deployment():
"""