mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
fix(router): leave a socket turn's cached input out of itpm
The native Responses WebSocket relay logs a turn with its list of events, so the deployment rate limit hook finds no usage on the response and falls back to the standard logging payload. That fallback read cached input from metadata.cache_read_input_tokens, a key the payload's cleaned metadata never carries, so every cached token of a socket turn was counted against itpm. The fallback now reads the cached count from the payload's usage_object, the same usage the logging path already built for cost, so a native socket turn counts only its uncached input. The managed relay and HTTP /v1/responses already counted correctly through the response's own usage
This commit is contained in:
parent
be078aa315
commit
4e2342d475
2 changed files with 91 additions and 5 deletions
|
|
@ -258,15 +258,13 @@ def _resolve_reconcile_usage_tokens(
|
|||
if isinstance(standard_logging_object, dict):
|
||||
prompt_tokens = int(standard_logging_object.get("prompt_tokens") or 0)
|
||||
completion_tokens = int(standard_logging_object.get("completion_tokens") or 0)
|
||||
cached_tokens = 0
|
||||
metadata: Final = standard_logging_object.get("metadata")
|
||||
if isinstance(metadata, dict):
|
||||
cached_tokens = int(metadata.get("cache_read_input_tokens") or 0)
|
||||
logged_usage: Final = _usage_field(standard_logging_object.get("metadata"), "usage_object")
|
||||
logged_cached_tokens: Final = _cached_prompt_tokens(logged_usage)
|
||||
# Same rationale as _usage_is_present: a bare total_tokens with no
|
||||
# prompt/completion breakdown can't be split, so it isn't treated
|
||||
# as resolved usage - the reservation is kept instead of refunded.
|
||||
if prompt_tokens or completion_tokens:
|
||||
return max(0, prompt_tokens - cached_tokens), completion_tokens, True
|
||||
return max(0, prompt_tokens - logged_cached_tokens), completion_tokens, True
|
||||
|
||||
return 0, 0, False
|
||||
|
||||
|
|
|
|||
|
|
@ -1743,6 +1743,56 @@ def _connection_settling_turns_through(
|
|||
return connection
|
||||
|
||||
|
||||
_ITPM_OPENED_ON: Final = "global_router:ws-dep:openai/gpt-5-mini:itpm:12-34"
|
||||
_ITPM_DEPLOYMENT: Final = {
|
||||
"litellm_params": {"model": "openai/gpt-5-mini", "itpm": 40_000},
|
||||
"model_info": {"id": "ws-dep"},
|
||||
}
|
||||
_TURN_INPUT_TOKENS: Final = 1_000
|
||||
|
||||
|
||||
def _turn_completed_reading_cache(cached_tokens: int) -> str:
|
||||
return json.dumps(
|
||||
{
|
||||
"type": "response.completed",
|
||||
"response": {
|
||||
"id": "resp_cached_turn",
|
||||
"status": "completed",
|
||||
"output": [],
|
||||
"usage": {
|
||||
"input_tokens": _TURN_INPUT_TOKENS,
|
||||
"input_tokens_details": {"cached_tokens": cached_tokens},
|
||||
"output_tokens": _OUTPUT_TOKENS_PER_TURN,
|
||||
"total_tokens": _TURN_INPUT_TOKENS + _OUTPUT_TOKENS_PER_TURN,
|
||||
},
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _connection_logged_through(rate_limits: "ModelRateLimitingCheck", opened_with: Mapping[str, object]) -> "Logging":
|
||||
import time
|
||||
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging
|
||||
from litellm.types.utils import CallTypes
|
||||
|
||||
connection: Final = Logging(
|
||||
model="gpt-5-mini",
|
||||
messages=[],
|
||||
stream=False,
|
||||
call_type=CallTypes.aresponses_websocket.value,
|
||||
start_time=time.time(),
|
||||
litellm_call_id="connection-call",
|
||||
function_id="connection-call",
|
||||
dynamic_async_success_callbacks=[rate_limits],
|
||||
)
|
||||
connection.update_environment_variables(
|
||||
litellm_params={"metadata": dict(opened_with), CallTypes.aresponses_websocket.value: True},
|
||||
optional_params={},
|
||||
)
|
||||
return connection
|
||||
|
||||
|
||||
class _TwoOutputTokenTurns(CustomLLM):
|
||||
def __init__(self, failing_turns: frozenset[int] = frozenset()) -> None:
|
||||
super().__init__()
|
||||
|
|
@ -1912,6 +1962,44 @@ class TestWebSocketTurnAdmission:
|
|||
assert await dual_cache.async_get_cache(key=_OPENED_ON) == _OTHER_USAGE + first_turn_tokens
|
||||
assert await dual_cache.async_get_cache(key=_TURNED_ON) == _OTHER_USAGE + 2 * _OUTPUT_TOKENS_PER_TURN
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("cached_tokens", "counted_input_tokens"),
|
||||
[
|
||||
pytest.param(800, _TURN_INPUT_TOKENS - 800, id="cached_input"),
|
||||
pytest.param(0, _TURN_INPUT_TOKENS, id="uncached_input"),
|
||||
],
|
||||
)
|
||||
async def test_native_relay_leaves_a_turn_s_cached_input_out_of_the_deployment_s_itpm(
|
||||
self, cached_tokens: int, counted_input_tokens: int
|
||||
) -> None:
|
||||
import websockets.exceptions # noqa: F401 (lazy submodule must be importable)
|
||||
|
||||
from litellm.caching.dual_cache import DualCache
|
||||
from litellm.responses.streaming_iterator import drain_logging_tasks
|
||||
from litellm.router_utils.pre_call_checks.io_token_rate_limit_check import async_admit_io_token_turn
|
||||
from litellm.router_utils.pre_call_checks.model_rate_limit_check import ModelRateLimitingCheck
|
||||
|
||||
dual_cache: Final = DualCache()
|
||||
opened_with: Final = await async_admit_io_token_turn(dual_cache, _ITPM_DEPLOYMENT, _OPENED_AT)
|
||||
first, _, _ = _turn_frames()
|
||||
backend_ws: Final = MagicMock()
|
||||
backend_ws.send = AsyncMock()
|
||||
backend_ws.recv = AsyncMock(side_effect=[_turn_completed_reading_cache(cached_tokens), Exception("stop")])
|
||||
handler: Final = _make_streaming(
|
||||
websocket=_ClientSocket(),
|
||||
backend_ws=backend_ws,
|
||||
request_data={},
|
||||
logging_obj=_connection_logged_through(ModelRateLimitingCheck(dual_cache), opened_with),
|
||||
first_message=first,
|
||||
)
|
||||
|
||||
await handler.client_to_backend()
|
||||
await handler.backend_to_client()
|
||||
await drain_logging_tasks()
|
||||
|
||||
assert await dual_cache.async_get_cache(key=_ITPM_OPENED_ON) == counted_input_tokens
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("failing_turns", "first_turn_tokens"),
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue