From 01340086289c23e25c450660023c00a9cabb869d Mon Sep 17 00:00:00 2001 From: quietjules <349048734+quietjules@users.noreply.github.com> Date: Mon, 16 Nov 2026 22:08:06 -0800 Subject: [PATCH] fix(responses): write one failure row per Responses WebSocket turn the deployment refuses (#50731) * fix(responses): write one failure row per refused Responses WebSocket turn A turn the deployment refuses on an open native or managed Responses WebSocket now writes its own spend log failure row when it is refused, with its own request id, the turn's start time and request, the socket's resolved model group and the websocket call type. Before, refused turns wrote nothing until the socket closed, and then only one row for the whole socket, carrying the socket's open time and first frame. Guardrail-blocked turns go through the same row builder, so their rows now carry the model group and call type too. Turns refused or crashed in the turn check before a session-limit cancel also keep their rows. The socket's close-time failure hooks still run once to settle the admission budget reservation, but when the close failure is the first refused turn the spend logger skips the duplicate row. * fix(responses): keep forged logging state out of refused-turn rows and wait for each row A refused turn's failure row now leaves out the top-level fields the HTTP path strips from a client body, plus standard_callback_dynamic_params and litellm_params, so a frame cannot forge its own logging state or turn off message redaction. The relay now waits for each refused turn's row before it reads the next frame, and the write still finishes if the socket is cancelled meanwhile * ci: start the unit workflow GitHub never created for the previous push * fix(responses): keep the client's request tags on refused-turn rows A refused turn's row now carries the key's tags plus the client's User-Agent tags, computed from the socket's request headers like its success rows, while the headers themselves stay out of the row * fix(responses): keep a refused turn's frame from forging its spend row and store no blocked prompt A refused turn's failure row now takes only the turn's model and call id from the frame, ignores frame metadata tags and model_group, and logs non-object metadata as empty instead of dropping the row. Guardrail-blocked frames no longer carry a stored request. * fix(proxy): keep the socket's session id, guardrail records, and router-key strip on refused WebSocket turn rows * fix(proxy): use the public router-reserved metadata field list, as main renamed it * fix(responses): give refused WebSocket turn rows the socket's session id When a socket's request carries no trace id, the router minted its own for the relayed turns, so success rows and refused-turn rows had different session ids. The endpoint now hands the router the socket's trace id The refused-turn row test's session-limit case jumps the loop clock past the limit once the relay is parked, instead of racing a 50 ms timer * test(responses): expect the parallel-slot refusal at the WebSocket handshake Main now refuses a cached key that is over its limits before the request body is read, so a second socket on a key whose only parallel slot an open socket holds is refused at the upgrade instead of with a 429 frame. The test still checks that the refused turn's failure row keeps the slot held until the first socket closes * fix(responses): take a refused WebSocket turn's agent and session fields from the socket A frame's metadata.agent_id, billing_agent_id or session_id no longer lands on the refused turn's spend row. The row copies those fields from the socket, like its success rows The integration test no longer pins the reopened socket's 1011 close, which is incidental to the held slot it checks * test(responses): expect a 403 when a held parallel slot refuses a second WebSocket handshake * test(responses): expect a held parallel slot to refuse a second socket's first turn with a 429 frame Main's #51675 skips the headers-only limit check for a bodyless GET, and every WebSocket upgrade is one, so a second socket on a key whose only parallel slot is held now opens and its first turn gets a 429 max_parallel_requests frame and close 1008 instead of a 403 at the handshake --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- .../proxy/hooks/proxy_track_cost_callback.py | 5 +- .../proxy/response_api_endpoints/endpoints.py | 164 +++++- litellm/proxy/utils.py | 4 + litellm/responses/streaming_iterator.py | 9 +- ...test_responses_websocket_turn_admission.py | 159 ++++-- .../hooks/test_proxy_track_cost_callback.py | 34 ++ .../response_api_endpoints/test_endpoints.py | 481 ++++++++++++++++-- .../test_responses_websocket_all_providers.py | 50 +- 8 files changed, 791 insertions(+), 115 deletions(-) diff --git a/litellm/proxy/hooks/proxy_track_cost_callback.py b/litellm/proxy/hooks/proxy_track_cost_callback.py index 7f37f8cb050..31c3b4a3676 100644 --- a/litellm/proxy/hooks/proxy_track_cost_callback.py +++ b/litellm/proxy/hooks/proxy_track_cost_callback.py @@ -72,6 +72,8 @@ from litellm.proxy.spend_tracking.spend_tracking_utils import ( # noqa: F401 # should_store_prompts_for_call, ) from litellm.proxy.utils import ( + FAILURE_ROW_ALREADY_WRITTEN, + FAILURE_ROW_ALREADY_WRITTEN_KEY, GUARDRAIL_ENDPOINT_FAILURE, GUARDRAIL_ENDPOINT_FAILURE_KEY, REJECTED_BEFORE_REQUEST_SETUP, @@ -506,7 +508,8 @@ class _ProxyDBLogger(CustomLogger): or request_fields.get(GUARDRAIL_ENDPOINT_FAILURE_KEY) is GUARDRAIL_ENDPOINT_FAILURE ) if ( - ProxyDBLogger._should_track_errors_in_db() is False + request_fields.get(FAILURE_ROW_ALREADY_WRITTEN_KEY) is FAILURE_ROW_ALREADY_WRITTEN + or ProxyDBLogger._should_track_errors_in_db() is False and not completed_cost or ProxyDBLogger._skips_pre_routing_rejections() and is_pre_routing_rejection(original_exception, _REQUEST_DATA.validate_python(request_data)) diff --git a/litellm/proxy/response_api_endpoints/endpoints.py b/litellm/proxy/response_api_endpoints/endpoints.py index 2f0e3f2bcfb..9a3c0b8eafb 100644 --- a/litellm/proxy/response_api_endpoints/endpoints.py +++ b/litellm/proxy/response_api_endpoints/endpoints.py @@ -5,6 +5,7 @@ import json import uuid from collections.abc import AsyncGenerator, AsyncIterator, Awaitable, Callable, Mapping, Sequence from dataclasses import dataclass +from datetime import datetime from enum import Enum from functools import partial from itertools import chain @@ -36,9 +37,19 @@ import litellm from litellm._logging import verbose_proxy_logger from litellm.anthropic_interface.exceptions.exception_mapping_utils import AnthropicExceptionMapping from litellm.caching.dual_cache import DualCache -from litellm.constants import EMPTY_MAPPING, MANAGED_BACKGROUND_RESPONSE_METADATA_KEY, TERMINAL_RESPONSE_STATUSES +from litellm.constants import ( + CALLER_SESSION_ID_METADATA_KEY, + EMPTY_MAPPING, + MANAGED_BACKGROUND_RESPONSE_METADATA_KEY, + SESSION_ID_OMITTED_METADATA_KEY, + TERMINAL_RESPONSE_STATUSES, +) from litellm.exceptions import EndpointNotSupportedByModelError, RateLimitError -from litellm.integrations.custom_guardrail import ModifyResponseException, guardrail_request_data_with_streaming +from litellm.integrations.custom_guardrail import ( + ModifyResponseException, + guardrail_request_data_with_streaming, + without_server_streaming_classification, +) from litellm.litellm_core_utils.core_helpers import as_str_mapping, get_metadata_variable_name_from_kwargs from litellm.litellm_core_utils.internal_call_metadata import is_background_response, requests_background from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj @@ -91,6 +102,8 @@ from litellm.proxy.hooks.responses_id_security import ResponsesIDSecurity, decry from litellm.proxy.litellm_pre_call_utils import ( _CLIENT_PRICING_METADATA_FIELDS, # pyright: ignore[reportPrivateUsage] # the HTTP path's own strip list _UNTRUSTED_METADATA_CONTROL_FIELDS, # pyright: ignore[reportPrivateUsage] # the HTTP path's own strip list + _UNTRUSTED_ROOT_CONTROL_FIELDS, # pyright: ignore[reportPrivateUsage] # the HTTP path's own strip list + ROUTER_RESERVED_METADATA_FIELDS, add_guardrails_from_policy_engine, is_chat_completions_body, ) @@ -98,7 +111,12 @@ from litellm.proxy.pass_through_endpoints.llm_provider_handlers.batch_attributio request_tags_from_request_data, ) from litellm.proxy.route_llm_request import raise_if_required_body_param_missing -from litellm.proxy.utils import count_request_input_tokens, handle_exception_on_proxy +from litellm.proxy.utils import ( + FAILURE_ROW_ALREADY_WRITTEN, + FAILURE_ROW_ALREADY_WRITTEN_KEY, + count_request_input_tokens, + handle_exception_on_proxy, +) from litellm.responses.litellm_completion_transformation.compaction_items import messages_since_newest_compaction from litellm.responses.litellm_completion_transformation.count_request import MODEL_FILE_ID_MAPPING_FIELD from litellm.responses.retrieve_stream_query import RETRIEVE_INCLUDE_QUERY_KEYS, RETRIEVE_STREAM_QUERY_KEYS @@ -117,7 +135,7 @@ from litellm.types.llms.openai import ( ) from litellm.types.responses.main import DeleteResponseResult from litellm.types.token_count import TokenCountSourceFields -from litellm.types.utils import ADDRESSED_RESPONSE_ID_FIELD, TokenCountResponse +from litellm.types.utils import ADDRESSED_RESPONSE_ID_FIELD, CallTypes, TokenCountResponse from litellm.utils import get_base_model_from_metadata if TYPE_CHECKING: @@ -2225,6 +2243,22 @@ _CLIENT_GUARDRAIL_SELECTORS: Final = ("guardrails", "guardrail_config", "policie _METADATA_GUARDRAIL_SELECTORS: Final = ("guardrails", "guardrail_config") _PROXY_ONLY_FRAME_FIELDS: Final = (*_CLIENT_GUARDRAIL_SELECTORS, "include_guardrail_response") _UNFORWARDED_FRAME_FIELDS: Final = (*_GUARDED_FRAME_FIELDS, *_PROXY_ONLY_FRAME_FIELDS, "litellm_metadata") +_UNTRUSTED_TURN_FIELDS: Final = frozenset( + (*_UNTRUSTED_ROOT_CONTROL_FIELDS, "standard_callback_dynamic_params", "litellm_params") +) +_SOCKET_ATTRIBUTION_METADATA_FIELDS: Final = ( + "agent_id", + "actor_agent_id", + "target_agent_id", + "billing_agent_id", + "agent_execution_mode", + "verified_human_user_id", + "session_id", + SESSION_ID_OMITTED_METADATA_KEY, + CALLER_SESSION_ID_METADATA_KEY, +) +_SOCKET_OWNED_METADATA_FIELDS: Final = frozenset(("tags", "model_group", *_SOCKET_ATTRIBUTION_METADATA_FIELDS)) +_TURN_ROW_FIELDS: Final = ("model", "litellm_call_id") _OBJECTS: Final = TypeAdapter(list[object]) _TURN_COPIED_CONTAINERS: Final = (list, dict) @@ -2307,9 +2341,30 @@ def _responses_ws_frame_guard( } model: Final = admitted.get("model") - # a blocked frame is billed as its own failed request; only the first turn settles the admission reservation + # a blocked or refused turn is billed as its own failed request; the socket settles the admission reservation rejected_frame_auth: Final = user_api_key_dict.model_copy(update={"budget_reservation": None}) + failure_row: Final = partial(_turn_failure_row, metadata_key, admitted) + + async def bill_refusal( + turn: Mapping[str, object], + guardrail_information: tuple[Mapping[str, object], ...], + started_at: datetime, + refusal: Exception, + ) -> None: + record_refused_turn(guardrail_information, refusal) + await asyncio.shield( + spawn_logging_task( + asyncio.get_running_loop(), + _bill_rejected_frame( + rejected_frame_auth, + refusal, + {**failure_row(turn, started_at), "proxy_server_request": _turn_stored_request(metadata_key, turn)}, + ), + task_name="Responses WebSocket refused turn row", + ) + ) + async def rejection( exception: Exception, data: dict[str, object], # mutable-ok: billed @@ -2319,6 +2374,7 @@ def _responses_ws_frame_guard( return ResponsesWsFrameRejection(proxy_exception.message, proxy_exception.provider_specific_fields) async def guard(frame: str) -> GuardedResponsesWsFrame: + started_at: Final = datetime.now() # noqa: DTZ005 # the request pipeline logs naive local start times frame_obj: Final = _WS_FRAME.validate_json(frame) nested: Final = frame_obj.get("response") body: Final = _WS_FRAME.validate_python(nested) if isinstance(nested, dict) else frame_obj @@ -2347,7 +2403,7 @@ def _responses_ws_frame_guard( ), } except Exception as e: - raise await rejection(e, turn_data) from e + raise await rejection(e, failure_row(turn_data, started_at)) from e try: guarded_data: Final = await proxy_logging_obj.pre_call_hook( user_api_key_dict=user_api_key_dict, data=frame_data, call_type="aresponses", guardrails_only=True @@ -2361,13 +2417,13 @@ def _responses_ws_frame_guard( data=guarded_data, user_api_key_dict=user_api_key_dict, call_type="aresponses", guardrails_only=True ) except Exception as e: - raise await rejection(e, frame_data) from e + raise await rejection(e, failure_row(frame_data, started_at)) from e if guarded_data.get("model") != model: rerouted: Final = ValueError( f"a guardrail rerouted this turn to {guarded_data.get('model')!r}; an open Responses WebSocket" f" is bound to {model!r}, send this turn over HTTP /v1/responses" ) - await _bill_rejected_frame(rejected_frame_auth, rerouted, frame_data) + await _bill_rejected_frame(rejected_frame_auth, rerouted, failure_row(frame_data, started_at)) raise rerouted rewritten: Final = ( {**_forwarded_frame_fields(frame_obj), "response": {**_forwarded_frame_fields(body), **guarded_fields}} @@ -2379,12 +2435,73 @@ def _responses_ws_frame_guard( frame=json.dumps(rewritten), guardrail_information=guardrail_information, pii_tokens=pii_tokens, - bill_refusal=partial(record_refused_turn, guardrail_information), + bill_refusal=partial(bill_refusal, guarded_data, guardrail_information, started_at), ) return guard +def _socket_trace_id(socket: Mapping[str, object]) -> Mapping[str, str]: + logging_obj: Final = socket.get("litellm_logging_obj") + return ( + MappingProxyType({"litellm_trace_id": logging_obj.litellm_trace_id}) + if isinstance(logging_obj, LiteLLMLoggingObj) + else MappingProxyType({}) + ) + + +def _turn_failure_row( + metadata_key: str, + socket: Mapping[str, object], + turn: Mapping[str, object], + started_at: datetime, +) -> dict[str, object]: # mutable-ok: the spend logger stamps the failure row on the request dict + socket_bucket: Final = socket.get(metadata_key) + socket_request: Final = socket.get("proxy_server_request") + turn_bucket: Final = turn.get(metadata_key) + turn_metadata: Final = _WS_FRAME.validate_python(turn_bucket) if isinstance(turn_bucket, dict) else {} + request_tags: Final = StandardLoggingPayloadSetup.get_request_tags( + {metadata_key: turn_metadata}, + _PROXY_SERVER_REQUEST_ADAPTER.validate_python(socket_request) if isinstance(socket_request, dict) else {}, + ) + socket_metadata: Final = _WS_FRAME.validate_python(socket_bucket) if isinstance(socket_bucket, dict) else {} + model_group: Final = socket_metadata.get("model_group") + client_metadata: Final = _without_untrusted_metadata("metadata", turn.get("metadata")) + return { + **{key: turn[key] for key in _TURN_ROW_FIELDS if key in turn}, + **_socket_trace_id(socket), + "metadata": { + **( + { + name: item + for name, item in _WS_FRAME.validate_python(client_metadata).items() + if name not in _SOCKET_OWNED_METADATA_FIELDS + } + if isinstance(client_metadata, dict) + else {} + ), + **{name: socket_metadata[name] for name in _SOCKET_ATTRIBUTION_METADATA_FIELDS if name in socket_metadata}, + }, + metadata_key: { + **turn_metadata, + **({"tags": request_tags} if request_tags else {}), + **({"model_group": model_group} if isinstance(model_group, str) else {}), + }, + "call_type": CallTypes.aresponses_websocket.value, + "start_time": started_at, + } + + +def _turn_stored_request(metadata_key: str, turn: Mapping[str, object]) -> Mapping[str, object]: + return { + "body": { + key: _without_untrusted_metadata(key, value) + for key, value in without_server_streaming_classification(turn).items() + if key not in _UNTRUSTED_TURN_FIELDS and key not in (metadata_key, "litellm_call_id") + } + } + + def _reject_guarded_field_beside_nested_response(frame: Mapping[str, object], nested: object) -> None: """Only the nested ``response`` body runs through the guardrails, so a guarded field beside it would go upstream unguarded""" @@ -2550,7 +2667,8 @@ def _turn_copy(value: object) -> object: def _without_untrusted_metadata(key: str, value: object) -> object: - """Like the HTTP path, drop the proxy-owned ``user_api_key_*`` and guardrail control keys a client may forge.""" + """Like the HTTP path, drop the proxy-owned ``user_api_key_*``, guardrail control and router keys a client may + forge.""" if key not in _CLIENT_METADATA_KEYS or not isinstance(value, dict): return value return { @@ -2559,6 +2677,7 @@ def _without_untrusted_metadata(key: str, value: object) -> object: if not name.startswith("user_api_key_") and name not in _UNTRUSTED_METADATA_CONTROL_FIELDS and name not in _CLIENT_PRICING_METADATA_FIELDS + and name not in ROUTER_RESERVED_METADATA_FIELDS and name not in ("headers", "pii_tokens", _GUARDRAIL_INFORMATION_KEY, *_CLIENT_GUARDRAIL_SELECTORS) } @@ -2591,9 +2710,10 @@ _NO_PII_TOKENS: Final[Mapping[str, str]] = MappingProxyType({}) @dataclass(slots=True) class _RefusedTurns: - """Turns the deployment refused after their guardrails passed. The socket's close-time failure row bills their - guardrail records, so its failure hooks, model group and call type cover those turns too, and the first refusal - is that row's exception when the session limit cancels the relay before it returns one.""" + """Each turn the deployment refused wrote its own failure row when refused. The first refusal is still the + socket's close-time failure, so its failure hooks run once, even when the session limit cancels the relay before + it returns one, and see every refused turn's guardrail records, but they write no second row for them. A close on + any other error writes that error's own row, which must not bill the refused turns' guardrail records again.""" first: Exception | None = None records: tuple[Mapping[str, object], ...] = () @@ -2603,8 +2723,8 @@ class _RefusedTurns: if self.first is None: self.first = refusal - def billed_on(self, request_data: Mapping[str, object]) -> Mapping[str, object]: - if not self.records: + def failure_hook_data(self, exception: Exception, request_data: Mapping[str, object]) -> Mapping[str, object]: + if exception is not self.first: return request_data metadata_key: Final = get_metadata_variable_name_from_kwargs(request_data) bucket: Final = request_data.get(metadata_key) @@ -2612,7 +2732,12 @@ class _RefusedTurns: recorded: Final = (*_recorded_guardrail_information(fields), *self.records) return { **request_data, - metadata_key: {**fields, _GUARDRAIL_INFORMATION_KEY: [dict(record) for record in recorded]}, + FAILURE_ROW_ALREADY_WRITTEN_KEY: FAILURE_ROW_ALREADY_WRITTEN, + **( + {metadata_key: {**fields, _GUARDRAIL_INFORMATION_KEY: [dict(record) for record in recorded]}} + if recorded + else {} + ), } @@ -2727,6 +2852,7 @@ async def _responses_websocket_session( refused_turns: Final = _RefusedTurns() routed_data: Final = dict( data, + **_socket_trace_id(data), user_api_key_dict=user_api_key_dict, responses_ws_frame_guard=_responses_ws_frame_guard( data, user_api_key_dict, proxy_logging_obj, refused_turns.add @@ -2734,16 +2860,16 @@ async def _responses_websocket_session( **_routing_hints_from_first_ws_frame(first_message), ) - async def write_failure_row(exception: Exception) -> None: + async def run_failure_hooks(exception: Exception) -> None: await proxy_logging_obj.post_call_failure_hook( user_api_key_dict=user_api_key_dict, original_exception=exception, - request_data=dict(refused_turns.billed_on(routed_data)), + request_data=dict(refused_turns.failure_hook_data(exception, routed_data)), ) def bill_socket_failure(exception: Exception) -> asyncio.Task[object]: return spawn_logging_task( - asyncio.get_running_loop(), write_failure_row(exception), task_name="Responses WebSocket failure row" + asyncio.get_running_loop(), run_failure_hooks(exception), task_name="Responses WebSocket failure row" ) async def relay_and_bill_failure() -> asyncio.Task[object] | None: diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 8104606db60..15b47fb04a4 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -1833,6 +1833,10 @@ object, meaning the request died (auth, body parsing, model resolution) before t object rather than ``True`` because ``request_data`` is the client body: JSON cannot forge this identity, so a client can neither hide a guardrail block nor force a row by sending the key or a ``litellm_call_id`` itself.""" +FAILURE_ROW_ALREADY_WRITTEN_KEY: Final = "litellm_failure_row_already_written" +FAILURE_ROW_ALREADY_WRITTEN: Final = object() +"""An object, like ``REJECTED_BEFORE_REQUEST_SETUP``, so the client body cannot forge it.""" + GUARDRAIL_ENDPOINT_FAILURE_KEY: Final = "litellm_guardrail_endpoint_failure" GUARDRAIL_ENDPOINT_FAILURE: Final = object() """Stored under ``GUARDRAIL_ENDPOINT_FAILURE_KEY`` by ``/apply_guardrail`` when a request it had set up fails, so the diff --git a/litellm/responses/streaming_iterator.py b/litellm/responses/streaming_iterator.py index dca93436efa..c872e456d6e 100644 --- a/litellm/responses/streaming_iterator.py +++ b/litellm/responses/streaming_iterator.py @@ -2323,12 +2323,12 @@ class GuardedResponsesWsFrame: """A ``response.create`` frame after the proxy's pre_call guardrails ran on it: the frame to forward (guardrails may rewrite ``input`` and ``instructions``), the guardrail records that bill against this turn only, the Presidio ``pii_tokens`` the walk masked with, which the handler needs to unmask the provider's reply, and how - to carry those guardrail records onto the connection's failure row when the deployment then refuses the turn.""" + to bill the turn as its own failed request when the deployment then refuses it.""" frame: str guardrail_information: tuple[Mapping[str, object], ...] = () pii_tokens: Mapping[str, str] = field(default_factory=lambda: MappingProxyType({})) - bill_refusal: Callable[[Exception], None] | None = None + bill_refusal: Callable[[Exception], Awaitable[None]] | None = None ResponsesWsFrameGuard: TypeAlias = Callable[[str], Awaitable[GuardedResponsesWsFrame]] @@ -2361,10 +2361,9 @@ def _frame_rejection_event(rejection: Exception) -> str: async def _reject_guarded_turn( websocket: ResponsesClientWebSocket, guarded: GuardedResponsesWsFrame, rejection: Exception ) -> None: - """Answers the refused turn and leaves its guardrail records to the connection's close-time failure row, which - runs every failure hook once per socket, like the first refusal it records.""" + """Bills the refused turn as its own failed request, then answers it on the open socket.""" if guarded.bill_refusal is not None: - guarded.bill_refusal(rejection) + await guarded.bill_refusal(rejection) with contextlib.suppress(Exception): await websocket.send_text(_turn_rejection_frame(rejection)) diff --git a/tests/integration/routing/test_responses_websocket_turn_admission.py b/tests/integration/routing/test_responses_websocket_turn_admission.py index 2de556e90e4..35647f4b256 100644 --- a/tests/integration/routing/test_responses_websocket_turn_admission.py +++ b/tests/integration/routing/test_responses_websocket_turn_admission.py @@ -2,6 +2,7 @@ import asyncio import hashlib import os import threading +import time import uuid from collections.abc import AsyncIterator, Callable, Iterator, Mapping from contextlib import asynccontextmanager, contextmanager @@ -26,6 +27,7 @@ from integration._support.client import ( list_value, object_value, string_value, + until_asserted, ) from integration._support.process import graceful_stop_seconds, owned_proxy from integration._support.responses_ws_peer import ( @@ -54,8 +56,10 @@ TURN_SPEND: Final = USAGE["input_tokens"] * INPUT_RATE + USAGE["output_tokens"] TURN_TOKENS: Final = USAGE["total_tokens"] MANAGED_PROVIDER: Final = "mistral" SPEND_ROWS_FOR_KEY: Final = ( - 'SELECT request_id, spend, status, model_group, metadata, request_tags FROM "LiteLLM_SpendLogs" WHERE api_key = %s' + "SELECT request_id, spend, status, model_group, call_type, metadata, request_tags, proxy_server_request," + ' extract(epoch from "startTime")::float8 AS started_at FROM "LiteLLM_SpendLogs" WHERE api_key = %s' ) +STORED_TIME_RESOLUTION_SECONDS: Final = 0.001 TURN_CHECK_FAULT: Final = "turn_check_fault" FAULTING_TURN_CHECK: Final = f""" from litellm.integrations.custom_logger import CustomLogger @@ -103,6 +107,7 @@ def _config(directory: Path, router_settings: Mapping[str, JsonValue], callbacks yaml.safe_dump( { **base, + "general_settings": {**object_value(base["general_settings"]), "store_prompts_in_spend_logs": True}, "litellm_settings": { **object_value(base["litellm_settings"]), **({"callbacks": list(callbacks)} if callbacks else {}), @@ -275,17 +280,54 @@ async def _until(read: Callable[[], float], satisfied: Callable[[float], bool]) return await asyncio.to_thread(eventually, read, satisfied, 15) -def _failure_rows(gateway: Gateway, key: str) -> list[dict[str, JsonValue]]: - rows: Final = eventually( - lambda: gateway.rows(SPEND_ROWS_FOR_KEY, (_key_hash(key),)), - lambda found: any(row["status"] == "failure" for row in found), - seconds=70, - ) - return [row for row in rows if row["status"] == "failure"] +def _failures_now(gateway: Gateway, key: str) -> list[dict[str, JsonValue]]: + return [row for row in gateway.rows(SPEND_ROWS_FOR_KEY, (_key_hash(key),)) if row["status"] == "failure"] + + +def _failure_rows(gateway: Gateway, key: str, count: int = 1) -> list[dict[str, JsonValue]]: + return eventually(lambda: _failures_now(gateway, key), lambda found: len(found) >= count, seconds=70) + + +def _settled_failure_rows(gateway: Gateway, key: str, count: int) -> list[dict[str, JsonValue]]: + return held(lambda: _failures_now(gateway, key), lambda found: len(found) == count, holding=5, seconds=70) + + +@dataclass(frozen=True, slots=True) +class AnsweredTurn: + text: str + sent_at: float + answered_at: float + received: tuple[dict[str, JsonValue], ...] + + +async def _timed_turn(connection: websockets.ClientConnection, model: str, label: str) -> AnsweredTurn: + text: Final = f"{label} {uuid.uuid4().hex}" + sent_at: Final = time.time() + received: Final = await _say(connection, model, text) + return AnsweredTurn(text, sent_at, time.time(), received) + + +def _stored_text(row: Mapping[str, JsonValue]) -> str: + message: Final = object_value(list_value(object_value(row["proxy_server_request"])["input"])[0]) + return string_value(object_value(list_value(message["content"])[0])["text"]) + + +def _assert_logged_as_its_own_failure( + row: Mapping[str, JsonValue], turn: AnsweredTurn, deployment: Deployment +) -> dict[str, JsonValue]: + assert _stored_text(row) == turn.text, row + started_at: Final = row["started_at"] + assert isinstance(started_at, float), row + assert turn.sent_at - STORED_TIME_RESOLUTION_SECONDS <= started_at, (turn, row) + assert started_at <= turn.answered_at + STORED_TIME_RESOLUTION_SECONDS, (turn, row) + assert row["model_group"] == deployment.name, row + assert row["call_type"] == "_aresponses_websocket", row + assert float(row["spend"]) == 0.0, row + return object_value(object_value(row["metadata"])["error_information"]) @pytest.mark.parametrize("relay", ("native", "managed")) -def test_a_turn_past_the_deployment_budget_is_refused_on_the_open_socket_and_logged_once_as_its_failure( +def test_each_turn_past_the_deployment_budget_is_refused_on_the_open_socket_and_logged_as_its_own_failure( limited: Gateway, cert: tuple[Path, Path], relay: str ) -> None: with responses_peer(cert) as peer, limited.scenario() as scenario: @@ -294,25 +336,71 @@ def test_a_turn_past_the_deployment_budget_is_refused_on_the_open_socket_and_log key: Final = scenario.key(models=[deployment.name]) tag: Final = f"ws-turn-{uuid.uuid4().hex}" - async def session() -> tuple[tuple[dict[str, JsonValue], ...], ...]: + async def session() -> tuple[tuple[dict[str, JsonValue], ...], tuple[AnsweredTurn, ...], list[str]]: async with _socket(limited, key, deployment.name, {"x-litellm-tags": tag}) as connection: first: Final = await _say(connection, deployment.name) await _until(lambda: _deployment_spend(deployment), lambda spent: spent > TURN_SPEND / 2) - return (first, await _say(connection, deployment.name), await _say(connection, deployment.name)) + refused: Final = ( + await _timed_turn(connection, deployment.name, "second"), + await _timed_turn(connection, deployment.name, "third"), + ) + logged_while_open: Final = await asyncio.to_thread(_failure_rows, limited, key, len(refused)) + return first, refused, [string_value(row["request_id"]) for row in logged_while_open] - first, second, third = asyncio.run(session()) + first, refused, logged_while_open = asyncio.run(session()) completed(first) - _assert_refused(second) - _assert_refused(third) + for answered in refused: + _assert_refused(answered.received) + assert deployment.forwarded_turns() == 1 + failures: Final = _settled_failure_rows(limited, key, len(refused)) + assert sorted(string_value(row["request_id"]) for row in failures) == sorted(logged_while_open), failures + assert len(set(logged_while_open)) == len(refused), failures + by_text: Final = {_stored_text(row): row for row in failures} + for answered in refused: + row = by_text[answered.text] + error = _assert_logged_as_its_own_failure(row, answered, deployment) + assert str(error["error_code"]) == "429", error + assert tag in list_value(row["request_tags"]), row + + +def test_a_refused_turn_s_failure_row_leaves_the_socket_s_parallel_slot_held_until_the_socket_closes( + limited: Gateway, cert: tuple[Path, Path] +) -> None: + with responses_peer(cert) as peer, limited.scenario() as scenario: + deployment: Final = _native(scenario, peer, max_budget=TURN_SPEND / 2, budget_duration="1d") + key: Final = scenario.key(models=[deployment.name], max_parallel_requests=1) + + async def opened_once() -> tuple[tuple[dict[str, JsonValue], ...], int | None]: + async with _socket(limited, key, deployment.name) as connection: + received: Final = await _say(connection, deployment.name) + await asyncio.wait_for(connection.wait_closed(), timeout=20) + return received, connection.close_code + + async def second_socket_while_the_first_is_open() -> None: + async with _socket(limited, key, deployment.name) as holding: + completed(await _say(holding, deployment.name)) + await _until(lambda: _deployment_spend(deployment), lambda spent: spent > TURN_SPEND / 2) + _assert_refused(await _say(holding, deployment.name)) + await asyncio.to_thread(_failure_rows, limited, key) + second, second_close_code = await opened_once() + _assert_refused(second) + assert "max_parallel_requests" in string_value(object_value(second[0]["error"])["message"]), second + assert second_close_code == 1008, second + + def reopened_once_the_slot_frees() -> tuple[tuple[dict[str, JsonValue], ...], int | None]: + try: + received, close_code = asyncio.run(opened_once()) + except websockets.exceptions.InvalidStatus as refused: + raise AssertionError(refused) from refused + assert close_code != 1008, received + return received, close_code + + asyncio.run(second_socket_while_the_first_is_open()) + reopened, _ = until_asserted(reopened_once_the_slot_frees, seconds=15) + + _assert_refused(reopened) assert deployment.forwarded_turns() == 1 - failures: Final = _failure_rows(limited, key) - assert len(failures) == 1, failures - assert failures[0]["model_group"] == deployment.name, failures - assert tag in list_value(failures[0]["request_tags"]), failures - assert float(failures[0]["spend"]) == 0.0, failures - error: Final = object_value(object_value(failures[0]["metadata"])["error_information"]) - assert str(error["error_code"]) == "429", error def _set_turn_check_fault(gateway: Gateway, deployment: Deployment, fault: str) -> None: @@ -354,27 +442,30 @@ def test_a_turn_whose_admission_check_fails_gets_a_500_frame_and_the_socket_serv key: Final = scenario.key(models=[deployment.name]) fault: Final = f"turn check unavailable {uuid.uuid4().hex}" - async def session() -> tuple[tuple[dict[str, JsonValue], ...], ...]: + async def session() -> tuple[tuple[dict[str, JsonValue], ...], tuple[AnsweredTurn, ...], AnsweredTurn]: async with _socket(faulting, key, deployment.name) as connection: first: Final = await _say(connection, deployment.name) await asyncio.to_thread(_set_turn_check_fault, faulting, deployment, fault) - second: Final = await _say(connection, deployment.name) - third: Final = await _say(connection, deployment.name) + failed: Final = ( + await _timed_turn(connection, deployment.name, "second"), + await _timed_turn(connection, deployment.name, "third"), + ) await asyncio.to_thread(_set_turn_check_fault, faulting, deployment, "") - return first, second, third, await _say(connection, deployment.name) + return first, failed, await _timed_turn(connection, deployment.name, "fourth") - first, second, third, fourth = asyncio.run(session()) + first, failed, fourth = asyncio.run(session()) completed(first) - _assert_turn_check_failed(second, fault) - _assert_turn_check_failed(third, fault) - completed(fourth) + for answered in failed: + _assert_turn_check_failed(answered.received, fault) + completed(fourth.received) assert deployment.forwarded_turns() == 2 - failures: Final = _failure_rows(faulting, key) - assert len(failures) == 1, failures - assert failures[0]["model_group"] == deployment.name, failures - recorded: Final = object_value(object_value(failures[0]["metadata"])["error_information"]) - assert fault in string_value(recorded["error_message"]), recorded + failures: Final = _settled_failure_rows(faulting, key, len(failed)) + assert len({row["request_id"] for row in failures}) == len(failed), failures + by_text: Final = {_stored_text(row): row for row in failures} + for answered in failed: + recorded = _assert_logged_as_its_own_failure(by_text[answered.text], answered, deployment) + assert fault in string_value(recorded["error_message"]), recorded def test_a_turn_still_streaming_when_the_next_turns_check_fails_completes_and_is_billed( diff --git a/tests/unit/proxy/hooks/test_proxy_track_cost_callback.py b/tests/unit/proxy/hooks/test_proxy_track_cost_callback.py index 1b5af737f1b..46fdde5fe21 100644 --- a/tests/unit/proxy/hooks/test_proxy_track_cost_callback.py +++ b/tests/unit/proxy/hooks/test_proxy_track_cost_callback.py @@ -43,6 +43,8 @@ from litellm.proxy.spend_tracking.spend_event import SpendEventDecodeError, buil from litellm.proxy.spend_tracking.spend_event_producer import SpendEventProducer, UnixAddress from litellm.proxy.spend_tracking.spend_tracking_utils import get_logging_payload from litellm.proxy.utils import ( + FAILURE_ROW_ALREADY_WRITTEN, + FAILURE_ROW_ALREADY_WRITTEN_KEY, GUARDRAIL_ENDPOINT_FAILURE, GUARDRAIL_ENDPOINT_FAILURE_KEY, REJECTED_BEFORE_REQUEST_SETUP, @@ -4457,6 +4459,38 @@ async def test_async_post_call_failure_hook_disable_pre_routing_error_logs( assert len(writer.rows) == rows_written +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("request_data", "rows_written"), + [ + ({**_ROUTED_REQUEST, FAILURE_ROW_ALREADY_WRITTEN_KEY: FAILURE_ROW_ALREADY_WRITTEN}, 0), + ({**_ROUTED_REQUEST, FAILURE_ROW_ALREADY_WRITTEN_KEY: True}, 1), + (_ROUTED_REQUEST, 1), + ], + ids=["already-written", "client-spoofed-marker", "unmarked"], +) +async def test_async_post_call_failure_hook_settles_the_reservation_but_writes_no_second_row_for_a_billed_failure( + request_data: Mapping[str, object], rows_written: int +) -> None: + writer: Final = _RecordingSpendWriter() + user_api_key_dict: Final = UserAPIKeyAuth( + api_key="test_api_key", + request_route="/v1/responses", + budget_reservation={"reserved_cost": 0.5, "entries": []}, + ) + with patch( + "litellm.proxy.spend_tracking.budget_reservation.release_budget_reservation", new_callable=AsyncMock + ) as release: + await ProxyDBLogger(spend_writer=lambda: cast(DBSpendUpdateWriter, writer)).async_post_call_failure_hook( + request_data={**request_data, "metadata": dict[str, object]()}, + original_exception=litellm.RateLimitError(message="crossed budget", llm_provider="", model="gpt-4o-mini"), + user_api_key_dict=user_api_key_dict, + ) + assert release.await_count == 1 + assert release.await_args.kwargs["budget_reservation"] is user_api_key_dict.budget_reservation + assert len(writer.rows) == rows_written + + class _RecordedCharges: def __init__(self, outcomes: tuple[bool, ...]) -> None: self.outcomes = outcomes diff --git a/tests/unit/proxy/response_api_endpoints/test_endpoints.py b/tests/unit/proxy/response_api_endpoints/test_endpoints.py index bbb09550e7a..c4b8d60931f 100644 --- a/tests/unit/proxy/response_api_endpoints/test_endpoints.py +++ b/tests/unit/proxy/response_api_endpoints/test_endpoints.py @@ -2492,6 +2492,18 @@ class TestResponsesWSFirstFrameValidation: ws.close.assert_awaited_once_with(code=1011, reason="Internal server error") +def _spend_log_row(write: Mapping[str, object]) -> Mapping[str, object]: + from litellm.proxy.spend_tracking.spend_tracking_utils import get_logging_payload + + payload = get_logging_payload( + kwargs=write["kwargs"], + response_obj=write["completion_response"], + start_time=write["start_time"], + end_time=write["end_time"], + ) + return {**payload, "spend": write["response_cost"]} + + class _FrameGuardrail(CustomGuardrail): def __init__( self, @@ -2871,45 +2883,273 @@ class TestResponsesWSFirstFrameModelAuth: ] @pytest.mark.asyncio - async def test_frame_guard_bills_a_rejected_frame_as_a_failed_request(self): + @pytest.mark.parametrize("outcome", ["blocked", "refused"]) + async def test_frame_guard_bills_a_blocked_or_refused_turn_as_its_own_failed_request(self, outcome: str): + from datetime import datetime + from unittest.mock import create_autospec + import litellm + from litellm.exceptions import RateLimitError + from litellm.litellm_core_utils.litellm_logging import Logging + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.db.db_spend_update_writer import DBSpendUpdateWriter from litellm.proxy.hooks.proxy_track_cost_callback import ProxyDBLogger from litellm.proxy.response_api_endpoints.endpoints import _responses_ws_frame_guard + from litellm.responses.streaming_iterator import drain_logging_tasks + + socket_logging_obj = Logging( + model="gpt-4o-mini", + messages=[], + stream=True, + call_type="_aresponses_websocket", + start_time=datetime.now(), + litellm_call_id="socket-call", + function_id="socket-call", + ) + blocked = ValueError("blocked") + refusal = RateLimitError(message="crossed budget", llm_provider="", model="") async def pre_call_hook(user_api_key_dict, data, call_type, guardrails_only): data["litellm_metadata"]["standard_logging_guardrail_information"] = [{"guardrail_cost": 0.5}] - raise ValueError("blocked") + if outcome == "blocked": + raise blocked + return {**data, "input": "my email is "} proxy_logging_obj = MagicMock(during_call_hook=AsyncMock()) proxy_logging_obj.pre_call_hook = AsyncMock(side_effect=pre_call_hook) proxy_logging_obj.post_call_failure_hook = AsyncMock() - spend_logger = MagicMock(spec=ProxyDBLogger) - spend_logger.async_post_call_failure_hook = AsyncMock() rate_limiter = MagicMock() rate_limiter.async_post_call_failure_hook = AsyncMock() - user_api_key_dict = MagicMock() + spend_writer = create_autospec(DBSpendUpdateWriter, instance=True) + socket_bucket = {"tags": ["ws-tag"], "standard_logging_guardrail_information": [{"guardrail_cost": 9.0}]} + refused_turns = [] guard = _responses_ws_frame_guard( - {"model": "gpt-4o-mini", "litellm_metadata": {}}, user_api_key_dict, proxy_logging_obj, MagicMock() + {"model": "gpt-4o-mini", "litellm_metadata": socket_bucket, "litellm_logging_obj": socket_logging_obj}, + UserAPIKeyAuth( + api_key="sk-socket", + request_route="/v1/responses", + budget_reservation={"reserved_cost": 0.5, "entries": []}, + ), + proxy_logging_obj, + lambda records, refusal: refused_turns.append((records, refusal)), ) + socket_bucket["model_group"] = "mini-group" + frame = json.dumps({"type": "response.create", "input": "my email is a@b.co", "metadata": {"trace": "t1"}}) with ( - patch.object(litellm, "callbacks", [rate_limiter, spend_logger]), - pytest.raises(ValueError, match="blocked"), + patch.object(litellm, "callbacks", [rate_limiter, ProxyDBLogger(spend_writer=lambda: spend_writer)]), + patch( # test-quality-ok: stored prompts are off by default and the stored request is under test + "litellm.proxy.proxy_server.general_settings", {"store_prompts_in_spend_logs": True} + ), + patch( # test-quality-ok: settling the reservation is a Redis write; only the socket's close may do it + "litellm.proxy.spend_tracking.budget_reservation.release_budget_reservation", new_callable=AsyncMock + ) as release, ): - await guard(json.dumps({"type": "response.create", "input": "hi"})) + sent_at = datetime.now() + if outcome == "blocked": + with pytest.raises(ValueError, match="blocked"): + await guard(frame) + else: + await (await guard(frame)).bill_refusal(refusal) + await drain_logging_tasks(timeout=1) + answered_at = datetime.now() + spend_writer.update_database.assert_awaited_once() + write = spend_writer.update_database.await_args.kwargs + row = _spend_log_row(write) proxy_logging_obj.post_call_failure_hook.assert_not_awaited() rate_limiter.async_post_call_failure_hook.assert_not_awaited() - spend_logger.async_post_call_failure_hook.assert_awaited_once() - failure = spend_logger.async_post_call_failure_hook.await_args.kwargs - user_api_key_dict.model_copy.assert_called_once_with(update={"budget_reservation": None}) - assert failure["user_api_key_dict"] is user_api_key_dict.model_copy.return_value - assert str(failure["original_exception"]) == "blocked" - assert failure["request_data"]["litellm_metadata"]["standard_logging_guardrail_information"] == [ - {"guardrail_cost": 0.5} - ] - assert failure["request_data"]["input"] == "hi" - assert len(failure["request_data"]["litellm_call_id"]) == 36 + release.assert_not_awaited() + assert refused_turns == ([(({"guardrail_cost": 0.5},), refusal)] if outcome == "refused" else []) + assert write["completion_response"] is (refusal if outcome == "refused" else blocked) + assert row["session_id"] == socket_logging_obj.litellm_trace_id + assert sent_at <= write["start_time"] <= answered_at + assert row["call_type"] == "_aresponses_websocket" + assert row["model_group"] == "mini-group" + assert json.loads(row["request_tags"]) == ["ws-tag"] + assert row["spend"] == 0.5 + assert row["request_id"] == write["kwargs"]["litellm_call_id"] and len(row["request_id"]) == 36 + assert json.loads(row["proxy_server_request"]) == ( + {"input": "my email is ", "metadata": {"trace": "t1"}, "model": "gpt-4o-mini"} + if outcome == "refused" + else {} + ) + + @pytest.mark.asyncio + @pytest.mark.parametrize( + "client_metadata", + [ + { + "tags": ["forged-tag"], + "model_group": "forged-group", + "attempted_fallbacks": 7, + "original_model_group": "forged-original-group", + "agent_id": "forged-agent", + "billing_agent_id": "forged-agent", + "session_id": "forged-session", + }, + "not an object", + ], + ) + async def test_frame_guard_bills_a_refused_turn_without_the_logging_state_its_frame_forges( + self, client_metadata: object + ): + from unittest.mock import create_autospec + + from litellm.exceptions import RateLimitError + from litellm.proxy.db.db_spend_update_writer import DBSpendUpdateWriter + from litellm.proxy.hooks.proxy_track_cost_callback import ProxyDBLogger + from litellm.proxy.response_api_endpoints.endpoints import _responses_ws_frame_guard + from litellm.responses.streaming_iterator import drain_logging_tasks + + refusal = RateLimitError(message="crossed budget", llm_provider="", model="") + proxy_logging_obj = MagicMock(during_call_hook=AsyncMock()) + proxy_logging_obj.pre_call_hook = AsyncMock(side_effect=lambda **kwargs: kwargs["data"]) + spend_writer = create_autospec(DBSpendUpdateWriter, instance=True) + guard = _responses_ws_frame_guard( + { + "model": "gpt-4o-mini", + "litellm_metadata": { + "tags": ["ws-tag"], + "model_group": "mini-group", + "agent_id": "socket-agent", + "billing_agent_id": "socket-agent", + "session_id": "socket-session", + }, + }, + UserAPIKeyAuth(api_key="sk-socket", request_route="/v1/responses"), + proxy_logging_obj, + MagicMock(), + ) + frame = json.dumps( + { + "type": "response.create", + "input": "my secret prompt", + "metadata": client_metadata, + "cache_hit": True, + "custom_llm_provider": "forged-provider", + "standard_logging_object": {}, + "standard_callback_dynamic_params": {"turn_off_message_logging": False}, + "litellm_params": {"metadata": {"tags": ["forged-tag"]}}, + } + ) + + with ( + patch.object(litellm, "callbacks", [ProxyDBLogger(spend_writer=lambda: spend_writer)]), + patch.object(litellm, "turn_off_message_logging", True), + patch( # test-quality-ok: stored prompts are off by default and the stored request is under test + "litellm.proxy.proxy_server.general_settings", + {"store_prompts_in_spend_logs": True, "missing_session_id": "omit"}, + ), + ): + await (await guard(frame)).bill_refusal(refusal) + await drain_logging_tasks(timeout=1) + spend_writer.update_database.assert_awaited_once() + row = _spend_log_row(spend_writer.update_database.await_args.kwargs) + + stored_request = json.loads(row["proxy_server_request"]) + row_metadata = json.loads(row["metadata"]) + assert row["model_group"] == "mini-group" + assert row["agent_id"] == "socket-agent" + assert row["billing_agent_id"] == "socket-agent" + assert row["session_id"] == "socket-session" + assert row_metadata.get("attempted_fallbacks") != 7 + assert row_metadata.get("original_model_group") != "forged-original-group" + assert json.loads(row["request_tags"]) == ["ws-tag"] + assert row["cache_hit"] == "False" + assert row["custom_llm_provider"] != "forged-provider" + assert "my secret prompt" not in row["proxy_server_request"] + assert stored_request.keys().isdisjoint( + {"standard_logging_object", "standard_callback_dynamic_params", "litellm_params"} + ) + + @pytest.mark.asyncio + async def test_frame_guard_bills_a_refused_turn_under_the_socket_client_but_with_its_own_request(self): + from unittest.mock import create_autospec + + from litellm.exceptions import RateLimitError + from litellm.proxy.db.db_spend_update_writer import DBSpendUpdateWriter + from litellm.proxy.hooks.proxy_track_cost_callback import ProxyDBLogger + from litellm.proxy.response_api_endpoints.endpoints import _responses_ws_frame_guard + from litellm.responses.streaming_iterator import drain_logging_tasks + + refusal = RateLimitError(message="crossed budget", llm_provider="", model="") + proxy_logging_obj = MagicMock(during_call_hook=AsyncMock()) + proxy_logging_obj.pre_call_hook = AsyncMock(side_effect=lambda **kwargs: kwargs["data"]) + spend_writer = create_autospec(DBSpendUpdateWriter, instance=True) + guard = _responses_ws_frame_guard( + { + "model": "gpt-4o-mini", + "litellm_metadata": {"tags": ["ws-tag"]}, + "proxy_server_request": { + "url": "http://proxy/v1/responses", + "method": "GET", + "headers": {"user-agent": "codex-tui/0.167.0"}, + "body": {"input": "first turn"}, + }, + }, + UserAPIKeyAuth(api_key="sk-socket", request_route="/v1/responses"), + proxy_logging_obj, + MagicMock(), + ) + + with ( + patch.object(litellm, "callbacks", [ProxyDBLogger(spend_writer=lambda: spend_writer)]), + patch( # test-quality-ok: stored prompts are off by default and the stored request is under test + "litellm.proxy.proxy_server.general_settings", {"store_prompts_in_spend_logs": True} + ), + ): + await (await guard(json.dumps({"type": "response.create", "input": "second turn"}))).bill_refusal(refusal) + await drain_logging_tasks(timeout=1) + spend_writer.update_database.assert_awaited_once() + row = _spend_log_row(spend_writer.update_database.await_args.kwargs) + + assert json.loads(row["request_tags"]) == ["ws-tag", "User-Agent: codex-tui", "User-Agent: codex-tui/0.167.0"] + assert json.loads(row["proxy_server_request"])["input"] == "second turn" + + @pytest.mark.asyncio + async def test_a_refused_turn_row_is_written_even_when_the_socket_is_cancelled_while_billing_it(self): + from unittest.mock import create_autospec + + from litellm.exceptions import RateLimitError + from litellm.proxy.db.db_spend_update_writer import DBSpendUpdateWriter + from litellm.proxy.hooks.proxy_track_cost_callback import ProxyDBLogger + from litellm.proxy.response_api_endpoints.endpoints import _responses_ws_frame_guard + from litellm.responses.streaming_iterator import drain_logging_tasks + + writing = asyncio.Event() + may_finish = asyncio.Event() + written = asyncio.Event() + + async def slow_update_database(**kwargs: object) -> None: + writing.set() + await may_finish.wait() + written.set() + + proxy_logging_obj = MagicMock(during_call_hook=AsyncMock()) + proxy_logging_obj.pre_call_hook = AsyncMock(side_effect=lambda **kwargs: kwargs["data"]) + spend_writer = create_autospec(DBSpendUpdateWriter, instance=True) + spend_writer.update_database.side_effect = slow_update_database + guard = _responses_ws_frame_guard( + {"model": "gpt-4o-mini", "litellm_metadata": {}}, + UserAPIKeyAuth(api_key="sk-socket", request_route="/v1/responses"), + proxy_logging_obj, + MagicMock(), + ) + guarded = await guard(json.dumps({"type": "response.create", "input": "hi"})) + + with patch.object(litellm, "callbacks", [ProxyDBLogger(spend_writer=lambda: spend_writer)]): + billing = asyncio.ensure_future( + guarded.bill_refusal(RateLimitError(message="crossed budget", llm_provider="", model="")) + ) + await asyncio.wait_for(writing.wait(), timeout=1) + billing.cancel() + with pytest.raises(asyncio.CancelledError): + await billing + may_finish.set() + await drain_logging_tasks(timeout=1) + + assert written.is_set() @pytest.mark.asyncio async def test_frame_guard_guards_the_nested_response_body_and_copies_back_tools(self): @@ -3490,7 +3730,8 @@ class TestResponsesWSFirstFrameModelAuth: await guard(json.dumps({"type": "response.create", field: "unguarded", "response": {"input": "hi"}})) proxy_logging_obj.pre_call_hook.assert_not_awaited() - assert spend_logger.async_post_call_failure_hook.await_args.kwargs["request_data"]["input"] == "hi" + billed = spend_logger.async_post_call_failure_hook.await_args.kwargs["request_data"] + assert (billed["model"], billed["call_type"]) == ("gpt-4o-mini", "_aresponses_websocket") @pytest.mark.asyncio @pytest.mark.parametrize( @@ -3833,9 +4074,16 @@ class TestResponsesWSFirstFrameModelAuth: @pytest.mark.asyncio @pytest.mark.parametrize("ending", ["refusal", "crash", "session_limit"]) - async def test_close_time_failure_row_bills_the_guardrails_of_the_turns_the_deployment_refused(self, ending: str): + async def test_each_refused_turn_writes_its_own_failure_row_while_the_socket_is_open(self, ending: str): + from datetime import datetime + from unittest.mock import create_autospec + from litellm.exceptions import RateLimitError + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.db.db_spend_update_writer import DBSpendUpdateWriter + from litellm.proxy.hooks.proxy_track_cost_callback import ProxyDBLogger from litellm.proxy.response_api_endpoints.endpoints import responses_websocket_endpoint + from litellm.responses.streaming_iterator import drain_logging_tasks ws = MagicMock() ws.headers = {} @@ -3849,12 +4097,24 @@ class TestResponsesWSFirstFrameModelAuth: ws.close = AsyncMock() processor = MagicMock() processor.common_processing_pre_call_logic = AsyncMock( - return_value=({"model": "gpt-4o-mini", "litellm_metadata": {"model_group": "gpt-4o-mini"}}, MagicMock()) + return_value=({"model": "gpt-4o-mini", "litellm_metadata": {"tags": ["ws-tag"]}}, MagicMock()) ) guardrail_costs = {"two": 0.2, "three": 0.3, "four": 0.4} refusal = RateLimitError(message="crossed budget", llm_provider="", model="") - later_refusal = RateLimitError(message="still over budget", llm_provider="", model="") + turn_check_crash = RuntimeError("turn check crashed") crash = RuntimeError("relay crashed") + rejections = {"two": refusal, "four": turn_check_crash} + spend_writer = create_autospec(DBSpendUpdateWriter, instance=True) + spend_logger = ProxyDBLogger(spend_writer=lambda: spend_writer) + budget_reservation = {"reserved_cost": 0.5, "entries": []} + user_api_key_dict = UserAPIKeyAuth( + api_key="sk-socket", request_route="/v1/responses", budget_reservation=budget_reservation + ) + sent_at = {} + rows_while_open = [] + loop = asyncio.get_running_loop() + loop_clock = loop.time + clock_jumps = [] async def pre_call_hook(user_api_key_dict, data, call_type, guardrails_only): recorded = [{"guardrail_cost": guardrail_costs[data["input"]]}] @@ -3865,27 +4125,41 @@ class TestResponsesWSFirstFrameModelAuth: def route_request(data, **kwargs): async def relay(): + data["litellm_metadata"]["model_group"] = "mini-group" for turn in ("two", "three", "four"): + sent_at[turn] = datetime.now() guarded = await data["responses_ws_frame_guard"]( json.dumps({"type": "response.create", "input": turn}) ) - if turn != "three": - guarded.bill_refusal(refusal if turn == "two" else later_refusal) + if turn in rejections: + await guarded.bill_refusal(rejections[turn]) + await drain_logging_tasks(timeout=1) + rows_while_open.extend(_spend_log_row(write.kwargs) for write in spend_writer.update_database.await_args_list) if ending == "crash": raise crash if ending == "session_limit": + clock_jumps.append(3601.0) await asyncio.Event().wait() return refusal return relay() - row_written = asyncio.Event() + async def run_failure_hooks(**kwargs): + await spend_logger.async_post_call_failure_hook(**kwargs) + proxy_logging_obj = MagicMock(during_call_hook=AsyncMock()) proxy_logging_obj.pre_call_hook = AsyncMock(side_effect=pre_call_hook) - proxy_logging_obj.post_call_failure_hook = AsyncMock(side_effect=lambda **kwargs: row_written.set()) + proxy_logging_obj.post_call_failure_hook = AsyncMock(side_effect=run_failure_hooks) proxy_logging_obj.arelease_responses_websocket_slot = AsyncMock(return_value=None) with ( + patch.object(litellm, "callbacks", [spend_logger]), + patch( # test-quality-ok: stored prompts are off by default and each row's stored request is under test + "litellm.proxy.proxy_server.general_settings", {"store_prompts_in_spend_logs": True} + ), + patch( # test-quality-ok: settling the reservation is a Redis write; the socket's close must do it once + "litellm.proxy.spend_tracking.budget_reservation.release_budget_reservation", new_callable=AsyncMock + ) as release, patch( # test-quality-ok: first-frame model auth needs a live router and key table and has its own tests above "litellm.proxy.response_api_endpoints.endpoints._enforce_responses_ws_first_frame_model_auth", new_callable=AsyncMock, @@ -3899,26 +4173,142 @@ class TestResponsesWSFirstFrameModelAuth: new_callable=AsyncMock, side_effect=route_request, ), - patch( # test-quality-ok: a short session limit stands in for the configured one so the cap fires mid-session + patch( # test-quality-ok: a known session limit stands in for the configured one "litellm.proxy.response_api_endpoints.endpoints._resolve_responses_ws_session_limit_seconds", - return_value=0.05 if ending == "session_limit" else 3600.0, + return_value=3600.0, + ), + patch.object( # test-quality-ok: the session limit is a loop timer, so jumping the loop clock past it once the relay is parked fires it with no wall-clock race + loop, "time", lambda: loop_clock() + sum(clock_jumps) ), patch( # test-quality-ok: the failure hook is the proxy's only path to a failed spend log row "litellm.proxy.proxy_server.proxy_logging_obj", proxy_logging_obj, ), ): - await responses_websocket_endpoint(websocket=ws, model=None, user_api_key_dict=MagicMock()) - await asyncio.wait_for(row_written.wait(), timeout=1) + await responses_websocket_endpoint(websocket=ws, model=None, user_api_key_dict=user_api_key_dict) + await drain_logging_tasks(timeout=1) + writes = [write.kwargs for write in spend_writer.update_database.await_args_list] proxy_logging_obj.post_call_failure_hook.assert_awaited_once() - proxy_logging_obj.arelease_responses_websocket_slot.assert_awaited_once() - booked = proxy_logging_obj.post_call_failure_hook.await_args.kwargs - assert booked["original_exception"] is (crash if ending == "crash" else refusal) - assert booked["request_data"]["litellm_metadata"] == { - "model_group": "gpt-4o-mini", - "standard_logging_guardrail_information": [{"guardrail_cost": 0.2}, {"guardrail_cost": 0.4}], - } + closed = proxy_logging_obj.post_call_failure_hook.await_args.kwargs + assert closed["original_exception"] is (crash if ending == "crash" else refusal) + assert closed["request_data"]["litellm_metadata"].get("standard_logging_guardrail_information") == ( + None if ending == "crash" else [{"guardrail_cost": 0.2}, {"guardrail_cost": 0.4}] + ) + assert release.await_count == 1 + assert release.await_args.kwargs["budget_reservation"] == budget_reservation + assert [row["request_id"] for row in rows_while_open] == [ + write["kwargs"]["litellm_call_id"] for write in writes[:2] + ] + assert len({row["request_id"] for row in rows_while_open}) == 2 + for turn, write, row in zip(("two", "four"), writes, rows_while_open): + assert write["completion_response"] is rejections[turn] + assert sent_at[turn] <= write["start_time"] + assert row["spend"] == guardrail_costs[turn] + assert row["call_type"] == "_aresponses_websocket" + assert row["model_group"] == "mini-group" + assert json.loads(row["request_tags"]) == ["ws-tag"] + assert json.loads(row["proxy_server_request"]) == {"input": turn, "model": "gpt-4o-mini"} + assert writes[0]["start_time"] < sent_at["four"] <= writes[1]["start_time"] + close_writes = writes[2:] + if ending != "crash": + assert close_writes == [] + return + assert [write["completion_response"] for write in close_writes] == [crash] + assert _spend_log_row(close_writes[0])["spend"] == 0.0 + + @pytest.mark.asyncio + async def test_a_refused_turn_row_shares_the_session_id_of_the_turns_the_router_relays(self): + from datetime import datetime + from unittest.mock import create_autospec + + from litellm.exceptions import RateLimitError + from litellm.litellm_core_utils.get_litellm_params import get_litellm_params + from litellm.litellm_core_utils.litellm_logging import Logging, StandardLoggingPayloadSetup + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.db.db_spend_update_writer import DBSpendUpdateWriter + from litellm.proxy.hooks.proxy_track_cost_callback import ProxyDBLogger + from litellm.proxy.response_api_endpoints.endpoints import responses_websocket_endpoint + from litellm.responses.streaming_iterator import drain_logging_tasks + from litellm.router_rust.invoke.endpoints import generic_plan, prepare_request + + socket_logging_obj = Logging( + model="gpt-4o-mini", + messages=[], + stream=True, + call_type="_aresponses_websocket", + start_time=datetime.now(), + litellm_call_id="socket-call", + function_id="socket-call", + ) + ws = MagicMock() + ws.headers = {} + ws.query_params = {} + ws.scope = {"headers": []} + ws.url = "ws://testserver/v1/responses" + ws.accept = AsyncMock() + ws.receive_text = AsyncMock( + return_value=json.dumps({"type": "response.create", "model": "gpt-4o-mini", "input": []}) + ) + ws.close = AsyncMock() + processor = MagicMock() + processor.common_processing_pre_call_logic = AsyncMock( + return_value=( + {"model": "gpt-4o-mini", "litellm_metadata": {}, "litellm_logging_obj": socket_logging_obj}, + socket_logging_obj, + ) + ) + refusal = RateLimitError(message="crossed budget", llm_provider="", model="") + spend_writer = create_autospec(DBSpendUpdateWriter, instance=True) + relayed_turn_session_ids = [] + + def route_request(data, **kwargs): + async def relay(): + relayed = prepare_request(generic_plan("_aresponses_websocket"), data, "gpt-4o-mini", {}) + relayed_turn_session_ids.append( + StandardLoggingPayloadSetup.get_standard_logging_payload_trace_id( + logging_obj=socket_logging_obj, litellm_params=get_litellm_params(**relayed) + ) + ) + guarded = await data["responses_ws_frame_guard"](json.dumps({"type": "response.create", "input": "hi"})) + await guarded.bill_refusal(refusal) + await drain_logging_tasks(timeout=1) + return None + + return relay() + + proxy_logging_obj = MagicMock(during_call_hook=AsyncMock()) + proxy_logging_obj.pre_call_hook = AsyncMock(side_effect=lambda **kwargs: kwargs["data"]) + proxy_logging_obj.post_call_failure_hook = AsyncMock() + proxy_logging_obj.arelease_responses_websocket_slot = AsyncMock(return_value=None) + + with ( + patch.object(litellm, "callbacks", [ProxyDBLogger(spend_writer=lambda: spend_writer)]), + patch( # test-quality-ok: first-frame model auth needs a live router and key table and has its own tests above + "litellm.proxy.response_api_endpoints.endpoints._enforce_responses_ws_first_frame_model_auth", + new_callable=AsyncMock, + ), + patch( # test-quality-ok: the pre-call processor needs a live proxy; the socket's logging object is its output + "litellm.proxy.response_api_endpoints.endpoints.ProxyBaseLLMRequestProcessing", + return_value=processor, + ), + patch( # test-quality-ok: routing is the seam where the router stamps the relayed turns' trace id + "litellm.proxy.route_llm_request.route_request", + new_callable=AsyncMock, + side_effect=route_request, + ), + patch( # test-quality-ok: the failure hook is the proxy's only path to a failed spend log row + "litellm.proxy.proxy_server.proxy_logging_obj", + proxy_logging_obj, + ), + ): + await responses_websocket_endpoint( + websocket=ws, model=None, user_api_key_dict=UserAPIKeyAuth(api_key="sk-socket") + ) + + spend_writer.update_database.assert_awaited_once() + refused_row = _spend_log_row(spend_writer.update_database.await_args.kwargs) + assert relayed_turn_session_ids == [refused_row["session_id"]] @pytest.mark.asyncio @pytest.mark.parametrize("ending", ["client_closed", "crash"]) @@ -3954,7 +4344,7 @@ class TestResponsesWSFirstFrameModelAuth: def route_request(data, **kwargs): async def relay(): guarded = await data["responses_ws_frame_guard"](json.dumps({"type": "response.create", "input": "two"})) - guarded.bill_refusal(refusal) + await guarded.bill_refusal(refusal) if ending == "crash": raise crash return refusal @@ -4003,7 +4393,7 @@ class TestResponsesWSFirstFrameModelAuth: assert booked["original_exception"] is (crash if ending == "crash" else refusal) @pytest.mark.asyncio - async def test_a_crashed_socket_whose_failure_row_cannot_be_built_still_gets_its_error_frame_and_close(self): + async def test_a_crashed_socket_whose_failure_hooks_fail_still_gets_its_error_frame_and_close(self): from litellm.exceptions import RateLimitError from litellm.proxy.response_api_endpoints.endpoints import responses_websocket_endpoint @@ -4029,20 +4419,16 @@ class TestResponsesWSFirstFrameModelAuth: def route_request(data, **kwargs): async def relay(): guarded = await data["responses_ws_frame_guard"](json.dumps({"type": "response.create", "input": "two"})) - guarded.bill_refusal(refusal) - data["litellm_metadata"]["standard_logging_guardrail_information"] = "written by a broken callback" + await guarded.bill_refusal(refusal) raise RuntimeError("relay crashed") return relay() proxy_logging_obj = MagicMock(during_call_hook=AsyncMock()) proxy_logging_obj.pre_call_hook = AsyncMock( - side_effect=lambda user_api_key_dict, data, call_type, guardrails_only: { - **data, - "litellm_metadata": {"standard_logging_guardrail_information": [{"guardrail_name": "turn-check"}]}, - } + side_effect=lambda user_api_key_dict, data, call_type, guardrails_only: data ) - proxy_logging_obj.post_call_failure_hook = AsyncMock() + proxy_logging_obj.post_call_failure_hook = AsyncMock(side_effect=RuntimeError("spend log db down")) proxy_logging_obj.arelease_responses_websocket_slot = AsyncMock(return_value=None) with ( @@ -4068,6 +4454,7 @@ class TestResponsesWSFirstFrameModelAuth: responses_websocket_endpoint(websocket=ws, model=None, user_api_key_dict=MagicMock()), timeout=1 ) + proxy_logging_obj.post_call_failure_hook.assert_awaited_once() assert [json.loads(frame)["error"]["message"] for frame in sent] == ["relay crashed"] assert closes == [(1011, "Internal server error")] diff --git a/tests/unit/responses/test_responses_websocket_all_providers.py b/tests/unit/responses/test_responses_websocket_all_providers.py index cb7208f9b9b..c72e32e01de 100644 --- a/tests/unit/responses/test_responses_websocket_all_providers.py +++ b/tests/unit/responses/test_responses_websocket_all_providers.py @@ -1694,7 +1694,7 @@ def _guard_blocking(word: str): return guard -def _guard_recording(bill_refusal: MagicMock): +def _guard_recording(bill_refusal: AsyncMock): from litellm.responses.streaming_iterator import GuardedResponsesWsFrame async def guard(frame: str) -> GuardedResponsesWsFrame: @@ -2568,14 +2568,14 @@ class TestWebSocketTurnAdmission: @pytest.mark.asyncio - async def test_native_relay_leaves_a_refused_turn_s_guardrail_records_to_the_close_time_failure_row(self): + async def test_native_relay_bills_a_refused_turn_and_returns_it_as_the_close_time_failure(self): from unittest.mock import AsyncMock from litellm.exceptions import RateLimitError first, second, third = _turn_frames() refusal: Final = RateLimitError(message="crossed budget", llm_provider="", model="") - bill_refusal: Final = MagicMock() + bill_refusal: Final = AsyncMock() backend_ws: Final = MagicMock() backend_ws.send = AsyncMock() backend_ws.close = AsyncMock() @@ -2592,9 +2592,41 @@ class TestWebSocketTurnAdmission: close_time_failure: Final = await handler.bidirectional_forward() assert [json.loads(sent.args[0])["input"] for sent in backend_ws.send.await_args_list] == ["one", "three"] - bill_refusal.assert_called_once_with(refusal) + bill_refusal.assert_awaited_once_with(refusal) assert close_time_failure is refusal + @pytest.mark.asyncio + async def test_native_relay_forwards_no_later_turn_until_the_refused_turn_is_billed(self): + from unittest.mock import AsyncMock + + from litellm.exceptions import RateLimitError + + first, second, third = _turn_frames() + events: Final[list[str]] = [] + + async def bill_refusal(rejection: Exception) -> None: + await asyncio.sleep(0.01) + events.append("billed") + + backend_ws: Final = MagicMock() + backend_ws.send = AsyncMock(side_effect=lambda frame: events.append(json.loads(frame)["input"])) + backend_ws.close = AsyncMock() + backend_ws.recv = AsyncMock(side_effect=asyncio.Event().wait) + handler: Final = _make_streaming( + websocket=_ClientSocket(second, third), + backend_ws=backend_ws, + request_data={}, + first_message=first, + frame_guard=_guard_recording(AsyncMock(side_effect=bill_refusal)), + turn_admission=AsyncMock( + side_effect=[RateLimitError(message="crossed budget", llm_provider="", model=""), {}] + ), + ) + + _ = await handler.bidirectional_forward() + + assert events == ["one", "billed", "three"] + @pytest.mark.asyncio async def test_a_refused_turn_is_recorded_even_when_the_relay_is_cancelled_while_answering_it(self): from unittest.mock import AsyncMock @@ -2603,7 +2635,7 @@ class TestWebSocketTurnAdmission: first, second, third = _turn_frames() refusal: Final = RateLimitError(message="crossed budget", llm_provider="", model="") - bill_refusal: Final = MagicMock() + bill_refusal: Final = AsyncMock() backend_ws: Final = MagicMock() backend_ws.send = AsyncMock() backend_ws.close = AsyncMock() @@ -2624,10 +2656,10 @@ class TestWebSocketTurnAdmission: with pytest.raises(asyncio.CancelledError): await relay - bill_refusal.assert_called_once_with(refusal) + bill_refusal.assert_awaited_once_with(refusal) @pytest.mark.asyncio - async def test_managed_relay_leaves_a_refused_turn_s_guardrail_records_to_the_close_time_failure_row( + async def test_managed_relay_bills_a_refused_turn_and_returns_it_as_the_close_time_failure( self, monkeypatch: pytest.MonkeyPatch ): from unittest.mock import AsyncMock @@ -2651,7 +2683,7 @@ class TestWebSocketTurnAdmission: monkeypatch.setattr(litellm, "aresponses", fake_aresponses) first, second, third = _turn_frames() refusal: Final = RateLimitError(message="crossed budget", llm_provider="", model="") - bill_refusal: Final = MagicMock() + bill_refusal: Final = AsyncMock() handler: Final = ManagedResponsesWebSocketHandler( websocket=_ClientSocket(second, third), model="test-model", @@ -2672,7 +2704,7 @@ class TestWebSocketTurnAdmission: close_time_failure: Final = await handler.run() assert answered == ["one", "three"] - bill_refusal.assert_called_once_with(refusal) + bill_refusal.assert_awaited_once_with(refusal) assert close_time_failure is refusal @pytest.mark.asyncio