diff --git a/litellm/proxy/hooks/proxy_track_cost_callback.py b/litellm/proxy/hooks/proxy_track_cost_callback.py index 7f37f8cb050..31c3b4a3676 100644 --- a/litellm/proxy/hooks/proxy_track_cost_callback.py +++ b/litellm/proxy/hooks/proxy_track_cost_callback.py @@ -72,6 +72,8 @@ from litellm.proxy.spend_tracking.spend_tracking_utils import ( # noqa: F401 # should_store_prompts_for_call, ) from litellm.proxy.utils import ( + FAILURE_ROW_ALREADY_WRITTEN, + FAILURE_ROW_ALREADY_WRITTEN_KEY, GUARDRAIL_ENDPOINT_FAILURE, GUARDRAIL_ENDPOINT_FAILURE_KEY, REJECTED_BEFORE_REQUEST_SETUP, @@ -506,7 +508,8 @@ class _ProxyDBLogger(CustomLogger): or request_fields.get(GUARDRAIL_ENDPOINT_FAILURE_KEY) is GUARDRAIL_ENDPOINT_FAILURE ) if ( - ProxyDBLogger._should_track_errors_in_db() is False + request_fields.get(FAILURE_ROW_ALREADY_WRITTEN_KEY) is FAILURE_ROW_ALREADY_WRITTEN + or ProxyDBLogger._should_track_errors_in_db() is False and not completed_cost or ProxyDBLogger._skips_pre_routing_rejections() and is_pre_routing_rejection(original_exception, _REQUEST_DATA.validate_python(request_data)) diff --git a/litellm/proxy/response_api_endpoints/endpoints.py b/litellm/proxy/response_api_endpoints/endpoints.py index 2f0e3f2bcfb..9a3c0b8eafb 100644 --- a/litellm/proxy/response_api_endpoints/endpoints.py +++ b/litellm/proxy/response_api_endpoints/endpoints.py @@ -5,6 +5,7 @@ import json import uuid from collections.abc import AsyncGenerator, AsyncIterator, Awaitable, Callable, Mapping, Sequence from dataclasses import dataclass +from datetime import datetime from enum import Enum from functools import partial from itertools import chain @@ -36,9 +37,19 @@ import litellm from litellm._logging import verbose_proxy_logger from litellm.anthropic_interface.exceptions.exception_mapping_utils import AnthropicExceptionMapping from litellm.caching.dual_cache import DualCache -from litellm.constants import EMPTY_MAPPING, MANAGED_BACKGROUND_RESPONSE_METADATA_KEY, TERMINAL_RESPONSE_STATUSES +from litellm.constants import ( + CALLER_SESSION_ID_METADATA_KEY, + EMPTY_MAPPING, + MANAGED_BACKGROUND_RESPONSE_METADATA_KEY, + SESSION_ID_OMITTED_METADATA_KEY, + TERMINAL_RESPONSE_STATUSES, +) from litellm.exceptions import EndpointNotSupportedByModelError, RateLimitError -from litellm.integrations.custom_guardrail import ModifyResponseException, guardrail_request_data_with_streaming +from litellm.integrations.custom_guardrail import ( + ModifyResponseException, + guardrail_request_data_with_streaming, + without_server_streaming_classification, +) from litellm.litellm_core_utils.core_helpers import as_str_mapping, get_metadata_variable_name_from_kwargs from litellm.litellm_core_utils.internal_call_metadata import is_background_response, requests_background from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj @@ -91,6 +102,8 @@ from litellm.proxy.hooks.responses_id_security import ResponsesIDSecurity, decry from litellm.proxy.litellm_pre_call_utils import ( _CLIENT_PRICING_METADATA_FIELDS, # pyright: ignore[reportPrivateUsage] # the HTTP path's own strip list _UNTRUSTED_METADATA_CONTROL_FIELDS, # pyright: ignore[reportPrivateUsage] # the HTTP path's own strip list + _UNTRUSTED_ROOT_CONTROL_FIELDS, # pyright: ignore[reportPrivateUsage] # the HTTP path's own strip list + ROUTER_RESERVED_METADATA_FIELDS, add_guardrails_from_policy_engine, is_chat_completions_body, ) @@ -98,7 +111,12 @@ from litellm.proxy.pass_through_endpoints.llm_provider_handlers.batch_attributio request_tags_from_request_data, ) from litellm.proxy.route_llm_request import raise_if_required_body_param_missing -from litellm.proxy.utils import count_request_input_tokens, handle_exception_on_proxy +from litellm.proxy.utils import ( + FAILURE_ROW_ALREADY_WRITTEN, + FAILURE_ROW_ALREADY_WRITTEN_KEY, + count_request_input_tokens, + handle_exception_on_proxy, +) from litellm.responses.litellm_completion_transformation.compaction_items import messages_since_newest_compaction from litellm.responses.litellm_completion_transformation.count_request import MODEL_FILE_ID_MAPPING_FIELD from litellm.responses.retrieve_stream_query import RETRIEVE_INCLUDE_QUERY_KEYS, RETRIEVE_STREAM_QUERY_KEYS @@ -117,7 +135,7 @@ from litellm.types.llms.openai import ( ) from litellm.types.responses.main import DeleteResponseResult from litellm.types.token_count import TokenCountSourceFields -from litellm.types.utils import ADDRESSED_RESPONSE_ID_FIELD, TokenCountResponse +from litellm.types.utils import ADDRESSED_RESPONSE_ID_FIELD, CallTypes, TokenCountResponse from litellm.utils import get_base_model_from_metadata if TYPE_CHECKING: @@ -2225,6 +2243,22 @@ _CLIENT_GUARDRAIL_SELECTORS: Final = ("guardrails", "guardrail_config", "policie _METADATA_GUARDRAIL_SELECTORS: Final = ("guardrails", "guardrail_config") _PROXY_ONLY_FRAME_FIELDS: Final = (*_CLIENT_GUARDRAIL_SELECTORS, "include_guardrail_response") _UNFORWARDED_FRAME_FIELDS: Final = (*_GUARDED_FRAME_FIELDS, *_PROXY_ONLY_FRAME_FIELDS, "litellm_metadata") +_UNTRUSTED_TURN_FIELDS: Final = frozenset( + (*_UNTRUSTED_ROOT_CONTROL_FIELDS, "standard_callback_dynamic_params", "litellm_params") +) +_SOCKET_ATTRIBUTION_METADATA_FIELDS: Final = ( + "agent_id", + "actor_agent_id", + "target_agent_id", + "billing_agent_id", + "agent_execution_mode", + "verified_human_user_id", + "session_id", + SESSION_ID_OMITTED_METADATA_KEY, + CALLER_SESSION_ID_METADATA_KEY, +) +_SOCKET_OWNED_METADATA_FIELDS: Final = frozenset(("tags", "model_group", *_SOCKET_ATTRIBUTION_METADATA_FIELDS)) +_TURN_ROW_FIELDS: Final = ("model", "litellm_call_id") _OBJECTS: Final = TypeAdapter(list[object]) _TURN_COPIED_CONTAINERS: Final = (list, dict) @@ -2307,9 +2341,30 @@ def _responses_ws_frame_guard( } model: Final = admitted.get("model") - # a blocked frame is billed as its own failed request; only the first turn settles the admission reservation + # a blocked or refused turn is billed as its own failed request; the socket settles the admission reservation rejected_frame_auth: Final = user_api_key_dict.model_copy(update={"budget_reservation": None}) + failure_row: Final = partial(_turn_failure_row, metadata_key, admitted) + + async def bill_refusal( + turn: Mapping[str, object], + guardrail_information: tuple[Mapping[str, object], ...], + started_at: datetime, + refusal: Exception, + ) -> None: + record_refused_turn(guardrail_information, refusal) + await asyncio.shield( + spawn_logging_task( + asyncio.get_running_loop(), + _bill_rejected_frame( + rejected_frame_auth, + refusal, + {**failure_row(turn, started_at), "proxy_server_request": _turn_stored_request(metadata_key, turn)}, + ), + task_name="Responses WebSocket refused turn row", + ) + ) + async def rejection( exception: Exception, data: dict[str, object], # mutable-ok: billed @@ -2319,6 +2374,7 @@ def _responses_ws_frame_guard( return ResponsesWsFrameRejection(proxy_exception.message, proxy_exception.provider_specific_fields) async def guard(frame: str) -> GuardedResponsesWsFrame: + started_at: Final = datetime.now() # noqa: DTZ005 # the request pipeline logs naive local start times frame_obj: Final = _WS_FRAME.validate_json(frame) nested: Final = frame_obj.get("response") body: Final = _WS_FRAME.validate_python(nested) if isinstance(nested, dict) else frame_obj @@ -2347,7 +2403,7 @@ def _responses_ws_frame_guard( ), } except Exception as e: - raise await rejection(e, turn_data) from e + raise await rejection(e, failure_row(turn_data, started_at)) from e try: guarded_data: Final = await proxy_logging_obj.pre_call_hook( user_api_key_dict=user_api_key_dict, data=frame_data, call_type="aresponses", guardrails_only=True @@ -2361,13 +2417,13 @@ def _responses_ws_frame_guard( data=guarded_data, user_api_key_dict=user_api_key_dict, call_type="aresponses", guardrails_only=True ) except Exception as e: - raise await rejection(e, frame_data) from e + raise await rejection(e, failure_row(frame_data, started_at)) from e if guarded_data.get("model") != model: rerouted: Final = ValueError( f"a guardrail rerouted this turn to {guarded_data.get('model')!r}; an open Responses WebSocket" f" is bound to {model!r}, send this turn over HTTP /v1/responses" ) - await _bill_rejected_frame(rejected_frame_auth, rerouted, frame_data) + await _bill_rejected_frame(rejected_frame_auth, rerouted, failure_row(frame_data, started_at)) raise rerouted rewritten: Final = ( {**_forwarded_frame_fields(frame_obj), "response": {**_forwarded_frame_fields(body), **guarded_fields}} @@ -2379,12 +2435,73 @@ def _responses_ws_frame_guard( frame=json.dumps(rewritten), guardrail_information=guardrail_information, pii_tokens=pii_tokens, - bill_refusal=partial(record_refused_turn, guardrail_information), + bill_refusal=partial(bill_refusal, guarded_data, guardrail_information, started_at), ) return guard +def _socket_trace_id(socket: Mapping[str, object]) -> Mapping[str, str]: + logging_obj: Final = socket.get("litellm_logging_obj") + return ( + MappingProxyType({"litellm_trace_id": logging_obj.litellm_trace_id}) + if isinstance(logging_obj, LiteLLMLoggingObj) + else MappingProxyType({}) + ) + + +def _turn_failure_row( + metadata_key: str, + socket: Mapping[str, object], + turn: Mapping[str, object], + started_at: datetime, +) -> dict[str, object]: # mutable-ok: the spend logger stamps the failure row on the request dict + socket_bucket: Final = socket.get(metadata_key) + socket_request: Final = socket.get("proxy_server_request") + turn_bucket: Final = turn.get(metadata_key) + turn_metadata: Final = _WS_FRAME.validate_python(turn_bucket) if isinstance(turn_bucket, dict) else {} + request_tags: Final = StandardLoggingPayloadSetup.get_request_tags( + {metadata_key: turn_metadata}, + _PROXY_SERVER_REQUEST_ADAPTER.validate_python(socket_request) if isinstance(socket_request, dict) else {}, + ) + socket_metadata: Final = _WS_FRAME.validate_python(socket_bucket) if isinstance(socket_bucket, dict) else {} + model_group: Final = socket_metadata.get("model_group") + client_metadata: Final = _without_untrusted_metadata("metadata", turn.get("metadata")) + return { + **{key: turn[key] for key in _TURN_ROW_FIELDS if key in turn}, + **_socket_trace_id(socket), + "metadata": { + **( + { + name: item + for name, item in _WS_FRAME.validate_python(client_metadata).items() + if name not in _SOCKET_OWNED_METADATA_FIELDS + } + if isinstance(client_metadata, dict) + else {} + ), + **{name: socket_metadata[name] for name in _SOCKET_ATTRIBUTION_METADATA_FIELDS if name in socket_metadata}, + }, + metadata_key: { + **turn_metadata, + **({"tags": request_tags} if request_tags else {}), + **({"model_group": model_group} if isinstance(model_group, str) else {}), + }, + "call_type": CallTypes.aresponses_websocket.value, + "start_time": started_at, + } + + +def _turn_stored_request(metadata_key: str, turn: Mapping[str, object]) -> Mapping[str, object]: + return { + "body": { + key: _without_untrusted_metadata(key, value) + for key, value in without_server_streaming_classification(turn).items() + if key not in _UNTRUSTED_TURN_FIELDS and key not in (metadata_key, "litellm_call_id") + } + } + + def _reject_guarded_field_beside_nested_response(frame: Mapping[str, object], nested: object) -> None: """Only the nested ``response`` body runs through the guardrails, so a guarded field beside it would go upstream unguarded""" @@ -2550,7 +2667,8 @@ def _turn_copy(value: object) -> object: def _without_untrusted_metadata(key: str, value: object) -> object: - """Like the HTTP path, drop the proxy-owned ``user_api_key_*`` and guardrail control keys a client may forge.""" + """Like the HTTP path, drop the proxy-owned ``user_api_key_*``, guardrail control and router keys a client may + forge.""" if key not in _CLIENT_METADATA_KEYS or not isinstance(value, dict): return value return { @@ -2559,6 +2677,7 @@ def _without_untrusted_metadata(key: str, value: object) -> object: if not name.startswith("user_api_key_") and name not in _UNTRUSTED_METADATA_CONTROL_FIELDS and name not in _CLIENT_PRICING_METADATA_FIELDS + and name not in ROUTER_RESERVED_METADATA_FIELDS and name not in ("headers", "pii_tokens", _GUARDRAIL_INFORMATION_KEY, *_CLIENT_GUARDRAIL_SELECTORS) } @@ -2591,9 +2710,10 @@ _NO_PII_TOKENS: Final[Mapping[str, str]] = MappingProxyType({}) @dataclass(slots=True) class _RefusedTurns: - """Turns the deployment refused after their guardrails passed. The socket's close-time failure row bills their - guardrail records, so its failure hooks, model group and call type cover those turns too, and the first refusal - is that row's exception when the session limit cancels the relay before it returns one.""" + """Each turn the deployment refused wrote its own failure row when refused. The first refusal is still the + socket's close-time failure, so its failure hooks run once, even when the session limit cancels the relay before + it returns one, and see every refused turn's guardrail records, but they write no second row for them. A close on + any other error writes that error's own row, which must not bill the refused turns' guardrail records again.""" first: Exception | None = None records: tuple[Mapping[str, object], ...] = () @@ -2603,8 +2723,8 @@ class _RefusedTurns: if self.first is None: self.first = refusal - def billed_on(self, request_data: Mapping[str, object]) -> Mapping[str, object]: - if not self.records: + def failure_hook_data(self, exception: Exception, request_data: Mapping[str, object]) -> Mapping[str, object]: + if exception is not self.first: return request_data metadata_key: Final = get_metadata_variable_name_from_kwargs(request_data) bucket: Final = request_data.get(metadata_key) @@ -2612,7 +2732,12 @@ class _RefusedTurns: recorded: Final = (*_recorded_guardrail_information(fields), *self.records) return { **request_data, - metadata_key: {**fields, _GUARDRAIL_INFORMATION_KEY: [dict(record) for record in recorded]}, + FAILURE_ROW_ALREADY_WRITTEN_KEY: FAILURE_ROW_ALREADY_WRITTEN, + **( + {metadata_key: {**fields, _GUARDRAIL_INFORMATION_KEY: [dict(record) for record in recorded]}} + if recorded + else {} + ), } @@ -2727,6 +2852,7 @@ async def _responses_websocket_session( refused_turns: Final = _RefusedTurns() routed_data: Final = dict( data, + **_socket_trace_id(data), user_api_key_dict=user_api_key_dict, responses_ws_frame_guard=_responses_ws_frame_guard( data, user_api_key_dict, proxy_logging_obj, refused_turns.add @@ -2734,16 +2860,16 @@ async def _responses_websocket_session( **_routing_hints_from_first_ws_frame(first_message), ) - async def write_failure_row(exception: Exception) -> None: + async def run_failure_hooks(exception: Exception) -> None: await proxy_logging_obj.post_call_failure_hook( user_api_key_dict=user_api_key_dict, original_exception=exception, - request_data=dict(refused_turns.billed_on(routed_data)), + request_data=dict(refused_turns.failure_hook_data(exception, routed_data)), ) def bill_socket_failure(exception: Exception) -> asyncio.Task[object]: return spawn_logging_task( - asyncio.get_running_loop(), write_failure_row(exception), task_name="Responses WebSocket failure row" + asyncio.get_running_loop(), run_failure_hooks(exception), task_name="Responses WebSocket failure row" ) async def relay_and_bill_failure() -> asyncio.Task[object] | None: diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 8104606db60..15b47fb04a4 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -1833,6 +1833,10 @@ object, meaning the request died (auth, body parsing, model resolution) before t object rather than ``True`` because ``request_data`` is the client body: JSON cannot forge this identity, so a client can neither hide a guardrail block nor force a row by sending the key or a ``litellm_call_id`` itself.""" +FAILURE_ROW_ALREADY_WRITTEN_KEY: Final = "litellm_failure_row_already_written" +FAILURE_ROW_ALREADY_WRITTEN: Final = object() +"""An object, like ``REJECTED_BEFORE_REQUEST_SETUP``, so the client body cannot forge it.""" + GUARDRAIL_ENDPOINT_FAILURE_KEY: Final = "litellm_guardrail_endpoint_failure" GUARDRAIL_ENDPOINT_FAILURE: Final = object() """Stored under ``GUARDRAIL_ENDPOINT_FAILURE_KEY`` by ``/apply_guardrail`` when a request it had set up fails, so the diff --git a/litellm/responses/streaming_iterator.py b/litellm/responses/streaming_iterator.py index dca93436efa..c872e456d6e 100644 --- a/litellm/responses/streaming_iterator.py +++ b/litellm/responses/streaming_iterator.py @@ -2323,12 +2323,12 @@ class GuardedResponsesWsFrame: """A ``response.create`` frame after the proxy's pre_call guardrails ran on it: the frame to forward (guardrails may rewrite ``input`` and ``instructions``), the guardrail records that bill against this turn only, the Presidio ``pii_tokens`` the walk masked with, which the handler needs to unmask the provider's reply, and how - to carry those guardrail records onto the connection's failure row when the deployment then refuses the turn.""" + to bill the turn as its own failed request when the deployment then refuses it.""" frame: str guardrail_information: tuple[Mapping[str, object], ...] = () pii_tokens: Mapping[str, str] = field(default_factory=lambda: MappingProxyType({})) - bill_refusal: Callable[[Exception], None] | None = None + bill_refusal: Callable[[Exception], Awaitable[None]] | None = None ResponsesWsFrameGuard: TypeAlias = Callable[[str], Awaitable[GuardedResponsesWsFrame]] @@ -2361,10 +2361,9 @@ def _frame_rejection_event(rejection: Exception) -> str: async def _reject_guarded_turn( websocket: ResponsesClientWebSocket, guarded: GuardedResponsesWsFrame, rejection: Exception ) -> None: - """Answers the refused turn and leaves its guardrail records to the connection's close-time failure row, which - runs every failure hook once per socket, like the first refusal it records.""" + """Bills the refused turn as its own failed request, then answers it on the open socket.""" if guarded.bill_refusal is not None: - guarded.bill_refusal(rejection) + await guarded.bill_refusal(rejection) with contextlib.suppress(Exception): await websocket.send_text(_turn_rejection_frame(rejection)) diff --git a/tests/integration/routing/test_responses_websocket_turn_admission.py b/tests/integration/routing/test_responses_websocket_turn_admission.py index 2de556e90e4..35647f4b256 100644 --- a/tests/integration/routing/test_responses_websocket_turn_admission.py +++ b/tests/integration/routing/test_responses_websocket_turn_admission.py @@ -2,6 +2,7 @@ import asyncio import hashlib import os import threading +import time import uuid from collections.abc import AsyncIterator, Callable, Iterator, Mapping from contextlib import asynccontextmanager, contextmanager @@ -26,6 +27,7 @@ from integration._support.client import ( list_value, object_value, string_value, + until_asserted, ) from integration._support.process import graceful_stop_seconds, owned_proxy from integration._support.responses_ws_peer import ( @@ -54,8 +56,10 @@ TURN_SPEND: Final = USAGE["input_tokens"] * INPUT_RATE + USAGE["output_tokens"] TURN_TOKENS: Final = USAGE["total_tokens"] MANAGED_PROVIDER: Final = "mistral" SPEND_ROWS_FOR_KEY: Final = ( - 'SELECT request_id, spend, status, model_group, metadata, request_tags FROM "LiteLLM_SpendLogs" WHERE api_key = %s' + "SELECT request_id, spend, status, model_group, call_type, metadata, request_tags, proxy_server_request," + ' extract(epoch from "startTime")::float8 AS started_at FROM "LiteLLM_SpendLogs" WHERE api_key = %s' ) +STORED_TIME_RESOLUTION_SECONDS: Final = 0.001 TURN_CHECK_FAULT: Final = "turn_check_fault" FAULTING_TURN_CHECK: Final = f""" from litellm.integrations.custom_logger import CustomLogger @@ -103,6 +107,7 @@ def _config(directory: Path, router_settings: Mapping[str, JsonValue], callbacks yaml.safe_dump( { **base, + "general_settings": {**object_value(base["general_settings"]), "store_prompts_in_spend_logs": True}, "litellm_settings": { **object_value(base["litellm_settings"]), **({"callbacks": list(callbacks)} if callbacks else {}), @@ -275,17 +280,54 @@ async def _until(read: Callable[[], float], satisfied: Callable[[float], bool]) return await asyncio.to_thread(eventually, read, satisfied, 15) -def _failure_rows(gateway: Gateway, key: str) -> list[dict[str, JsonValue]]: - rows: Final = eventually( - lambda: gateway.rows(SPEND_ROWS_FOR_KEY, (_key_hash(key),)), - lambda found: any(row["status"] == "failure" for row in found), - seconds=70, - ) - return [row for row in rows if row["status"] == "failure"] +def _failures_now(gateway: Gateway, key: str) -> list[dict[str, JsonValue]]: + return [row for row in gateway.rows(SPEND_ROWS_FOR_KEY, (_key_hash(key),)) if row["status"] == "failure"] + + +def _failure_rows(gateway: Gateway, key: str, count: int = 1) -> list[dict[str, JsonValue]]: + return eventually(lambda: _failures_now(gateway, key), lambda found: len(found) >= count, seconds=70) + + +def _settled_failure_rows(gateway: Gateway, key: str, count: int) -> list[dict[str, JsonValue]]: + return held(lambda: _failures_now(gateway, key), lambda found: len(found) == count, holding=5, seconds=70) + + +@dataclass(frozen=True, slots=True) +class AnsweredTurn: + text: str + sent_at: float + answered_at: float + received: tuple[dict[str, JsonValue], ...] + + +async def _timed_turn(connection: websockets.ClientConnection, model: str, label: str) -> AnsweredTurn: + text: Final = f"{label} {uuid.uuid4().hex}" + sent_at: Final = time.time() + received: Final = await _say(connection, model, text) + return AnsweredTurn(text, sent_at, time.time(), received) + + +def _stored_text(row: Mapping[str, JsonValue]) -> str: + message: Final = object_value(list_value(object_value(row["proxy_server_request"])["input"])[0]) + return string_value(object_value(list_value(message["content"])[0])["text"]) + + +def _assert_logged_as_its_own_failure( + row: Mapping[str, JsonValue], turn: AnsweredTurn, deployment: Deployment +) -> dict[str, JsonValue]: + assert _stored_text(row) == turn.text, row + started_at: Final = row["started_at"] + assert isinstance(started_at, float), row + assert turn.sent_at - STORED_TIME_RESOLUTION_SECONDS <= started_at, (turn, row) + assert started_at <= turn.answered_at + STORED_TIME_RESOLUTION_SECONDS, (turn, row) + assert row["model_group"] == deployment.name, row + assert row["call_type"] == "_aresponses_websocket", row + assert float(row["spend"]) == 0.0, row + return object_value(object_value(row["metadata"])["error_information"]) @pytest.mark.parametrize("relay", ("native", "managed")) -def test_a_turn_past_the_deployment_budget_is_refused_on_the_open_socket_and_logged_once_as_its_failure( +def test_each_turn_past_the_deployment_budget_is_refused_on_the_open_socket_and_logged_as_its_own_failure( limited: Gateway, cert: tuple[Path, Path], relay: str ) -> None: with responses_peer(cert) as peer, limited.scenario() as scenario: @@ -294,25 +336,71 @@ def test_a_turn_past_the_deployment_budget_is_refused_on_the_open_socket_and_log key: Final = scenario.key(models=[deployment.name]) tag: Final = f"ws-turn-{uuid.uuid4().hex}" - async def session() -> tuple[tuple[dict[str, JsonValue], ...], ...]: + async def session() -> tuple[tuple[dict[str, JsonValue], ...], tuple[AnsweredTurn, ...], list[str]]: async with _socket(limited, key, deployment.name, {"x-litellm-tags": tag}) as connection: first: Final = await _say(connection, deployment.name) await _until(lambda: _deployment_spend(deployment), lambda spent: spent > TURN_SPEND / 2) - return (first, await _say(connection, deployment.name), await _say(connection, deployment.name)) + refused: Final = ( + await _timed_turn(connection, deployment.name, "second"), + await _timed_turn(connection, deployment.name, "third"), + ) + logged_while_open: Final = await asyncio.to_thread(_failure_rows, limited, key, len(refused)) + return first, refused, [string_value(row["request_id"]) for row in logged_while_open] - first, second, third = asyncio.run(session()) + first, refused, logged_while_open = asyncio.run(session()) completed(first) - _assert_refused(second) - _assert_refused(third) + for answered in refused: + _assert_refused(answered.received) + assert deployment.forwarded_turns() == 1 + failures: Final = _settled_failure_rows(limited, key, len(refused)) + assert sorted(string_value(row["request_id"]) for row in failures) == sorted(logged_while_open), failures + assert len(set(logged_while_open)) == len(refused), failures + by_text: Final = {_stored_text(row): row for row in failures} + for answered in refused: + row = by_text[answered.text] + error = _assert_logged_as_its_own_failure(row, answered, deployment) + assert str(error["error_code"]) == "429", error + assert tag in list_value(row["request_tags"]), row + + +def test_a_refused_turn_s_failure_row_leaves_the_socket_s_parallel_slot_held_until_the_socket_closes( + limited: Gateway, cert: tuple[Path, Path] +) -> None: + with responses_peer(cert) as peer, limited.scenario() as scenario: + deployment: Final = _native(scenario, peer, max_budget=TURN_SPEND / 2, budget_duration="1d") + key: Final = scenario.key(models=[deployment.name], max_parallel_requests=1) + + async def opened_once() -> tuple[tuple[dict[str, JsonValue], ...], int | None]: + async with _socket(limited, key, deployment.name) as connection: + received: Final = await _say(connection, deployment.name) + await asyncio.wait_for(connection.wait_closed(), timeout=20) + return received, connection.close_code + + async def second_socket_while_the_first_is_open() -> None: + async with _socket(limited, key, deployment.name) as holding: + completed(await _say(holding, deployment.name)) + await _until(lambda: _deployment_spend(deployment), lambda spent: spent > TURN_SPEND / 2) + _assert_refused(await _say(holding, deployment.name)) + await asyncio.to_thread(_failure_rows, limited, key) + second, second_close_code = await opened_once() + _assert_refused(second) + assert "max_parallel_requests" in string_value(object_value(second[0]["error"])["message"]), second + assert second_close_code == 1008, second + + def reopened_once_the_slot_frees() -> tuple[tuple[dict[str, JsonValue], ...], int | None]: + try: + received, close_code = asyncio.run(opened_once()) + except websockets.exceptions.InvalidStatus as refused: + raise AssertionError(refused) from refused + assert close_code != 1008, received + return received, close_code + + asyncio.run(second_socket_while_the_first_is_open()) + reopened, _ = until_asserted(reopened_once_the_slot_frees, seconds=15) + + _assert_refused(reopened) assert deployment.forwarded_turns() == 1 - failures: Final = _failure_rows(limited, key) - assert len(failures) == 1, failures - assert failures[0]["model_group"] == deployment.name, failures - assert tag in list_value(failures[0]["request_tags"]), failures - assert float(failures[0]["spend"]) == 0.0, failures - error: Final = object_value(object_value(failures[0]["metadata"])["error_information"]) - assert str(error["error_code"]) == "429", error def _set_turn_check_fault(gateway: Gateway, deployment: Deployment, fault: str) -> None: @@ -354,27 +442,30 @@ def test_a_turn_whose_admission_check_fails_gets_a_500_frame_and_the_socket_serv key: Final = scenario.key(models=[deployment.name]) fault: Final = f"turn check unavailable {uuid.uuid4().hex}" - async def session() -> tuple[tuple[dict[str, JsonValue], ...], ...]: + async def session() -> tuple[tuple[dict[str, JsonValue], ...], tuple[AnsweredTurn, ...], AnsweredTurn]: async with _socket(faulting, key, deployment.name) as connection: first: Final = await _say(connection, deployment.name) await asyncio.to_thread(_set_turn_check_fault, faulting, deployment, fault) - second: Final = await _say(connection, deployment.name) - third: Final = await _say(connection, deployment.name) + failed: Final = ( + await _timed_turn(connection, deployment.name, "second"), + await _timed_turn(connection, deployment.name, "third"), + ) await asyncio.to_thread(_set_turn_check_fault, faulting, deployment, "") - return first, second, third, await _say(connection, deployment.name) + return first, failed, await _timed_turn(connection, deployment.name, "fourth") - first, second, third, fourth = asyncio.run(session()) + first, failed, fourth = asyncio.run(session()) completed(first) - _assert_turn_check_failed(second, fault) - _assert_turn_check_failed(third, fault) - completed(fourth) + for answered in failed: + _assert_turn_check_failed(answered.received, fault) + completed(fourth.received) assert deployment.forwarded_turns() == 2 - failures: Final = _failure_rows(faulting, key) - assert len(failures) == 1, failures - assert failures[0]["model_group"] == deployment.name, failures - recorded: Final = object_value(object_value(failures[0]["metadata"])["error_information"]) - assert fault in string_value(recorded["error_message"]), recorded + failures: Final = _settled_failure_rows(faulting, key, len(failed)) + assert len({row["request_id"] for row in failures}) == len(failed), failures + by_text: Final = {_stored_text(row): row for row in failures} + for answered in failed: + recorded = _assert_logged_as_its_own_failure(by_text[answered.text], answered, deployment) + assert fault in string_value(recorded["error_message"]), recorded def test_a_turn_still_streaming_when_the_next_turns_check_fails_completes_and_is_billed( diff --git a/tests/unit/proxy/hooks/test_proxy_track_cost_callback.py b/tests/unit/proxy/hooks/test_proxy_track_cost_callback.py index 1b5af737f1b..46fdde5fe21 100644 --- a/tests/unit/proxy/hooks/test_proxy_track_cost_callback.py +++ b/tests/unit/proxy/hooks/test_proxy_track_cost_callback.py @@ -43,6 +43,8 @@ from litellm.proxy.spend_tracking.spend_event import SpendEventDecodeError, buil from litellm.proxy.spend_tracking.spend_event_producer import SpendEventProducer, UnixAddress from litellm.proxy.spend_tracking.spend_tracking_utils import get_logging_payload from litellm.proxy.utils import ( + FAILURE_ROW_ALREADY_WRITTEN, + FAILURE_ROW_ALREADY_WRITTEN_KEY, GUARDRAIL_ENDPOINT_FAILURE, GUARDRAIL_ENDPOINT_FAILURE_KEY, REJECTED_BEFORE_REQUEST_SETUP, @@ -4457,6 +4459,38 @@ async def test_async_post_call_failure_hook_disable_pre_routing_error_logs( assert len(writer.rows) == rows_written +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("request_data", "rows_written"), + [ + ({**_ROUTED_REQUEST, FAILURE_ROW_ALREADY_WRITTEN_KEY: FAILURE_ROW_ALREADY_WRITTEN}, 0), + ({**_ROUTED_REQUEST, FAILURE_ROW_ALREADY_WRITTEN_KEY: True}, 1), + (_ROUTED_REQUEST, 1), + ], + ids=["already-written", "client-spoofed-marker", "unmarked"], +) +async def test_async_post_call_failure_hook_settles_the_reservation_but_writes_no_second_row_for_a_billed_failure( + request_data: Mapping[str, object], rows_written: int +) -> None: + writer: Final = _RecordingSpendWriter() + user_api_key_dict: Final = UserAPIKeyAuth( + api_key="test_api_key", + request_route="/v1/responses", + budget_reservation={"reserved_cost": 0.5, "entries": []}, + ) + with patch( + "litellm.proxy.spend_tracking.budget_reservation.release_budget_reservation", new_callable=AsyncMock + ) as release: + await ProxyDBLogger(spend_writer=lambda: cast(DBSpendUpdateWriter, writer)).async_post_call_failure_hook( + request_data={**request_data, "metadata": dict[str, object]()}, + original_exception=litellm.RateLimitError(message="crossed budget", llm_provider="", model="gpt-4o-mini"), + user_api_key_dict=user_api_key_dict, + ) + assert release.await_count == 1 + assert release.await_args.kwargs["budget_reservation"] is user_api_key_dict.budget_reservation + assert len(writer.rows) == rows_written + + class _RecordedCharges: def __init__(self, outcomes: tuple[bool, ...]) -> None: self.outcomes = outcomes diff --git a/tests/unit/proxy/response_api_endpoints/test_endpoints.py b/tests/unit/proxy/response_api_endpoints/test_endpoints.py index bbb09550e7a..c4b8d60931f 100644 --- a/tests/unit/proxy/response_api_endpoints/test_endpoints.py +++ b/tests/unit/proxy/response_api_endpoints/test_endpoints.py @@ -2492,6 +2492,18 @@ class TestResponsesWSFirstFrameValidation: ws.close.assert_awaited_once_with(code=1011, reason="Internal server error") +def _spend_log_row(write: Mapping[str, object]) -> Mapping[str, object]: + from litellm.proxy.spend_tracking.spend_tracking_utils import get_logging_payload + + payload = get_logging_payload( + kwargs=write["kwargs"], + response_obj=write["completion_response"], + start_time=write["start_time"], + end_time=write["end_time"], + ) + return {**payload, "spend": write["response_cost"]} + + class _FrameGuardrail(CustomGuardrail): def __init__( self, @@ -2871,45 +2883,273 @@ class TestResponsesWSFirstFrameModelAuth: ] @pytest.mark.asyncio - async def test_frame_guard_bills_a_rejected_frame_as_a_failed_request(self): + @pytest.mark.parametrize("outcome", ["blocked", "refused"]) + async def test_frame_guard_bills_a_blocked_or_refused_turn_as_its_own_failed_request(self, outcome: str): + from datetime import datetime + from unittest.mock import create_autospec + import litellm + from litellm.exceptions import RateLimitError + from litellm.litellm_core_utils.litellm_logging import Logging + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.db.db_spend_update_writer import DBSpendUpdateWriter from litellm.proxy.hooks.proxy_track_cost_callback import ProxyDBLogger from litellm.proxy.response_api_endpoints.endpoints import _responses_ws_frame_guard + from litellm.responses.streaming_iterator import drain_logging_tasks + + socket_logging_obj = Logging( + model="gpt-4o-mini", + messages=[], + stream=True, + call_type="_aresponses_websocket", + start_time=datetime.now(), + litellm_call_id="socket-call", + function_id="socket-call", + ) + blocked = ValueError("blocked") + refusal = RateLimitError(message="crossed budget", llm_provider="", model="") async def pre_call_hook(user_api_key_dict, data, call_type, guardrails_only): data["litellm_metadata"]["standard_logging_guardrail_information"] = [{"guardrail_cost": 0.5}] - raise ValueError("blocked") + if outcome == "blocked": + raise blocked + return {**data, "input": "my email is "} proxy_logging_obj = MagicMock(during_call_hook=AsyncMock()) proxy_logging_obj.pre_call_hook = AsyncMock(side_effect=pre_call_hook) proxy_logging_obj.post_call_failure_hook = AsyncMock() - spend_logger = MagicMock(spec=ProxyDBLogger) - spend_logger.async_post_call_failure_hook = AsyncMock() rate_limiter = MagicMock() rate_limiter.async_post_call_failure_hook = AsyncMock() - user_api_key_dict = MagicMock() + spend_writer = create_autospec(DBSpendUpdateWriter, instance=True) + socket_bucket = {"tags": ["ws-tag"], "standard_logging_guardrail_information": [{"guardrail_cost": 9.0}]} + refused_turns = [] guard = _responses_ws_frame_guard( - {"model": "gpt-4o-mini", "litellm_metadata": {}}, user_api_key_dict, proxy_logging_obj, MagicMock() + {"model": "gpt-4o-mini", "litellm_metadata": socket_bucket, "litellm_logging_obj": socket_logging_obj}, + UserAPIKeyAuth( + api_key="sk-socket", + request_route="/v1/responses", + budget_reservation={"reserved_cost": 0.5, "entries": []}, + ), + proxy_logging_obj, + lambda records, refusal: refused_turns.append((records, refusal)), ) + socket_bucket["model_group"] = "mini-group" + frame = json.dumps({"type": "response.create", "input": "my email is a@b.co", "metadata": {"trace": "t1"}}) with ( - patch.object(litellm, "callbacks", [rate_limiter, spend_logger]), - pytest.raises(ValueError, match="blocked"), + patch.object(litellm, "callbacks", [rate_limiter, ProxyDBLogger(spend_writer=lambda: spend_writer)]), + patch( # test-quality-ok: stored prompts are off by default and the stored request is under test + "litellm.proxy.proxy_server.general_settings", {"store_prompts_in_spend_logs": True} + ), + patch( # test-quality-ok: settling the reservation is a Redis write; only the socket's close may do it + "litellm.proxy.spend_tracking.budget_reservation.release_budget_reservation", new_callable=AsyncMock + ) as release, ): - await guard(json.dumps({"type": "response.create", "input": "hi"})) + sent_at = datetime.now() + if outcome == "blocked": + with pytest.raises(ValueError, match="blocked"): + await guard(frame) + else: + await (await guard(frame)).bill_refusal(refusal) + await drain_logging_tasks(timeout=1) + answered_at = datetime.now() + spend_writer.update_database.assert_awaited_once() + write = spend_writer.update_database.await_args.kwargs + row = _spend_log_row(write) proxy_logging_obj.post_call_failure_hook.assert_not_awaited() rate_limiter.async_post_call_failure_hook.assert_not_awaited() - spend_logger.async_post_call_failure_hook.assert_awaited_once() - failure = spend_logger.async_post_call_failure_hook.await_args.kwargs - user_api_key_dict.model_copy.assert_called_once_with(update={"budget_reservation": None}) - assert failure["user_api_key_dict"] is user_api_key_dict.model_copy.return_value - assert str(failure["original_exception"]) == "blocked" - assert failure["request_data"]["litellm_metadata"]["standard_logging_guardrail_information"] == [ - {"guardrail_cost": 0.5} - ] - assert failure["request_data"]["input"] == "hi" - assert len(failure["request_data"]["litellm_call_id"]) == 36 + release.assert_not_awaited() + assert refused_turns == ([(({"guardrail_cost": 0.5},), refusal)] if outcome == "refused" else []) + assert write["completion_response"] is (refusal if outcome == "refused" else blocked) + assert row["session_id"] == socket_logging_obj.litellm_trace_id + assert sent_at <= write["start_time"] <= answered_at + assert row["call_type"] == "_aresponses_websocket" + assert row["model_group"] == "mini-group" + assert json.loads(row["request_tags"]) == ["ws-tag"] + assert row["spend"] == 0.5 + assert row["request_id"] == write["kwargs"]["litellm_call_id"] and len(row["request_id"]) == 36 + assert json.loads(row["proxy_server_request"]) == ( + {"input": "my email is ", "metadata": {"trace": "t1"}, "model": "gpt-4o-mini"} + if outcome == "refused" + else {} + ) + + @pytest.mark.asyncio + @pytest.mark.parametrize( + "client_metadata", + [ + { + "tags": ["forged-tag"], + "model_group": "forged-group", + "attempted_fallbacks": 7, + "original_model_group": "forged-original-group", + "agent_id": "forged-agent", + "billing_agent_id": "forged-agent", + "session_id": "forged-session", + }, + "not an object", + ], + ) + async def test_frame_guard_bills_a_refused_turn_without_the_logging_state_its_frame_forges( + self, client_metadata: object + ): + from unittest.mock import create_autospec + + from litellm.exceptions import RateLimitError + from litellm.proxy.db.db_spend_update_writer import DBSpendUpdateWriter + from litellm.proxy.hooks.proxy_track_cost_callback import ProxyDBLogger + from litellm.proxy.response_api_endpoints.endpoints import _responses_ws_frame_guard + from litellm.responses.streaming_iterator import drain_logging_tasks + + refusal = RateLimitError(message="crossed budget", llm_provider="", model="") + proxy_logging_obj = MagicMock(during_call_hook=AsyncMock()) + proxy_logging_obj.pre_call_hook = AsyncMock(side_effect=lambda **kwargs: kwargs["data"]) + spend_writer = create_autospec(DBSpendUpdateWriter, instance=True) + guard = _responses_ws_frame_guard( + { + "model": "gpt-4o-mini", + "litellm_metadata": { + "tags": ["ws-tag"], + "model_group": "mini-group", + "agent_id": "socket-agent", + "billing_agent_id": "socket-agent", + "session_id": "socket-session", + }, + }, + UserAPIKeyAuth(api_key="sk-socket", request_route="/v1/responses"), + proxy_logging_obj, + MagicMock(), + ) + frame = json.dumps( + { + "type": "response.create", + "input": "my secret prompt", + "metadata": client_metadata, + "cache_hit": True, + "custom_llm_provider": "forged-provider", + "standard_logging_object": {}, + "standard_callback_dynamic_params": {"turn_off_message_logging": False}, + "litellm_params": {"metadata": {"tags": ["forged-tag"]}}, + } + ) + + with ( + patch.object(litellm, "callbacks", [ProxyDBLogger(spend_writer=lambda: spend_writer)]), + patch.object(litellm, "turn_off_message_logging", True), + patch( # test-quality-ok: stored prompts are off by default and the stored request is under test + "litellm.proxy.proxy_server.general_settings", + {"store_prompts_in_spend_logs": True, "missing_session_id": "omit"}, + ), + ): + await (await guard(frame)).bill_refusal(refusal) + await drain_logging_tasks(timeout=1) + spend_writer.update_database.assert_awaited_once() + row = _spend_log_row(spend_writer.update_database.await_args.kwargs) + + stored_request = json.loads(row["proxy_server_request"]) + row_metadata = json.loads(row["metadata"]) + assert row["model_group"] == "mini-group" + assert row["agent_id"] == "socket-agent" + assert row["billing_agent_id"] == "socket-agent" + assert row["session_id"] == "socket-session" + assert row_metadata.get("attempted_fallbacks") != 7 + assert row_metadata.get("original_model_group") != "forged-original-group" + assert json.loads(row["request_tags"]) == ["ws-tag"] + assert row["cache_hit"] == "False" + assert row["custom_llm_provider"] != "forged-provider" + assert "my secret prompt" not in row["proxy_server_request"] + assert stored_request.keys().isdisjoint( + {"standard_logging_object", "standard_callback_dynamic_params", "litellm_params"} + ) + + @pytest.mark.asyncio + async def test_frame_guard_bills_a_refused_turn_under_the_socket_client_but_with_its_own_request(self): + from unittest.mock import create_autospec + + from litellm.exceptions import RateLimitError + from litellm.proxy.db.db_spend_update_writer import DBSpendUpdateWriter + from litellm.proxy.hooks.proxy_track_cost_callback import ProxyDBLogger + from litellm.proxy.response_api_endpoints.endpoints import _responses_ws_frame_guard + from litellm.responses.streaming_iterator import drain_logging_tasks + + refusal = RateLimitError(message="crossed budget", llm_provider="", model="") + proxy_logging_obj = MagicMock(during_call_hook=AsyncMock()) + proxy_logging_obj.pre_call_hook = AsyncMock(side_effect=lambda **kwargs: kwargs["data"]) + spend_writer = create_autospec(DBSpendUpdateWriter, instance=True) + guard = _responses_ws_frame_guard( + { + "model": "gpt-4o-mini", + "litellm_metadata": {"tags": ["ws-tag"]}, + "proxy_server_request": { + "url": "http://proxy/v1/responses", + "method": "GET", + "headers": {"user-agent": "codex-tui/0.167.0"}, + "body": {"input": "first turn"}, + }, + }, + UserAPIKeyAuth(api_key="sk-socket", request_route="/v1/responses"), + proxy_logging_obj, + MagicMock(), + ) + + with ( + patch.object(litellm, "callbacks", [ProxyDBLogger(spend_writer=lambda: spend_writer)]), + patch( # test-quality-ok: stored prompts are off by default and the stored request is under test + "litellm.proxy.proxy_server.general_settings", {"store_prompts_in_spend_logs": True} + ), + ): + await (await guard(json.dumps({"type": "response.create", "input": "second turn"}))).bill_refusal(refusal) + await drain_logging_tasks(timeout=1) + spend_writer.update_database.assert_awaited_once() + row = _spend_log_row(spend_writer.update_database.await_args.kwargs) + + assert json.loads(row["request_tags"]) == ["ws-tag", "User-Agent: codex-tui", "User-Agent: codex-tui/0.167.0"] + assert json.loads(row["proxy_server_request"])["input"] == "second turn" + + @pytest.mark.asyncio + async def test_a_refused_turn_row_is_written_even_when_the_socket_is_cancelled_while_billing_it(self): + from unittest.mock import create_autospec + + from litellm.exceptions import RateLimitError + from litellm.proxy.db.db_spend_update_writer import DBSpendUpdateWriter + from litellm.proxy.hooks.proxy_track_cost_callback import ProxyDBLogger + from litellm.proxy.response_api_endpoints.endpoints import _responses_ws_frame_guard + from litellm.responses.streaming_iterator import drain_logging_tasks + + writing = asyncio.Event() + may_finish = asyncio.Event() + written = asyncio.Event() + + async def slow_update_database(**kwargs: object) -> None: + writing.set() + await may_finish.wait() + written.set() + + proxy_logging_obj = MagicMock(during_call_hook=AsyncMock()) + proxy_logging_obj.pre_call_hook = AsyncMock(side_effect=lambda **kwargs: kwargs["data"]) + spend_writer = create_autospec(DBSpendUpdateWriter, instance=True) + spend_writer.update_database.side_effect = slow_update_database + guard = _responses_ws_frame_guard( + {"model": "gpt-4o-mini", "litellm_metadata": {}}, + UserAPIKeyAuth(api_key="sk-socket", request_route="/v1/responses"), + proxy_logging_obj, + MagicMock(), + ) + guarded = await guard(json.dumps({"type": "response.create", "input": "hi"})) + + with patch.object(litellm, "callbacks", [ProxyDBLogger(spend_writer=lambda: spend_writer)]): + billing = asyncio.ensure_future( + guarded.bill_refusal(RateLimitError(message="crossed budget", llm_provider="", model="")) + ) + await asyncio.wait_for(writing.wait(), timeout=1) + billing.cancel() + with pytest.raises(asyncio.CancelledError): + await billing + may_finish.set() + await drain_logging_tasks(timeout=1) + + assert written.is_set() @pytest.mark.asyncio async def test_frame_guard_guards_the_nested_response_body_and_copies_back_tools(self): @@ -3490,7 +3730,8 @@ class TestResponsesWSFirstFrameModelAuth: await guard(json.dumps({"type": "response.create", field: "unguarded", "response": {"input": "hi"}})) proxy_logging_obj.pre_call_hook.assert_not_awaited() - assert spend_logger.async_post_call_failure_hook.await_args.kwargs["request_data"]["input"] == "hi" + billed = spend_logger.async_post_call_failure_hook.await_args.kwargs["request_data"] + assert (billed["model"], billed["call_type"]) == ("gpt-4o-mini", "_aresponses_websocket") @pytest.mark.asyncio @pytest.mark.parametrize( @@ -3833,9 +4074,16 @@ class TestResponsesWSFirstFrameModelAuth: @pytest.mark.asyncio @pytest.mark.parametrize("ending", ["refusal", "crash", "session_limit"]) - async def test_close_time_failure_row_bills_the_guardrails_of_the_turns_the_deployment_refused(self, ending: str): + async def test_each_refused_turn_writes_its_own_failure_row_while_the_socket_is_open(self, ending: str): + from datetime import datetime + from unittest.mock import create_autospec + from litellm.exceptions import RateLimitError + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.db.db_spend_update_writer import DBSpendUpdateWriter + from litellm.proxy.hooks.proxy_track_cost_callback import ProxyDBLogger from litellm.proxy.response_api_endpoints.endpoints import responses_websocket_endpoint + from litellm.responses.streaming_iterator import drain_logging_tasks ws = MagicMock() ws.headers = {} @@ -3849,12 +4097,24 @@ class TestResponsesWSFirstFrameModelAuth: ws.close = AsyncMock() processor = MagicMock() processor.common_processing_pre_call_logic = AsyncMock( - return_value=({"model": "gpt-4o-mini", "litellm_metadata": {"model_group": "gpt-4o-mini"}}, MagicMock()) + return_value=({"model": "gpt-4o-mini", "litellm_metadata": {"tags": ["ws-tag"]}}, MagicMock()) ) guardrail_costs = {"two": 0.2, "three": 0.3, "four": 0.4} refusal = RateLimitError(message="crossed budget", llm_provider="", model="") - later_refusal = RateLimitError(message="still over budget", llm_provider="", model="") + turn_check_crash = RuntimeError("turn check crashed") crash = RuntimeError("relay crashed") + rejections = {"two": refusal, "four": turn_check_crash} + spend_writer = create_autospec(DBSpendUpdateWriter, instance=True) + spend_logger = ProxyDBLogger(spend_writer=lambda: spend_writer) + budget_reservation = {"reserved_cost": 0.5, "entries": []} + user_api_key_dict = UserAPIKeyAuth( + api_key="sk-socket", request_route="/v1/responses", budget_reservation=budget_reservation + ) + sent_at = {} + rows_while_open = [] + loop = asyncio.get_running_loop() + loop_clock = loop.time + clock_jumps = [] async def pre_call_hook(user_api_key_dict, data, call_type, guardrails_only): recorded = [{"guardrail_cost": guardrail_costs[data["input"]]}] @@ -3865,27 +4125,41 @@ class TestResponsesWSFirstFrameModelAuth: def route_request(data, **kwargs): async def relay(): + data["litellm_metadata"]["model_group"] = "mini-group" for turn in ("two", "three", "four"): + sent_at[turn] = datetime.now() guarded = await data["responses_ws_frame_guard"]( json.dumps({"type": "response.create", "input": turn}) ) - if turn != "three": - guarded.bill_refusal(refusal if turn == "two" else later_refusal) + if turn in rejections: + await guarded.bill_refusal(rejections[turn]) + await drain_logging_tasks(timeout=1) + rows_while_open.extend(_spend_log_row(write.kwargs) for write in spend_writer.update_database.await_args_list) if ending == "crash": raise crash if ending == "session_limit": + clock_jumps.append(3601.0) await asyncio.Event().wait() return refusal return relay() - row_written = asyncio.Event() + async def run_failure_hooks(**kwargs): + await spend_logger.async_post_call_failure_hook(**kwargs) + proxy_logging_obj = MagicMock(during_call_hook=AsyncMock()) proxy_logging_obj.pre_call_hook = AsyncMock(side_effect=pre_call_hook) - proxy_logging_obj.post_call_failure_hook = AsyncMock(side_effect=lambda **kwargs: row_written.set()) + proxy_logging_obj.post_call_failure_hook = AsyncMock(side_effect=run_failure_hooks) proxy_logging_obj.arelease_responses_websocket_slot = AsyncMock(return_value=None) with ( + patch.object(litellm, "callbacks", [spend_logger]), + patch( # test-quality-ok: stored prompts are off by default and each row's stored request is under test + "litellm.proxy.proxy_server.general_settings", {"store_prompts_in_spend_logs": True} + ), + patch( # test-quality-ok: settling the reservation is a Redis write; the socket's close must do it once + "litellm.proxy.spend_tracking.budget_reservation.release_budget_reservation", new_callable=AsyncMock + ) as release, patch( # test-quality-ok: first-frame model auth needs a live router and key table and has its own tests above "litellm.proxy.response_api_endpoints.endpoints._enforce_responses_ws_first_frame_model_auth", new_callable=AsyncMock, @@ -3899,26 +4173,142 @@ class TestResponsesWSFirstFrameModelAuth: new_callable=AsyncMock, side_effect=route_request, ), - patch( # test-quality-ok: a short session limit stands in for the configured one so the cap fires mid-session + patch( # test-quality-ok: a known session limit stands in for the configured one "litellm.proxy.response_api_endpoints.endpoints._resolve_responses_ws_session_limit_seconds", - return_value=0.05 if ending == "session_limit" else 3600.0, + return_value=3600.0, + ), + patch.object( # test-quality-ok: the session limit is a loop timer, so jumping the loop clock past it once the relay is parked fires it with no wall-clock race + loop, "time", lambda: loop_clock() + sum(clock_jumps) ), patch( # test-quality-ok: the failure hook is the proxy's only path to a failed spend log row "litellm.proxy.proxy_server.proxy_logging_obj", proxy_logging_obj, ), ): - await responses_websocket_endpoint(websocket=ws, model=None, user_api_key_dict=MagicMock()) - await asyncio.wait_for(row_written.wait(), timeout=1) + await responses_websocket_endpoint(websocket=ws, model=None, user_api_key_dict=user_api_key_dict) + await drain_logging_tasks(timeout=1) + writes = [write.kwargs for write in spend_writer.update_database.await_args_list] proxy_logging_obj.post_call_failure_hook.assert_awaited_once() - proxy_logging_obj.arelease_responses_websocket_slot.assert_awaited_once() - booked = proxy_logging_obj.post_call_failure_hook.await_args.kwargs - assert booked["original_exception"] is (crash if ending == "crash" else refusal) - assert booked["request_data"]["litellm_metadata"] == { - "model_group": "gpt-4o-mini", - "standard_logging_guardrail_information": [{"guardrail_cost": 0.2}, {"guardrail_cost": 0.4}], - } + closed = proxy_logging_obj.post_call_failure_hook.await_args.kwargs + assert closed["original_exception"] is (crash if ending == "crash" else refusal) + assert closed["request_data"]["litellm_metadata"].get("standard_logging_guardrail_information") == ( + None if ending == "crash" else [{"guardrail_cost": 0.2}, {"guardrail_cost": 0.4}] + ) + assert release.await_count == 1 + assert release.await_args.kwargs["budget_reservation"] == budget_reservation + assert [row["request_id"] for row in rows_while_open] == [ + write["kwargs"]["litellm_call_id"] for write in writes[:2] + ] + assert len({row["request_id"] for row in rows_while_open}) == 2 + for turn, write, row in zip(("two", "four"), writes, rows_while_open): + assert write["completion_response"] is rejections[turn] + assert sent_at[turn] <= write["start_time"] + assert row["spend"] == guardrail_costs[turn] + assert row["call_type"] == "_aresponses_websocket" + assert row["model_group"] == "mini-group" + assert json.loads(row["request_tags"]) == ["ws-tag"] + assert json.loads(row["proxy_server_request"]) == {"input": turn, "model": "gpt-4o-mini"} + assert writes[0]["start_time"] < sent_at["four"] <= writes[1]["start_time"] + close_writes = writes[2:] + if ending != "crash": + assert close_writes == [] + return + assert [write["completion_response"] for write in close_writes] == [crash] + assert _spend_log_row(close_writes[0])["spend"] == 0.0 + + @pytest.mark.asyncio + async def test_a_refused_turn_row_shares_the_session_id_of_the_turns_the_router_relays(self): + from datetime import datetime + from unittest.mock import create_autospec + + from litellm.exceptions import RateLimitError + from litellm.litellm_core_utils.get_litellm_params import get_litellm_params + from litellm.litellm_core_utils.litellm_logging import Logging, StandardLoggingPayloadSetup + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.db.db_spend_update_writer import DBSpendUpdateWriter + from litellm.proxy.hooks.proxy_track_cost_callback import ProxyDBLogger + from litellm.proxy.response_api_endpoints.endpoints import responses_websocket_endpoint + from litellm.responses.streaming_iterator import drain_logging_tasks + from litellm.router_rust.invoke.endpoints import generic_plan, prepare_request + + socket_logging_obj = Logging( + model="gpt-4o-mini", + messages=[], + stream=True, + call_type="_aresponses_websocket", + start_time=datetime.now(), + litellm_call_id="socket-call", + function_id="socket-call", + ) + ws = MagicMock() + ws.headers = {} + ws.query_params = {} + ws.scope = {"headers": []} + ws.url = "ws://testserver/v1/responses" + ws.accept = AsyncMock() + ws.receive_text = AsyncMock( + return_value=json.dumps({"type": "response.create", "model": "gpt-4o-mini", "input": []}) + ) + ws.close = AsyncMock() + processor = MagicMock() + processor.common_processing_pre_call_logic = AsyncMock( + return_value=( + {"model": "gpt-4o-mini", "litellm_metadata": {}, "litellm_logging_obj": socket_logging_obj}, + socket_logging_obj, + ) + ) + refusal = RateLimitError(message="crossed budget", llm_provider="", model="") + spend_writer = create_autospec(DBSpendUpdateWriter, instance=True) + relayed_turn_session_ids = [] + + def route_request(data, **kwargs): + async def relay(): + relayed = prepare_request(generic_plan("_aresponses_websocket"), data, "gpt-4o-mini", {}) + relayed_turn_session_ids.append( + StandardLoggingPayloadSetup.get_standard_logging_payload_trace_id( + logging_obj=socket_logging_obj, litellm_params=get_litellm_params(**relayed) + ) + ) + guarded = await data["responses_ws_frame_guard"](json.dumps({"type": "response.create", "input": "hi"})) + await guarded.bill_refusal(refusal) + await drain_logging_tasks(timeout=1) + return None + + return relay() + + proxy_logging_obj = MagicMock(during_call_hook=AsyncMock()) + proxy_logging_obj.pre_call_hook = AsyncMock(side_effect=lambda **kwargs: kwargs["data"]) + proxy_logging_obj.post_call_failure_hook = AsyncMock() + proxy_logging_obj.arelease_responses_websocket_slot = AsyncMock(return_value=None) + + with ( + patch.object(litellm, "callbacks", [ProxyDBLogger(spend_writer=lambda: spend_writer)]), + patch( # test-quality-ok: first-frame model auth needs a live router and key table and has its own tests above + "litellm.proxy.response_api_endpoints.endpoints._enforce_responses_ws_first_frame_model_auth", + new_callable=AsyncMock, + ), + patch( # test-quality-ok: the pre-call processor needs a live proxy; the socket's logging object is its output + "litellm.proxy.response_api_endpoints.endpoints.ProxyBaseLLMRequestProcessing", + return_value=processor, + ), + patch( # test-quality-ok: routing is the seam where the router stamps the relayed turns' trace id + "litellm.proxy.route_llm_request.route_request", + new_callable=AsyncMock, + side_effect=route_request, + ), + patch( # test-quality-ok: the failure hook is the proxy's only path to a failed spend log row + "litellm.proxy.proxy_server.proxy_logging_obj", + proxy_logging_obj, + ), + ): + await responses_websocket_endpoint( + websocket=ws, model=None, user_api_key_dict=UserAPIKeyAuth(api_key="sk-socket") + ) + + spend_writer.update_database.assert_awaited_once() + refused_row = _spend_log_row(spend_writer.update_database.await_args.kwargs) + assert relayed_turn_session_ids == [refused_row["session_id"]] @pytest.mark.asyncio @pytest.mark.parametrize("ending", ["client_closed", "crash"]) @@ -3954,7 +4344,7 @@ class TestResponsesWSFirstFrameModelAuth: def route_request(data, **kwargs): async def relay(): guarded = await data["responses_ws_frame_guard"](json.dumps({"type": "response.create", "input": "two"})) - guarded.bill_refusal(refusal) + await guarded.bill_refusal(refusal) if ending == "crash": raise crash return refusal @@ -4003,7 +4393,7 @@ class TestResponsesWSFirstFrameModelAuth: assert booked["original_exception"] is (crash if ending == "crash" else refusal) @pytest.mark.asyncio - async def test_a_crashed_socket_whose_failure_row_cannot_be_built_still_gets_its_error_frame_and_close(self): + async def test_a_crashed_socket_whose_failure_hooks_fail_still_gets_its_error_frame_and_close(self): from litellm.exceptions import RateLimitError from litellm.proxy.response_api_endpoints.endpoints import responses_websocket_endpoint @@ -4029,20 +4419,16 @@ class TestResponsesWSFirstFrameModelAuth: def route_request(data, **kwargs): async def relay(): guarded = await data["responses_ws_frame_guard"](json.dumps({"type": "response.create", "input": "two"})) - guarded.bill_refusal(refusal) - data["litellm_metadata"]["standard_logging_guardrail_information"] = "written by a broken callback" + await guarded.bill_refusal(refusal) raise RuntimeError("relay crashed") return relay() proxy_logging_obj = MagicMock(during_call_hook=AsyncMock()) proxy_logging_obj.pre_call_hook = AsyncMock( - side_effect=lambda user_api_key_dict, data, call_type, guardrails_only: { - **data, - "litellm_metadata": {"standard_logging_guardrail_information": [{"guardrail_name": "turn-check"}]}, - } + side_effect=lambda user_api_key_dict, data, call_type, guardrails_only: data ) - proxy_logging_obj.post_call_failure_hook = AsyncMock() + proxy_logging_obj.post_call_failure_hook = AsyncMock(side_effect=RuntimeError("spend log db down")) proxy_logging_obj.arelease_responses_websocket_slot = AsyncMock(return_value=None) with ( @@ -4068,6 +4454,7 @@ class TestResponsesWSFirstFrameModelAuth: responses_websocket_endpoint(websocket=ws, model=None, user_api_key_dict=MagicMock()), timeout=1 ) + proxy_logging_obj.post_call_failure_hook.assert_awaited_once() assert [json.loads(frame)["error"]["message"] for frame in sent] == ["relay crashed"] assert closes == [(1011, "Internal server error")] diff --git a/tests/unit/responses/test_responses_websocket_all_providers.py b/tests/unit/responses/test_responses_websocket_all_providers.py index cb7208f9b9b..c72e32e01de 100644 --- a/tests/unit/responses/test_responses_websocket_all_providers.py +++ b/tests/unit/responses/test_responses_websocket_all_providers.py @@ -1694,7 +1694,7 @@ def _guard_blocking(word: str): return guard -def _guard_recording(bill_refusal: MagicMock): +def _guard_recording(bill_refusal: AsyncMock): from litellm.responses.streaming_iterator import GuardedResponsesWsFrame async def guard(frame: str) -> GuardedResponsesWsFrame: @@ -2568,14 +2568,14 @@ class TestWebSocketTurnAdmission: @pytest.mark.asyncio - async def test_native_relay_leaves_a_refused_turn_s_guardrail_records_to_the_close_time_failure_row(self): + async def test_native_relay_bills_a_refused_turn_and_returns_it_as_the_close_time_failure(self): from unittest.mock import AsyncMock from litellm.exceptions import RateLimitError first, second, third = _turn_frames() refusal: Final = RateLimitError(message="crossed budget", llm_provider="", model="") - bill_refusal: Final = MagicMock() + bill_refusal: Final = AsyncMock() backend_ws: Final = MagicMock() backend_ws.send = AsyncMock() backend_ws.close = AsyncMock() @@ -2592,9 +2592,41 @@ class TestWebSocketTurnAdmission: close_time_failure: Final = await handler.bidirectional_forward() assert [json.loads(sent.args[0])["input"] for sent in backend_ws.send.await_args_list] == ["one", "three"] - bill_refusal.assert_called_once_with(refusal) + bill_refusal.assert_awaited_once_with(refusal) assert close_time_failure is refusal + @pytest.mark.asyncio + async def test_native_relay_forwards_no_later_turn_until_the_refused_turn_is_billed(self): + from unittest.mock import AsyncMock + + from litellm.exceptions import RateLimitError + + first, second, third = _turn_frames() + events: Final[list[str]] = [] + + async def bill_refusal(rejection: Exception) -> None: + await asyncio.sleep(0.01) + events.append("billed") + + backend_ws: Final = MagicMock() + backend_ws.send = AsyncMock(side_effect=lambda frame: events.append(json.loads(frame)["input"])) + backend_ws.close = AsyncMock() + backend_ws.recv = AsyncMock(side_effect=asyncio.Event().wait) + handler: Final = _make_streaming( + websocket=_ClientSocket(second, third), + backend_ws=backend_ws, + request_data={}, + first_message=first, + frame_guard=_guard_recording(AsyncMock(side_effect=bill_refusal)), + turn_admission=AsyncMock( + side_effect=[RateLimitError(message="crossed budget", llm_provider="", model=""), {}] + ), + ) + + _ = await handler.bidirectional_forward() + + assert events == ["one", "billed", "three"] + @pytest.mark.asyncio async def test_a_refused_turn_is_recorded_even_when_the_relay_is_cancelled_while_answering_it(self): from unittest.mock import AsyncMock @@ -2603,7 +2635,7 @@ class TestWebSocketTurnAdmission: first, second, third = _turn_frames() refusal: Final = RateLimitError(message="crossed budget", llm_provider="", model="") - bill_refusal: Final = MagicMock() + bill_refusal: Final = AsyncMock() backend_ws: Final = MagicMock() backend_ws.send = AsyncMock() backend_ws.close = AsyncMock() @@ -2624,10 +2656,10 @@ class TestWebSocketTurnAdmission: with pytest.raises(asyncio.CancelledError): await relay - bill_refusal.assert_called_once_with(refusal) + bill_refusal.assert_awaited_once_with(refusal) @pytest.mark.asyncio - async def test_managed_relay_leaves_a_refused_turn_s_guardrail_records_to_the_close_time_failure_row( + async def test_managed_relay_bills_a_refused_turn_and_returns_it_as_the_close_time_failure( self, monkeypatch: pytest.MonkeyPatch ): from unittest.mock import AsyncMock @@ -2651,7 +2683,7 @@ class TestWebSocketTurnAdmission: monkeypatch.setattr(litellm, "aresponses", fake_aresponses) first, second, third = _turn_frames() refusal: Final = RateLimitError(message="crossed budget", llm_provider="", model="") - bill_refusal: Final = MagicMock() + bill_refusal: Final = AsyncMock() handler: Final = ManagedResponsesWebSocketHandler( websocket=_ClientSocket(second, third), model="test-model", @@ -2672,7 +2704,7 @@ class TestWebSocketTurnAdmission: close_time_failure: Final = await handler.run() assert answered == ["one", "three"] - bill_refusal.assert_called_once_with(refusal) + bill_refusal.assert_awaited_once_with(refusal) assert close_time_failure is refusal @pytest.mark.asyncio