From 13f1bf6343bbc17a0910d9281db3cd1a7e01c830 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 28 Oct 2026 11:22:25 -0700 Subject: [PATCH] fix(responses): authorize proxy-issued comparison ids on WebSocket frames and pin a new socket to the compared response --- litellm/proxy/hooks/responses_id_security.py | 4 +- .../proxy/response_api_endpoints/endpoints.py | 46 ++++- litellm/responses/main.py | 2 +- .../response_api_endpoints/test_endpoints.py | 159 ++++++++++++++++++ .../test_responses_api_request_body.py | 2 + 5 files changed, 206 insertions(+), 7 deletions(-) diff --git a/litellm/proxy/hooks/responses_id_security.py b/litellm/proxy/hooks/responses_id_security.py index 90252baaede..9c3ba2c502b 100644 --- a/litellm/proxy/hooks/responses_id_security.py +++ b/litellm/proxy/hooks/responses_id_security.py @@ -198,7 +198,7 @@ class ResponsesIDSecurity(CustomLogger): if comparison_id is None: return _NOTHING_AUTHORIZED authorized_options: Final = _with_comparison_response_id( - options, self._authorized_comparison_id(comparison_id, user_api_key_dict) + options, self.authorize_comparison_id(comparison_id, user_api_key_dict) ) placed_options: Final = ( {_PROMPT_CACHE_OPTIONS: authorized_options} @@ -207,7 +207,7 @@ class ResponsesIDSecurity(CustomLogger): ) return {**placed_options, COMPARISON_RESPONSE_ID_FIELD: comparison_id} - def _authorized_comparison_id(self, comparison_id: str, user_api_key_dict: "UserAPIKeyAuth") -> str: + def authorize_comparison_id(self, comparison_id: str, user_api_key_dict: "UserAPIKeyAuth") -> str: if not self._is_encrypted_response_id(comparison_id): return comparison_id return self.authorize_response_id(comparison_id, user_api_key_dict) diff --git a/litellm/proxy/response_api_endpoints/endpoints.py b/litellm/proxy/response_api_endpoints/endpoints.py index 97bbf893e3e..77c75c61572 100644 --- a/litellm/proxy/response_api_endpoints/endpoints.py +++ b/litellm/proxy/response_api_endpoints/endpoints.py @@ -1490,6 +1490,29 @@ def _with_authorized_previous_response( return {**data, "previous_response_id": authorized_id, ADDRESSED_RESPONSE_ID_FIELD: previous_response_id} +def _with_authorized_comparison_id( + request: Mapping[str, object], security: ResponsesIDSecurity | None, user_api_key_dict: UserAPIKeyAuth +) -> Mapping[str, object]: + options: Final = as_str_mapping(request.get("prompt_cache_options")) + comparison_id: Final = None if options is None else options.get("comparison_response_id") + if security is None or options is None or not isinstance(comparison_id, str) or not comparison_id: + return request + try: + authorized_id: Final = security.authorize_comparison_id(comparison_id, user_api_key_dict) + except HTTPException as e: + raise ProxyException(message=str(e.detail), type="permission_error", param=None, code=e.status_code) from e + return {**request, "prompt_cache_options": {**options, "comparison_response_id": authorized_id}} + + +def _with_routable_comparison_id( + hints: Mapping[str, object], security: ResponsesIDSecurity | None, user_api_key_dict: UserAPIKeyAuth +) -> Mapping[str, object]: + try: + return _with_authorized_comparison_id(hints, security, user_api_key_dict) + except ProxyException: + return {key: value for key, value in hints.items() if key != "prompt_cache_options"} + + def _responses_invalid_request_response(message: str, param: str | None, code: str | None) -> JSONResponse: body: Final[_ResponsesApiErrorBody] = { "error": { @@ -1939,6 +1962,7 @@ class _ResponseCreateRoutingHints(LiteLLMBaseModel): input: str | Sequence[object] | None = None previous_response_id: str | None = None + prompt_cache_options: Mapping[str, object] | None = None response: "_ResponseCreateRoutingHints | None" = None @@ -1953,6 +1977,9 @@ def _routing_hints_from_first_ws_frame(first_message: str) -> Mapping[str, objec "previous_response_id": ( frame.previous_response_id if nested.previous_response_id is None else nested.previous_response_id ), + "prompt_cache_options": ( + frame.prompt_cache_options if nested.prompt_cache_options is None else nested.prompt_cache_options + ), } return MappingProxyType({key: value for key, value in hints.items() if value is not None}) @@ -2037,6 +2064,7 @@ def _responses_ws_frame_guard( user_api_key_dict: UserAPIKeyAuth, proxy_logging_obj: _PreCallGuardrailWalk, record_refused_turn: Callable[[tuple[Mapping[str, object], ...], Exception], None], + response_id_security: ResponsesIDSecurity | None = None, ) -> ResponsesWsFrameGuard: """Socket admission skips guardrails (the first frame's ``input`` is not part of the admitted request), so every ``response.create`` frame gets the pre_call guardrail walk the HTTP ``/v1/responses`` path runs on its body.""" @@ -2074,6 +2102,7 @@ def _responses_ws_frame_guard( metadata_key: {**bucket}, } try: + authorized_body: Final = _with_authorized_comparison_id(body, response_id_security, user_api_key_dict) guarded_data: Final = await proxy_logging_obj.pre_call_hook( user_api_key_dict=user_api_key_dict, data=frame_data, call_type="aresponses", guardrails_only=True ) @@ -2089,7 +2118,7 @@ def _responses_ws_frame_guard( raise rerouted # a guardrail may delete a field (the Responses translation pops ``instructions``), so start without them guarded_fields: Final = {key: guarded_data[key] for key in _GUARDED_FRAME_FIELDS if key in guarded_data} - kept: Final = {key: value for key, value in body.items() if key not in _GUARDED_FRAME_FIELDS} + kept: Final = {key: value for key, value in authorized_body.items() if key not in _GUARDED_FRAME_FIELDS} rewritten: Final = ( {**frame_obj, "response": {**kept, **guarded_fields}} if isinstance(nested, dict) @@ -2184,6 +2213,7 @@ async def _responses_websocket_session( websocket: WebSocket, model: str | None, user_api_key_dict: UserAPIKeyAuth, + response_id_security: ResponsesIDSecurity | None, ) -> None: from litellm.proxy.proxy_server import ( general_settings, @@ -2281,9 +2311,11 @@ async def _responses_websocket_session( data, user_api_key_dict=user_api_key_dict, responses_ws_frame_guard=_responses_ws_frame_guard( - data, user_api_key_dict, proxy_logging_obj, refused_turns.add + data, user_api_key_dict, proxy_logging_obj, refused_turns.add, response_id_security + ), + **_with_routable_comparison_id( + _routing_hints_from_first_ws_frame(first_message), response_id_security, user_api_key_dict ), - **_routing_hints_from_first_ws_frame(first_message), ) async def write_failure_row(exception: Exception) -> None: @@ -2336,6 +2368,7 @@ async def responses_websocket_endpoint( websocket: WebSocket, model: str | None = fastapi.Query(None, description="The model to use for the responses WebSocket session."), user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth_websocket), + response_id_security: ResponsesIDSecurity | None = _response_id_security_dep, ): """ Responses API WebSocket mode endpoint. @@ -2366,7 +2399,12 @@ async def responses_websocket_endpoint( limit_seconds: Final = _resolve_responses_ws_session_limit_seconds() session_task: Final = asyncio.ensure_future( - _responses_websocket_session(websocket=websocket, model=model, user_api_key_dict=user_api_key_dict) + _responses_websocket_session( + websocket=websocket, + model=model, + user_api_key_dict=user_api_key_dict, + response_id_security=response_id_security, + ) ) try: await asyncio.wait_for(asyncio.shield(session_task), timeout=limit_seconds) diff --git a/litellm/responses/main.py b/litellm/responses/main.py index 8451834eab2..027d21ba54f 100644 --- a/litellm/responses/main.py +++ b/litellm/responses/main.py @@ -2718,7 +2718,7 @@ def _deployment_reasoning_default(kwargs: Mapping[str, object]) -> Reasoning | d return LiteLLMResponsesTransformationHandler().map_reasoning_effort(reasoning_effort) -_RESPONSES_WS_ROUTING_HINT_KEYS: Final = frozenset({"input", "previous_response_id"}) +_RESPONSES_WS_ROUTING_HINT_KEYS: Final = frozenset({"input", "previous_response_id", "prompt_cache_options"}) def _first_ws_frame_with_routed_input(first_message: str, routed_input: object) -> str: diff --git a/tests/unit/proxy/response_api_endpoints/test_endpoints.py b/tests/unit/proxy/response_api_endpoints/test_endpoints.py index 985aee061bc..89c295c30a2 100644 --- a/tests/unit/proxy/response_api_endpoints/test_endpoints.py +++ b/tests/unit/proxy/response_api_endpoints/test_endpoints.py @@ -1963,6 +1963,24 @@ class TestResponsesWSFirstFrameValidation: ws.close.assert_awaited_once_with(code=1011, reason="Internal server error") +_COMPARISON_ID_SALT: Final = "comparison-id-unit-test-salt" + + +def _comparison_id_security_and_owners_id(monkeypatch: pytest.MonkeyPatch, original_id: str): + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.hooks.responses_id_security import ResponsesIDSecurity + + monkeypatch.setenv("LITELLM_SALT_KEY", _COMPARISON_ID_SALT) + security: Final = ResponsesIDSecurity( + general_settings_reader=lambda: {}, signing_key_reader=lambda: _COMPARISON_ID_SALT + ) + issued: Final = security._encrypt_response_id( + ResponsesAPIResponse(id=original_id, object="response", status="completed", created_at=1, output=[]), + UserAPIKeyAuth(user_id="owner"), + ) + return security, issued.id + + class TestResponsesWSFirstFrameModelAuth: @pytest.mark.asyncio async def test_endpoint_enforces_auth_after_model_from_first_frame(self): @@ -2082,6 +2100,78 @@ class TestResponsesWSFirstFrameModelAuth: assert mock_route_request.await_args.kwargs["route_type"] == "_aresponses_websocket" ws.close.assert_not_awaited() + @pytest.mark.asyncio + @pytest.mark.parametrize("nested", [False, True]) + @pytest.mark.parametrize( + "caller_id, routed_options", + [ + ("owner", {"ttl": "12h", "comparison_response_id": "resp_compared"}), + ("stranger", None), + ], + ) + async def test_endpoint_routes_a_first_frame_comparison_id_only_for_its_owner( + self, + monkeypatch: pytest.MonkeyPatch, + nested: bool, + caller_id: str, + routed_options: Mapping[str, object] | None, + ): + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.response_api_endpoints.endpoints import responses_websocket_endpoint + + security, issued_id = _comparison_id_security_and_owners_id(monkeypatch, "resp_compared") + payload = { + "model": "gpt-4o-mini", + "input": "hi", + "prompt_cache_options": {"ttl": "12h", "comparison_response_id": issued_id}, + } + first_frame = ( + {"type": "response.create", "response": payload} if nested else {"type": "response.create", **payload} + ) + + ws = MagicMock() + ws.headers = {} + ws.query_params = {} + ws.scope = {"headers": []} + ws.url = "ws://testserver/v1/responses" + ws.accept = AsyncMock() + ws.receive_text = AsyncMock(return_value=json.dumps(first_frame)) + ws.close = AsyncMock() + + processor = MagicMock() + processor.common_processing_pre_call_logic = AsyncMock( + return_value=({"model": "gpt-4o-mini", "litellm_metadata": {}}, MagicMock()) + ) + + async def fake_llm_call(): + return None + + with ( + patch( # test-quality-ok: first-frame model auth needs a live router and key table and has its own tests + "litellm.proxy.response_api_endpoints.endpoints._enforce_responses_ws_first_frame_model_auth", + new_callable=AsyncMock, + ), + patch( # test-quality-ok: the pre-call processor needs a live proxy; the routed payload is under test + "litellm.proxy.response_api_endpoints.endpoints.ProxyBaseLLMRequestProcessing", + return_value=processor, + ), + patch( # test-quality-ok: routing is the seam where the first frame's comparison id becomes observable + "litellm.proxy.route_llm_request.route_request", + new_callable=AsyncMock, + return_value=fake_llm_call(), + ) as mock_route_request, + ): + await responses_websocket_endpoint( + websocket=ws, + model=None, + user_api_key_dict=UserAPIKeyAuth(user_id=caller_id), + response_id_security=security, + ) + + routed = mock_route_request.await_args.kwargs["data"] + assert routed.get("prompt_cache_options") == routed_options + assert routed["input"] == "hi" + @pytest.mark.asyncio async def test_frame_guard_runs_the_guardrail_walk_on_the_frame_body_and_returns_its_records(self): from litellm.proxy.response_api_endpoints.endpoints import _responses_ws_frame_guard @@ -2178,6 +2268,75 @@ class TestResponsesWSFirstFrameModelAuth: "response": {"input": "hi [guarded]", "tools": tools[:1]}, } + @pytest.mark.asyncio + @pytest.mark.parametrize("nested", [False, True]) + @pytest.mark.parametrize("sent_id, forwarded_id", [(None, "resp_compared"), ("resp_raw_provider_id", None)]) + async def test_frame_guard_forwards_the_compared_response_id_the_provider_issued( + self, monkeypatch: pytest.MonkeyPatch, nested: bool, sent_id: str | None, forwarded_id: str | None + ): + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.response_api_endpoints.endpoints import _responses_ws_frame_guard + + security, issued_id = _comparison_id_security_and_owners_id(monkeypatch, "resp_compared") + comparison_id = sent_id or issued_id + proxy_logging_obj = MagicMock() + proxy_logging_obj.pre_call_hook = AsyncMock(side_effect=lambda **kwargs: kwargs["data"]) + guard = _responses_ws_frame_guard( + {"model": "gpt-4o-mini", "litellm_metadata": {}}, + UserAPIKeyAuth(user_id="owner"), + proxy_logging_obj, + MagicMock(), + security, + ) + body = {"input": "hi", "prompt_cache_options": {"ttl": "12h", "comparison_response_id": comparison_id}} + frame = {"type": "response.create", "response": body} if nested else {"type": "response.create", **body} + + guarded = await guard(json.dumps(frame)) + + sent = json.loads(guarded.frame) + assert (sent["response"] if nested else sent)["prompt_cache_options"] == { + "ttl": "12h", + "comparison_response_id": forwarded_id or comparison_id, + } + + @pytest.mark.asyncio + async def test_frame_guard_refuses_and_bills_a_comparison_id_issued_to_another_user( + self, monkeypatch: pytest.MonkeyPatch + ): + import litellm + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.hooks.proxy_track_cost_callback import ProxyDBLogger + from litellm.proxy.response_api_endpoints.endpoints import _responses_ws_frame_guard + + security, issued_id = _comparison_id_security_and_owners_id(monkeypatch, "resp_compared") + proxy_logging_obj = MagicMock() + proxy_logging_obj.pre_call_hook = AsyncMock(side_effect=lambda **kwargs: kwargs["data"]) + spend_logger = MagicMock(spec=ProxyDBLogger) + spend_logger.async_post_call_failure_hook = AsyncMock() + guard = _responses_ws_frame_guard( + {"model": "gpt-4o-mini", "litellm_metadata": {}}, + UserAPIKeyAuth(user_id="stranger"), + proxy_logging_obj, + MagicMock(), + security, + ) + + with patch.object(litellm, "callbacks", [spend_logger]), pytest.raises(ProxyException) as refused: + await guard( + json.dumps( + { + "type": "response.create", + "input": "hi", + "prompt_cache_options": {"comparison_response_id": issued_id}, + } + ) + ) + + assert (refused.value.type, refused.value.code) == ("permission_error", "403") + proxy_logging_obj.pre_call_hook.assert_not_awaited() + spend_logger.async_post_call_failure_hook.assert_awaited_once() + assert spend_logger.async_post_call_failure_hook.await_args.kwargs["original_exception"] is refused.value + @pytest.mark.asyncio async def test_frame_guard_strips_the_proxy_owned_metadata_keys_a_client_sends(self): from litellm.proxy.response_api_endpoints.endpoints import _responses_ws_frame_guard diff --git a/tests/unit/responses/test_responses_api_request_body.py b/tests/unit/responses/test_responses_api_request_body.py index bf571c74e22..0ba907536ca 100644 --- a/tests/unit/responses/test_responses_api_request_body.py +++ b/tests/unit/responses/test_responses_api_request_body.py @@ -662,11 +662,13 @@ async def test_aresponses_websocket_keeps_routing_hints_out_of_the_relay_kwargs( litellm_logging_obj=MagicMock(), input=[{"type": "message", "role": "user", "content": "hi"}], previous_response_id="resp_prev", + prompt_cache_options={"comparison_response_id": "resp_compared"}, ) mock_ws.assert_awaited_once() assert "input" not in mock_ws.call_args.kwargs assert "previous_response_id" not in mock_ws.call_args.kwargs + assert "prompt_cache_options" not in mock_ws.call_args.kwargs _STRIPPED_WS_INPUT = [{"role": "user", "content": "hi"}]