fix(responses): authorize proxy-issued comparison ids on WebSocket frames and pin a new socket to the compared response

This commit is contained in:
mateo-berri 2026-10-28 11:22:25 -07:00
parent 25c42c4229
commit 13f1bf6343
5 changed files with 206 additions and 7 deletions

View file

@ -198,7 +198,7 @@ class ResponsesIDSecurity(CustomLogger):
if comparison_id is None:
return _NOTHING_AUTHORIZED
authorized_options: Final = _with_comparison_response_id(
options, self._authorized_comparison_id(comparison_id, user_api_key_dict)
options, self.authorize_comparison_id(comparison_id, user_api_key_dict)
)
placed_options: Final = (
{_PROMPT_CACHE_OPTIONS: authorized_options}
@ -207,7 +207,7 @@ class ResponsesIDSecurity(CustomLogger):
)
return {**placed_options, COMPARISON_RESPONSE_ID_FIELD: comparison_id}
def _authorized_comparison_id(self, comparison_id: str, user_api_key_dict: "UserAPIKeyAuth") -> str:
def authorize_comparison_id(self, comparison_id: str, user_api_key_dict: "UserAPIKeyAuth") -> str:
if not self._is_encrypted_response_id(comparison_id):
return comparison_id
return self.authorize_response_id(comparison_id, user_api_key_dict)

View file

@ -1490,6 +1490,29 @@ def _with_authorized_previous_response(
return {**data, "previous_response_id": authorized_id, ADDRESSED_RESPONSE_ID_FIELD: previous_response_id}
def _with_authorized_comparison_id(
request: Mapping[str, object], security: ResponsesIDSecurity | None, user_api_key_dict: UserAPIKeyAuth
) -> Mapping[str, object]:
options: Final = as_str_mapping(request.get("prompt_cache_options"))
comparison_id: Final = None if options is None else options.get("comparison_response_id")
if security is None or options is None or not isinstance(comparison_id, str) or not comparison_id:
return request
try:
authorized_id: Final = security.authorize_comparison_id(comparison_id, user_api_key_dict)
except HTTPException as e:
raise ProxyException(message=str(e.detail), type="permission_error", param=None, code=e.status_code) from e
return {**request, "prompt_cache_options": {**options, "comparison_response_id": authorized_id}}
def _with_routable_comparison_id(
hints: Mapping[str, object], security: ResponsesIDSecurity | None, user_api_key_dict: UserAPIKeyAuth
) -> Mapping[str, object]:
try:
return _with_authorized_comparison_id(hints, security, user_api_key_dict)
except ProxyException:
return {key: value for key, value in hints.items() if key != "prompt_cache_options"}
def _responses_invalid_request_response(message: str, param: str | None, code: str | None) -> JSONResponse:
body: Final[_ResponsesApiErrorBody] = {
"error": {
@ -1939,6 +1962,7 @@ class _ResponseCreateRoutingHints(LiteLLMBaseModel):
input: str | Sequence[object] | None = None
previous_response_id: str | None = None
prompt_cache_options: Mapping[str, object] | None = None
response: "_ResponseCreateRoutingHints | None" = None
@ -1953,6 +1977,9 @@ def _routing_hints_from_first_ws_frame(first_message: str) -> Mapping[str, objec
"previous_response_id": (
frame.previous_response_id if nested.previous_response_id is None else nested.previous_response_id
),
"prompt_cache_options": (
frame.prompt_cache_options if nested.prompt_cache_options is None else nested.prompt_cache_options
),
}
return MappingProxyType({key: value for key, value in hints.items() if value is not None})
@ -2037,6 +2064,7 @@ def _responses_ws_frame_guard(
user_api_key_dict: UserAPIKeyAuth,
proxy_logging_obj: _PreCallGuardrailWalk,
record_refused_turn: Callable[[tuple[Mapping[str, object], ...], Exception], None],
response_id_security: ResponsesIDSecurity | None = None,
) -> ResponsesWsFrameGuard:
"""Socket admission skips guardrails (the first frame's ``input`` is not part of the admitted request), so every
``response.create`` frame gets the pre_call guardrail walk the HTTP ``/v1/responses`` path runs on its body."""
@ -2074,6 +2102,7 @@ def _responses_ws_frame_guard(
metadata_key: {**bucket},
}
try:
authorized_body: Final = _with_authorized_comparison_id(body, response_id_security, user_api_key_dict)
guarded_data: Final = await proxy_logging_obj.pre_call_hook(
user_api_key_dict=user_api_key_dict, data=frame_data, call_type="aresponses", guardrails_only=True
)
@ -2089,7 +2118,7 @@ def _responses_ws_frame_guard(
raise rerouted
# a guardrail may delete a field (the Responses translation pops ``instructions``), so start without them
guarded_fields: Final = {key: guarded_data[key] for key in _GUARDED_FRAME_FIELDS if key in guarded_data}
kept: Final = {key: value for key, value in body.items() if key not in _GUARDED_FRAME_FIELDS}
kept: Final = {key: value for key, value in authorized_body.items() if key not in _GUARDED_FRAME_FIELDS}
rewritten: Final = (
{**frame_obj, "response": {**kept, **guarded_fields}}
if isinstance(nested, dict)
@ -2184,6 +2213,7 @@ async def _responses_websocket_session(
websocket: WebSocket,
model: str | None,
user_api_key_dict: UserAPIKeyAuth,
response_id_security: ResponsesIDSecurity | None,
) -> None:
from litellm.proxy.proxy_server import (
general_settings,
@ -2281,9 +2311,11 @@ async def _responses_websocket_session(
data,
user_api_key_dict=user_api_key_dict,
responses_ws_frame_guard=_responses_ws_frame_guard(
data, user_api_key_dict, proxy_logging_obj, refused_turns.add
data, user_api_key_dict, proxy_logging_obj, refused_turns.add, response_id_security
),
**_with_routable_comparison_id(
_routing_hints_from_first_ws_frame(first_message), response_id_security, user_api_key_dict
),
**_routing_hints_from_first_ws_frame(first_message),
)
async def write_failure_row(exception: Exception) -> None:
@ -2336,6 +2368,7 @@ async def responses_websocket_endpoint(
websocket: WebSocket,
model: str | None = fastapi.Query(None, description="The model to use for the responses WebSocket session."),
user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth_websocket),
response_id_security: ResponsesIDSecurity | None = _response_id_security_dep,
):
"""
Responses API WebSocket mode endpoint.
@ -2366,7 +2399,12 @@ async def responses_websocket_endpoint(
limit_seconds: Final = _resolve_responses_ws_session_limit_seconds()
session_task: Final = asyncio.ensure_future(
_responses_websocket_session(websocket=websocket, model=model, user_api_key_dict=user_api_key_dict)
_responses_websocket_session(
websocket=websocket,
model=model,
user_api_key_dict=user_api_key_dict,
response_id_security=response_id_security,
)
)
try:
await asyncio.wait_for(asyncio.shield(session_task), timeout=limit_seconds)

View file

@ -2718,7 +2718,7 @@ def _deployment_reasoning_default(kwargs: Mapping[str, object]) -> Reasoning | d
return LiteLLMResponsesTransformationHandler().map_reasoning_effort(reasoning_effort)
_RESPONSES_WS_ROUTING_HINT_KEYS: Final = frozenset({"input", "previous_response_id"})
_RESPONSES_WS_ROUTING_HINT_KEYS: Final = frozenset({"input", "previous_response_id", "prompt_cache_options"})
def _first_ws_frame_with_routed_input(first_message: str, routed_input: object) -> str:

View file

@ -1963,6 +1963,24 @@ class TestResponsesWSFirstFrameValidation:
ws.close.assert_awaited_once_with(code=1011, reason="Internal server error")
_COMPARISON_ID_SALT: Final = "comparison-id-unit-test-salt"
def _comparison_id_security_and_owners_id(monkeypatch: pytest.MonkeyPatch, original_id: str):
from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy.hooks.responses_id_security import ResponsesIDSecurity
monkeypatch.setenv("LITELLM_SALT_KEY", _COMPARISON_ID_SALT)
security: Final = ResponsesIDSecurity(
general_settings_reader=lambda: {}, signing_key_reader=lambda: _COMPARISON_ID_SALT
)
issued: Final = security._encrypt_response_id(
ResponsesAPIResponse(id=original_id, object="response", status="completed", created_at=1, output=[]),
UserAPIKeyAuth(user_id="owner"),
)
return security, issued.id
class TestResponsesWSFirstFrameModelAuth:
@pytest.mark.asyncio
async def test_endpoint_enforces_auth_after_model_from_first_frame(self):
@ -2082,6 +2100,78 @@ class TestResponsesWSFirstFrameModelAuth:
assert mock_route_request.await_args.kwargs["route_type"] == "_aresponses_websocket"
ws.close.assert_not_awaited()
@pytest.mark.asyncio
@pytest.mark.parametrize("nested", [False, True])
@pytest.mark.parametrize(
"caller_id, routed_options",
[
("owner", {"ttl": "12h", "comparison_response_id": "resp_compared"}),
("stranger", None),
],
)
async def test_endpoint_routes_a_first_frame_comparison_id_only_for_its_owner(
self,
monkeypatch: pytest.MonkeyPatch,
nested: bool,
caller_id: str,
routed_options: Mapping[str, object] | None,
):
from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy.response_api_endpoints.endpoints import responses_websocket_endpoint
security, issued_id = _comparison_id_security_and_owners_id(monkeypatch, "resp_compared")
payload = {
"model": "gpt-4o-mini",
"input": "hi",
"prompt_cache_options": {"ttl": "12h", "comparison_response_id": issued_id},
}
first_frame = (
{"type": "response.create", "response": payload} if nested else {"type": "response.create", **payload}
)
ws = MagicMock()
ws.headers = {}
ws.query_params = {}
ws.scope = {"headers": []}
ws.url = "ws://testserver/v1/responses"
ws.accept = AsyncMock()
ws.receive_text = AsyncMock(return_value=json.dumps(first_frame))
ws.close = AsyncMock()
processor = MagicMock()
processor.common_processing_pre_call_logic = AsyncMock(
return_value=({"model": "gpt-4o-mini", "litellm_metadata": {}}, MagicMock())
)
async def fake_llm_call():
return None
with (
patch( # test-quality-ok: first-frame model auth needs a live router and key table and has its own tests
"litellm.proxy.response_api_endpoints.endpoints._enforce_responses_ws_first_frame_model_auth",
new_callable=AsyncMock,
),
patch( # test-quality-ok: the pre-call processor needs a live proxy; the routed payload is under test
"litellm.proxy.response_api_endpoints.endpoints.ProxyBaseLLMRequestProcessing",
return_value=processor,
),
patch( # test-quality-ok: routing is the seam where the first frame's comparison id becomes observable
"litellm.proxy.route_llm_request.route_request",
new_callable=AsyncMock,
return_value=fake_llm_call(),
) as mock_route_request,
):
await responses_websocket_endpoint(
websocket=ws,
model=None,
user_api_key_dict=UserAPIKeyAuth(user_id=caller_id),
response_id_security=security,
)
routed = mock_route_request.await_args.kwargs["data"]
assert routed.get("prompt_cache_options") == routed_options
assert routed["input"] == "hi"
@pytest.mark.asyncio
async def test_frame_guard_runs_the_guardrail_walk_on_the_frame_body_and_returns_its_records(self):
from litellm.proxy.response_api_endpoints.endpoints import _responses_ws_frame_guard
@ -2178,6 +2268,75 @@ class TestResponsesWSFirstFrameModelAuth:
"response": {"input": "hi [guarded]", "tools": tools[:1]},
}
@pytest.mark.asyncio
@pytest.mark.parametrize("nested", [False, True])
@pytest.mark.parametrize("sent_id, forwarded_id", [(None, "resp_compared"), ("resp_raw_provider_id", None)])
async def test_frame_guard_forwards_the_compared_response_id_the_provider_issued(
self, monkeypatch: pytest.MonkeyPatch, nested: bool, sent_id: str | None, forwarded_id: str | None
):
from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy.response_api_endpoints.endpoints import _responses_ws_frame_guard
security, issued_id = _comparison_id_security_and_owners_id(monkeypatch, "resp_compared")
comparison_id = sent_id or issued_id
proxy_logging_obj = MagicMock()
proxy_logging_obj.pre_call_hook = AsyncMock(side_effect=lambda **kwargs: kwargs["data"])
guard = _responses_ws_frame_guard(
{"model": "gpt-4o-mini", "litellm_metadata": {}},
UserAPIKeyAuth(user_id="owner"),
proxy_logging_obj,
MagicMock(),
security,
)
body = {"input": "hi", "prompt_cache_options": {"ttl": "12h", "comparison_response_id": comparison_id}}
frame = {"type": "response.create", "response": body} if nested else {"type": "response.create", **body}
guarded = await guard(json.dumps(frame))
sent = json.loads(guarded.frame)
assert (sent["response"] if nested else sent)["prompt_cache_options"] == {
"ttl": "12h",
"comparison_response_id": forwarded_id or comparison_id,
}
@pytest.mark.asyncio
async def test_frame_guard_refuses_and_bills_a_comparison_id_issued_to_another_user(
self, monkeypatch: pytest.MonkeyPatch
):
import litellm
from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy.hooks.proxy_track_cost_callback import ProxyDBLogger
from litellm.proxy.response_api_endpoints.endpoints import _responses_ws_frame_guard
security, issued_id = _comparison_id_security_and_owners_id(monkeypatch, "resp_compared")
proxy_logging_obj = MagicMock()
proxy_logging_obj.pre_call_hook = AsyncMock(side_effect=lambda **kwargs: kwargs["data"])
spend_logger = MagicMock(spec=ProxyDBLogger)
spend_logger.async_post_call_failure_hook = AsyncMock()
guard = _responses_ws_frame_guard(
{"model": "gpt-4o-mini", "litellm_metadata": {}},
UserAPIKeyAuth(user_id="stranger"),
proxy_logging_obj,
MagicMock(),
security,
)
with patch.object(litellm, "callbacks", [spend_logger]), pytest.raises(ProxyException) as refused:
await guard(
json.dumps(
{
"type": "response.create",
"input": "hi",
"prompt_cache_options": {"comparison_response_id": issued_id},
}
)
)
assert (refused.value.type, refused.value.code) == ("permission_error", "403")
proxy_logging_obj.pre_call_hook.assert_not_awaited()
spend_logger.async_post_call_failure_hook.assert_awaited_once()
assert spend_logger.async_post_call_failure_hook.await_args.kwargs["original_exception"] is refused.value
@pytest.mark.asyncio
async def test_frame_guard_strips_the_proxy_owned_metadata_keys_a_client_sends(self):
from litellm.proxy.response_api_endpoints.endpoints import _responses_ws_frame_guard

View file

@ -662,11 +662,13 @@ async def test_aresponses_websocket_keeps_routing_hints_out_of_the_relay_kwargs(
litellm_logging_obj=MagicMock(),
input=[{"type": "message", "role": "user", "content": "hi"}],
previous_response_id="resp_prev",
prompt_cache_options={"comparison_response_id": "resp_compared"},
)
mock_ws.assert_awaited_once()
assert "input" not in mock_ws.call_args.kwargs
assert "previous_response_id" not in mock_ws.call_args.kwargs
assert "prompt_cache_options" not in mock_ws.call_args.kwargs
_STRIPPED_WS_INPUT = [{"role": "user", "content": "hi"}]