mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-17 09:34:48 +00:00
fix(responses): authorize proxy-issued comparison ids on WebSocket frames and pin a new socket to the compared response
This commit is contained in:
parent
25c42c4229
commit
13f1bf6343
5 changed files with 206 additions and 7 deletions
|
|
@ -198,7 +198,7 @@ class ResponsesIDSecurity(CustomLogger):
|
|||
if comparison_id is None:
|
||||
return _NOTHING_AUTHORIZED
|
||||
authorized_options: Final = _with_comparison_response_id(
|
||||
options, self._authorized_comparison_id(comparison_id, user_api_key_dict)
|
||||
options, self.authorize_comparison_id(comparison_id, user_api_key_dict)
|
||||
)
|
||||
placed_options: Final = (
|
||||
{_PROMPT_CACHE_OPTIONS: authorized_options}
|
||||
|
|
@ -207,7 +207,7 @@ class ResponsesIDSecurity(CustomLogger):
|
|||
)
|
||||
return {**placed_options, COMPARISON_RESPONSE_ID_FIELD: comparison_id}
|
||||
|
||||
def _authorized_comparison_id(self, comparison_id: str, user_api_key_dict: "UserAPIKeyAuth") -> str:
|
||||
def authorize_comparison_id(self, comparison_id: str, user_api_key_dict: "UserAPIKeyAuth") -> str:
|
||||
if not self._is_encrypted_response_id(comparison_id):
|
||||
return comparison_id
|
||||
return self.authorize_response_id(comparison_id, user_api_key_dict)
|
||||
|
|
|
|||
|
|
@ -1490,6 +1490,29 @@ def _with_authorized_previous_response(
|
|||
return {**data, "previous_response_id": authorized_id, ADDRESSED_RESPONSE_ID_FIELD: previous_response_id}
|
||||
|
||||
|
||||
def _with_authorized_comparison_id(
|
||||
request: Mapping[str, object], security: ResponsesIDSecurity | None, user_api_key_dict: UserAPIKeyAuth
|
||||
) -> Mapping[str, object]:
|
||||
options: Final = as_str_mapping(request.get("prompt_cache_options"))
|
||||
comparison_id: Final = None if options is None else options.get("comparison_response_id")
|
||||
if security is None or options is None or not isinstance(comparison_id, str) or not comparison_id:
|
||||
return request
|
||||
try:
|
||||
authorized_id: Final = security.authorize_comparison_id(comparison_id, user_api_key_dict)
|
||||
except HTTPException as e:
|
||||
raise ProxyException(message=str(e.detail), type="permission_error", param=None, code=e.status_code) from e
|
||||
return {**request, "prompt_cache_options": {**options, "comparison_response_id": authorized_id}}
|
||||
|
||||
|
||||
def _with_routable_comparison_id(
|
||||
hints: Mapping[str, object], security: ResponsesIDSecurity | None, user_api_key_dict: UserAPIKeyAuth
|
||||
) -> Mapping[str, object]:
|
||||
try:
|
||||
return _with_authorized_comparison_id(hints, security, user_api_key_dict)
|
||||
except ProxyException:
|
||||
return {key: value for key, value in hints.items() if key != "prompt_cache_options"}
|
||||
|
||||
|
||||
def _responses_invalid_request_response(message: str, param: str | None, code: str | None) -> JSONResponse:
|
||||
body: Final[_ResponsesApiErrorBody] = {
|
||||
"error": {
|
||||
|
|
@ -1939,6 +1962,7 @@ class _ResponseCreateRoutingHints(LiteLLMBaseModel):
|
|||
|
||||
input: str | Sequence[object] | None = None
|
||||
previous_response_id: str | None = None
|
||||
prompt_cache_options: Mapping[str, object] | None = None
|
||||
response: "_ResponseCreateRoutingHints | None" = None
|
||||
|
||||
|
||||
|
|
@ -1953,6 +1977,9 @@ def _routing_hints_from_first_ws_frame(first_message: str) -> Mapping[str, objec
|
|||
"previous_response_id": (
|
||||
frame.previous_response_id if nested.previous_response_id is None else nested.previous_response_id
|
||||
),
|
||||
"prompt_cache_options": (
|
||||
frame.prompt_cache_options if nested.prompt_cache_options is None else nested.prompt_cache_options
|
||||
),
|
||||
}
|
||||
return MappingProxyType({key: value for key, value in hints.items() if value is not None})
|
||||
|
||||
|
|
@ -2037,6 +2064,7 @@ def _responses_ws_frame_guard(
|
|||
user_api_key_dict: UserAPIKeyAuth,
|
||||
proxy_logging_obj: _PreCallGuardrailWalk,
|
||||
record_refused_turn: Callable[[tuple[Mapping[str, object], ...], Exception], None],
|
||||
response_id_security: ResponsesIDSecurity | None = None,
|
||||
) -> ResponsesWsFrameGuard:
|
||||
"""Socket admission skips guardrails (the first frame's ``input`` is not part of the admitted request), so every
|
||||
``response.create`` frame gets the pre_call guardrail walk the HTTP ``/v1/responses`` path runs on its body."""
|
||||
|
|
@ -2074,6 +2102,7 @@ def _responses_ws_frame_guard(
|
|||
metadata_key: {**bucket},
|
||||
}
|
||||
try:
|
||||
authorized_body: Final = _with_authorized_comparison_id(body, response_id_security, user_api_key_dict)
|
||||
guarded_data: Final = await proxy_logging_obj.pre_call_hook(
|
||||
user_api_key_dict=user_api_key_dict, data=frame_data, call_type="aresponses", guardrails_only=True
|
||||
)
|
||||
|
|
@ -2089,7 +2118,7 @@ def _responses_ws_frame_guard(
|
|||
raise rerouted
|
||||
# a guardrail may delete a field (the Responses translation pops ``instructions``), so start without them
|
||||
guarded_fields: Final = {key: guarded_data[key] for key in _GUARDED_FRAME_FIELDS if key in guarded_data}
|
||||
kept: Final = {key: value for key, value in body.items() if key not in _GUARDED_FRAME_FIELDS}
|
||||
kept: Final = {key: value for key, value in authorized_body.items() if key not in _GUARDED_FRAME_FIELDS}
|
||||
rewritten: Final = (
|
||||
{**frame_obj, "response": {**kept, **guarded_fields}}
|
||||
if isinstance(nested, dict)
|
||||
|
|
@ -2184,6 +2213,7 @@ async def _responses_websocket_session(
|
|||
websocket: WebSocket,
|
||||
model: str | None,
|
||||
user_api_key_dict: UserAPIKeyAuth,
|
||||
response_id_security: ResponsesIDSecurity | None,
|
||||
) -> None:
|
||||
from litellm.proxy.proxy_server import (
|
||||
general_settings,
|
||||
|
|
@ -2281,9 +2311,11 @@ async def _responses_websocket_session(
|
|||
data,
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
responses_ws_frame_guard=_responses_ws_frame_guard(
|
||||
data, user_api_key_dict, proxy_logging_obj, refused_turns.add
|
||||
data, user_api_key_dict, proxy_logging_obj, refused_turns.add, response_id_security
|
||||
),
|
||||
**_with_routable_comparison_id(
|
||||
_routing_hints_from_first_ws_frame(first_message), response_id_security, user_api_key_dict
|
||||
),
|
||||
**_routing_hints_from_first_ws_frame(first_message),
|
||||
)
|
||||
|
||||
async def write_failure_row(exception: Exception) -> None:
|
||||
|
|
@ -2336,6 +2368,7 @@ async def responses_websocket_endpoint(
|
|||
websocket: WebSocket,
|
||||
model: str | None = fastapi.Query(None, description="The model to use for the responses WebSocket session."),
|
||||
user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth_websocket),
|
||||
response_id_security: ResponsesIDSecurity | None = _response_id_security_dep,
|
||||
):
|
||||
"""
|
||||
Responses API WebSocket mode endpoint.
|
||||
|
|
@ -2366,7 +2399,12 @@ async def responses_websocket_endpoint(
|
|||
|
||||
limit_seconds: Final = _resolve_responses_ws_session_limit_seconds()
|
||||
session_task: Final = asyncio.ensure_future(
|
||||
_responses_websocket_session(websocket=websocket, model=model, user_api_key_dict=user_api_key_dict)
|
||||
_responses_websocket_session(
|
||||
websocket=websocket,
|
||||
model=model,
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
response_id_security=response_id_security,
|
||||
)
|
||||
)
|
||||
try:
|
||||
await asyncio.wait_for(asyncio.shield(session_task), timeout=limit_seconds)
|
||||
|
|
|
|||
|
|
@ -2718,7 +2718,7 @@ def _deployment_reasoning_default(kwargs: Mapping[str, object]) -> Reasoning | d
|
|||
return LiteLLMResponsesTransformationHandler().map_reasoning_effort(reasoning_effort)
|
||||
|
||||
|
||||
_RESPONSES_WS_ROUTING_HINT_KEYS: Final = frozenset({"input", "previous_response_id"})
|
||||
_RESPONSES_WS_ROUTING_HINT_KEYS: Final = frozenset({"input", "previous_response_id", "prompt_cache_options"})
|
||||
|
||||
|
||||
def _first_ws_frame_with_routed_input(first_message: str, routed_input: object) -> str:
|
||||
|
|
|
|||
|
|
@ -1963,6 +1963,24 @@ class TestResponsesWSFirstFrameValidation:
|
|||
ws.close.assert_awaited_once_with(code=1011, reason="Internal server error")
|
||||
|
||||
|
||||
_COMPARISON_ID_SALT: Final = "comparison-id-unit-test-salt"
|
||||
|
||||
|
||||
def _comparison_id_security_and_owners_id(monkeypatch: pytest.MonkeyPatch, original_id: str):
|
||||
from litellm.proxy._types import UserAPIKeyAuth
|
||||
from litellm.proxy.hooks.responses_id_security import ResponsesIDSecurity
|
||||
|
||||
monkeypatch.setenv("LITELLM_SALT_KEY", _COMPARISON_ID_SALT)
|
||||
security: Final = ResponsesIDSecurity(
|
||||
general_settings_reader=lambda: {}, signing_key_reader=lambda: _COMPARISON_ID_SALT
|
||||
)
|
||||
issued: Final = security._encrypt_response_id(
|
||||
ResponsesAPIResponse(id=original_id, object="response", status="completed", created_at=1, output=[]),
|
||||
UserAPIKeyAuth(user_id="owner"),
|
||||
)
|
||||
return security, issued.id
|
||||
|
||||
|
||||
class TestResponsesWSFirstFrameModelAuth:
|
||||
@pytest.mark.asyncio
|
||||
async def test_endpoint_enforces_auth_after_model_from_first_frame(self):
|
||||
|
|
@ -2082,6 +2100,78 @@ class TestResponsesWSFirstFrameModelAuth:
|
|||
assert mock_route_request.await_args.kwargs["route_type"] == "_aresponses_websocket"
|
||||
ws.close.assert_not_awaited()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("nested", [False, True])
|
||||
@pytest.mark.parametrize(
|
||||
"caller_id, routed_options",
|
||||
[
|
||||
("owner", {"ttl": "12h", "comparison_response_id": "resp_compared"}),
|
||||
("stranger", None),
|
||||
],
|
||||
)
|
||||
async def test_endpoint_routes_a_first_frame_comparison_id_only_for_its_owner(
|
||||
self,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
nested: bool,
|
||||
caller_id: str,
|
||||
routed_options: Mapping[str, object] | None,
|
||||
):
|
||||
from litellm.proxy._types import UserAPIKeyAuth
|
||||
from litellm.proxy.response_api_endpoints.endpoints import responses_websocket_endpoint
|
||||
|
||||
security, issued_id = _comparison_id_security_and_owners_id(monkeypatch, "resp_compared")
|
||||
payload = {
|
||||
"model": "gpt-4o-mini",
|
||||
"input": "hi",
|
||||
"prompt_cache_options": {"ttl": "12h", "comparison_response_id": issued_id},
|
||||
}
|
||||
first_frame = (
|
||||
{"type": "response.create", "response": payload} if nested else {"type": "response.create", **payload}
|
||||
)
|
||||
|
||||
ws = MagicMock()
|
||||
ws.headers = {}
|
||||
ws.query_params = {}
|
||||
ws.scope = {"headers": []}
|
||||
ws.url = "ws://testserver/v1/responses"
|
||||
ws.accept = AsyncMock()
|
||||
ws.receive_text = AsyncMock(return_value=json.dumps(first_frame))
|
||||
ws.close = AsyncMock()
|
||||
|
||||
processor = MagicMock()
|
||||
processor.common_processing_pre_call_logic = AsyncMock(
|
||||
return_value=({"model": "gpt-4o-mini", "litellm_metadata": {}}, MagicMock())
|
||||
)
|
||||
|
||||
async def fake_llm_call():
|
||||
return None
|
||||
|
||||
with (
|
||||
patch( # test-quality-ok: first-frame model auth needs a live router and key table and has its own tests
|
||||
"litellm.proxy.response_api_endpoints.endpoints._enforce_responses_ws_first_frame_model_auth",
|
||||
new_callable=AsyncMock,
|
||||
),
|
||||
patch( # test-quality-ok: the pre-call processor needs a live proxy; the routed payload is under test
|
||||
"litellm.proxy.response_api_endpoints.endpoints.ProxyBaseLLMRequestProcessing",
|
||||
return_value=processor,
|
||||
),
|
||||
patch( # test-quality-ok: routing is the seam where the first frame's comparison id becomes observable
|
||||
"litellm.proxy.route_llm_request.route_request",
|
||||
new_callable=AsyncMock,
|
||||
return_value=fake_llm_call(),
|
||||
) as mock_route_request,
|
||||
):
|
||||
await responses_websocket_endpoint(
|
||||
websocket=ws,
|
||||
model=None,
|
||||
user_api_key_dict=UserAPIKeyAuth(user_id=caller_id),
|
||||
response_id_security=security,
|
||||
)
|
||||
|
||||
routed = mock_route_request.await_args.kwargs["data"]
|
||||
assert routed.get("prompt_cache_options") == routed_options
|
||||
assert routed["input"] == "hi"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_frame_guard_runs_the_guardrail_walk_on_the_frame_body_and_returns_its_records(self):
|
||||
from litellm.proxy.response_api_endpoints.endpoints import _responses_ws_frame_guard
|
||||
|
|
@ -2178,6 +2268,75 @@ class TestResponsesWSFirstFrameModelAuth:
|
|||
"response": {"input": "hi [guarded]", "tools": tools[:1]},
|
||||
}
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("nested", [False, True])
|
||||
@pytest.mark.parametrize("sent_id, forwarded_id", [(None, "resp_compared"), ("resp_raw_provider_id", None)])
|
||||
async def test_frame_guard_forwards_the_compared_response_id_the_provider_issued(
|
||||
self, monkeypatch: pytest.MonkeyPatch, nested: bool, sent_id: str | None, forwarded_id: str | None
|
||||
):
|
||||
from litellm.proxy._types import UserAPIKeyAuth
|
||||
from litellm.proxy.response_api_endpoints.endpoints import _responses_ws_frame_guard
|
||||
|
||||
security, issued_id = _comparison_id_security_and_owners_id(monkeypatch, "resp_compared")
|
||||
comparison_id = sent_id or issued_id
|
||||
proxy_logging_obj = MagicMock()
|
||||
proxy_logging_obj.pre_call_hook = AsyncMock(side_effect=lambda **kwargs: kwargs["data"])
|
||||
guard = _responses_ws_frame_guard(
|
||||
{"model": "gpt-4o-mini", "litellm_metadata": {}},
|
||||
UserAPIKeyAuth(user_id="owner"),
|
||||
proxy_logging_obj,
|
||||
MagicMock(),
|
||||
security,
|
||||
)
|
||||
body = {"input": "hi", "prompt_cache_options": {"ttl": "12h", "comparison_response_id": comparison_id}}
|
||||
frame = {"type": "response.create", "response": body} if nested else {"type": "response.create", **body}
|
||||
|
||||
guarded = await guard(json.dumps(frame))
|
||||
|
||||
sent = json.loads(guarded.frame)
|
||||
assert (sent["response"] if nested else sent)["prompt_cache_options"] == {
|
||||
"ttl": "12h",
|
||||
"comparison_response_id": forwarded_id or comparison_id,
|
||||
}
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_frame_guard_refuses_and_bills_a_comparison_id_issued_to_another_user(
|
||||
self, monkeypatch: pytest.MonkeyPatch
|
||||
):
|
||||
import litellm
|
||||
from litellm.proxy._types import UserAPIKeyAuth
|
||||
from litellm.proxy.hooks.proxy_track_cost_callback import ProxyDBLogger
|
||||
from litellm.proxy.response_api_endpoints.endpoints import _responses_ws_frame_guard
|
||||
|
||||
security, issued_id = _comparison_id_security_and_owners_id(monkeypatch, "resp_compared")
|
||||
proxy_logging_obj = MagicMock()
|
||||
proxy_logging_obj.pre_call_hook = AsyncMock(side_effect=lambda **kwargs: kwargs["data"])
|
||||
spend_logger = MagicMock(spec=ProxyDBLogger)
|
||||
spend_logger.async_post_call_failure_hook = AsyncMock()
|
||||
guard = _responses_ws_frame_guard(
|
||||
{"model": "gpt-4o-mini", "litellm_metadata": {}},
|
||||
UserAPIKeyAuth(user_id="stranger"),
|
||||
proxy_logging_obj,
|
||||
MagicMock(),
|
||||
security,
|
||||
)
|
||||
|
||||
with patch.object(litellm, "callbacks", [spend_logger]), pytest.raises(ProxyException) as refused:
|
||||
await guard(
|
||||
json.dumps(
|
||||
{
|
||||
"type": "response.create",
|
||||
"input": "hi",
|
||||
"prompt_cache_options": {"comparison_response_id": issued_id},
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
assert (refused.value.type, refused.value.code) == ("permission_error", "403")
|
||||
proxy_logging_obj.pre_call_hook.assert_not_awaited()
|
||||
spend_logger.async_post_call_failure_hook.assert_awaited_once()
|
||||
assert spend_logger.async_post_call_failure_hook.await_args.kwargs["original_exception"] is refused.value
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_frame_guard_strips_the_proxy_owned_metadata_keys_a_client_sends(self):
|
||||
from litellm.proxy.response_api_endpoints.endpoints import _responses_ws_frame_guard
|
||||
|
|
|
|||
|
|
@ -662,11 +662,13 @@ async def test_aresponses_websocket_keeps_routing_hints_out_of_the_relay_kwargs(
|
|||
litellm_logging_obj=MagicMock(),
|
||||
input=[{"type": "message", "role": "user", "content": "hi"}],
|
||||
previous_response_id="resp_prev",
|
||||
prompt_cache_options={"comparison_response_id": "resp_compared"},
|
||||
)
|
||||
|
||||
mock_ws.assert_awaited_once()
|
||||
assert "input" not in mock_ws.call_args.kwargs
|
||||
assert "previous_response_id" not in mock_ws.call_args.kwargs
|
||||
assert "prompt_cache_options" not in mock_ws.call_args.kwargs
|
||||
|
||||
|
||||
_STRIPPED_WS_INPUT = [{"role": "user", "content": "hi"}]
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue