fix(responses): trust the WebSocket comparison routing hint only as an internal typed value

A JSON body can no longer set the hint key to pin routing to a deployment.
This commit is contained in:
mateo-berri 2026-10-28 15:25:21 -07:00
parent d423432a1e
commit e7eddf08ef
5 changed files with 41 additions and 23 deletions

View file

@ -84,7 +84,7 @@ from litellm.proxy.utils import count_request_input_tokens, handle_exception_on_
from litellm.responses.litellm_completion_transformation.compaction_items import messages_since_newest_compaction
from litellm.responses.litellm_completion_transformation.count_request import MODEL_FILE_ID_MAPPING_FIELD
from litellm.responses.streaming_iterator import GuardedResponsesWsFrame, ResponsesWsFrameGuard, spawn_logging_task
from litellm.responses.utils import COMPARISON_RESPONSE_ID_ROUTING_HINT
from litellm.responses.utils import COMPARISON_RESPONSE_ID_ROUTING_HINT, ComparisonRoutingHint
from litellm.router import Router
from litellm.types.llms.base import LiteLLMBaseModel
from litellm.types.llms.openai import (
@ -1508,14 +1508,14 @@ def _with_authorized_comparison_id(
def _with_routable_comparison_id(
hints: Mapping[str, object], security: ResponsesIDSecurity | None, user_api_key_dict: UserAPIKeyAuth
) -> Mapping[str, object]:
comparison_id: Final = hints.get(COMPARISON_RESPONSE_ID_ROUTING_HINT)
if security is None or not isinstance(comparison_id, str):
routing_hint: Final = hints.get(COMPARISON_RESPONSE_ID_ROUTING_HINT)
if security is None or not isinstance(routing_hint, ComparisonRoutingHint):
return hints
try:
authorized_id: Final = security.authorize_comparison_id(comparison_id, user_api_key_dict)
authorized_id: Final = security.authorize_comparison_id(routing_hint.response_id, user_api_key_dict)
except HTTPException:
return {key: value for key, value in hints.items() if key != COMPARISON_RESPONSE_ID_ROUTING_HINT}
return {**hints, COMPARISON_RESPONSE_ID_ROUTING_HINT: authorized_id}
return {**hints, COMPARISON_RESPONSE_ID_ROUTING_HINT: ComparisonRoutingHint(authorized_id)}
def _responses_invalid_request_response(message: str, param: str | None, code: str | None) -> JSONResponse:
@ -1986,7 +1986,9 @@ def _routing_hints_from_first_ws_frame(first_message: str) -> Mapping[str, objec
"previous_response_id": (
frame.previous_response_id if nested.previous_response_id is None else nested.previous_response_id
),
COMPARISON_RESPONSE_ID_ROUTING_HINT: comparison_id if isinstance(comparison_id, str) else None,
COMPARISON_RESPONSE_ID_ROUTING_HINT: (
ComparisonRoutingHint(comparison_id) if isinstance(comparison_id, str) else None
),
}
return MappingProxyType({key: value for key, value in hints.items() if value is not None})

View file

@ -1,6 +1,7 @@
import base64
import re
from collections.abc import Callable, Collection, Iterable, Mapping, Sequence
from dataclasses import dataclass
from functools import reduce
from typing import Any, Final, Optional, TypeVar, Union, cast, get_type_hints, overload
@ -38,6 +39,12 @@ from litellm.types.utils import (
COMPARISON_RESPONSE_ID_ROUTING_HINT: Final = "comparison_response_id_routing_hint"
@dataclass(frozen=True, slots=True)
class ComparisonRoutingHint:
response_id: str
_PROVIDER_SPECIFIC_HEADER_ADAPTER: Final[
TypeAdapter[ProviderSpecificHeader | Sequence[ProviderSpecificHeader] | None]
] = TypeAdapter(ProviderSpecificHeader | Sequence[ProviderSpecificHeader] | None)
@ -807,8 +814,8 @@ class ResponsesAPIRequestUtils:
@staticmethod
def comparison_response_id(request: Mapping[str, object]) -> str | None:
routing_hint: Final = request.get(COMPARISON_RESPONSE_ID_ROUTING_HINT)
if isinstance(routing_hint, str) and routing_hint:
return routing_hint
if isinstance(routing_hint, ComparisonRoutingHint) and routing_hint.response_id:
return routing_hint.response_id
extra_body: Final = as_str_mapping(request.get("extra_body"))
container: Final = extra_body if extra_body is not None and "prompt_cache_options" in extra_body else request
options: Final = as_str_mapping(container.get("prompt_cache_options"))

View file

@ -20,7 +20,7 @@ from httpx import Response
import litellm
from litellm.proxy._types import ProxyException
from litellm.responses.utils import COMPARISON_RESPONSE_ID_ROUTING_HINT
from litellm.responses.utils import COMPARISON_RESPONSE_ID_ROUTING_HINT, ComparisonRoutingHint
from litellm.proxy.proxy_server import app
from litellm.types.llms.openai import ResponseAPIUsage, ResponsesAPIResponse
from litellm.types.router import RouterErrors
@ -2106,7 +2106,7 @@ class TestResponsesWSFirstFrameModelAuth:
@pytest.mark.parametrize(
"caller_id, routed_comparison_id",
[
("owner", "resp_compared"),
("owner", ComparisonRoutingHint("resp_compared")),
("stranger", None),
],
)
@ -2115,7 +2115,7 @@ class TestResponsesWSFirstFrameModelAuth:
monkeypatch: pytest.MonkeyPatch,
nested: bool,
caller_id: str,
routed_comparison_id: str | None,
routed_comparison_id: ComparisonRoutingHint | None,
):
from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy.response_api_endpoints.endpoints import responses_websocket_endpoint

View file

@ -23,7 +23,7 @@ from pydantic import TypeAdapter
import litellm
from litellm._logging import verbose_logger
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
from litellm.responses.utils import COMPARISON_RESPONSE_ID_ROUTING_HINT
from litellm.responses.utils import COMPARISON_RESPONSE_ID_ROUTING_HINT, ComparisonRoutingHint
from litellm.types.llms.openai import ResponsesAPIResponse
from tests.unit.llms.bedrock.slow_upstream import (
STREAM_TIMEOUT_SECONDS,
@ -663,7 +663,7 @@ async def test_aresponses_websocket_keeps_routing_hints_out_of_the_relay_kwargs(
litellm_logging_obj=MagicMock(),
input=[{"type": "message", "role": "user", "content": "hi"}],
previous_response_id="resp_prev",
**{COMPARISON_RESPONSE_ID_ROUTING_HINT: "resp_compared"},
**{COMPARISON_RESPONSE_ID_ROUTING_HINT: ComparisonRoutingHint("resp_compared")},
)
mock_ws.assert_awaited_once()
@ -689,7 +689,7 @@ async def test_aresponses_websocket_fills_a_deployment_prompt_cache_options_next
api_key="sk-test",
litellm_logging_obj=MagicMock(),
prompt_cache_options={"ttl": "12h"},
**{COMPARISON_RESPONSE_ID_ROUTING_HINT: "resp_compared"},
**{COMPARISON_RESPONSE_ID_ROUTING_HINT: ComparisonRoutingHint("resp_compared")},
)
request_defaults: Final = mock_ws.call_args.kwargs["request_defaults"]

View file

@ -9,7 +9,11 @@ import pytest
import litellm
from litellm.caching.dual_cache import DualCache
from litellm.responses.utils import COMPARISON_RESPONSE_ID_ROUTING_HINT, ResponsesAPIRequestUtils
from litellm.responses.utils import (
COMPARISON_RESPONSE_ID_ROUTING_HINT,
ComparisonRoutingHint,
ResponsesAPIRequestUtils,
)
from litellm.router_utils.pre_call_checks.deployment_affinity_check import (
DeploymentAffinityCheck,
)
@ -1309,11 +1313,11 @@ def _response_id_from(model_id: str) -> str:
@pytest.mark.asyncio
@pytest.mark.parametrize(
("request_kwargs", "pinned_model_id"),
("request_kwargs", "pinned_model_ids"),
[
pytest.param(
{"prompt_cache_options": {"comparison_response_id": _response_id_from("B")}},
"B",
("B",),
id="top-level-comparison",
),
pytest.param(
@ -1321,7 +1325,7 @@ def _response_id_from(model_id: str) -> str:
"prompt_cache_options": {"comparison_response_id": _response_id_from("A")},
"extra_body": {"prompt_cache_options": {"comparison_response_id": _response_id_from("B")}},
},
"B",
("B",),
id="extra-body-comparison-wins",
),
pytest.param(
@ -1329,18 +1333,23 @@ def _response_id_from(model_id: str) -> str:
"previous_response_id": _response_id_from("A"),
"prompt_cache_options": {"comparison_response_id": _response_id_from("B")},
},
"A",
("A",),
id="previous-response-wins",
),
pytest.param(
{COMPARISON_RESPONSE_ID_ROUTING_HINT: _response_id_from("C")},
"C",
{COMPARISON_RESPONSE_ID_ROUTING_HINT: ComparisonRoutingHint(_response_id_from("C"))},
("C",),
id="websocket-first-frame-routing-hint",
),
pytest.param(
{COMPARISON_RESPONSE_ID_ROUTING_HINT: _response_id_from("C")},
("A", "B", "C"),
id="request-body-key-named-like-the-routing-hint",
),
],
)
async def test_comparison_response_id_pins_its_deployment_unless_a_previous_response_id_does(
request_kwargs: dict[str, object], pinned_model_id: str
request_kwargs: dict[str, object], pinned_model_ids: tuple[str, ...]
) -> None:
callback: Final = DeploymentAffinityCheck(
cache=DualCache(),
@ -1361,7 +1370,7 @@ async def test_comparison_response_id_pins_its_deployment_unless_a_previous_resp
parent_otel_span=None,
)
assert [deployment["model_info"]["id"] for deployment in pinned] == [pinned_model_id]
assert tuple(deployment["model_info"]["id"] for deployment in pinned) == pinned_model_ids
@pytest.mark.asyncio