diff --git a/litellm/proxy/response_api_endpoints/endpoints.py b/litellm/proxy/response_api_endpoints/endpoints.py index 22b74834eb4..4729ae35970 100644 --- a/litellm/proxy/response_api_endpoints/endpoints.py +++ b/litellm/proxy/response_api_endpoints/endpoints.py @@ -84,7 +84,7 @@ from litellm.proxy.utils import count_request_input_tokens, handle_exception_on_ from litellm.responses.litellm_completion_transformation.compaction_items import messages_since_newest_compaction from litellm.responses.litellm_completion_transformation.count_request import MODEL_FILE_ID_MAPPING_FIELD from litellm.responses.streaming_iterator import GuardedResponsesWsFrame, ResponsesWsFrameGuard, spawn_logging_task -from litellm.responses.utils import COMPARISON_RESPONSE_ID_ROUTING_HINT +from litellm.responses.utils import COMPARISON_RESPONSE_ID_ROUTING_HINT, ComparisonRoutingHint from litellm.router import Router from litellm.types.llms.base import LiteLLMBaseModel from litellm.types.llms.openai import ( @@ -1508,14 +1508,14 @@ def _with_authorized_comparison_id( def _with_routable_comparison_id( hints: Mapping[str, object], security: ResponsesIDSecurity | None, user_api_key_dict: UserAPIKeyAuth ) -> Mapping[str, object]: - comparison_id: Final = hints.get(COMPARISON_RESPONSE_ID_ROUTING_HINT) - if security is None or not isinstance(comparison_id, str): + routing_hint: Final = hints.get(COMPARISON_RESPONSE_ID_ROUTING_HINT) + if security is None or not isinstance(routing_hint, ComparisonRoutingHint): return hints try: - authorized_id: Final = security.authorize_comparison_id(comparison_id, user_api_key_dict) + authorized_id: Final = security.authorize_comparison_id(routing_hint.response_id, user_api_key_dict) except HTTPException: return {key: value for key, value in hints.items() if key != COMPARISON_RESPONSE_ID_ROUTING_HINT} - return {**hints, COMPARISON_RESPONSE_ID_ROUTING_HINT: authorized_id} + return {**hints, COMPARISON_RESPONSE_ID_ROUTING_HINT: ComparisonRoutingHint(authorized_id)} def _responses_invalid_request_response(message: str, param: str | None, code: str | None) -> JSONResponse: @@ -1986,7 +1986,9 @@ def _routing_hints_from_first_ws_frame(first_message: str) -> Mapping[str, objec "previous_response_id": ( frame.previous_response_id if nested.previous_response_id is None else nested.previous_response_id ), - COMPARISON_RESPONSE_ID_ROUTING_HINT: comparison_id if isinstance(comparison_id, str) else None, + COMPARISON_RESPONSE_ID_ROUTING_HINT: ( + ComparisonRoutingHint(comparison_id) if isinstance(comparison_id, str) else None + ), } return MappingProxyType({key: value for key, value in hints.items() if value is not None}) diff --git a/litellm/responses/utils.py b/litellm/responses/utils.py index ef54f3d0a1c..fc375b4b9b6 100644 --- a/litellm/responses/utils.py +++ b/litellm/responses/utils.py @@ -1,6 +1,7 @@ import base64 import re from collections.abc import Callable, Collection, Iterable, Mapping, Sequence +from dataclasses import dataclass from functools import reduce from typing import Any, Final, Optional, TypeVar, Union, cast, get_type_hints, overload @@ -38,6 +39,12 @@ from litellm.types.utils import ( COMPARISON_RESPONSE_ID_ROUTING_HINT: Final = "comparison_response_id_routing_hint" + +@dataclass(frozen=True, slots=True) +class ComparisonRoutingHint: + response_id: str + + _PROVIDER_SPECIFIC_HEADER_ADAPTER: Final[ TypeAdapter[ProviderSpecificHeader | Sequence[ProviderSpecificHeader] | None] ] = TypeAdapter(ProviderSpecificHeader | Sequence[ProviderSpecificHeader] | None) @@ -807,8 +814,8 @@ class ResponsesAPIRequestUtils: @staticmethod def comparison_response_id(request: Mapping[str, object]) -> str | None: routing_hint: Final = request.get(COMPARISON_RESPONSE_ID_ROUTING_HINT) - if isinstance(routing_hint, str) and routing_hint: - return routing_hint + if isinstance(routing_hint, ComparisonRoutingHint) and routing_hint.response_id: + return routing_hint.response_id extra_body: Final = as_str_mapping(request.get("extra_body")) container: Final = extra_body if extra_body is not None and "prompt_cache_options" in extra_body else request options: Final = as_str_mapping(container.get("prompt_cache_options")) diff --git a/tests/unit/proxy/response_api_endpoints/test_endpoints.py b/tests/unit/proxy/response_api_endpoints/test_endpoints.py index 2166e8340c7..5ed1058be55 100644 --- a/tests/unit/proxy/response_api_endpoints/test_endpoints.py +++ b/tests/unit/proxy/response_api_endpoints/test_endpoints.py @@ -20,7 +20,7 @@ from httpx import Response import litellm from litellm.proxy._types import ProxyException -from litellm.responses.utils import COMPARISON_RESPONSE_ID_ROUTING_HINT +from litellm.responses.utils import COMPARISON_RESPONSE_ID_ROUTING_HINT, ComparisonRoutingHint from litellm.proxy.proxy_server import app from litellm.types.llms.openai import ResponseAPIUsage, ResponsesAPIResponse from litellm.types.router import RouterErrors @@ -2106,7 +2106,7 @@ class TestResponsesWSFirstFrameModelAuth: @pytest.mark.parametrize( "caller_id, routed_comparison_id", [ - ("owner", "resp_compared"), + ("owner", ComparisonRoutingHint("resp_compared")), ("stranger", None), ], ) @@ -2115,7 +2115,7 @@ class TestResponsesWSFirstFrameModelAuth: monkeypatch: pytest.MonkeyPatch, nested: bool, caller_id: str, - routed_comparison_id: str | None, + routed_comparison_id: ComparisonRoutingHint | None, ): from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.response_api_endpoints.endpoints import responses_websocket_endpoint diff --git a/tests/unit/responses/test_responses_api_request_body.py b/tests/unit/responses/test_responses_api_request_body.py index 1e5c9639847..1c135407236 100644 --- a/tests/unit/responses/test_responses_api_request_body.py +++ b/tests/unit/responses/test_responses_api_request_body.py @@ -23,7 +23,7 @@ from pydantic import TypeAdapter import litellm from litellm._logging import verbose_logger from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler -from litellm.responses.utils import COMPARISON_RESPONSE_ID_ROUTING_HINT +from litellm.responses.utils import COMPARISON_RESPONSE_ID_ROUTING_HINT, ComparisonRoutingHint from litellm.types.llms.openai import ResponsesAPIResponse from tests.unit.llms.bedrock.slow_upstream import ( STREAM_TIMEOUT_SECONDS, @@ -663,7 +663,7 @@ async def test_aresponses_websocket_keeps_routing_hints_out_of_the_relay_kwargs( litellm_logging_obj=MagicMock(), input=[{"type": "message", "role": "user", "content": "hi"}], previous_response_id="resp_prev", - **{COMPARISON_RESPONSE_ID_ROUTING_HINT: "resp_compared"}, + **{COMPARISON_RESPONSE_ID_ROUTING_HINT: ComparisonRoutingHint("resp_compared")}, ) mock_ws.assert_awaited_once() @@ -689,7 +689,7 @@ async def test_aresponses_websocket_fills_a_deployment_prompt_cache_options_next api_key="sk-test", litellm_logging_obj=MagicMock(), prompt_cache_options={"ttl": "12h"}, - **{COMPARISON_RESPONSE_ID_ROUTING_HINT: "resp_compared"}, + **{COMPARISON_RESPONSE_ID_ROUTING_HINT: ComparisonRoutingHint("resp_compared")}, ) request_defaults: Final = mock_ws.call_args.kwargs["request_defaults"] diff --git a/tests/unit/router_utils/pre_call_checks/test_deployment_affinity_check.py b/tests/unit/router_utils/pre_call_checks/test_deployment_affinity_check.py index 5fa974ea634..32dd075de6b 100644 --- a/tests/unit/router_utils/pre_call_checks/test_deployment_affinity_check.py +++ b/tests/unit/router_utils/pre_call_checks/test_deployment_affinity_check.py @@ -9,7 +9,11 @@ import pytest import litellm from litellm.caching.dual_cache import DualCache -from litellm.responses.utils import COMPARISON_RESPONSE_ID_ROUTING_HINT, ResponsesAPIRequestUtils +from litellm.responses.utils import ( + COMPARISON_RESPONSE_ID_ROUTING_HINT, + ComparisonRoutingHint, + ResponsesAPIRequestUtils, +) from litellm.router_utils.pre_call_checks.deployment_affinity_check import ( DeploymentAffinityCheck, ) @@ -1309,11 +1313,11 @@ def _response_id_from(model_id: str) -> str: @pytest.mark.asyncio @pytest.mark.parametrize( - ("request_kwargs", "pinned_model_id"), + ("request_kwargs", "pinned_model_ids"), [ pytest.param( {"prompt_cache_options": {"comparison_response_id": _response_id_from("B")}}, - "B", + ("B",), id="top-level-comparison", ), pytest.param( @@ -1321,7 +1325,7 @@ def _response_id_from(model_id: str) -> str: "prompt_cache_options": {"comparison_response_id": _response_id_from("A")}, "extra_body": {"prompt_cache_options": {"comparison_response_id": _response_id_from("B")}}, }, - "B", + ("B",), id="extra-body-comparison-wins", ), pytest.param( @@ -1329,18 +1333,23 @@ def _response_id_from(model_id: str) -> str: "previous_response_id": _response_id_from("A"), "prompt_cache_options": {"comparison_response_id": _response_id_from("B")}, }, - "A", + ("A",), id="previous-response-wins", ), pytest.param( - {COMPARISON_RESPONSE_ID_ROUTING_HINT: _response_id_from("C")}, - "C", + {COMPARISON_RESPONSE_ID_ROUTING_HINT: ComparisonRoutingHint(_response_id_from("C"))}, + ("C",), id="websocket-first-frame-routing-hint", ), + pytest.param( + {COMPARISON_RESPONSE_ID_ROUTING_HINT: _response_id_from("C")}, + ("A", "B", "C"), + id="request-body-key-named-like-the-routing-hint", + ), ], ) async def test_comparison_response_id_pins_its_deployment_unless_a_previous_response_id_does( - request_kwargs: dict[str, object], pinned_model_id: str + request_kwargs: dict[str, object], pinned_model_ids: tuple[str, ...] ) -> None: callback: Final = DeploymentAffinityCheck( cache=DualCache(), @@ -1361,7 +1370,7 @@ async def test_comparison_response_id_pins_its_deployment_unless_a_previous_resp parent_otel_span=None, ) - assert [deployment["model_info"]["id"] for deployment in pinned] == [pinned_model_id] + assert tuple(deployment["model_info"]["id"] for deployment in pinned) == pinned_model_ids @pytest.mark.asyncio