diff --git a/litellm/responses/main.py b/litellm/responses/main.py index d993b05f5b7..8451834eab2 100644 --- a/litellm/responses/main.py +++ b/litellm/responses/main.py @@ -6,7 +6,7 @@ from contextlib import contextmanager from dataclasses import dataclass from functools import partial from types import MappingProxyType -from typing import TYPE_CHECKING, Annotated, Any, Final, Literal, NoReturn, Optional, TypeAlias, cast +from typing import TYPE_CHECKING, Annotated, Any, Final, Literal, NoReturn, Optional, TypeAlias, cast, get_type_hints import httpx from pydantic import BaseModel, ConfigDict, Field, TypeAdapter, ValidationError @@ -22,6 +22,7 @@ from litellm.completion_extras.litellm_responses_transformation.transformation i LiteLLMResponsesTransformationHandler, ) from litellm.constants import DEFAULT_CHAT_COMPLETION_PARAM_VALUES, request_timeout +from litellm.exceptions import UnsupportedParamsError from litellm.integrations.anthropic_cache_control_hook import ( CARRY_UNMATCHED_MESSAGE_POINTS, AnthropicCacheControlHook, @@ -29,7 +30,7 @@ from litellm.integrations.anthropic_cache_control_hook import ( ) from litellm.litellm_core_utils.asyncify import run_async_function from litellm.litellm_core_utils.completion_timeout import CompletionTimeout, StreamKind -from litellm.litellm_core_utils.core_helpers import normalize_drop_params +from litellm.litellm_core_utils.core_helpers import drops_unsupported_params, normalize_drop_params from litellm.litellm_core_utils.get_llm_provider_logic import get_provider_affinity_header from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup @@ -616,16 +617,16 @@ def previous_response_client_ids(previous_response_id: str | None, kwargs: Mappi ) -def _bridge_kwargs( - kwargs: Mapping[str, object], - responses_api_provider_config: BaseResponsesAPIConfig | None, - allowed_openai_params: Sequence[str] | None, -) -> Mapping[str, object]: - if responses_api_provider_config is None: - return MappingProxyType( - {key: value for key, value in kwargs.items() if key not in _RESPONSES_ONLY_REQUEST_FIELDS_NEVER_BRIDGED} - ) - forwarded_keys: Final = frozenset( +_RESPONSES_REQUEST_FIELDS: Final = frozenset(get_type_hints(ResponsesAPIOptionalRequestParams)) + + +@dataclass(frozen=True, slots=True) +class _UntranslatableResponsesFields: + fields: tuple[str, ...] + + +def _chat_completion_request_fields(allowed_openai_params: Sequence[str] | None) -> frozenset[str]: + return frozenset( ( *litellm.OPENAI_CHAT_COMPLETION_PARAMS, *DEFAULT_CHAT_COMPLETION_PARAM_VALUES, @@ -634,7 +635,49 @@ def _bridge_kwargs( *(allowed_openai_params or ()), ) ).difference(_RESPONSES_ONLY_REQUEST_FIELDS_NEVER_BRIDGED) - return MappingProxyType({key: value for key, value in kwargs.items() if key in forwarded_keys}) + + +_ADDITIONAL_DROP_PARAMS: Final = TypeAdapter(list[str]) + + +def _explicitly_dropped_fields(kwargs: Mapping[str, object]) -> frozenset[str]: + try: + return frozenset(_ADDITIONAL_DROP_PARAMS.validate_python(kwargs.get("additional_drop_params"))) + except ValidationError: + return frozenset() + + +def _bridge_kwargs( + kwargs: Mapping[str, object], + responses_api_provider_config: BaseResponsesAPIConfig | None, + allowed_openai_params: Sequence[str] | None, +) -> Mapping[str, object] | _UntranslatableResponsesFields: + forwarded_keys: Final = _chat_completion_request_fields(allowed_openai_params) + if responses_api_provider_config is not None: + return MappingProxyType({key: value for key, value in kwargs.items() if key in forwarded_keys}) + candidate_keys: Final = _RESPONSES_REQUEST_FIELDS.difference(forwarded_keys, _explicitly_dropped_fields(kwargs)) + untranslatable: Final = tuple(sorted(key for key in candidate_keys if kwargs.get(key) is not None)) + if untranslatable and not drops_unsupported_params(kwargs.get("drop_params")): + return _UntranslatableResponsesFields(untranslatable) + dropped_keys: Final = _RESPONSES_ONLY_REQUEST_FIELDS_NEVER_BRIDGED.union(untranslatable) + return MappingProxyType({key: value for key, value in kwargs.items() if key not in dropped_keys}) + + +def _raise_untranslatable_responses_fields( + failure: _UntranslatableResponsesFields, model: str, custom_llm_provider: str | None +) -> NoReturn: + fields: Final = list(failure.fields) + raise UnsupportedParamsError( + status_code=400, + model=model, + llm_provider=custom_llm_provider, + message=( + f"{custom_llm_provider} does not support parameters: {fields}, for model={model}. " + "The Responses API to chat completions bridge cannot translate them. " + "To drop these, set `litellm.drop_params=True` or for proxy:\n\n`litellm_settings:\n drop_params: true`\n. \n " + f"If you want to use these params dynamically send allowed_openai_params={fields} in your request." + ), + ) _ResponsesCompatibilityFailure: TypeAlias = Literal["encrypted_task_unsupported"] @@ -1709,6 +1752,8 @@ def responses( if _bridges_to_chat_completions(responses_api_provider_config, use_chat_completions_api): bridge_kwargs: Final = _bridge_kwargs(kwargs, responses_api_provider_config, allowed_openai_params) + if isinstance(bridge_kwargs, _UntranslatableResponsesFields): + _raise_untranslatable_responses_fields(bridge_kwargs, model, custom_llm_provider) return litellm_completion_transformation_handler.response_api_handler( previous_response_client_ids(previous_response_id, kwargs), model=model, diff --git a/tests/unit/responses/test_responses_api_bridge_flag.py b/tests/unit/responses/test_responses_api_bridge_flag.py index 278bd4c1e52..de96e995dd4 100644 --- a/tests/unit/responses/test_responses_api_bridge_flag.py +++ b/tests/unit/responses/test_responses_api_bridge_flag.py @@ -423,6 +423,149 @@ class TestUseResponsesApiBridgeFlag: assert request.headers["authorization"] == "Bearer fake-azure-ad-token" assert "chat_template_kwargs" not in json.loads(request.read()) + def test_bridge_rejects_a_responses_field_it_cannot_translate_when_drop_params_is_off( + self, monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter + ): + monkeypatch.setattr(litellm, "drop_params", False) + upstream: Final = _databricks_chat_upstream(respx_mock) + + with pytest.raises(litellm.UnsupportedParamsError) as raised: + litellm.responses( + model="databricks/my-custom-model", + input="Hello", + max_tool_calls=5, + api_base="https://example.databricks.test/serving-endpoints", + api_key="fake-databricks-api-key", + num_retries=0, + ) + + assert upstream.call_count == 0, "the provider must not be called for a field the bridge cannot translate" + assert raised.value.status_code == 400 + assert "max_tool_calls" in str(raised.value) + assert "drop_params" in str(raised.value) + + @pytest.mark.parametrize("drop_params_source", ["request", "global"]) + def test_bridge_drops_a_responses_field_it_cannot_translate_when_drop_params_is_on( + self, drop_params_source: str, monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter + ): + monkeypatch.setattr(litellm, "drop_params", drop_params_source == "global") + upstream: Final = _databricks_chat_upstream(respx_mock) + + response: Final = litellm.responses( + model="databricks/my-custom-model", + input="Hello", + max_tool_calls=5, + chat_template_kwargs={"thinking": True}, + api_base="https://example.databricks.test/serving-endpoints", + api_key="fake-databricks-api-key", + num_retries=0, + **({"drop_params": True} if drop_params_source == "request" else {}), + ) + + assert upstream.call_count == 1 + assert json.loads(upstream.calls[0].request.read()) == { + "model": "my-custom-model", + "messages": [{"role": "user", "content": "Hello"}], + "chat_template_kwargs": {"thinking": True}, + } + assert response.output[0].content[0].text == "Answer" + + def test_bridge_drops_a_responses_field_it_cannot_translate_when_the_deployment_sets_drop_params( + self, monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter + ): + monkeypatch.setattr(litellm, "drop_params", False) + upstream: Final = _databricks_chat_upstream(respx_mock) + router: Final = Router( + model_list=[ + { + "model_name": "bridged", + "litellm_params": { + "model": "databricks/my-custom-model", + "api_base": "https://example.databricks.test/serving-endpoints", + "api_key": "fake-databricks-api-key", + "drop_params": True, + }, + } + ], + num_retries=0, + ) + + response: Final = router.responses(model="bridged", input="Hello", max_tool_calls=5) + + assert upstream.call_count == 1 + request_body: Final = json.loads(upstream.calls[0].request.read()) + assert "max_tool_calls" not in request_body + assert request_body["messages"] == [{"role": "user", "content": "Hello"}] + assert response.output[0].content[0].text == "Answer" + + def test_allowed_openai_params_sends_a_responses_field_through_the_bridge( + self, monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter + ): + monkeypatch.setattr(litellm, "drop_params", False) + upstream: Final = _databricks_chat_upstream(respx_mock) + + litellm.responses( + model="databricks/my-custom-model", + input="Hello", + max_tool_calls=5, + allowed_openai_params=["max_tool_calls"], + api_base="https://example.databricks.test/serving-endpoints", + api_key="fake-databricks-api-key", + num_retries=0, + ) + + assert upstream.call_count == 1 + assert json.loads(upstream.calls[0].request.read())["max_tool_calls"] == 5 + + def test_bridge_still_honors_additional_drop_params_for_a_responses_field_it_cannot_translate( + self, monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter + ): + monkeypatch.setattr(litellm, "drop_params", False) + upstream: Final = _databricks_chat_upstream(respx_mock) + + response: Final = litellm.responses( + model="databricks/my-custom-model", + input="Hello", + max_tool_calls=5, + additional_drop_params=["max_tool_calls"], + api_base="https://example.databricks.test/serving-endpoints", + api_key="fake-databricks-api-key", + num_retries=0, + ) + + assert upstream.call_count == 1 + assert json.loads(upstream.calls[0].request.read()) == { + "model": "my-custom-model", + "messages": [{"role": "user", "content": "Hello"}], + } + assert response.output[0].content[0].text == "Answer" + + def test_bridge_still_quietly_drops_responses_fields_it_already_handles( + self, monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter + ): + monkeypatch.setattr(litellm, "drop_params", False) + upstream: Final = _databricks_chat_upstream(respx_mock) + + litellm.responses( + model="databricks/my-custom-model", + input="Hello", + store=False, + include=["reasoning.encrypted_content"], + truncation="auto", + metadata={"k": "v"}, + background=False, + text={"verbosity": "low"}, + api_base="https://example.databricks.test/serving-endpoints", + api_key="fake-databricks-api-key", + num_retries=0, + ) + + assert upstream.call_count == 1 + assert json.loads(upstream.calls[0].request.read()) == { + "model": "my-custom-model", + "messages": [{"role": "user", "content": "Hello"}], + } + @patch.object(import_module("litellm.responses.file_search.emulated_handler"), "_call_aresponses") @patch.object(import_module("litellm.responses.main").ProviderConfigManager, "get_provider_responses_api_config") async def test_bridge_flag_forwarded_to_file_search_emulation(self, mock_get_config, mock_call_aresponses): @@ -576,6 +719,24 @@ _CHAT: Final[Mapping[str, JsonValue]] = {"id": "dep-1", "mode": "chat"} _RESPONSES: Final[Mapping[str, JsonValue]] = {"id": "dep-1", "mode": "responses"} +def _databricks_chat_upstream(respx_mock: respx.MockRouter) -> respx.Route: + return respx_mock.post("https://example.databricks.test/serving-endpoints/chat/completions").mock( + return_value=httpx.Response( + status_code=200, + json={ + "id": "chatcmpl-123", + "object": "chat.completion", + "created": 1677652288, + "model": "my-custom-model", + "choices": [ + {"index": 0, "message": {"role": "assistant", "content": "Answer"}, "finish_reason": "stop"} + ], + "usage": {"prompt_tokens": 9, "completion_tokens": 1, "total_tokens": 10}, + }, + ) + ) + + def _pin_shared_mode(monkeypatch: pytest.MonkeyPatch, model: str, mode: str) -> None: cost_map: Final = cast(dict[str, dict[str, object]], litellm.model_cost) monkeypatch.setitem(cost_map, model, {**cost_map.get(model, {}), "mode": mode})