From 4e29aea11c8eb5fadd96c78de2bd9d10b0381727 Mon Sep 17 00:00:00 2001 From: tavren27 <348969968+tavren27@users.noreply.github.com> Date: Sat, 14 Nov 2026 01:32:38 -0800 Subject: [PATCH] fix(responses): accept a string or tuple allowed_openai_params on Responses models instead of answering 500 (#51751) * fix(responses): accept any sequence for allowed_openai_params and give UnsupportedParamsError a response A deployment whose allowed_openai_params is a string (or an SDK caller passing a tuple) made every Responses-mode request answer 500 "can only concatenate list". The Responses path now builds the allowlist from any sequence, matching the chat path. UnsupportedParamsError now runs BadRequestError's constructor, so it carries response, body and request like every other 400 instead of raising AttributeError on .response. * fix(params): read a bare-string allowed_openai_params as one param name on every path A deployment or request with allowed_openai_params: "presence_penalty" now allowlists presence_penalty on chat completions, embeddings, the Responses API, the chat-to-Responses bridge, the router's tier and precall filters, and Mistral's prompt_cache_key check. Before, chat and Responses read the string letter by letter, so it allowlisted nothing, and the router ignored it. UnsupportedParamsError's args now hold its own message instead of the BadRequestError one. * fix(params): read a dict or comma-separated allowed_openai_params the way chat and embeddings did A dict allowlist is read for its keys and a string is split on commas, so configs that worked on chat or embeddings before keep working, and the router reads them the same way. The fallback errors header names an error by its class when its type is empty, so UnsupportedParamsError still shows up there by name. * fix(params): read every param name in a string allowed_openai_params A string allowlist now yields every name in it, whether the names are split by commas or spaces or written as a list inside the string, so embeddings configs like "dimensions encoding_format" keep working. A request with no allowlist returns early instead of raising and catching a validation error. * fix(fallbacks): keep the fallback errors header type for errors other than UnsupportedParamsError The header named an error by its class whenever its type was empty, which changed "None" to the class name for timeouts and other errors built without a provider body. Only UnsupportedParamsError needs its class name now that it inherits an empty type, so every other error keeps the type it carries. * perf(params): read a string allowed_openai_params with one regex scan The string reader walked every character in Python and ran twice per request, so a 1 MB allowlist added about 300 ms of event-loop time. `\w+` matches exactly the alphanumeric-or-underscore runs the old loop kept, checked over every Unicode code point, and runs in C. * test(params): cover an int allowed_openai_params and an admin request for a model two teams share An allowlist that is no string, mapping, or sequence now allowlists nothing instead of raising TypeError, and an admin request with no team for a model name two teams share answers the multiple-teams 400 with the model name --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- litellm/exceptions.py | 28 ++++----- .../add_retry_fallback_headers.py | 7 ++- litellm/litellm_core_utils/core_helpers.py | 19 ++++++ .../routing_common_utils.py | 3 +- litellm/llms/mistral/chat/transformation.py | 5 +- litellm/main.py | 3 +- litellm/responses/main.py | 9 ++- litellm/responses/utils.py | 7 ++- litellm/router_rust/invoke/shaping.py | 8 +-- litellm/router_rust/steps/precall.py | 5 +- litellm/router_rust/steps/prerouting.py | 7 +-- litellm/utils.py | 25 ++++---- .../test_mistral_chat_transformation.py | 11 +++- tests/unit/responses/test_responses_utils.py | 24 ++++++++ .../test_add_retry_fallback_headers.py | 37 ++++++++++++ .../test_router_utils_common_utils.py | 18 +++++- tests/unit/test_main.py | 35 +++++++++++ ...uction_precall_characterization_precall.py | 13 ++-- .../core/test_gaps_l14000_selection.py | 26 +++++++- tests/unit/test_utils_get_optional_params.py | 59 +++++++++++++++++++ 20 files changed, 287 insertions(+), 62 deletions(-) diff --git a/litellm/exceptions.py b/litellm/exceptions.py index 61810f650af..9350396ee88 100644 --- a/litellm/exceptions.py +++ b/litellm/exceptions.py @@ -277,9 +277,9 @@ class NotFoundError(openai.NotFoundError): class BadRequestError(openai.BadRequestError): def __init__( self, - message, - model, - llm_provider, + message: str, + model: str | None, + llm_provider: str | None, response: httpx.Response | None = None, litellm_debug_info: str | None = None, max_retries: int | None = None, @@ -1072,7 +1072,7 @@ class OpenAIError(openai.OpenAIError): class UnsupportedParamsError(BadRequestError): def __init__( self, - message, + message: str, llm_provider: str | None = None, model: str | None = None, status_code: int = 400, @@ -1081,17 +1081,17 @@ class UnsupportedParamsError(BadRequestError): max_retries: int | None = None, num_retries: int | None = None, ): - self.status_code = 400 - self.message = f"litellm.UnsupportedParamsError: {message}" - self.model = model - self.llm_provider = llm_provider - self.litellm_debug_info = litellm_debug_info - response = response or httpx.Response( - status_code=self.status_code, - request=httpx.Request(method="GET", url="https://litellm.ai"), # mock request object + super().__init__( + message=message, + model=model, + llm_provider=llm_provider, + response=response, + litellm_debug_info=litellm_debug_info, + max_retries=max_retries, + num_retries=num_retries, ) - self.max_retries = max_retries - self.num_retries = num_retries + self.message = f"litellm.UnsupportedParamsError: {message}" + self.args = (self.message,) LITELLM_EXCEPTION_TYPES: Final = [ diff --git a/litellm/litellm_core_utils/add_retry_fallback_headers.py b/litellm/litellm_core_utils/add_retry_fallback_headers.py index 4cf2d1d58d8..ebd71b20fba 100644 --- a/litellm/litellm_core_utils/add_retry_fallback_headers.py +++ b/litellm/litellm_core_utils/add_retry_fallback_headers.py @@ -6,6 +6,7 @@ from typing import Final, Protocol, TypedDict, cast from pydantic import BaseModel, TypeAdapter, ValidationError +from litellm.exceptions import UnsupportedParamsError from litellm.litellm_core_utils.hidden_params import ( HIDDEN_PARAMS_ATTR as _HIDDEN_PARAMS_ATTR, ) @@ -249,7 +250,11 @@ def _ensure_additional_headers_dict( def get_fallback_error_info(error: Exception) -> FallbackErrorInfo: message: Final = cast(object, getattr(error, "message", str(error))) - error_type: Final = cast(object, getattr(error, "type", error.__class__.__name__)) + error_type: Final = ( + error.__class__.__name__ + if isinstance(error, UnsupportedParamsError) + else cast(object, getattr(error, "type", error.__class__.__name__)) + ) param: Final = cast(object, getattr(error, "param", None)) code: Final = cast(object, getattr(error, "status_code", getattr(error, "code", None))) return FallbackErrorInfo( diff --git a/litellm/litellm_core_utils/core_helpers.py b/litellm/litellm_core_utils/core_helpers.py index 1a34b8f01ed..656fadc483d 100644 --- a/litellm/litellm_core_utils/core_helpers.py +++ b/litellm/litellm_core_utils/core_helpers.py @@ -109,6 +109,25 @@ def as_object_list(value: object) -> Sequence[object] | None: return value if isinstance(value, list) else None +_OBJECT_ENTRIES: Final = TypeAdapter(list[object]) +_OBJECT_KEYS: Final = TypeAdapter(dict[object, object]) +_PARAM_NAME: Final = re.compile(r"\w+") + + +def allowed_openai_param_names(value: object) -> tuple[str, ...]: + if value is None: + return () + if isinstance(value, str): + return tuple(_PARAM_NAME.findall(value)) + if isinstance(value, Mapping): + return tuple(key for key in _OBJECT_KEYS.validate_python(value) if isinstance(key, str)) + try: + entries: Final = _OBJECT_ENTRIES.validate_python(value) + except ValidationError: + return () + return tuple(entry for entry in entries if isinstance(entry, str)) + + _Key = TypeVar("_Key") # rebind-ok: a TypeVar cannot be Final _Value = TypeVar("_Value") # rebind-ok: a TypeVar cannot be Final diff --git a/litellm/litellm_core_utils/routing_common_utils.py b/litellm/litellm_core_utils/routing_common_utils.py index 363854f70de..8d0f642eb0a 100644 --- a/litellm/litellm_core_utils/routing_common_utils.py +++ b/litellm/litellm_core_utils/routing_common_utils.py @@ -401,9 +401,10 @@ def filter_team_based_models( litellm_metadata: Final = request_kwargs.get("litellm_metadata") or {} request_team_id: Final = get_request_team_id(request_kwargs) if request_team_id is None and is_proxy_admin_request(request_kwargs) and isinstance(healthy_deployments, list): - requested_model: Final = ( + requested_model_value: Final = ( request_kwargs.get("model") or metadata.get("model_group") or litellm_metadata.get("model_group") ) + requested_model: Final = requested_model_value if isinstance(requested_model_value, str) else None candidate_deployments: Final = tuple( (deployment.get("model_name"), deployment.get("model_info") or {}) for deployment in healthy_deployments ) diff --git a/litellm/llms/mistral/chat/transformation.py b/litellm/llms/mistral/chat/transformation.py index c454c37c01b..7942680e161 100644 --- a/litellm/llms/mistral/chat/transformation.py +++ b/litellm/llms/mistral/chat/transformation.py @@ -19,7 +19,7 @@ from typing_extensions import TypeIs, assert_never # noqa: TID251 # these are import litellm from litellm._logging import verbose_logger from litellm.exceptions import UnsupportedParamsError -from litellm.litellm_core_utils.core_helpers import as_object_list, as_str_mapping +from litellm.litellm_core_utils.core_helpers import allowed_openai_param_names, as_object_list, as_str_mapping from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.litellm_core_utils.llm_cost_calc.guardrail_cost import is_str_keyed_dict from litellm.litellm_core_utils.prompt_templates.common_utils import ( @@ -355,8 +355,7 @@ class MistralConfig(OpenAIGPTConfig): def _prompt_cache_key_withheld(self, model: str, prompt_cache_key: object, routing: MistralRequestRouting) -> bool: requested_api_base: Final = routing.get("api_base") api_base: Final = requested_api_base if isinstance(requested_api_base, str) else None - allowed_openai_params: Final = routing.get("allowed_openai_params") - if isinstance(allowed_openai_params, (list, tuple)) and "prompt_cache_key" in allowed_openai_params: + if "prompt_cache_key" in allowed_openai_param_names(routing.get("allowed_openai_params")): return False rejection: Final = self._prompt_cache_key_rejection(prompt_cache_key, api_base) if rejection is None: diff --git a/litellm/main.py b/litellm/main.py index 553c7a95411..2cd9f12c649 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -233,6 +233,7 @@ from litellm.utils import ( from ._logging import WarningLogger, verbose_logger from .caching.caching import disable_cache, enable_cache, update_cache from .litellm_core_utils.core_helpers import ( + allowed_openai_param_names, as_str_mapping, drops_unsupported_params, normalize_drop_params, @@ -6085,7 +6086,7 @@ def completion( responses_api_model_info.get("mode") == "responses" and not skip_responses_api_bridge ) allowed_openai_params: Final[list[str] | None] = ( - [*(kwargs.get("allowed_openai_params") or []), "reasoning_effort"] + [*allowed_openai_param_names(kwargs.get("allowed_openai_params")), "reasoning_effort"] if bridges_to_responses_api or reasoning_effort_needs_allowing( chat_reasoning.effort, diff --git a/litellm/responses/main.py b/litellm/responses/main.py index 78a3966f3b0..db5ac5a264b 100644 --- a/litellm/responses/main.py +++ b/litellm/responses/main.py @@ -35,7 +35,12 @@ from litellm.litellm_core_utils.allowed_tools_tool_choice import ( ) from litellm.litellm_core_utils.asyncify import run_async_function from litellm.litellm_core_utils.completion_timeout import CompletionTimeout, StreamKind -from litellm.litellm_core_utils.core_helpers import drops_unsupported_params, normalize_drop_params, require_attributes +from litellm.litellm_core_utils.core_helpers import ( + allowed_openai_param_names, + drops_unsupported_params, + normalize_drop_params, + require_attributes, +) from litellm.litellm_core_utils.get_llm_provider_logic import ( declared_authenticating_provider, get_provider_affinity_header, @@ -664,7 +669,7 @@ def _chat_completion_request_fields(allowed_openai_params: Sequence[str] | None) *DEFAULT_CHAT_COMPLETION_PARAM_VALUES, *all_litellm_params, *GenericLiteLLMParams.model_fields, - *(allowed_openai_params or ()), + *allowed_openai_param_names(allowed_openai_params), ) ).difference(_RESPONSES_ONLY_REQUEST_FIELDS_NEVER_BRIDGED) diff --git a/litellm/responses/utils.py b/litellm/responses/utils.py index dd3bd1c7e35..7f8c442c986 100644 --- a/litellm/responses/utils.py +++ b/litellm/responses/utils.py @@ -14,6 +14,7 @@ from typing_extensions import TypeIs # noqa: TID251 # narrows untyped wire pay import litellm from litellm._logging import verbose_logger from litellm.litellm_core_utils.core_helpers import ( + allowed_openai_param_names, as_object_list, as_str_mapping, attribute_or_none, @@ -314,7 +315,7 @@ class ResponsesAPIRequestUtils: model: str, responses_api_provider_config: BaseResponsesAPIConfig, response_api_optional_params: ResponsesAPIOptionalRequestParams, - allowed_openai_params: list[str] | None = None, + allowed_openai_params: Sequence[str] | None = None, drop_params: bool | None = None, declared_reasoning_efforts: Collection[str] | None = None, ) -> dict: @@ -340,7 +341,7 @@ class ResponsesAPIRequestUtils: non_default_params: Final = cast(dict, response_api_optional_params) # Check for unsupported parameters ResponsesAPIRequestUtils._check_valid_arg( - supported_params=supported_params + (allowed_openai_params or []), + supported_params=[*supported_params, *allowed_openai_param_names(allowed_openai_params)], non_default_params=non_default_params, drop_params=should_drop_params, custom_llm_provider=responses_api_provider_config.custom_llm_provider, @@ -369,7 +370,7 @@ class ResponsesAPIRequestUtils: return apply_openai_param_overrides( optional_params=params_with_normalized_stream_options, non_default_params=non_default_params, - allowed_openai_params=allowed_openai_params or [], + allowed_openai_params=allowed_openai_params, ) @staticmethod diff --git a/litellm/router_rust/invoke/shaping.py b/litellm/router_rust/invoke/shaping.py index 5fa6fc2543c..d3e00d67c5b 100644 --- a/litellm/router_rust/invoke/shaping.py +++ b/litellm/router_rust/invoke/shaping.py @@ -28,7 +28,7 @@ from litellm.litellm_core_utils.clientside_credential_handler import ( is_clientside_credential, # pyright: ignore[reportUnknownVariableType] # litellm's untyped helpers and request values ) from litellm.litellm_core_utils.completion_timeout import CompletionTimeout -from litellm.litellm_core_utils.core_helpers import without +from litellm.litellm_core_utils.core_helpers import allowed_openai_param_names, without from litellm.litellm_core_utils.get_litellm_params import SUPPLEMENTAL_KWARGS_KEYS from litellm.litellm_core_utils.get_llm_provider_logic import declared_authenticating_provider, inferred_provider from litellm.litellm_core_utils.request_timeout_resolver import get_service_tier_timeout @@ -242,11 +242,7 @@ def _is_classifier_internal_call(kwargs: Mapping[str, object]) -> bool: def _declared_param_allowlist(params: Mapping[str, object]) -> frozenset[str]: - declared: Final = params.get("allowed_openai_params") - if not isinstance(declared, (list, tuple, set, frozenset)): - return frozenset() - entries: Final[Sequence[object]] = list(declared) # pyright: ignore[reportUnknownArgumentType] # the deployment's untyped allowlist - return frozenset(entry for entry in entries if isinstance(entry, str)) + return frozenset(allowed_openai_param_names(params.get("allowed_openai_params"))) def _deployment_supports_param(deployment: Mapping[str, object], group: str, param: str, host: "PythonHost") -> bool: diff --git a/litellm/router_rust/steps/precall.py b/litellm/router_rust/steps/precall.py index 9de5c47b820..77f7b94784f 100644 --- a/litellm/router_rust/steps/precall.py +++ b/litellm/router_rust/steps/precall.py @@ -20,7 +20,7 @@ from types import MappingProxyType from typing import TYPE_CHECKING, Final, Protocol from litellm._logging import verbose_router_logger -from litellm.litellm_core_utils.core_helpers import as_str_mapping +from litellm.litellm_core_utils.core_helpers import allowed_openai_param_names, as_str_mapping from litellm.litellm_core_utils.deployment_pricing import deployment_backend_models from litellm.litellm_core_utils.get_llm_provider_logic import declared_authenticating_provider from litellm.litellm_core_utils.prompt_templates.tpm_media_utils import is_plain_embedding_text @@ -204,8 +204,7 @@ def _is_list(value: object) -> bool: def _declared_param_allowlist(params: Mapping[str, object]) -> frozenset[str]: - declared: Final = params.get("allowed_openai_params") - return frozenset(entry for entry in _items(declared) if isinstance(entry, str)) + return frozenset(allowed_openai_param_names(params.get("allowed_openai_params"))) def _capability_model_and_provider(deployment_params: Mapping[str, object], group: str) -> tuple[str, str]: diff --git a/litellm/router_rust/steps/prerouting.py b/litellm/router_rust/steps/prerouting.py index 44a4800af8d..9e5d5ede967 100644 --- a/litellm/router_rust/steps/prerouting.py +++ b/litellm/router_rust/steps/prerouting.py @@ -35,7 +35,7 @@ from litellm.constants import ( ) from litellm.integrations.otel.routing import routing_decision_attributes from litellm.integrations.otel.runtime import phase_attributes -from litellm.litellm_core_utils.core_helpers import get_or_create_metadata_bucket, without +from litellm.litellm_core_utils.core_helpers import allowed_openai_param_names, get_or_create_metadata_bucket, without from litellm.litellm_core_utils.credential_accessor import reject_api_key_secret_reference from litellm.router_api import deployment_id from litellm.router_host import router_host @@ -1171,10 +1171,7 @@ def _tags_of(params: Mapping[str, object]) -> Sequence[object]: def _allowlist(params: Mapping[str, object]) -> frozenset[str]: - declared: Final = params.get("allowed_openai_params") - if not isinstance(declared, list | tuple | set | frozenset): - return frozenset() - return frozenset(entry for entry in declared if isinstance(entry, str)) + return frozenset(allowed_openai_param_names(params.get("allowed_openai_params"))) __all__ = ["PreRouter", "PreRoutingStrategies", "as_output_cap", "is_strategy_marker", "stamped_client_ceiling"] diff --git a/litellm/utils.py b/litellm/utils.py index 3f4ebf7a4fb..c015290726e 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -91,6 +91,7 @@ from litellm.litellm_core_utils.allowed_tools_tool_choice import ( ) from litellm.litellm_core_utils.attempt_settlement import mark_attempt_answered from litellm.litellm_core_utils.core_helpers import ( + allowed_openai_param_names, as_str_mapping, bind_budget_reservation_to_callbacks, has_attributes, @@ -1870,9 +1871,10 @@ async def async_pre_call_deployment_hook( route: Final[object] = getattr(kwargs.get("litellm_logging_obj"), "call_type", None) resolved: Final = resolve_deployment_tool_fields(kwargs, route) if isinstance(resolved, MalformedField): + model: Final = kwargs.get("model") raise MalformedRequestFieldError( message=resolved.message, - model=kwargs.get("model"), + model=model if isinstance(model, str) else None, llm_provider=_deployment_provider(kwargs), body={"type": "invalid_request_error", "param": resolved.field}, ) @@ -4854,7 +4856,6 @@ def get_optional_params_embeddings( drop_params = normalize_drop_params(drop_params) passed_params.pop("additional_drop_params", None) passed_params.pop("allowed_openai_params", None) - allowed_openai_params = allowed_openai_params or [] # Remove function objects from passed_params to avoid JSON serialization errors passed_params.pop("get_supported_openai_params", None) @@ -4918,7 +4919,7 @@ def get_optional_params_embeddings( model is not None and "text-embedding-3" not in model and "dimensions" in non_default_params - and "dimensions" not in (allowed_openai_params or []) + and "dimensions" not in allowed_openai_param_names(allowed_openai_params) ): # Honor drop_params (per-call) and litellm.drop_params (global) the same # way `_check_valid_arg` does above. The raised error message itself @@ -5772,8 +5773,7 @@ def get_optional_params( supported_params = get_supported_openai_params(model=model, custom_llm_provider="openai") supported_params = supported_params or [] - allowed_openai_params = allowed_openai_params or [] - supported_params.extend(allowed_openai_params) + supported_params.extend(allowed_openai_param_names(allowed_openai_params)) _check_valid_arg( supported_params=supported_params or [], @@ -6370,7 +6370,7 @@ def add_provider_specific_params_to_optional_params( def apply_openai_param_overrides( optional_params: dict[str, object], non_default_params: dict[str, object], - allowed_openai_params: list[str], + allowed_openai_params: Sequence[str] | None, ) -> dict[str, object]: """ If user passes in allowed_openai_params, apply them to optional_params @@ -6384,13 +6384,12 @@ def apply_openai_param_overrides( `AsyncCompletions.create() got an unexpected keyword argument 'enable_thinking'`). See https://github.com/BerriAI/litellm/issues/25697 """ - if allowed_openai_params: - for param in allowed_openai_params: - if param in optional_params: - continue - if param not in non_default_params: - continue - optional_params[param] = non_default_params.pop(param) + for param in allowed_openai_param_names(allowed_openai_params): + if param in optional_params: + continue + if param not in non_default_params: + continue + optional_params[param] = non_default_params.pop(param) return optional_params diff --git a/tests/unit/llms/mistral/test_mistral_chat_transformation.py b/tests/unit/llms/mistral/test_mistral_chat_transformation.py index cca07cce9c4..a8fcfc6775b 100644 --- a/tests/unit/llms/mistral/test_mistral_chat_transformation.py +++ b/tests/unit/llms/mistral/test_mistral_chat_transformation.py @@ -1846,7 +1846,10 @@ def _chat_completion_reply() -> httpx.Response: ) -def test_allowlisted_prompt_cache_key_reaches_a_gateway_in_front_of_mistral(respx_mock: respx.MockRouter) -> None: +@pytest.mark.parametrize("allowlist", [["prompt_cache_key"], "prompt_cache_key"], ids=["list", "bare-string"]) +def test_allowlisted_prompt_cache_key_reaches_a_gateway_in_front_of_mistral( + respx_mock: respx.MockRouter, allowlist: list[str] | str +) -> None: import litellm gateway_route: Final = respx_mock.post("https://gateway.example.com/mistral/v1/chat/completions").mock( @@ -1859,7 +1862,7 @@ def test_allowlisted_prompt_cache_key_reaches_a_gateway_in_front_of_mistral(resp api_base="https://gateway.example.com/mistral/v1", api_key="gateway-key", prompt_cache_key="session-1", - allowed_openai_params=["prompt_cache_key"], + allowed_openai_params=allowlist, max_retries=0, ) @@ -1959,7 +1962,7 @@ def test_prompt_cache_key_rejection_for_a_sync_codestral_call_is_an_unsupported_ with pytest.raises( litellm.UnsupportedParamsError, match=r"^litellm.UnsupportedParamsError: codestral does not support" - ): + ) as rejection: litellm.completion( model="codestral/codestral-latest", messages=[{"role": "user", "content": "hi"}], @@ -1970,6 +1973,8 @@ def test_prompt_cache_key_rejection_for_a_sync_codestral_call_is_an_unsupported_ ) assert not azure_route.called + assert rejection.value.response.status_code == rejection.value.status_code == 400 + assert rejection.value.args == (rejection.value.message,) def test_prompt_cache_key_for_another_host_is_dropped_under_drop_params() -> None: diff --git a/tests/unit/responses/test_responses_utils.py b/tests/unit/responses/test_responses_utils.py index 94f60fa5b30..e84afc9c609 100644 --- a/tests/unit/responses/test_responses_utils.py +++ b/tests/unit/responses/test_responses_utils.py @@ -46,6 +46,30 @@ class TestResponsesAPIRequestUtils: assert "prompt" in result assert result["prompt"] == {"id": "pmpt_123"} + def test_get_optional_params_responses_api_forwards_a_param_a_tuple_allowlist_names(self): + result: Final = ResponsesAPIRequestUtils.get_optional_params_responses_api( + model="gpt-4o", + responses_api_provider_config=OpenAIResponsesAPIConfig(), + response_api_optional_params=ResponsesAPIOptionalRequestParams( + {"temperature": 0.7, "custom_flag": "on"} # pyright: ignore[reportCallIssue] # an unsupported key is the input under test + ), + allowed_openai_params=("custom_flag",), + ) + + assert result == {"temperature": 0.7, "custom_flag": "on"} + + def test_get_optional_params_responses_api_reads_a_bare_string_allowlist_as_one_param_name(self): + result: Final = ResponsesAPIRequestUtils.get_optional_params_responses_api( + model="gpt-4o", + responses_api_provider_config=OpenAIResponsesAPIConfig(), + response_api_optional_params=ResponsesAPIOptionalRequestParams( + {"temperature": 0.7, "custom_flag": "on"} # pyright: ignore[reportCallIssue] # an unsupported key is the input under test + ), + allowed_openai_params="custom_flag", + ) + + assert result == {"temperature": 0.7, "custom_flag": "on"} + def test_get_optional_params_responses_api_unsupported_param(self): """Test that unsupported parameters raise an error""" # Setup diff --git a/tests/unit/router_utils/test_add_retry_fallback_headers.py b/tests/unit/router_utils/test_add_retry_fallback_headers.py index 0f65e0ecf48..f700a8a84b9 100644 --- a/tests/unit/router_utils/test_add_retry_fallback_headers.py +++ b/tests/unit/router_utils/test_add_retry_fallback_headers.py @@ -10,10 +10,12 @@ from litellm.litellm_core_utils.add_retry_fallback_headers import ( add_retry_headers_to_response, complexity_router_decision_headers, ensure_response_additional_headers, + get_fallback_error_info, get_fallback_errors_from_headers, get_hidden_params_dict, replace_complexity_router_headers, ) +from litellm.exceptions import AuthenticationError, Timeout, UnsupportedParamsError from litellm.types.decisions import DecisionsResponse @@ -208,6 +210,41 @@ def test_add_fallback_headers_serializes_fallback_errors(): ) +class _ProviderTypedError(Exception): + type = "invalid_request_error" + + +class _UntypedError(Exception): + pass + + +def test_get_fallback_error_info_names_an_unsupported_params_error_by_its_class(): + error: Final = UnsupportedParamsError( + message="mistral does not support parameters: ['presence_penalty']", + llm_provider="mistral", + model="mistral-large-latest", + ) + + assert get_fallback_error_info(error)["type"] == "UnsupportedParamsError" + + +@pytest.mark.parametrize( + "error", + [ + _ProviderTypedError("bad request"), + Timeout(message="timed out", model="gpt-5.6", llm_provider="openai"), + AuthenticationError(message="bad key", llm_provider="openai", model="gpt-5.6"), + ], + ids=["provider-type", "timeout-without-provider-type", "auth-without-provider-type"], +) +def test_get_fallback_error_info_keeps_the_type_other_errors_carry(error: Exception): + assert get_fallback_error_info(error)["type"] == str(getattr(error, "type", None)) + + +def test_get_fallback_error_info_names_an_error_without_a_type_by_its_class(): + assert get_fallback_error_info(_UntypedError("boom"))["type"] == "_UntypedError" + + def test_add_retry_headers_to_streaming_wrapper(): response = StreamingWrapper() diff --git a/tests/unit/router_utils/test_router_utils_common_utils.py b/tests/unit/router_utils/test_router_utils_common_utils.py index ea384d837fd..cecc27daefd 100644 --- a/tests/unit/router_utils/test_router_utils_common_utils.py +++ b/tests/unit/router_utils/test_router_utils_common_utils.py @@ -9,7 +9,7 @@ import litellm from litellm import Router from litellm.constants import ROUTER_FALLBACK_ERROR_DETAIL_MAX_CHARS from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap -from litellm.proxy._types import UserAPIKeyAuth +from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth from litellm.litellm_core_utils.routing_common_utils import ( _deployment_supports_web_search, add_model_file_id_mappings, @@ -53,6 +53,22 @@ class TestFilterTeamBasedModels: {"model_info": {"id": "deployment-2"}}, ] + def test_an_admin_without_a_team_gets_a_400_for_a_model_name_two_teams_share(self) -> None: + deployments: Final = [ + {"model_name": "shared-gpt", "model_info": {"id": "deployment-a", "team_id": "team-a"}}, + {"model_name": "shared-gpt", "model_info": {"id": "deployment-b", "team_id": "team-b"}}, + ] + request_kwargs: Final = { + "model": "shared-gpt", + "metadata": {"user_api_key_auth": UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN)}, + } + + with pytest.raises(litellm.BadRequestError, match="'shared-gpt' matches deployments from multiple teams") as error: + filter_team_based_models(deployments, request_kwargs) + + assert error.value.model == "shared-gpt" + assert error.value.status_code == 400 + def test_filter_team_based_models_none_request_kwargs(self, sample_deployments_with_teams): """Test that when request_kwargs is None, all deployments are returned unchanged""" result = filter_team_based_models(sample_deployments_with_teams, None) diff --git a/tests/unit/test_main.py b/tests/unit/test_main.py index 20d4f164cdb..a44e14f662d 100644 --- a/tests/unit/test_main.py +++ b/tests/unit/test_main.py @@ -943,6 +943,41 @@ def test_completion_drops_unanswered_claude_tool_search_calls_before_the_respons ] +def test_completion_through_the_responses_bridge_honors_a_string_allowlist( + respx_mock: respx.MockRouter, +): + route: Final = respx_mock.post("https://api.openai.com/v1/responses").respond( + json={ + "id": "resp_allowlist", + "object": "response", + "created_at": 1, + "status": "completed", + "model": "gpt-5.6", + "output": [ + { + "type": "message", + "id": "msg_allowlist", + "role": "assistant", + "status": "completed", + "content": [{"type": "output_text", "text": "hi", "annotations": []}], + } + ], + "usage": {"input_tokens": 1, "output_tokens": 1, "total_tokens": 2}, + } + ) + + litellm.completion( + model="openai/responses/gpt-5.6", + api_key="sk-test", + api_base="https://api.openai.com/v1", + messages=[{"role": "user", "content": "hi"}], + presence_penalty=0.1, + allowed_openai_params="presence_penalty", + ) + + assert route.call_count == 1 + + def _openai_called_names(body: Mapping[str, object]) -> tuple[str, ...]: messages: Final = body["messages"] assert isinstance(messages, list) diff --git a/tests/unit/test_router/core/test_construction_precall_characterization_precall.py b/tests/unit/test_router/core/test_construction_precall_characterization_precall.py index ab53f01662f..7a384e14518 100644 --- a/tests/unit/test_router/core/test_construction_precall_characterization_precall.py +++ b/tests/unit/test_router/core/test_construction_precall_characterization_precall.py @@ -581,13 +581,14 @@ def test_when_no_deployment_supports_the_param_sync_completion_raises_without_ca REQUEST_ESCAPES: Final = [ {"allowed_openai_params": ["response_format"]}, + {"allowed_openai_params": "response_format"}, {"drop_params": True}, {"additional_drop_params": ["response_format"]}, ] @pytest.mark.asyncio -@pytest.mark.parametrize("escape", REQUEST_ESCAPES, ids=["allowlist", "drop", "additional-drop"]) +@pytest.mark.parametrize("escape", REQUEST_ESCAPES, ids=["allowlist", "allowlist-string", "drop", "additional-drop"]) async def test_a_request_that_allows_or_drops_the_param_keeps_the_only_deployment( make_core: MakeCore, host: ScriptedHost, escape: dict[str, object] ) -> None: @@ -597,7 +598,7 @@ async def test_a_request_that_allows_or_drops_the_param_keeps_the_only_deploymen @pytest.mark.asyncio -@pytest.mark.parametrize("escape", REQUEST_ESCAPES, ids=["allowlist", "drop", "additional-drop"]) +@pytest.mark.parametrize("escape", REQUEST_ESCAPES, ids=["allowlist", "allowlist-string", "drop", "additional-drop"]) async def test_a_deployment_that_takes_the_param_is_preferred_over_dropping_it( make_core: MakeCore, host: ScriptedHost, escape: dict[str, object] ) -> None: @@ -609,8 +610,12 @@ async def test_a_deployment_that_takes_the_param_is_preferred_over_dropping_it( @pytest.mark.asyncio @pytest.mark.parametrize( "deployment_escape", - [{"drop_params": True}, {"allowed_openai_params": ["response_format"]}], - ids=["drop", "allowlist"], + [ + {"drop_params": True}, + {"allowed_openai_params": ["response_format"]}, + {"allowed_openai_params": "response_format"}, + ], + ids=["drop", "allowlist", "allowlist-string"], ) async def test_a_deployment_that_allows_or_drops_the_param_is_kept( make_core: MakeCore, host: ScriptedHost, deployment_escape: dict[str, object] diff --git a/tests/unit/test_router/core/test_gaps_l14000_selection.py b/tests/unit/test_router/core/test_gaps_l14000_selection.py index 8cff6bcdbe5..f2e3e02dfa0 100644 --- a/tests/unit/test_router/core/test_gaps_l14000_selection.py +++ b/tests/unit/test_router/core/test_gaps_l14000_selection.py @@ -338,7 +338,12 @@ async def test_a_tier_param_no_deployment_supports_is_dropped(make_core: MakeCor @pytest.mark.asyncio -async def test_a_tier_param_the_request_allowlists_is_kept(make_core: MakeCore) -> None: +@pytest.mark.parametrize( + "allowlist", + [["logit_bias"], "logit_bias", "seed, logit_bias", {"logit_bias": True}], + ids=["list", "bare-string", "comma-separated", "dict"], +) +async def test_a_tier_param_the_request_allowlists_is_kept(make_core: MakeCore, allowlist: object) -> None: core: Final = make_core( model_list=[ deployment("small", "s", model="anthropic/claude-3-5-sonnet-20240620"), @@ -346,7 +351,24 @@ async def test_a_tier_param_the_request_allowlists_is_kept(make_core: MakeCore) _complexity_router(tiers=_pinned(logit_bias={"1": 2})), ] ) - request: Final[dict[str, object]] = {"allowed_openai_params": ["logit_bias"]} + request: Final[dict[str, object]] = {"allowed_openai_params": allowlist} + await core.select("smart", request, messages=MESSAGES) # pyright: ignore[reportCallIssue] + assert request["logit_bias"] == {"1": 2} + + +@pytest.mark.asyncio +@pytest.mark.parametrize("allowlist", [["logit_bias"], "logit_bias"], ids=["list", "bare-string"]) +async def test_a_tier_param_the_deployment_allowlists_is_kept(make_core: MakeCore, allowlist: object) -> None: + core: Final = make_core( + model_list=[ + deployment( + "small", "s", model="anthropic/claude-3-5-sonnet-20240620", allowed_openai_params=allowlist + ), + deployment("big", "b"), + _complexity_router(tiers=_pinned(logit_bias={"1": 2})), + ] + ) + request: Final[dict[str, object]] = {} await core.select("smart", request, messages=MESSAGES) # pyright: ignore[reportCallIssue] assert request["logit_bias"] == {"1": 2} diff --git a/tests/unit/test_utils_get_optional_params.py b/tests/unit/test_utils_get_optional_params.py index 05b776cb873..7b69f097e3e 100644 --- a/tests/unit/test_utils_get_optional_params.py +++ b/tests/unit/test_utils_get_optional_params.py @@ -128,6 +128,65 @@ def test_get_optional_params_with_allowed_openai_params(): assert optional_params["reasoning_effort"] == reasoning_effort +@pytest.mark.parametrize( + "allowlist", + ["user", "seed, user", "seed user", '["user"]', {"user": True}], + ids=["bare-string", "comma-separated", "space-separated", "json-list-string", "dict"], +) +def test_get_optional_params_reads_param_names_from_a_string_or_dict_allowlist( + monkeypatch: pytest.MonkeyPatch, allowlist: str | dict[str, bool] +): + monkeypatch.setattr(litellm, "drop_params", False) + optional_params: Final = get_optional_params( + model="cf/llama-3.1-70b-instruct", + custom_llm_provider="cloudflare", + allowed_openai_params=allowlist, + user="end-user-1", + ) + + assert optional_params["user"] == "end-user-1" + + +def test_get_optional_params_an_allowlist_that_is_no_string_mapping_or_sequence_allowlists_nothing( + monkeypatch: pytest.MonkeyPatch, +): + monkeypatch.setattr(litellm, "drop_params", False) + optional_params: Final = get_optional_params( + model="cf/llama-3.1-70b-instruct", + custom_llm_provider="cloudflare", + allowed_openai_params=5, + user="end-user-1", + ) + + assert "user" not in optional_params + + +@pytest.mark.parametrize( + "allowlist", + [ + ["dimensions"], + "dimensions", + "dimensions,user", + "dimensions encoding_format", + '["dimensions"]', + {"dimensions": True}, + ], + ids=["list", "bare-string", "comma-separated", "space-separated", "json-list-string", "dict"], +) +def test_get_optional_params_embeddings_keeps_dimensions_the_allowlist_names( + allowlist: list[str] | str | dict[str, bool], +): + optional_params: Final = get_optional_params_embeddings( + model="text-embedding-ada-002", + custom_llm_provider="openai", + dimensions=256, + allowed_openai_params=allowlist, + drop_params=False, + ) + + assert optional_params["dimensions"] == 256 + + def test_allowed_openai_params_does_not_forward_unset_params(): """ Regression test for https://github.com/BerriAI/litellm/issues/25697