mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-15 09:12:18 +00:00
fix(responses): accept a string or tuple allowed_openai_params on Responses models instead of answering 500 (#51751)
* fix(responses): accept any sequence for allowed_openai_params and give UnsupportedParamsError a response A deployment whose allowed_openai_params is a string (or an SDK caller passing a tuple) made every Responses-mode request answer 500 "can only concatenate list". The Responses path now builds the allowlist from any sequence, matching the chat path. UnsupportedParamsError now runs BadRequestError's constructor, so it carries response, body and request like every other 400 instead of raising AttributeError on .response. * fix(params): read a bare-string allowed_openai_params as one param name on every path A deployment or request with allowed_openai_params: "presence_penalty" now allowlists presence_penalty on chat completions, embeddings, the Responses API, the chat-to-Responses bridge, the router's tier and precall filters, and Mistral's prompt_cache_key check. Before, chat and Responses read the string letter by letter, so it allowlisted nothing, and the router ignored it. UnsupportedParamsError's args now hold its own message instead of the BadRequestError one. * fix(params): read a dict or comma-separated allowed_openai_params the way chat and embeddings did A dict allowlist is read for its keys and a string is split on commas, so configs that worked on chat or embeddings before keep working, and the router reads them the same way. The fallback errors header names an error by its class when its type is empty, so UnsupportedParamsError still shows up there by name. * fix(params): read every param name in a string allowed_openai_params A string allowlist now yields every name in it, whether the names are split by commas or spaces or written as a list inside the string, so embeddings configs like "dimensions encoding_format" keep working. A request with no allowlist returns early instead of raising and catching a validation error. * fix(fallbacks): keep the fallback errors header type for errors other than UnsupportedParamsError The header named an error by its class whenever its type was empty, which changed "None" to the class name for timeouts and other errors built without a provider body. Only UnsupportedParamsError needs its class name now that it inherits an empty type, so every other error keeps the type it carries. * perf(params): read a string allowed_openai_params with one regex scan The string reader walked every character in Python and ran twice per request, so a 1 MB allowlist added about 300 ms of event-loop time. `\w+` matches exactly the alphanumeric-or-underscore runs the old loop kept, checked over every Unicode code point, and runs in C. * test(params): cover an int allowed_openai_params and an admin request for a model two teams share An allowlist that is no string, mapping, or sequence now allowlists nothing instead of raising TypeError, and an admin request with no team for a model name two teams share answers the multiple-teams 400 with the model name --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
This commit is contained in:
parent
ad24ca9f31
commit
4e29aea11c
20 changed files with 287 additions and 62 deletions
|
|
@ -277,9 +277,9 @@ class NotFoundError(openai.NotFoundError):
|
|||
class BadRequestError(openai.BadRequestError):
|
||||
def __init__(
|
||||
self,
|
||||
message,
|
||||
model,
|
||||
llm_provider,
|
||||
message: str,
|
||||
model: str | None,
|
||||
llm_provider: str | None,
|
||||
response: httpx.Response | None = None,
|
||||
litellm_debug_info: str | None = None,
|
||||
max_retries: int | None = None,
|
||||
|
|
@ -1072,7 +1072,7 @@ class OpenAIError(openai.OpenAIError):
|
|||
class UnsupportedParamsError(BadRequestError):
|
||||
def __init__(
|
||||
self,
|
||||
message,
|
||||
message: str,
|
||||
llm_provider: str | None = None,
|
||||
model: str | None = None,
|
||||
status_code: int = 400,
|
||||
|
|
@ -1081,17 +1081,17 @@ class UnsupportedParamsError(BadRequestError):
|
|||
max_retries: int | None = None,
|
||||
num_retries: int | None = None,
|
||||
):
|
||||
self.status_code = 400
|
||||
self.message = f"litellm.UnsupportedParamsError: {message}"
|
||||
self.model = model
|
||||
self.llm_provider = llm_provider
|
||||
self.litellm_debug_info = litellm_debug_info
|
||||
response = response or httpx.Response(
|
||||
status_code=self.status_code,
|
||||
request=httpx.Request(method="GET", url="https://litellm.ai"), # mock request object
|
||||
super().__init__(
|
||||
message=message,
|
||||
model=model,
|
||||
llm_provider=llm_provider,
|
||||
response=response,
|
||||
litellm_debug_info=litellm_debug_info,
|
||||
max_retries=max_retries,
|
||||
num_retries=num_retries,
|
||||
)
|
||||
self.max_retries = max_retries
|
||||
self.num_retries = num_retries
|
||||
self.message = f"litellm.UnsupportedParamsError: {message}"
|
||||
self.args = (self.message,)
|
||||
|
||||
|
||||
LITELLM_EXCEPTION_TYPES: Final = [
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ from typing import Final, Protocol, TypedDict, cast
|
|||
|
||||
from pydantic import BaseModel, TypeAdapter, ValidationError
|
||||
|
||||
from litellm.exceptions import UnsupportedParamsError
|
||||
from litellm.litellm_core_utils.hidden_params import (
|
||||
HIDDEN_PARAMS_ATTR as _HIDDEN_PARAMS_ATTR,
|
||||
)
|
||||
|
|
@ -249,7 +250,11 @@ def _ensure_additional_headers_dict(
|
|||
|
||||
def get_fallback_error_info(error: Exception) -> FallbackErrorInfo:
|
||||
message: Final = cast(object, getattr(error, "message", str(error)))
|
||||
error_type: Final = cast(object, getattr(error, "type", error.__class__.__name__))
|
||||
error_type: Final = (
|
||||
error.__class__.__name__
|
||||
if isinstance(error, UnsupportedParamsError)
|
||||
else cast(object, getattr(error, "type", error.__class__.__name__))
|
||||
)
|
||||
param: Final = cast(object, getattr(error, "param", None))
|
||||
code: Final = cast(object, getattr(error, "status_code", getattr(error, "code", None)))
|
||||
return FallbackErrorInfo(
|
||||
|
|
|
|||
|
|
@ -109,6 +109,25 @@ def as_object_list(value: object) -> Sequence[object] | None:
|
|||
return value if isinstance(value, list) else None
|
||||
|
||||
|
||||
_OBJECT_ENTRIES: Final = TypeAdapter(list[object])
|
||||
_OBJECT_KEYS: Final = TypeAdapter(dict[object, object])
|
||||
_PARAM_NAME: Final = re.compile(r"\w+")
|
||||
|
||||
|
||||
def allowed_openai_param_names(value: object) -> tuple[str, ...]:
|
||||
if value is None:
|
||||
return ()
|
||||
if isinstance(value, str):
|
||||
return tuple(_PARAM_NAME.findall(value))
|
||||
if isinstance(value, Mapping):
|
||||
return tuple(key for key in _OBJECT_KEYS.validate_python(value) if isinstance(key, str))
|
||||
try:
|
||||
entries: Final = _OBJECT_ENTRIES.validate_python(value)
|
||||
except ValidationError:
|
||||
return ()
|
||||
return tuple(entry for entry in entries if isinstance(entry, str))
|
||||
|
||||
|
||||
_Key = TypeVar("_Key") # rebind-ok: a TypeVar cannot be Final
|
||||
_Value = TypeVar("_Value") # rebind-ok: a TypeVar cannot be Final
|
||||
|
||||
|
|
|
|||
|
|
@ -401,9 +401,10 @@ def filter_team_based_models(
|
|||
litellm_metadata: Final = request_kwargs.get("litellm_metadata") or {}
|
||||
request_team_id: Final = get_request_team_id(request_kwargs)
|
||||
if request_team_id is None and is_proxy_admin_request(request_kwargs) and isinstance(healthy_deployments, list):
|
||||
requested_model: Final = (
|
||||
requested_model_value: Final = (
|
||||
request_kwargs.get("model") or metadata.get("model_group") or litellm_metadata.get("model_group")
|
||||
)
|
||||
requested_model: Final = requested_model_value if isinstance(requested_model_value, str) else None
|
||||
candidate_deployments: Final = tuple(
|
||||
(deployment.get("model_name"), deployment.get("model_info") or {}) for deployment in healthy_deployments
|
||||
)
|
||||
|
|
|
|||
|
|
@ -19,7 +19,7 @@ from typing_extensions import TypeIs, assert_never # noqa: TID251 # these are
|
|||
import litellm
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.exceptions import UnsupportedParamsError
|
||||
from litellm.litellm_core_utils.core_helpers import as_object_list, as_str_mapping
|
||||
from litellm.litellm_core_utils.core_helpers import allowed_openai_param_names, as_object_list, as_str_mapping
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
from litellm.litellm_core_utils.llm_cost_calc.guardrail_cost import is_str_keyed_dict
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
||||
|
|
@ -355,8 +355,7 @@ class MistralConfig(OpenAIGPTConfig):
|
|||
def _prompt_cache_key_withheld(self, model: str, prompt_cache_key: object, routing: MistralRequestRouting) -> bool:
|
||||
requested_api_base: Final = routing.get("api_base")
|
||||
api_base: Final = requested_api_base if isinstance(requested_api_base, str) else None
|
||||
allowed_openai_params: Final = routing.get("allowed_openai_params")
|
||||
if isinstance(allowed_openai_params, (list, tuple)) and "prompt_cache_key" in allowed_openai_params:
|
||||
if "prompt_cache_key" in allowed_openai_param_names(routing.get("allowed_openai_params")):
|
||||
return False
|
||||
rejection: Final = self._prompt_cache_key_rejection(prompt_cache_key, api_base)
|
||||
if rejection is None:
|
||||
|
|
|
|||
|
|
@ -233,6 +233,7 @@ from litellm.utils import (
|
|||
from ._logging import WarningLogger, verbose_logger
|
||||
from .caching.caching import disable_cache, enable_cache, update_cache
|
||||
from .litellm_core_utils.core_helpers import (
|
||||
allowed_openai_param_names,
|
||||
as_str_mapping,
|
||||
drops_unsupported_params,
|
||||
normalize_drop_params,
|
||||
|
|
@ -6085,7 +6086,7 @@ def completion(
|
|||
responses_api_model_info.get("mode") == "responses" and not skip_responses_api_bridge
|
||||
)
|
||||
allowed_openai_params: Final[list[str] | None] = (
|
||||
[*(kwargs.get("allowed_openai_params") or []), "reasoning_effort"]
|
||||
[*allowed_openai_param_names(kwargs.get("allowed_openai_params")), "reasoning_effort"]
|
||||
if bridges_to_responses_api
|
||||
or reasoning_effort_needs_allowing(
|
||||
chat_reasoning.effort,
|
||||
|
|
|
|||
|
|
@ -35,7 +35,12 @@ from litellm.litellm_core_utils.allowed_tools_tool_choice import (
|
|||
)
|
||||
from litellm.litellm_core_utils.asyncify import run_async_function
|
||||
from litellm.litellm_core_utils.completion_timeout import CompletionTimeout, StreamKind
|
||||
from litellm.litellm_core_utils.core_helpers import drops_unsupported_params, normalize_drop_params, require_attributes
|
||||
from litellm.litellm_core_utils.core_helpers import (
|
||||
allowed_openai_param_names,
|
||||
drops_unsupported_params,
|
||||
normalize_drop_params,
|
||||
require_attributes,
|
||||
)
|
||||
from litellm.litellm_core_utils.get_llm_provider_logic import (
|
||||
declared_authenticating_provider,
|
||||
get_provider_affinity_header,
|
||||
|
|
@ -664,7 +669,7 @@ def _chat_completion_request_fields(allowed_openai_params: Sequence[str] | None)
|
|||
*DEFAULT_CHAT_COMPLETION_PARAM_VALUES,
|
||||
*all_litellm_params,
|
||||
*GenericLiteLLMParams.model_fields,
|
||||
*(allowed_openai_params or ()),
|
||||
*allowed_openai_param_names(allowed_openai_params),
|
||||
)
|
||||
).difference(_RESPONSES_ONLY_REQUEST_FIELDS_NEVER_BRIDGED)
|
||||
|
||||
|
|
|
|||
|
|
@ -14,6 +14,7 @@ from typing_extensions import TypeIs # noqa: TID251 # narrows untyped wire pay
|
|||
import litellm
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.litellm_core_utils.core_helpers import (
|
||||
allowed_openai_param_names,
|
||||
as_object_list,
|
||||
as_str_mapping,
|
||||
attribute_or_none,
|
||||
|
|
@ -314,7 +315,7 @@ class ResponsesAPIRequestUtils:
|
|||
model: str,
|
||||
responses_api_provider_config: BaseResponsesAPIConfig,
|
||||
response_api_optional_params: ResponsesAPIOptionalRequestParams,
|
||||
allowed_openai_params: list[str] | None = None,
|
||||
allowed_openai_params: Sequence[str] | None = None,
|
||||
drop_params: bool | None = None,
|
||||
declared_reasoning_efforts: Collection[str] | None = None,
|
||||
) -> dict:
|
||||
|
|
@ -340,7 +341,7 @@ class ResponsesAPIRequestUtils:
|
|||
non_default_params: Final = cast(dict, response_api_optional_params)
|
||||
# Check for unsupported parameters
|
||||
ResponsesAPIRequestUtils._check_valid_arg(
|
||||
supported_params=supported_params + (allowed_openai_params or []),
|
||||
supported_params=[*supported_params, *allowed_openai_param_names(allowed_openai_params)],
|
||||
non_default_params=non_default_params,
|
||||
drop_params=should_drop_params,
|
||||
custom_llm_provider=responses_api_provider_config.custom_llm_provider,
|
||||
|
|
@ -369,7 +370,7 @@ class ResponsesAPIRequestUtils:
|
|||
return apply_openai_param_overrides(
|
||||
optional_params=params_with_normalized_stream_options,
|
||||
non_default_params=non_default_params,
|
||||
allowed_openai_params=allowed_openai_params or [],
|
||||
allowed_openai_params=allowed_openai_params,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
|
|
|
|||
|
|
@ -28,7 +28,7 @@ from litellm.litellm_core_utils.clientside_credential_handler import (
|
|||
is_clientside_credential, # pyright: ignore[reportUnknownVariableType] # litellm's untyped helpers and request values
|
||||
)
|
||||
from litellm.litellm_core_utils.completion_timeout import CompletionTimeout
|
||||
from litellm.litellm_core_utils.core_helpers import without
|
||||
from litellm.litellm_core_utils.core_helpers import allowed_openai_param_names, without
|
||||
from litellm.litellm_core_utils.get_litellm_params import SUPPLEMENTAL_KWARGS_KEYS
|
||||
from litellm.litellm_core_utils.get_llm_provider_logic import declared_authenticating_provider, inferred_provider
|
||||
from litellm.litellm_core_utils.request_timeout_resolver import get_service_tier_timeout
|
||||
|
|
@ -242,11 +242,7 @@ def _is_classifier_internal_call(kwargs: Mapping[str, object]) -> bool:
|
|||
|
||||
|
||||
def _declared_param_allowlist(params: Mapping[str, object]) -> frozenset[str]:
|
||||
declared: Final = params.get("allowed_openai_params")
|
||||
if not isinstance(declared, (list, tuple, set, frozenset)):
|
||||
return frozenset()
|
||||
entries: Final[Sequence[object]] = list(declared) # pyright: ignore[reportUnknownArgumentType] # the deployment's untyped allowlist
|
||||
return frozenset(entry for entry in entries if isinstance(entry, str))
|
||||
return frozenset(allowed_openai_param_names(params.get("allowed_openai_params")))
|
||||
|
||||
|
||||
def _deployment_supports_param(deployment: Mapping[str, object], group: str, param: str, host: "PythonHost") -> bool:
|
||||
|
|
|
|||
|
|
@ -20,7 +20,7 @@ from types import MappingProxyType
|
|||
from typing import TYPE_CHECKING, Final, Protocol
|
||||
|
||||
from litellm._logging import verbose_router_logger
|
||||
from litellm.litellm_core_utils.core_helpers import as_str_mapping
|
||||
from litellm.litellm_core_utils.core_helpers import allowed_openai_param_names, as_str_mapping
|
||||
from litellm.litellm_core_utils.deployment_pricing import deployment_backend_models
|
||||
from litellm.litellm_core_utils.get_llm_provider_logic import declared_authenticating_provider
|
||||
from litellm.litellm_core_utils.prompt_templates.tpm_media_utils import is_plain_embedding_text
|
||||
|
|
@ -204,8 +204,7 @@ def _is_list(value: object) -> bool:
|
|||
|
||||
|
||||
def _declared_param_allowlist(params: Mapping[str, object]) -> frozenset[str]:
|
||||
declared: Final = params.get("allowed_openai_params")
|
||||
return frozenset(entry for entry in _items(declared) if isinstance(entry, str))
|
||||
return frozenset(allowed_openai_param_names(params.get("allowed_openai_params")))
|
||||
|
||||
|
||||
def _capability_model_and_provider(deployment_params: Mapping[str, object], group: str) -> tuple[str, str]:
|
||||
|
|
|
|||
|
|
@ -35,7 +35,7 @@ from litellm.constants import (
|
|||
)
|
||||
from litellm.integrations.otel.routing import routing_decision_attributes
|
||||
from litellm.integrations.otel.runtime import phase_attributes
|
||||
from litellm.litellm_core_utils.core_helpers import get_or_create_metadata_bucket, without
|
||||
from litellm.litellm_core_utils.core_helpers import allowed_openai_param_names, get_or_create_metadata_bucket, without
|
||||
from litellm.litellm_core_utils.credential_accessor import reject_api_key_secret_reference
|
||||
from litellm.router_api import deployment_id
|
||||
from litellm.router_host import router_host
|
||||
|
|
@ -1171,10 +1171,7 @@ def _tags_of(params: Mapping[str, object]) -> Sequence[object]:
|
|||
|
||||
|
||||
def _allowlist(params: Mapping[str, object]) -> frozenset[str]:
|
||||
declared: Final = params.get("allowed_openai_params")
|
||||
if not isinstance(declared, list | tuple | set | frozenset):
|
||||
return frozenset()
|
||||
return frozenset(entry for entry in declared if isinstance(entry, str))
|
||||
return frozenset(allowed_openai_param_names(params.get("allowed_openai_params")))
|
||||
|
||||
|
||||
__all__ = ["PreRouter", "PreRoutingStrategies", "as_output_cap", "is_strategy_marker", "stamped_client_ceiling"]
|
||||
|
|
|
|||
|
|
@ -91,6 +91,7 @@ from litellm.litellm_core_utils.allowed_tools_tool_choice import (
|
|||
)
|
||||
from litellm.litellm_core_utils.attempt_settlement import mark_attempt_answered
|
||||
from litellm.litellm_core_utils.core_helpers import (
|
||||
allowed_openai_param_names,
|
||||
as_str_mapping,
|
||||
bind_budget_reservation_to_callbacks,
|
||||
has_attributes,
|
||||
|
|
@ -1870,9 +1871,10 @@ async def async_pre_call_deployment_hook(
|
|||
route: Final[object] = getattr(kwargs.get("litellm_logging_obj"), "call_type", None)
|
||||
resolved: Final = resolve_deployment_tool_fields(kwargs, route)
|
||||
if isinstance(resolved, MalformedField):
|
||||
model: Final = kwargs.get("model")
|
||||
raise MalformedRequestFieldError(
|
||||
message=resolved.message,
|
||||
model=kwargs.get("model"),
|
||||
model=model if isinstance(model, str) else None,
|
||||
llm_provider=_deployment_provider(kwargs),
|
||||
body={"type": "invalid_request_error", "param": resolved.field},
|
||||
)
|
||||
|
|
@ -4854,7 +4856,6 @@ def get_optional_params_embeddings(
|
|||
drop_params = normalize_drop_params(drop_params)
|
||||
passed_params.pop("additional_drop_params", None)
|
||||
passed_params.pop("allowed_openai_params", None)
|
||||
allowed_openai_params = allowed_openai_params or []
|
||||
# Remove function objects from passed_params to avoid JSON serialization errors
|
||||
passed_params.pop("get_supported_openai_params", None)
|
||||
|
||||
|
|
@ -4918,7 +4919,7 @@ def get_optional_params_embeddings(
|
|||
model is not None
|
||||
and "text-embedding-3" not in model
|
||||
and "dimensions" in non_default_params
|
||||
and "dimensions" not in (allowed_openai_params or [])
|
||||
and "dimensions" not in allowed_openai_param_names(allowed_openai_params)
|
||||
):
|
||||
# Honor drop_params (per-call) and litellm.drop_params (global) the same
|
||||
# way `_check_valid_arg` does above. The raised error message itself
|
||||
|
|
@ -5772,8 +5773,7 @@ def get_optional_params(
|
|||
supported_params = get_supported_openai_params(model=model, custom_llm_provider="openai")
|
||||
|
||||
supported_params = supported_params or []
|
||||
allowed_openai_params = allowed_openai_params or []
|
||||
supported_params.extend(allowed_openai_params)
|
||||
supported_params.extend(allowed_openai_param_names(allowed_openai_params))
|
||||
|
||||
_check_valid_arg(
|
||||
supported_params=supported_params or [],
|
||||
|
|
@ -6370,7 +6370,7 @@ def add_provider_specific_params_to_optional_params(
|
|||
def apply_openai_param_overrides(
|
||||
optional_params: dict[str, object],
|
||||
non_default_params: dict[str, object],
|
||||
allowed_openai_params: list[str],
|
||||
allowed_openai_params: Sequence[str] | None,
|
||||
) -> dict[str, object]:
|
||||
"""
|
||||
If user passes in allowed_openai_params, apply them to optional_params
|
||||
|
|
@ -6384,13 +6384,12 @@ def apply_openai_param_overrides(
|
|||
`AsyncCompletions.create() got an unexpected keyword argument 'enable_thinking'`).
|
||||
See https://github.com/BerriAI/litellm/issues/25697
|
||||
"""
|
||||
if allowed_openai_params:
|
||||
for param in allowed_openai_params:
|
||||
if param in optional_params:
|
||||
continue
|
||||
if param not in non_default_params:
|
||||
continue
|
||||
optional_params[param] = non_default_params.pop(param)
|
||||
for param in allowed_openai_param_names(allowed_openai_params):
|
||||
if param in optional_params:
|
||||
continue
|
||||
if param not in non_default_params:
|
||||
continue
|
||||
optional_params[param] = non_default_params.pop(param)
|
||||
return optional_params
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -1846,7 +1846,10 @@ def _chat_completion_reply() -> httpx.Response:
|
|||
)
|
||||
|
||||
|
||||
def test_allowlisted_prompt_cache_key_reaches_a_gateway_in_front_of_mistral(respx_mock: respx.MockRouter) -> None:
|
||||
@pytest.mark.parametrize("allowlist", [["prompt_cache_key"], "prompt_cache_key"], ids=["list", "bare-string"])
|
||||
def test_allowlisted_prompt_cache_key_reaches_a_gateway_in_front_of_mistral(
|
||||
respx_mock: respx.MockRouter, allowlist: list[str] | str
|
||||
) -> None:
|
||||
import litellm
|
||||
|
||||
gateway_route: Final = respx_mock.post("https://gateway.example.com/mistral/v1/chat/completions").mock(
|
||||
|
|
@ -1859,7 +1862,7 @@ def test_allowlisted_prompt_cache_key_reaches_a_gateway_in_front_of_mistral(resp
|
|||
api_base="https://gateway.example.com/mistral/v1",
|
||||
api_key="gateway-key",
|
||||
prompt_cache_key="session-1",
|
||||
allowed_openai_params=["prompt_cache_key"],
|
||||
allowed_openai_params=allowlist,
|
||||
max_retries=0,
|
||||
)
|
||||
|
||||
|
|
@ -1959,7 +1962,7 @@ def test_prompt_cache_key_rejection_for_a_sync_codestral_call_is_an_unsupported_
|
|||
|
||||
with pytest.raises(
|
||||
litellm.UnsupportedParamsError, match=r"^litellm.UnsupportedParamsError: codestral does not support"
|
||||
):
|
||||
) as rejection:
|
||||
litellm.completion(
|
||||
model="codestral/codestral-latest",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
|
|
@ -1970,6 +1973,8 @@ def test_prompt_cache_key_rejection_for_a_sync_codestral_call_is_an_unsupported_
|
|||
)
|
||||
|
||||
assert not azure_route.called
|
||||
assert rejection.value.response.status_code == rejection.value.status_code == 400
|
||||
assert rejection.value.args == (rejection.value.message,)
|
||||
|
||||
|
||||
def test_prompt_cache_key_for_another_host_is_dropped_under_drop_params() -> None:
|
||||
|
|
|
|||
|
|
@ -46,6 +46,30 @@ class TestResponsesAPIRequestUtils:
|
|||
assert "prompt" in result
|
||||
assert result["prompt"] == {"id": "pmpt_123"}
|
||||
|
||||
def test_get_optional_params_responses_api_forwards_a_param_a_tuple_allowlist_names(self):
|
||||
result: Final = ResponsesAPIRequestUtils.get_optional_params_responses_api(
|
||||
model="gpt-4o",
|
||||
responses_api_provider_config=OpenAIResponsesAPIConfig(),
|
||||
response_api_optional_params=ResponsesAPIOptionalRequestParams(
|
||||
{"temperature": 0.7, "custom_flag": "on"} # pyright: ignore[reportCallIssue] # an unsupported key is the input under test
|
||||
),
|
||||
allowed_openai_params=("custom_flag",),
|
||||
)
|
||||
|
||||
assert result == {"temperature": 0.7, "custom_flag": "on"}
|
||||
|
||||
def test_get_optional_params_responses_api_reads_a_bare_string_allowlist_as_one_param_name(self):
|
||||
result: Final = ResponsesAPIRequestUtils.get_optional_params_responses_api(
|
||||
model="gpt-4o",
|
||||
responses_api_provider_config=OpenAIResponsesAPIConfig(),
|
||||
response_api_optional_params=ResponsesAPIOptionalRequestParams(
|
||||
{"temperature": 0.7, "custom_flag": "on"} # pyright: ignore[reportCallIssue] # an unsupported key is the input under test
|
||||
),
|
||||
allowed_openai_params="custom_flag",
|
||||
)
|
||||
|
||||
assert result == {"temperature": 0.7, "custom_flag": "on"}
|
||||
|
||||
def test_get_optional_params_responses_api_unsupported_param(self):
|
||||
"""Test that unsupported parameters raise an error"""
|
||||
# Setup
|
||||
|
|
|
|||
|
|
@ -10,10 +10,12 @@ from litellm.litellm_core_utils.add_retry_fallback_headers import (
|
|||
add_retry_headers_to_response,
|
||||
complexity_router_decision_headers,
|
||||
ensure_response_additional_headers,
|
||||
get_fallback_error_info,
|
||||
get_fallback_errors_from_headers,
|
||||
get_hidden_params_dict,
|
||||
replace_complexity_router_headers,
|
||||
)
|
||||
from litellm.exceptions import AuthenticationError, Timeout, UnsupportedParamsError
|
||||
from litellm.types.decisions import DecisionsResponse
|
||||
|
||||
|
||||
|
|
@ -208,6 +210,41 @@ def test_add_fallback_headers_serializes_fallback_errors():
|
|||
)
|
||||
|
||||
|
||||
class _ProviderTypedError(Exception):
|
||||
type = "invalid_request_error"
|
||||
|
||||
|
||||
class _UntypedError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
def test_get_fallback_error_info_names_an_unsupported_params_error_by_its_class():
|
||||
error: Final = UnsupportedParamsError(
|
||||
message="mistral does not support parameters: ['presence_penalty']",
|
||||
llm_provider="mistral",
|
||||
model="mistral-large-latest",
|
||||
)
|
||||
|
||||
assert get_fallback_error_info(error)["type"] == "UnsupportedParamsError"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"error",
|
||||
[
|
||||
_ProviderTypedError("bad request"),
|
||||
Timeout(message="timed out", model="gpt-5.6", llm_provider="openai"),
|
||||
AuthenticationError(message="bad key", llm_provider="openai", model="gpt-5.6"),
|
||||
],
|
||||
ids=["provider-type", "timeout-without-provider-type", "auth-without-provider-type"],
|
||||
)
|
||||
def test_get_fallback_error_info_keeps_the_type_other_errors_carry(error: Exception):
|
||||
assert get_fallback_error_info(error)["type"] == str(getattr(error, "type", None))
|
||||
|
||||
|
||||
def test_get_fallback_error_info_names_an_error_without_a_type_by_its_class():
|
||||
assert get_fallback_error_info(_UntypedError("boom"))["type"] == "_UntypedError"
|
||||
|
||||
|
||||
def test_add_retry_headers_to_streaming_wrapper():
|
||||
response = StreamingWrapper()
|
||||
|
||||
|
|
|
|||
|
|
@ -9,7 +9,7 @@ import litellm
|
|||
from litellm import Router
|
||||
from litellm.constants import ROUTER_FALLBACK_ERROR_DETAIL_MAX_CHARS
|
||||
from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap
|
||||
from litellm.proxy._types import UserAPIKeyAuth
|
||||
from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth
|
||||
from litellm.litellm_core_utils.routing_common_utils import (
|
||||
_deployment_supports_web_search,
|
||||
add_model_file_id_mappings,
|
||||
|
|
@ -53,6 +53,22 @@ class TestFilterTeamBasedModels:
|
|||
{"model_info": {"id": "deployment-2"}},
|
||||
]
|
||||
|
||||
def test_an_admin_without_a_team_gets_a_400_for_a_model_name_two_teams_share(self) -> None:
|
||||
deployments: Final = [
|
||||
{"model_name": "shared-gpt", "model_info": {"id": "deployment-a", "team_id": "team-a"}},
|
||||
{"model_name": "shared-gpt", "model_info": {"id": "deployment-b", "team_id": "team-b"}},
|
||||
]
|
||||
request_kwargs: Final = {
|
||||
"model": "shared-gpt",
|
||||
"metadata": {"user_api_key_auth": UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN)},
|
||||
}
|
||||
|
||||
with pytest.raises(litellm.BadRequestError, match="'shared-gpt' matches deployments from multiple teams") as error:
|
||||
filter_team_based_models(deployments, request_kwargs)
|
||||
|
||||
assert error.value.model == "shared-gpt"
|
||||
assert error.value.status_code == 400
|
||||
|
||||
def test_filter_team_based_models_none_request_kwargs(self, sample_deployments_with_teams):
|
||||
"""Test that when request_kwargs is None, all deployments are returned unchanged"""
|
||||
result = filter_team_based_models(sample_deployments_with_teams, None)
|
||||
|
|
|
|||
|
|
@ -943,6 +943,41 @@ def test_completion_drops_unanswered_claude_tool_search_calls_before_the_respons
|
|||
]
|
||||
|
||||
|
||||
def test_completion_through_the_responses_bridge_honors_a_string_allowlist(
|
||||
respx_mock: respx.MockRouter,
|
||||
):
|
||||
route: Final = respx_mock.post("https://api.openai.com/v1/responses").respond(
|
||||
json={
|
||||
"id": "resp_allowlist",
|
||||
"object": "response",
|
||||
"created_at": 1,
|
||||
"status": "completed",
|
||||
"model": "gpt-5.6",
|
||||
"output": [
|
||||
{
|
||||
"type": "message",
|
||||
"id": "msg_allowlist",
|
||||
"role": "assistant",
|
||||
"status": "completed",
|
||||
"content": [{"type": "output_text", "text": "hi", "annotations": []}],
|
||||
}
|
||||
],
|
||||
"usage": {"input_tokens": 1, "output_tokens": 1, "total_tokens": 2},
|
||||
}
|
||||
)
|
||||
|
||||
litellm.completion(
|
||||
model="openai/responses/gpt-5.6",
|
||||
api_key="sk-test",
|
||||
api_base="https://api.openai.com/v1",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
presence_penalty=0.1,
|
||||
allowed_openai_params="presence_penalty",
|
||||
)
|
||||
|
||||
assert route.call_count == 1
|
||||
|
||||
|
||||
def _openai_called_names(body: Mapping[str, object]) -> tuple[str, ...]:
|
||||
messages: Final = body["messages"]
|
||||
assert isinstance(messages, list)
|
||||
|
|
|
|||
|
|
@ -581,13 +581,14 @@ def test_when_no_deployment_supports_the_param_sync_completion_raises_without_ca
|
|||
|
||||
REQUEST_ESCAPES: Final = [
|
||||
{"allowed_openai_params": ["response_format"]},
|
||||
{"allowed_openai_params": "response_format"},
|
||||
{"drop_params": True},
|
||||
{"additional_drop_params": ["response_format"]},
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("escape", REQUEST_ESCAPES, ids=["allowlist", "drop", "additional-drop"])
|
||||
@pytest.mark.parametrize("escape", REQUEST_ESCAPES, ids=["allowlist", "allowlist-string", "drop", "additional-drop"])
|
||||
async def test_a_request_that_allows_or_drops_the_param_keeps_the_only_deployment(
|
||||
make_core: MakeCore, host: ScriptedHost, escape: dict[str, object]
|
||||
) -> None:
|
||||
|
|
@ -597,7 +598,7 @@ async def test_a_request_that_allows_or_drops_the_param_keeps_the_only_deploymen
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("escape", REQUEST_ESCAPES, ids=["allowlist", "drop", "additional-drop"])
|
||||
@pytest.mark.parametrize("escape", REQUEST_ESCAPES, ids=["allowlist", "allowlist-string", "drop", "additional-drop"])
|
||||
async def test_a_deployment_that_takes_the_param_is_preferred_over_dropping_it(
|
||||
make_core: MakeCore, host: ScriptedHost, escape: dict[str, object]
|
||||
) -> None:
|
||||
|
|
@ -609,8 +610,12 @@ async def test_a_deployment_that_takes_the_param_is_preferred_over_dropping_it(
|
|||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"deployment_escape",
|
||||
[{"drop_params": True}, {"allowed_openai_params": ["response_format"]}],
|
||||
ids=["drop", "allowlist"],
|
||||
[
|
||||
{"drop_params": True},
|
||||
{"allowed_openai_params": ["response_format"]},
|
||||
{"allowed_openai_params": "response_format"},
|
||||
],
|
||||
ids=["drop", "allowlist", "allowlist-string"],
|
||||
)
|
||||
async def test_a_deployment_that_allows_or_drops_the_param_is_kept(
|
||||
make_core: MakeCore, host: ScriptedHost, deployment_escape: dict[str, object]
|
||||
|
|
|
|||
|
|
@ -338,7 +338,12 @@ async def test_a_tier_param_no_deployment_supports_is_dropped(make_core: MakeCor
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_tier_param_the_request_allowlists_is_kept(make_core: MakeCore) -> None:
|
||||
@pytest.mark.parametrize(
|
||||
"allowlist",
|
||||
[["logit_bias"], "logit_bias", "seed, logit_bias", {"logit_bias": True}],
|
||||
ids=["list", "bare-string", "comma-separated", "dict"],
|
||||
)
|
||||
async def test_a_tier_param_the_request_allowlists_is_kept(make_core: MakeCore, allowlist: object) -> None:
|
||||
core: Final = make_core(
|
||||
model_list=[
|
||||
deployment("small", "s", model="anthropic/claude-3-5-sonnet-20240620"),
|
||||
|
|
@ -346,7 +351,24 @@ async def test_a_tier_param_the_request_allowlists_is_kept(make_core: MakeCore)
|
|||
_complexity_router(tiers=_pinned(logit_bias={"1": 2})),
|
||||
]
|
||||
)
|
||||
request: Final[dict[str, object]] = {"allowed_openai_params": ["logit_bias"]}
|
||||
request: Final[dict[str, object]] = {"allowed_openai_params": allowlist}
|
||||
await core.select("smart", request, messages=MESSAGES) # pyright: ignore[reportCallIssue]
|
||||
assert request["logit_bias"] == {"1": 2}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("allowlist", [["logit_bias"], "logit_bias"], ids=["list", "bare-string"])
|
||||
async def test_a_tier_param_the_deployment_allowlists_is_kept(make_core: MakeCore, allowlist: object) -> None:
|
||||
core: Final = make_core(
|
||||
model_list=[
|
||||
deployment(
|
||||
"small", "s", model="anthropic/claude-3-5-sonnet-20240620", allowed_openai_params=allowlist
|
||||
),
|
||||
deployment("big", "b"),
|
||||
_complexity_router(tiers=_pinned(logit_bias={"1": 2})),
|
||||
]
|
||||
)
|
||||
request: Final[dict[str, object]] = {}
|
||||
await core.select("smart", request, messages=MESSAGES) # pyright: ignore[reportCallIssue]
|
||||
assert request["logit_bias"] == {"1": 2}
|
||||
|
||||
|
|
|
|||
|
|
@ -128,6 +128,65 @@ def test_get_optional_params_with_allowed_openai_params():
|
|||
assert optional_params["reasoning_effort"] == reasoning_effort
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"allowlist",
|
||||
["user", "seed, user", "seed user", '["user"]', {"user": True}],
|
||||
ids=["bare-string", "comma-separated", "space-separated", "json-list-string", "dict"],
|
||||
)
|
||||
def test_get_optional_params_reads_param_names_from_a_string_or_dict_allowlist(
|
||||
monkeypatch: pytest.MonkeyPatch, allowlist: str | dict[str, bool]
|
||||
):
|
||||
monkeypatch.setattr(litellm, "drop_params", False)
|
||||
optional_params: Final = get_optional_params(
|
||||
model="cf/llama-3.1-70b-instruct",
|
||||
custom_llm_provider="cloudflare",
|
||||
allowed_openai_params=allowlist,
|
||||
user="end-user-1",
|
||||
)
|
||||
|
||||
assert optional_params["user"] == "end-user-1"
|
||||
|
||||
|
||||
def test_get_optional_params_an_allowlist_that_is_no_string_mapping_or_sequence_allowlists_nothing(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
monkeypatch.setattr(litellm, "drop_params", False)
|
||||
optional_params: Final = get_optional_params(
|
||||
model="cf/llama-3.1-70b-instruct",
|
||||
custom_llm_provider="cloudflare",
|
||||
allowed_openai_params=5,
|
||||
user="end-user-1",
|
||||
)
|
||||
|
||||
assert "user" not in optional_params
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"allowlist",
|
||||
[
|
||||
["dimensions"],
|
||||
"dimensions",
|
||||
"dimensions,user",
|
||||
"dimensions encoding_format",
|
||||
'["dimensions"]',
|
||||
{"dimensions": True},
|
||||
],
|
||||
ids=["list", "bare-string", "comma-separated", "space-separated", "json-list-string", "dict"],
|
||||
)
|
||||
def test_get_optional_params_embeddings_keeps_dimensions_the_allowlist_names(
|
||||
allowlist: list[str] | str | dict[str, bool],
|
||||
):
|
||||
optional_params: Final = get_optional_params_embeddings(
|
||||
model="text-embedding-ada-002",
|
||||
custom_llm_provider="openai",
|
||||
dimensions=256,
|
||||
allowed_openai_params=allowlist,
|
||||
drop_params=False,
|
||||
)
|
||||
|
||||
assert optional_params["dimensions"] == 256
|
||||
|
||||
|
||||
def test_allowed_openai_params_does_not_forward_unset_params():
|
||||
"""
|
||||
Regression test for https://github.com/BerriAI/litellm/issues/25697
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue