diff --git a/litellm/router.py b/litellm/router.py index facb4978ae5..e00b495725b 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -229,6 +229,7 @@ from litellm.types.utils import ( StandardLoggingPayload, StandardLoggingRoutingDecision, Usage, + all_litellm_params, shared_backend_model_info, ) from litellm.types.utils import ModelInfo as ModelMapInfo @@ -243,6 +244,7 @@ from litellm.utils import ( get_secret, get_utc_datetime, is_region_allowed, + provider_rejectable_params, set_live_deployment_replay, ) @@ -10832,6 +10834,68 @@ class Router: } return {**deployment, "model_info": model_info} # mutable-ok: DeploymentTypedDict rows are plain dicts + TIER_PARAMS_NEVER_DROPPED: Final = frozenset(all_litellm_params) | frozenset( + {"additional_drop_params", "drop_params", "messages", "model", "extra_headers"} + ) + + @staticmethod + def _deployment_accepts_param(deployment: DeploymentTypedDict, group: str, param: str) -> bool: + deployment_params: Final = deployment.get("litellm_params") + if not deployment_params: + return True + try: + model, custom_llm_provider, _, _ = litellm.get_llm_provider( + model=deployment_params.get("model") or group, + custom_llm_provider=deployment_params.get("custom_llm_provider"), + ) + supported: Final = litellm.get_supported_openai_params(model=model, custom_llm_provider=custom_llm_provider) + except Exception as e: # noqa: BLE001 # best-effort filter: an unresolvable provider must not narrow the request + verbose_router_logger.debug( + "litellm.router.py::_deployment_accepts_param: keeping %s for model=%s. Got - %s", param, group, e + ) + return True + return supported is None or param in supported + + def _tier_params_the_target_accepts(self, model: str, tier_params: Mapping[str, object]) -> Mapping[str, object]: + """Drop an OpenAI param that no deployment behind ``model`` declares. + + A tier's litellm_params are an operator override applied to every request the tier routes, + so one the target cannot take turns that whole tier into a 400 raised before the request + leaves the proxy. The candidates are exactly what get_optional_params can reject, asked of + the module that raises, so credentials and endpoint controls are never at risk. + + TIER_PARAMS_NEVER_DROPPED is excluded on top of that, for two reasons. No provider lists a + litellm control among its supported params, so "no deployment declares it" means litellm + consumes it rather than that the target refuses it, and dropping one changes litellm's own + behavior: dropping drop_params or additional_drop_params silently disables the sanitization + the operator configured. Providers do list extra_headers, but it carries auth, tenancy and + routing information, so sending fewer headers than configured is worse than today's error. + + The trade this filter makes is a param for a working request, which is right for one that + only shapes how the model answers and wrong for anything else. + + A param survives if ANY deployment could take it, because routing has not chosen one yet, + and it survives both an unresolvable provider and a group with no deployments, because a + best-effort filter must never narrow what the request already did. + """ + deployments: Final = self.get_model_list(model_name=model) or () + if not deployments: + return tier_params + candidates: Final = provider_rejectable_params(tier_params) - self.TIER_PARAMS_NEVER_DROPPED + unsupported: Final = frozenset( + param + for param in candidates + if not any(self._deployment_accepts_param(deployment, model, param) for deployment in deployments) + ) + if not unsupported: + return tier_params + verbose_router_logger.warning( + "litellm.router.py: dropping tier params %s for model=%s, no deployment behind it declares them", + ", ".join(sorted(unsupported)), + model, + ) + return MappingProxyType({key: value for key, value in tier_params.items() if key not in unsupported}) + def get_model_list( self, model_name: str | None = None, team_id: str | None = None ) -> list[DeploymentTypedDict] | None: @@ -11887,7 +11951,9 @@ class Router: model = pre_routing_hook_response.model messages = pre_routing_hook_response.messages if pre_routing_hook_response.litellm_params: - request_kwargs.update(pre_routing_hook_response.litellm_params) + request_kwargs.update( + self._tier_params_the_target_accepts(model, pre_routing_hook_response.litellm_params) + ) ######################################################### # Resolve the strategy and logger AFTER the pre-routing hook, since @@ -11998,7 +12064,9 @@ class Router: model = pre_routing_hook_response.model messages = pre_routing_hook_response.messages if pre_routing_hook_response.litellm_params: - request_kwargs.update(pre_routing_hook_response.litellm_params) + request_kwargs.update( + self._tier_params_the_target_accepts(model, pre_routing_hook_response.litellm_params) + ) # 2. Get healthy deployments healthy_deployments: Final = await self.async_get_healthy_deployments( diff --git a/litellm/utils.py b/litellm/utils.py index 520c40f67c0..54b5448b7e9 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -4150,17 +4150,11 @@ def get_optional_params( unsupported_params: Final = {} for k in non_default_params: if k not in supported_params: - if k == "user" or k == "stream_options" or k == "stream": + if k in PROVIDER_UNVALIDATED_PARAMS: continue if k == "n" and n == 1: # langchain sends n=1 as a default value continue # skip this param - if ( - k == "max_retries" - ): # TODO: This is a patch. We support max retries for OpenAI, Azure. For non OpenAI LLMs we need to add support for max retries - continue # skip this param - # Always keeps this in elif code blocks - else: - unsupported_params[k] = non_default_params[k] + unsupported_params[k] = non_default_params[k] if unsupported_params: if litellm.drop_params is True or (drop_params is not None and drop_params is True): @@ -4729,6 +4723,22 @@ def _apply_openai_param_overrides(optional_params: dict, non_default_params: dic return optional_params +PROVIDER_UNVALIDATED_PARAMS: Final = frozenset({"user", "stream_options", "stream", "max_retries"}) + + +def provider_rejectable_params(passed_params: Mapping[str, object]) -> frozenset[str]: + """The params a provider can actually be rejected for, i.e. the ones _check_valid_arg compares + against its supported list. + + Anything outside this set never reaches that comparison. Endpoint and transport controls such as + base_url, timeout, default_headers, organization and deployment_id are not chat completion + params at all, so a caller filtering on "is this an OpenAI param" would discard configuration the + request needs while never touching what the provider would have rejected. + """ + params: Final = dict(passed_params) # mutable-ok: get_non_default_params takes a dict + return frozenset(get_non_default_params(params)) - PROVIDER_UNVALIDATED_PARAMS + + def get_non_default_params(passed_params: dict) -> dict: # filter out those parameters that were passed with non-default values non_default_params: Final = { diff --git a/tests/test_litellm/router_strategy/test_complexity_router.py b/tests/test_litellm/router_strategy/test_complexity_router.py index ae9e3907aeb..6f76be18256 100644 --- a/tests/test_litellm/router_strategy/test_complexity_router.py +++ b/tests/test_litellm/router_strategy/test_complexity_router.py @@ -2213,14 +2213,14 @@ class TestRouterPreRoutingAliasOverrides: "complexity_router_config": { "tiers": { "SIMPLE": { - "model_name": "gpt-4o-mini", + "model_name": "gpt-5-mini", "litellm_params": {"reasoning_effort": "xhigh"}, } } }, }, }, - {"model_name": "gpt-4o-mini", "litellm_params": {"model": "openai/gpt-4o-mini"}}, + {"model_name": "gpt-5-mini", "litellm_params": {"model": "openai/gpt-5-mini"}}, ] ) request_kwargs: Dict = {"reasoning_effort": "low"} @@ -2231,7 +2231,7 @@ class TestRouterPreRoutingAliasOverrides: messages=[{"role": "user", "content": "hi"}], ) - assert deployment["model_name"] == "gpt-4o-mini" + assert deployment["model_name"] == "gpt-5-mini" assert request_kwargs["reasoning_effort"] == "xhigh" @pytest.mark.asyncio diff --git a/tests/test_litellm/test_router.py b/tests/test_litellm/test_router.py index 2948568198f..24941e88185 100644 --- a/tests/test_litellm/test_router.py +++ b/tests/test_litellm/test_router.py @@ -11194,3 +11194,128 @@ def test_resolved_litellm_models_answers_through_every_channel_a_request_uses( result is not "the call fails", so what to do about it stays each caller's policy. """ assert set(_resolution_router().resolved_litellm_models(model_name)) == set(expected) + + +class TestTierParamsTheTargetAccepts: + """A tier's litellm_params are applied to every request that tier routes, so one the target + cannot take raised UnsupportedParamsError before the request left the proxy, turning the whole + tier into a 400.""" + + @pytest.fixture(autouse=True) + def force_local_model_cost(self, monkeypatch): + from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap + + monkeypatch.setattr(litellm, "model_cost", GetModelCostMap.load_local_model_cost_map()) + + @staticmethod + def _router(model: str) -> litellm.Router: + return litellm.Router( + model_list=[{"model_name": "tiered", "litellm_params": {"model": model, "api_key": "sk-x"}}] + ) + + def test_drops_a_param_no_deployment_declares(self): + router = self._router("novita/moonshotai/kimi-k3") + + accepted = router._tier_params_the_target_accepts("tiered", {"reasoning_effort": "max"}) + + assert accepted == {} + + def test_keeps_a_param_the_deployment_declares(self): + router = self._router("fireworks_ai/kimi-k3") + + accepted = router._tier_params_the_target_accepts("tiered", {"reasoning_effort": "max"}) + + assert accepted == {"reasoning_effort": "max"} + + @pytest.mark.parametrize( + "control, value", + [ + ("api_base", "https://example.invalid"), + ("api_key", "sk-tier"), + ("base_url", "https://example.invalid"), + ("timeout", 30), + ("default_headers", {"x-tier": "1"}), + ("organization", "org-tier"), + ("deployment_id", "dep-tier"), + ], + ) + def test_keeps_credentials_and_transport_controls(self, control, value): + """These are not chat completion params, so get_optional_params never compares them against + a provider's supported list. Filtering on "is this an OpenAI param" would discard the + configuration the request needs while never touching what the provider would reject.""" + router = self._router("novita/moonshotai/kimi-k3") + + accepted = router._tier_params_the_target_accepts("tiered", {control: value, "reasoning_effort": "max"}) + + assert accepted == {control: value} + + @pytest.mark.parametrize( + "control, value", + [ + ("additional_drop_params", ["seed"]), + ("drop_params", True), + ("allowed_openai_params", ["reasoning_effort"]), + ("api_version", "2024-02-01"), + ("metadata", {"tier": "complex"}), + ], + ) + def test_keeps_litellm_controls_the_provider_never_lists(self, control, value): + """No provider lists a litellm control among its supported params, so "no deployment + declares it" means litellm consumes it, not that the target refuses it. Dropping + drop_params or additional_drop_params would silently disable the operator's sanitization.""" + router = self._router("novita/moonshotai/kimi-k3") + + accepted = router._tier_params_the_target_accepts("tiered", {control: value, "reasoning_effort": "max"}) + + assert accepted == {control: value} + + def test_keeps_extra_headers_even_when_the_provider_omits_it(self): + """Several providers leave extra_headers out of their supported params, so the filter would + drop it. Headers carry auth and tenancy, so sending fewer than the operator configured is + worse than the error they already get.""" + router = self._router("ai21/jamba-1.5-mini") + + accepted = router._tier_params_the_target_accepts( + "tiered", {"extra_headers": {"x-tenant": "acme"}, "reasoning_effort": "max"} + ) + + assert accepted == {"extra_headers": {"x-tenant": "acme"}} + + def test_keeps_a_param_any_deployment_in_the_group_declares(self): + """Routing has not picked a deployment yet, so one capable member keeps the param alive.""" + router = litellm.Router( + model_list=[ + {"model_name": "tiered", "litellm_params": {"model": "novita/moonshotai/kimi-k3", "api_key": "k"}}, + {"model_name": "tiered", "litellm_params": {"model": "fireworks_ai/kimi-k3", "api_key": "k"}}, + ] + ) + + accepted = router._tier_params_the_target_accepts("tiered", {"reasoning_effort": "max"}) + + assert accepted == {"reasoning_effort": "max"} + + def test_deployment_accepts_param_reads_the_provider(self): + deployment = {"model_name": "x", "litellm_params": {"model": "fireworks_ai/kimi-k3"}} + + assert litellm.Router._deployment_accepts_param(deployment, "x", "reasoning_effort") is True + + def test_deployment_accepts_param_is_false_when_the_provider_omits_it(self): + deployment = {"model_name": "x", "litellm_params": {"model": "novita/moonshotai/kimi-k3"}} + + assert litellm.Router._deployment_accepts_param(deployment, "x", "reasoning_effort") is False + + @pytest.mark.parametrize( + "deployment", + [{"model_name": "x"}, {"model_name": "x", "litellm_params": {}}, {"model_name": "x", "litellm_params": {"model": "not-a-real-provider/nope"}}], + ) + def test_deployment_accepts_param_fails_open(self, deployment): + """An unresolvable deployment must not be the reason a param is dropped.""" + assert litellm.Router._deployment_accepts_param(deployment, "x", "reasoning_effort") is True + + def test_keeps_everything_for_an_unknown_group(self): + """An unresolvable target must never narrow what the request already did.""" + router = self._router("fireworks_ai/kimi-k3") + + accepted = router._tier_params_the_target_accepts("no-such-group", {"reasoning_effort": "max"}) + + assert accepted == {"reasoning_effort": "max"}