diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index ef7615f7ba3..97a62456b77 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -193,6 +193,7 @@ from litellm.responses.streaming_iterator import partial_responses_incomplete_ev from litellm.router import Router from litellm.router_api import RoutingContext from litellm.router_strategy.auto_router.auto_router_model_naming import AUTO_ROUTER_MODEL_PREFIX +from litellm.router_utils.default_litellm_params import params_sent_to_deployment from litellm.types.extraction import ExtractionResponse from litellm.types.guardrails import GuardrailEventHooks from litellm.types.router import RouterRateLimitError @@ -869,14 +870,16 @@ def _router_defaults(llm_router: Router | None) -> Mapping[str, object]: def counted_drop_policies( data: Mapping[str, object], llm_router: Router | None, team_id: str | None ) -> tuple[DropPolicy, ...]: - requested: Final = {**_router_defaults(llm_router), **data} + router_defaults: Final = _router_defaults(llm_router) model: Final = data.get("model") deployments: Final = ( _named_deployments(llm_router, model, team_id) if llm_router is not None and isinstance(model, str) else () ) if not deployments: - return (drop_policy(requested), _UNSEEN_DEPLOYMENT) - return tuple(drop_policy({**deployment, **requested}) for deployment in deployments) + return (drop_policy({**router_defaults, **data}), _UNSEEN_DEPLOYMENT) + return tuple( + drop_policy(params_sent_to_deployment(deployment, router_defaults, data)) for deployment in deployments + ) def resolve_counted_tool_fields( diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 6a171c5fe53..c076bd49d5b 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -207,6 +207,7 @@ from litellm.router_strategy.auto_router.auto_router_tuning_baseline import ( snapshot_tuning_baselines, tuning_limit_violation, ) +from litellm.router_utils.default_litellm_params import params_sent_to_deployment from litellm.rust_bridge.catalog import Target from litellm.types.utils import ( PRICING_OVERRIDES_KEY, @@ -15969,7 +15970,8 @@ def _count_context_management( ) ) sent_params: Final = tuple( - {**params, **router_defaults, **request_litellm_params} for params in deployment_params or ({},) + params_sent_to_deployment(params, router_defaults, request_litellm_params) + for params in deployment_params or ({},) ) if any(context_management_explicitly_dropped(params.get("additional_drop_params")) for params in sent_params): return None @@ -16072,7 +16074,7 @@ def _responses_routing_kwargs(responses_request: Mapping[str, object] | None) -> def _inference_params( litellm_params: Mapping[str, object], responses_request: Mapping[str, object] ) -> Mapping[str, object]: - return {**litellm_params, **_router_default_litellm_params(llm_router), **responses_request} + return params_sent_to_deployment(litellm_params, _router_default_litellm_params(llm_router), responses_request) def _count_drop_policy(deployment: Mapping[str, object] | None, responses_request: Mapping[str, object]) -> DropPolicy: @@ -16314,12 +16316,17 @@ async def _count_request_tokens( if _model is not None: model_to_use = _model - sent_params: Final[Mapping[str, object]] = {**_router_default_litellm_params(llm_router), **request_litellm_params} + router_defaults: Final = _router_default_litellm_params(llm_router) + sent_params: Final = params_sent_to_deployment( + _DEPLOYMENT_LITELLM_PARAMS.validate_python((deployment or {}).get("litellm_params") or {}), + router_defaults, + request_litellm_params, + ) count_deployment: Final = _deployment_with_request_drop_settings(deployment, sent_params) count_litellm_params: Final = _DEPLOYMENT_LITELLM_PARAMS.validate_python( (count_deployment or {}).get("litellm_params") or {} ) - deployment_defaults: Final = {**count_litellm_params, **_router_default_litellm_params(llm_router)} + deployment_defaults: Final = params_sent_to_deployment(count_litellm_params, router_defaults, {}) conversation_empty: Final = not messages and not contents if prompt and not conversation_empty and not deployment_drops_params(count_deployment): raise ProxyException( diff --git a/litellm/router_rust/core.py b/litellm/router_rust/core.py index 08afb2cbfb9..10485757eea 100644 --- a/litellm/router_rust/core.py +++ b/litellm/router_rust/core.py @@ -242,6 +242,7 @@ from litellm.router_strategy.least_busy import ( picks_reserve, release_started_attempt, ) +from litellm.router_utils import default_litellm_params as default_params from litellm.router_utils.cooldown_handlers import get_cooldown_deployment_info, mark_deployment_selection_failure from litellm.router_utils.endpoint_eligibility import endpoint_for_call, selected_endpoint from litellm.router_utils.kubernetes_pod_discovery import ( @@ -3203,13 +3204,11 @@ class RustCore( ) def params_sent_to_deployment(self, request: Mapping[str, object], deployment_id: object) -> Mapping[str, object]: - """The request as a deployment received it: its params, the router defaults, its tool choice, then + """The request as a deployment received it: its params, the router defaults it leaves unset, then the request (``_params_sent_to_deployment``).""" record: Final = self._handle.deployment(deployment_id) if isinstance(deployment_id, str) else None params: Final = as_str_mapping((as_str_mapping(record) or {}).get("litellm_params")) or {} - defaults: Final = {k: v for k, v in self._resolved.default_litellm_params.items() if v is not None} - tool_choice: Final = {"tool_choice": params["tool_choice"]} if params.get("tool_choice") is not None else {} - return MappingProxyType({**params, **defaults, **tool_choice, **request}) + return default_params.params_sent_to_deployment(params, self._resolved.default_litellm_params, request) def _drive_sync_steps(self, call: "RouterCall[_FailureReportSteps | _SuccessReportSteps]") -> Mapping[str, object]: """Drive a call whose only steps are cache ops (``report_failure``) to its end.""" @@ -3434,15 +3433,17 @@ class RustCore( ) -> list[str]: # mutable-ok: the step's answer """The candidates that refuse the audio or videos a Decisions call carries.""" group: Final = str(step.get("model_group")) - defaults: Final = {k: v for k, v in self._resolved.default_litellm_params.items() if v is not None} - sent: Final = {**defaults, **request} audio: Final = bool(request.get("audio")) videos: Final = bool(request.get("videos")) return [ str(id) for id in _as_list(step.get("candidates")) if refuses_decisions_media( - as_str_mapping(self._handle.deployment(str(id))) or {}, group, sent, audio=audio, videos=videos + as_str_mapping(self._handle.deployment(str(id))) or {}, + group, + self.params_sent_to_deployment(request, str(id)), + audio=audio, + videos=videos, ) ] diff --git a/litellm/router_rust/invoke/endpoints.py b/litellm/router_rust/invoke/endpoints.py index 496f8093706..caf37697919 100644 --- a/litellm/router_rust/invoke/endpoints.py +++ b/litellm/router_rust/invoke/endpoints.py @@ -52,6 +52,7 @@ from litellm.router_strategy.complexity_router.context_compaction import ( initialize_compaction_state, surface_for_call, ) +from litellm.router_utils.default_litellm_params import router_defaults_for_deployment from litellm.router_utils.endpoint_eligibility import endpoint_for_call, endpoint_selection from litellm.vector_stores.search_query import store_search_params @@ -2176,7 +2177,7 @@ class RustEndpoints: return await self._direct("amoderation", prepared) def text_completion(self, request: Mapping[str, object]) -> object: - """The sync `text_completion`: one pick, no retries, the group name as the model.""" + """The sync `text_completion`: one pick, no retries.""" from litellm.litellm_core_utils.routing_common_utils import ( deployment_model_id_pin, metadata_defaults_under_request, @@ -2197,17 +2198,21 @@ class RustEndpoints: request_kwargs=kwargs, ) data: Final = as_dict(deployment.get("litellm_params")) - for key, value in self._core.resolved.default_litellm_params.items(): - if key not in kwargs: - kwargs[key] = value - elif key == "metadata": - metadata_bucket(kwargs, key).update( - metadata_defaults_under_request(defaults=as_dict(value), request_metadata=as_dict(kwargs[key])) + defaults: Final = self._core.resolved.default_litellm_params + for key, value in router_defaults_for_deployment(without(defaults, "metadata"), data).items(): + kwargs.setdefault(key, value) + if "metadata" in defaults: + metadata_bucket(kwargs, "metadata").update( + metadata_defaults_under_request( + defaults=as_dict(defaults["metadata"]), request_metadata=as_dict(kwargs["metadata"]) ) + ) pinned: Final = deployment_model_id_pin(data) if pinned is not None: kwargs["model_id"] = pinned - return self._invoke("text_completion")(**{**data, "prompt": prompt, "caching": self._cache_responses, **kwargs}) + return self._invoke("text_completion")( + **{**data, "prompt": prompt, "caching": self._cache_responses, **without(kwargs, "model")} + ) async def aadapter_completion( self, diff --git a/litellm/router_rust/invoke/shaping.py b/litellm/router_rust/invoke/shaping.py index 5fa6fc2543c..eb3f881a846 100644 --- a/litellm/router_rust/invoke/shaping.py +++ b/litellm/router_rust/invoke/shaping.py @@ -41,6 +41,11 @@ from litellm.router_strategy.tag_based_routing import ( get_tags_from_request_kwargs, tags_with_serving_deployment_tags, ) +from litellm.router_utils.default_litellm_params import ( + deployment_sets_litellm_param, + params_sent_to_deployment, + router_defaults_for_deployment, +) from litellm.router_utils.pre_call_checks.io_token_rate_limit_check import ( deployment_has_io_token_limits, # pyright: ignore[reportUnknownVariableType] # litellm's untyped helpers and request values refund_stale_reservation_before_retry, @@ -119,8 +124,7 @@ def _drops_malformed_request_tools( ) -> bool: if isinstance(request.get("tools") or [], (list, tuple)) or not deployment_params.get("tools"): return False - set_defaults: Final = {name: value for name, value in defaults.items() if value is not None} - return drop_policy({**deployment_params, **set_defaults, **request}).drops("tools") + return drop_policy(params_sent_to_deployment(deployment_params, defaults, request)).drops("tools") def _malformed_request_params_the_deployment_sets( @@ -136,8 +140,7 @@ def _malformed_request_params_the_deployment_sets( try: validate_litellm_params_strictly(GenericLiteLLMParams, overrides) except ValidationError as error: - set_defaults: Final = {name: value for name, value in defaults.items() if value is not None} - policy: Final = drop_policy({**deployment_params, **set_defaults, **request}) + policy: Final = drop_policy(params_sent_to_deployment(deployment_params, defaults, request)) malformed: Final = (str(detail["loc"][0]) for detail in error.errors() if detail["loc"]) return frozenset(key for key in malformed if policy.drops(key)) return frozenset() @@ -175,21 +178,13 @@ def merge_tools_from_deployment( ) -def _deployment_sets_litellm_param(deployment_params: Mapping[str, object], key: str) -> bool: - value: Final = deployment_params.get(key) - if value is None: - return False - field: Final = LiteLLM_Params.model_fields.get(key) - return field is None or value != field.default # pyright: ignore[reportAny] # litellm's untyped helpers and request values - - def _forwarded_alias_marker_keys_the_deployment_sets( deployment_params: Mapping[str, object], forwarded_keys: object ) -> tuple[str, ...]: if not isinstance(forwarded_keys, tuple): return () keys: Final[tuple[object, ...]] = forwarded_keys # pyright: ignore[reportUnknownVariableType] # the router's own marker - return tuple(key for key in keys if isinstance(key, str) and _deployment_sets_litellm_param(deployment_params, key)) + return tuple(key for key in keys if isinstance(key, str) and deployment_sets_litellm_param(deployment_params, key)) # --- reasoning effort ------------------------------------------------------------------------- @@ -355,19 +350,18 @@ def call_timeout(shaping: Shaping, kwargs: Mapping[str, object], data: Mapping[s def update_kwargs_with_default_litellm_params( shaping: Shaping, + deployment_params: Mapping[str, object], kwargs: MutableMapping[str, object], # mutable-ok: the request kwargs the Python router mutates in place metadata_variable_name: str, ) -> None: - """``_update_kwargs_with_default_litellm_params``: unset non-None defaults fill the request; - the default metadata fills the keys the request leaves unset, the router's caching groups excepted.""" - defaults: Final = dict(without_retrieve_stream_defaults(shaping.default_litellm_params, kwargs)) - metadata_defaults: Final = defaults.pop("metadata", {}) or {} # pyright: ignore[reportUnknownVariableType] # litellm's untyped helpers and request values - for key, value in defaults.items(): - if value is None: - continue + """``_update_kwargs_with_default_litellm_params``: the non-None defaults the request leaves unset and + the deployment does not set itself fill the request; the default metadata fills the keys the request + leaves unset, the router's caching groups excepted.""" + defaults: Final = without_retrieve_stream_defaults(shaping.default_litellm_params, kwargs) + for key, value in router_defaults_for_deployment(without(defaults, "metadata"), deployment_params).items(): kwargs.setdefault(key, value) bucket: Final = metadata_bucket(kwargs, metadata_variable_name) - bucket.update(metadata_defaults_under_request(defaults=as_dict(metadata_defaults), request_metadata=bucket)) # pyright: ignore[reportUnknownArgumentType] # litellm's untyped helpers and request values + bucket.update(metadata_defaults_under_request(defaults=as_dict(defaults.get("metadata")), request_metadata=bucket)) # --- clientside credentials ------------------------------------------------------------------- @@ -515,11 +509,11 @@ def update_kwargs_with_deployment( ) _set_timeouts(shaping, deployment_params, kwargs, function_name, splits_stream_timeout) - if router_config_requests_stream_usage(kwargs, (deployment_params, defaults)): + if router_config_requests_stream_usage(kwargs, (params_sent_to_deployment(deployment_params, defaults, {}),)): kwargs[ # rebind-ok: the request kwargs are shaped in place, as the Python router shapes them STREAM_USAGE_INJECTED_FIELD ] = True - update_kwargs_with_default_litellm_params(shaping, kwargs, metadata_variable_name) + update_kwargs_with_default_litellm_params(shaping, deployment_params, kwargs, metadata_variable_name) pinned_model_id: Final = deployment_model_id_pin(deployment_params) if pinned_model_id is not None: kwargs["model_id"] = ( # rebind-ok: the request kwargs are shaped in place, as the Python router shapes them diff --git a/litellm/router_rust/surface/legacy_members.py b/litellm/router_rust/surface/legacy_members.py index 9b436d6ce3e..931ef0fe390 100644 --- a/litellm/router_rust/surface/legacy_members.py +++ b/litellm/router_rust/surface/legacy_members.py @@ -18,7 +18,6 @@ import traceback import uuid from collections.abc import Awaitable, Callable, Iterable, Mapping from functools import partial -from types import MappingProxyType from typing import TYPE_CHECKING, Final from typing_extensions import NotRequired, ReadOnly, TypedDict @@ -46,6 +45,7 @@ from litellm.router_strategy.auto_router.auto_router_model_naming import ( count_capability_routers, ) from litellm.router_utils.cooldown_handlers import set_cooldown_deployments +from litellm.router_utils.default_litellm_params import params_sent_to_deployment from litellm.router_utils.endpoint_eligibility import endpoint_selection from litellm.router_utils.pre_call_checks.io_token_rate_limit_check import async_io_token_refund_failure from litellm.router_utils.router_callbacks.track_deployment_metrics import ( @@ -538,9 +538,7 @@ class RustLegacyMembers(RustRegistryMembers, RustOptionalChecks): """``_params_sent_to_deployment`` of a ``Deployment``, which ``RoutingContext.deployment_params`` hands in.""" params: Final = deployment.litellm_params.model_dump(exclude_none=True) if deployment else {} configured: Final = as_str_mapping(RoutingContext(self).settings().default_litellm_params) # pyright: ignore[reportArgumentType, reportUnknownMemberType, reportUnknownArgumentType] # the core's RoutingContext - defaults: Final = {k: v for k, v in (configured or {}).items() if v is not None} - tool_choice: Final = {"tool_choice": params["tool_choice"]} if params.get("tool_choice") is not None else {} - return MappingProxyType({**params, **defaults, **tool_choice, **request_kwargs}) + return params_sent_to_deployment(params, configured or {}, request_kwargs) __all__ = ["RustLegacyMembers"] diff --git a/litellm/router_utils/cooldown_handlers.py b/litellm/router_utils/cooldown_handlers.py index 9e3e0189fd0..3627efcdda4 100644 --- a/litellm/router_utils/cooldown_handlers.py +++ b/litellm/router_utils/cooldown_handlers.py @@ -30,6 +30,7 @@ from litellm.litellm_core_utils.litellm_logging import mask_api_base_credentials from litellm.router_deprecated import renamed_router_param from litellm.router_utils.cooldown_cache import CooldownCacheValue from litellm.router_utils.cooldown_callbacks import router_cooldown_event_callback +from litellm.router_utils.default_litellm_params import params_sent_to_deployment from litellm.types.llms.anthropic import MESSAGE_THREAD_NOT_FOUND_ERROR_CODE from litellm.types.router import Deployment from litellm.types.utils import ( @@ -888,8 +889,9 @@ def is_caller_entra_credential_failure( credential_name: Final = deployment.litellm_params.litellm_credential_name configured_by_precedence: Final[tuple[Mapping[str, object], ...]] = ( - router_defaults, - _CREDENTIAL_VALUES_ADAPTER.validate_python(deployment.litellm_params.model_dump(exclude_none=True)), + _CREDENTIAL_VALUES_ADAPTER.validate_python( + params_sent_to_deployment(deployment.litellm_params.model_dump(exclude_none=True), router_defaults, {}) + ), _CREDENTIAL_VALUES_ADAPTER.validate_python( CredentialAccessor.get_credential_values(credential_name) if credential_name is not None else {} ), diff --git a/litellm/router_utils/default_litellm_params.py b/litellm/router_utils/default_litellm_params.py new file mode 100644 index 00000000000..64eb41f8b39 --- /dev/null +++ b/litellm/router_utils/default_litellm_params.py @@ -0,0 +1,33 @@ +from collections.abc import Mapping +from types import MappingProxyType +from typing import Final + +from litellm.types.router import LiteLLM_Params + + +def deployment_sets_litellm_param(deployment_params: Mapping[str, object], key: str) -> bool: + value: Final = deployment_params.get(key) + if value is None: + return False + field: Final = LiteLLM_Params.model_fields.get(key) + return field is None or value != field.default # pyright: ignore[reportAny] # pydantic types FieldInfo.default as Any + + +def router_defaults_for_deployment( + router_defaults: Mapping[str, object], deployment_params: Mapping[str, object] +) -> Mapping[str, object]: + return MappingProxyType( + { + key: value + for key, value in router_defaults.items() + if value is not None and not deployment_sets_litellm_param(deployment_params, key) + } + ) + + +def params_sent_to_deployment( + deployment_params: Mapping[str, object], router_defaults: Mapping[str, object], request: Mapping[str, object] +) -> Mapping[str, object]: + return MappingProxyType( + {**deployment_params, **router_defaults_for_deployment(router_defaults, deployment_params), **request} + ) diff --git a/litellm/router_utils/request_priority.py b/litellm/router_utils/request_priority.py index 61627d4b0cf..c9493d69b41 100644 --- a/litellm/router_utils/request_priority.py +++ b/litellm/router_utils/request_priority.py @@ -6,6 +6,7 @@ from pydantic import TypeAdapter, ValidationError import litellm from litellm.litellm_core_utils.core_helpers import normalize_drop_params +from litellm.router_utils.default_litellm_params import params_sent_to_deployment @dataclass(frozen=True, slots=True) @@ -91,7 +92,8 @@ def request_drops_params( request_flag: Final = _explicit_drop_params(kwargs) if request_flag is not None: return request_flag - router_flag: Final = _explicit_drop_params(router_defaults) - if router_flag is not None: - return router_flag - return any(_explicit_drop_params(params) is True for params in deployments) + if not deployments: + return _explicit_drop_params(router_defaults) is True + return any( + _explicit_drop_params(params_sent_to_deployment(params, router_defaults, {})) is True for params in deployments + ) diff --git a/tests/unit/decisions/test_main.py b/tests/unit/decisions/test_main.py index 8b470248be7..8eaa72217d1 100644 --- a/tests/unit/decisions/test_main.py +++ b/tests/unit/decisions/test_main.py @@ -4174,7 +4174,7 @@ async def test_clips_to_a_group_whose_only_taker_is_cooling_down_are_refused_rat @pytest.mark.asyncio @pytest.mark.parametrize("surface", tuple(_MEDIA_SURFACES)) -async def test_a_router_default_drop_params_overrides_a_deployment_when_choosing_who_takes_clips( +async def test_a_deployment_drop_params_beats_a_router_default_when_choosing_who_takes_clips( respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, local_model_cost_map: None, surface: str ) -> None: monkeypatch.setattr(litellm, "drop_params", False) @@ -4194,13 +4194,13 @@ async def test_a_router_default_drop_params_overrides_a_deployment_when_choosing return await response if asyncio.iscoroutine(response) else response await send() - assert respx_mock.calls.last.request.headers["authorization"] == "Bearer takes" - assert json.loads(respx_mock.calls.last.request.content)["audio"] == list(_AUDIO_CLIPS) - - await send(drop_params=True) assert respx_mock.calls.last.request.headers["authorization"] == "Bearer drops" assert "audio" not in json.loads(respx_mock.calls.last.request.content) + await send(drop_params=False) + assert respx_mock.calls.last.request.headers["authorization"] == "Bearer takes" + assert json.loads(respx_mock.calls.last.request.content)["audio"] == list(_AUDIO_CLIPS) + @pytest.mark.asyncio async def test_a_deployment_that_turns_audio_off_is_refused_or_dropped_even_when_the_cost_map_allows_it( diff --git a/tests/unit/proxy/response_api_endpoints/test_endpoints.py b/tests/unit/proxy/response_api_endpoints/test_endpoints.py index 6abf439dfa9..fb12a2039a6 100644 --- a/tests/unit/proxy/response_api_endpoints/test_endpoints.py +++ b/tests/unit/proxy/response_api_endpoints/test_endpoints.py @@ -6000,6 +6000,28 @@ class TestResponsesInputTokens: assert counter.await_args.kwargs["request"].tools is None assert "tools" not in counter.await_args.kwargs["responses_request"] + @pytest.mark.parametrize( + ("deployment_drop_params", "expected_status"), + [({}, 200), ({"drop_params": False}, 400)], + ids=["router-default-drops", "deployment-keeps-over-router-default"], + ) + def test_malformed_tools_follow_the_deployment_drop_params_over_the_router_default( + self, deployment_drop_params: dict[str, bool], expected_status: int + ) -> None: + router: Final = litellm.Router( + model_list=[ + { + "model_name": "gpt-5.6", + "litellm_params": {"model": "openai/gpt-5.6", "api_key": "sk-test", **deployment_drop_params}, + } + ], + default_litellm_params={"drop_params": True}, + ) + + response, _ = self._post_input_tokens({"model": "gpt-5.6", "input": "hi", "tools": 5}, llm_router=router) + + assert response.status_code == expected_status, response.text + def test_malformed_tools_for_a_model_outside_the_router_get_a_clean_400(self) -> None: router: Final = litellm.Router( model_list=[{"model_name": "gpt-5.2", "litellm_params": {"model": "openai/gpt-5.2", "api_key": "sk-test"}}] diff --git a/tests/unit/proxy/test_proxy_token_counter.py b/tests/unit/proxy/test_proxy_token_counter.py index d9ddf0f131a..384741c84a3 100644 --- a/tests/unit/proxy/test_proxy_token_counter.py +++ b/tests/unit/proxy/test_proxy_token_counter.py @@ -1582,7 +1582,10 @@ def _build_log_messages(log_text: JsonValue) -> list[dict[str, JsonValue]]: async def _count_on_a_model_without_a_count_api( - monkeypatch: pytest.MonkeyPatch, body: Mapping[str, object], router_defaults: dict[str, JsonValue] | None = None + monkeypatch: pytest.MonkeyPatch, + body: Mapping[str, object], + router_defaults: dict[str, JsonValue] | None = None, + deployment_params: Mapping[str, JsonValue] = MappingProxyType({}), ) -> JSONResponse | dict[str, JsonValue]: import litellm.proxy.proxy_server as proxy_server @@ -1590,7 +1593,7 @@ async def _count_on_a_model_without_a_count_api( model_list=[ { "model_name": "mistral-tool-result", - "litellm_params": {"model": "mistral/mistral-small-latest", "api_key": "sk-test"}, + "litellm_params": {"model": "mistral/mistral-small-latest", "api_key": "sk-test", **deployment_params}, } ], default_litellm_params=router_defaults, @@ -1633,14 +1636,22 @@ async def test_count_tokens_without_a_count_api_counts_a_non_string_tool_result_ @pytest.mark.asyncio -@pytest.mark.parametrize(("body_drop_params", "expected_status"), (({}, None), ({"drop_params": False}, 400))) -async def test_count_tokens_honors_router_default_drop_params_unless_the_body_overrides_it( - monkeypatch: pytest.MonkeyPatch, body_drop_params: dict[str, JsonValue], expected_status: int | None +@pytest.mark.parametrize( + ("deployment_params", "body_drop_params", "expected_status"), + (({}, {}, None), ({}, {"drop_params": False}, 400), ({"drop_params": False}, {}, 400)), + ids=("router-default", "body-over-router-default", "deployment-over-router-default"), +) +async def test_count_tokens_honors_router_default_drop_params_unless_the_body_or_deployment_overrides_it( + monkeypatch: pytest.MonkeyPatch, + deployment_params: dict[str, JsonValue], + body_drop_params: dict[str, JsonValue], + expected_status: int | None, ) -> None: response: Final = await _count_on_a_model_without_a_count_api( monkeypatch, {"model": "mistral-tool-result", "messages": _build_log_messages(None), **body_drop_params}, router_defaults={"drop_params": True}, + deployment_params=deployment_params, ) match response: @@ -2992,6 +3003,7 @@ _NATIVE_RESPONSES_BASE: Final = { _FIREWORKS_DEEPSEEK_V4_1: "https://unused.invalid", _XAI_GROK_4_3: "https://unused.invalid", } +_BRIDGED_RESPONSES_BASE: Final = "https://unused.cognitiveservices.azure.com/models" def _responses_function(tool: Mapping[str, object], **strict: bool | None) -> dict[str, object]: @@ -3300,26 +3312,34 @@ async def test_a_responses_count_that_asks_for_the_chat_completions_bridge_frame @pytest.mark.asyncio -async def test_a_responses_count_frames_for_the_api_base_router_defaults_send_inference_to( - monkeypatch: pytest.MonkeyPatch, +@pytest.mark.parametrize( + ("deployment_params", "inference_api_base"), + [ + pytest.param({}, _BRIDGED_RESPONSES_BASE, id="router-default-fills-an-unset-api-base"), + pytest.param( + {"api_base": _NATIVE_RESPONSES_BASE[_AZURE_GROK_4_1]}, + _NATIVE_RESPONSES_BASE[_AZURE_GROK_4_1], + id="deployment-api-base-beats-the-router-default", + ), + ], +) +async def test_a_responses_count_frames_for_the_api_base_inference_is_sent_to( + monkeypatch: pytest.MonkeyPatch, deployment_params: dict[str, object], inference_api_base: str ): router: Final = Router( model_list=[ { "model_name": "counted", - "litellm_params": { - "api_key": "unused", - "model": _AZURE_GROK_4_1, - "api_base": _NATIVE_RESPONSES_BASE[_AZURE_GROK_4_1], - }, + "litellm_params": {"api_key": "unused", "model": _AZURE_GROK_4_1, **deployment_params}, "model_info": {}, } ], - default_litellm_params={"api_base": "https://unused.cognitiveservices.azure.com/models"}, + default_litellm_params={"api_base": _BRIDGED_RESPONSES_BASE}, ) - chat_count: Final = await _count_with_router(monkeypatch, router, (_GET_WEATHER,), responses=False) + sent_alone: Final = _single_deployment_router({"model": _AZURE_GROK_4_1, "api_base": inference_api_base}, {}) + expected: Final = await _count_with_router(monkeypatch, sent_alone, (_GET_WEATHER,), responses=True) - assert await _count_with_router(monkeypatch, router, (_GET_WEATHER,), responses=True) == chat_count + assert await _count_with_router(monkeypatch, router, (_GET_WEATHER,), responses=True) == expected @pytest.mark.asyncio @@ -4157,10 +4177,20 @@ async def test_messages_count_tokens_for_a_model_with_no_deployment_drops_an_inv @pytest.mark.asyncio -async def test_gemini_messages_count_tokens_rejects_an_invalid_edit_without_drop_params() -> None: +@pytest.mark.parametrize( + ("deployment_params", "router_defaults"), + (({}, {}), ({"drop_params": False}, {"drop_params": True})), + ids=("no-drop-params", "deployment-over-router-default"), +) +async def test_gemini_messages_count_tokens_rejects_an_invalid_edit_without_drop_params( + deployment_params: dict[str, JsonValue], router_defaults: dict[str, JsonValue] +) -> None: with patch.object(litellm, "drop_params", False): response, sent_bodies = await _count_through_the_messages_endpoint( - Router(model_list=[_gemini_deployment("gemini-agent", {})]), + Router( + model_list=[_gemini_deployment("gemini-agent", deployment_params)], + default_litellm_params=router_defaults, + ), {"model": "gemini-agent", "context_management": _INVALID_COMPACT_EDIT}, ) @@ -5480,6 +5510,7 @@ async def _claude_count( monkeypatch: pytest.MonkeyPatch, deployment_params: Mapping[str, JsonValue], responses_request: Mapping[str, object], + router_defaults: dict[str, JsonValue] | None = None, ) -> tuple[TokenCountResponse, list[dict[str, JsonValue]]]: router: Final = Router( model_list=[ @@ -5492,7 +5523,8 @@ async def _claude_count( **deployment_params, }, } - ] + ], + default_litellm_params=router_defaults, ) monkeypatch.setattr(litellm.proxy.proxy_server, "llm_router", router) bodies: Final[list[dict[str, JsonValue]]] = [] # mutable-ok: collects what the mock peer receives @@ -5516,14 +5548,15 @@ async def _claude_count( @pytest.mark.asyncio @pytest.mark.parametrize( - ("deployment_params", "request_params"), - [({"drop_params": True}, {}), ({}, {"drop_params": True})], - ids=["deployment", "request"], + ("deployment_params", "request_params", "router_defaults"), + [({"drop_params": True}, {}, None), ({}, {"drop_params": True}, None), ({}, {}, {"drop_params": True})], + ids=["deployment", "request", "router-default"], ) async def test_responses_count_on_claude_downgrades_forced_tool_choice_under_drop_params( monkeypatch: pytest.MonkeyPatch, deployment_params: Mapping[str, JsonValue], request_params: Mapping[str, JsonValue], + router_defaults: dict[str, JsonValue] | None, ) -> None: monkeypatch.setitem(litellm.model_cost, "claude-zeta-9", {"supports_forced_tool_use": False}) @@ -5531,6 +5564,7 @@ async def test_responses_count_on_claude_downgrades_forced_tool_choice_under_dro monkeypatch, deployment_params, {"input": "Weather?", "tools": [_WEATHER_TOOL], "tool_choice": "required", **request_params}, + router_defaults, ) assert (result.error, result.total_tokens) == (False, 77) @@ -5539,13 +5573,25 @@ async def test_responses_count_on_claude_downgrades_forced_tool_choice_under_dro @pytest.mark.asyncio +@pytest.mark.parametrize( + ("deployment_params", "router_defaults"), + [({}, None), ({"drop_params": False}, {"drop_params": True})], + ids=["no-drop-params", "deployment-over-router-default"], +) async def test_responses_count_on_claude_rejects_forced_tool_choice_without_drop_params( monkeypatch: pytest.MonkeyPatch, + deployment_params: Mapping[str, JsonValue], + router_defaults: dict[str, JsonValue] | None, ) -> None: monkeypatch.setitem(litellm.model_cost, "claude-zeta-9", {"supports_forced_tool_use": False}) with pytest.raises(ProxyException) as rejected: - await _claude_count(monkeypatch, {}, {"input": "Weather?", "tools": [_WEATHER_TOOL], "tool_choice": "required"}) + await _claude_count( + monkeypatch, + deployment_params, + {"input": "Weather?", "tools": [_WEATHER_TOOL], "tool_choice": "required"}, + router_defaults, + ) assert (rejected.value.code, "forced tool use" in rejected.value.message) == ("400", True) diff --git a/tests/unit/router_strategy/complexity_router/test_jev_classifier.py b/tests/unit/router_strategy/complexity_router/test_jev_classifier.py index bcbf69e14a6..d8927de5721 100644 --- a/tests/unit/router_strategy/complexity_router/test_jev_classifier.py +++ b/tests/unit/router_strategy/complexity_router/test_jev_classifier.py @@ -1138,7 +1138,7 @@ async def test_jev_bounds_complete_input_without_changing_forwarded_messages( [ ("false", None, None, None, "truncated"), ("true", None, None, None, "within_budget"), - ("true", "false", None, None, "truncated"), + ("true", "false", None, None, "within_budget"), ("true", None, 8192, None, "truncated"), ("true", None, 8192, "default", "truncated"), ("true", None, None, "default", "truncated"), @@ -1199,7 +1199,7 @@ async def test_jev_budget_uses_effective_headers_and_every_deployment( assert telemetry["classifier_input_tokens"] <= telemetry["classifier_input_budget_tokens"] sent: Final = JevSystemOneRequest.model_validate_json(route.calls[0].request.content) if other_member is None: - assert route.calls[0].request.headers[header] == (default_header or mode) + assert route.calls[0].request.headers[header] == mode if explicit_cap is not None: assert telemetry["classifier_input_budget_tokens"] < explicit_cap assert (ask in sent.state) is (expected_status == "within_budget") diff --git a/tests/unit/router_utils/test_default_litellm_params.py b/tests/unit/router_utils/test_default_litellm_params.py new file mode 100644 index 00000000000..0f39e589a83 --- /dev/null +++ b/tests/unit/router_utils/test_default_litellm_params.py @@ -0,0 +1,34 @@ +from collections.abc import Mapping +from typing import Final + +import pytest + +from litellm.router_utils.default_litellm_params import params_sent_to_deployment + +_ROUTER_DEFAULTS: Final = {"vertex_location": "us-east5", "use_in_pass_through": True, "timeout": None} + + +@pytest.mark.parametrize( + ("deployment", "request_params", "sent"), + [ + ({"vertex_location": "eu"}, {}, {"vertex_location": "eu", "use_in_pass_through": True}), + ({}, {}, {"vertex_location": "us-east5", "use_in_pass_through": True}), + ({"vertex_location": None}, {}, {"vertex_location": "us-east5", "use_in_pass_through": True}), + ({"vertex_location": "eu"}, {"vertex_location": "asia-northeast1"}, {"vertex_location": "asia-northeast1"}), + ({"use_in_pass_through": False}, {}, {"vertex_location": "us-east5", "use_in_pass_through": True}), + ], + ids=[ + "deployment-over-router-default", + "router-default-fills-unset", + "router-default-fills-none", + "request-over-both", + "deployment-field-default-counts-as-unset", + ], +) +def test_params_sent_to_deployment_prefer_the_request_then_the_deployment_then_the_router_default( + deployment: Mapping[str, object], request_params: Mapping[str, object], sent: Mapping[str, object] +) -> None: + merged: Final = params_sent_to_deployment(deployment, _ROUTER_DEFAULTS, request_params) + + assert {key: merged[key] for key in sent} == sent + assert "timeout" not in merged diff --git a/tests/unit/router_utils/test_request_priority.py b/tests/unit/router_utils/test_request_priority.py index 71ab441d199..00aa43e30c5 100644 --- a/tests/unit/router_utils/test_request_priority.py +++ b/tests/unit/router_utils/test_request_priority.py @@ -55,7 +55,7 @@ def test_the_drop_params_decision_is_consulted_only_for_a_non_integer_priority() assert resolve_request_priority("1", 3, drops_params=_never) == 1 -def test_the_global_drop_params_wins_then_the_request_then_the_router_defaults_then_the_deployments( +def test_the_global_drop_params_wins_then_the_request_then_each_deployment_over_the_router_defaults( monkeypatch: pytest.MonkeyPatch, ) -> None: monkeypatch.setattr(litellm, "drop_params", False) @@ -67,7 +67,10 @@ def test_the_global_drop_params_wins_then_the_request_then_the_router_defaults_t assert request_drops_params({}, {}, ({"drop_params": True},)) is True assert request_drops_params({}, {}, ({"drop_params": True}, {"model": "openai/gpt-5.4-nano"})) is True assert request_drops_params({}, {}, ({"drop_params": False}, {"model": "openai/gpt-5.4-nano"})) is False - assert request_drops_params({}, {"drop_params": False}, ({"drop_params": True},)) is False + assert request_drops_params({}, {"drop_params": False}, ({"drop_params": True},)) is True + assert request_drops_params({}, {"drop_params": True}, ({"drop_params": False},)) is False + one_keeps_one_inherits: Final = ({"drop_params": False}, {"model": "openai/gpt-5.4-nano"}) + assert request_drops_params({}, {"drop_params": True}, one_keeps_one_inherits) is True monkeypatch.setattr(litellm, "drop_params", True) assert request_drops_params({}, {}, ()) is True assert request_drops_params({}, {}, ({"model": "openai/gpt-5.4-nano"},)) is True diff --git a/tests/unit/test_router/core/test_endpoints_characterization_inference.py b/tests/unit/test_router/core/test_endpoints_characterization_inference.py index c975e45b1d3..cb68d05234e 100644 --- a/tests/unit/test_router/core/test_endpoints_characterization_inference.py +++ b/tests/unit/test_router/core/test_endpoints_characterization_inference.py @@ -216,18 +216,17 @@ async def test_sync_text_completion_keeps_the_request_metadata_over_the_default_ assert _bucket(_sent(host))["requester_metadata"] == {"env": "dev", "region": "us", "team": "t1"} -async def test_sync_text_completion_sends_the_model_group_name_not_the_deployment_model( +async def test_sync_text_completion_sends_the_deployment_model_not_the_model_group_name( make_core: MakeCore, host: EndpointHost ): - """The request's model="g" is spread after the deployment params, so it replaces "openai/d1".""" core: Final = make_core(model_list=[deployment("g", "d1", api_base="https://d1.example")]) core.text_completion(model="g", prompt="p") call: Final = host.calls[0] - assert (call.model, call.kwargs["api_base"]) == ("g", "https://d1.example") + assert (call.model, call.kwargs["api_base"]) == ("openai/d1", "https://d1.example") async def test_sync_text_completion_is_not_retried(make_core: MakeCore, host: EndpointHost): - host.answer("g", litellm.InternalServerError(message="boom", llm_provider="openai", model="d1")) + host.answer("openai/d1", litellm.InternalServerError(message="boom", llm_provider="openai", model="d1")) core: Final = make_core(model_list=[deployment("g", "d1")], num_retries=3) with pytest.raises(litellm.InternalServerError): core.text_completion(model="g", prompt="p") diff --git a/tests/unit/test_router/core/test_legacy_members.py b/tests/unit/test_router/core/test_legacy_members.py index f81349db0e4..646a53729ae 100644 --- a/tests/unit/test_router/core/test_legacy_members.py +++ b/tests/unit/test_router/core/test_legacy_members.py @@ -640,7 +640,7 @@ def test_deployment_params_merge_in_the_order_the_deployment_receives_them(make_ params: Final = RoutingContext(core).deployment_params({"top_p": 0.7}, target) # pyright: ignore[reportArgumentType] # the core a context wraps assert params["model"] == "openai/a" - assert params["temperature"] == 0.9 + assert params["temperature"] == 0.1 assert params["tool_choice"] == "auto" assert params["top_p"] == 0.7 assert RoutingContext(core).deployment_params({}, None).get("temperature") == 0.9 # pyright: ignore[reportArgumentType] # see above diff --git a/tests/unit/test_router/test_router.py b/tests/unit/test_router/test_router.py index 814036e37f4..ca05cb2e6ee 100644 --- a/tests/unit/test_router/test_router.py +++ b/tests/unit/test_router/test_router.py @@ -15107,7 +15107,21 @@ async def test_router_caller_entra_field_that_changes_nothing_still_cools_down_a @pytest.mark.usefixtures("_vcr_outcome_gate", "isolate_litellm_state", "setup_and_teardown") -async def test_router_caller_repeating_the_secret_an_empty_router_default_selects_still_cools_down(monkeypatch): +@pytest.mark.parametrize( + ("deployment_secret", "router_default_secret", "caller_secret"), + [ + pytest.param({}, "", "environment-secret", id="empty-router-default-selects-the-environment"), + pytest.param( + {"client_secret": "configured-secret"}, + "router-default-secret", + "configured-secret", + id="deployment-secret-beats-the-router-default", + ), + ], +) +async def test_router_caller_repeating_the_configured_secret_still_cools_down( + monkeypatch, deployment_secret, router_default_secret, caller_secret +): monkeypatch.setenv("AZURE_CLIENT_SECRET", "environment-secret") router = Router( model_list=[ @@ -15119,12 +15133,12 @@ async def test_router_caller_repeating_the_secret_an_empty_router_default_select "api_version": "2025-04-01-preview", "tenant_id": "configured-tenant", "client_id": "configured-client", - "client_secret": "configured-secret", + **deployment_secret, }, "model_info": {"id": "configured-deployment"}, } ], - default_litellm_params={"client_secret": ""}, + default_litellm_params={"client_secret": router_default_secret}, ) with pytest.raises(litellm.AuthenticationError): @@ -15138,7 +15152,7 @@ async def test_router_caller_repeating_the_secret_an_empty_router_default_select model="gpt-4.1", ) ), - client_secret="environment-secret", + client_secret=caller_secret, ) with pytest.raises(litellm.RouterRateLimitError) as excinfo: @@ -20514,3 +20528,161 @@ def test_registry_finds_a_team_deployment_by_public_and_internal_name_for_its_te assert [row["model_info"]["id"] for row in registry.deployments(internal_name, team_id="team-a")] == ["team-row"] assert list(registry.deployments("public-chat", team_id="team-b")) == [] assert list(registry.deployments("public-chat")) == [] + + +_ROUTER_DEFAULT_REGION: Final = "us-east-1" +_BEDROCK_URL: Final = r"https://bedrock-runtime\.[a-z0-9-]+\.amazonaws\.com/model/.+" +_BEDROCK_CONVERSE_REPLY: Final = { + "output": {"message": {"role": "assistant", "content": [{"text": "hi"}]}}, + "stopReason": "end_turn", + "usage": {"inputTokens": 1, "outputTokens": 1, "totalTokens": 2}, +} +_REGION_PRECEDENCE: Final = pytest.mark.parametrize( + ("deployment_region", "request_region", "called_region"), + [ + ("eu-west-1", None, "eu-west-1"), + (None, None, _ROUTER_DEFAULT_REGION), + ("eu-west-1", "ap-northeast-1", "ap-northeast-1"), + ], + ids=["deployment-over-router-default", "router-default-fills-unset", "request-over-both"], +) + + +def _bedrock_router_with_a_default_region(model: str, deployment_region: str | None) -> Router: + region: Final = {} if deployment_region is None else {"aws_region_name": deployment_region} + return Router( + model_list=[ + { + "model_name": "regional", + "litellm_params": { + "model": model, + "aws_access_key_id": "unit-access-key", + "aws_secret_access_key": "unit-secret-key", + **region, + }, + } + ], + default_litellm_params={"aws_region_name": _ROUTER_DEFAULT_REGION}, + num_retries=0, + ) + + +def _request_region(request_region: str | None) -> Mapping[str, str]: + return {} if request_region is None else {"aws_region_name": request_region} + + +def _called_regions(route: respx.Route) -> list[str]: + return [call.request.url.host.split(".")[1] for call in route.calls] + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("_router_httpx_transport") +@_REGION_PRECEDENCE +async def test_acompletion_calls_the_deployment_region_over_a_conflicting_router_default( + deployment_region: str | None, request_region: str | None, called_region: str +) -> None: + router: Final = _bedrock_router_with_a_default_region( + "bedrock/converse/anthropic.claude-haiku-5-5", deployment_region + ) + + with respx.mock(assert_all_called=True) as upstream: + route: Final = upstream.post(url__regex=_BEDROCK_URL).respond(json=_BEDROCK_CONVERSE_REPLY) + await router.acompletion( + model="regional", messages=[{"role": "user", "content": "Say hi"}], **_request_region(request_region) + ) + + assert _called_regions(route) == [called_region] + + +@_REGION_PRECEDENCE +def test_text_completion_calls_the_deployment_region_over_a_conflicting_router_default( + deployment_region: str | None, request_region: str | None, called_region: str +) -> None: + router: Final = _bedrock_router_with_a_default_region( + "bedrock/converse/anthropic.claude-haiku-5-5", deployment_region + ) + + with respx.mock(assert_all_called=True) as upstream: + route: Final = upstream.post(url__regex=_BEDROCK_URL).respond(json=_BEDROCK_CONVERSE_REPLY) + router.text_completion(model="regional", prompt="Say hi", **_request_region(request_region)) + + assert _called_regions(route) == [called_region] + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("_router_httpx_transport") +@_REGION_PRECEDENCE +async def test_aembedding_calls_the_deployment_region_over_a_conflicting_router_default( + deployment_region: str | None, request_region: str | None, called_region: str +) -> None: + router: Final = _bedrock_router_with_a_default_region("bedrock/amazon.titan-embed-text-v2:0", deployment_region) + + with respx.mock(assert_all_called=True) as upstream: + route: Final = upstream.post(url__regex=_BEDROCK_URL).respond( + json={"embedding": [0.1, 0.2], "inputTextTokenCount": 2} + ) + await router.aembedding(model="regional", input=["Say hi"], **_request_region(request_region)) + + assert _called_regions(route) == [called_region] + + +def test_text_completion_calls_the_provider_with_the_deployment_model_not_the_group_name() -> None: + router: Final = Router( + model_list=[ + { + "model_name": "text-group", + "litellm_params": { + "model": "text-completion-openai/unit-text-model", + "api_base": "https://sync-text-group.invalid/v1", + "api_key": "unit-provider-key", + }, + } + ], + num_retries=0, + ) + with respx.mock(assert_all_called=True) as upstream: + provider: Final = upstream.post("https://sync-text-group.invalid/v1/completions").respond( + 200, + json={ + "id": "cmpl-unit", + "object": "text_completion", + "created": 1, + "model": "unit-text-model", + "choices": [{"text": "provider called", "index": 0, "finish_reason": "stop", "logprobs": None}], + "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, + }, + ) + response: Final = router.text_completion(model="text-group", prompt="Say hi") + + assert response.choices[0].text == "provider called" + assert json.loads(provider.calls.last.request.content)["model"] == "unit-text-model" + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("_router_httpx_transport") +@pytest.mark.parametrize( + ("deployment_params", "provider_hits"), + [({"max_retries": 1}, 2), ({"max_retries": 3}, 4)], + ids=["deployment-max-retries-1", "deployment-max-retries-3"], +) +async def test_aembedding_retries_at_the_provider_as_often_as_the_deployment_sets( + deployment_params: Mapping[str, int], provider_hits: int +) -> None: + router: Final = Router( + model_list=[ + { + "model_name": "embed", + "litellm_params": {"model": "openai/text-embedding-3-large", "api_key": "sk-test", **deployment_params}, + } + ], + num_retries=0, + ) + + with respx.mock(assert_all_called=True) as upstream: + route: Final = upstream.post("https://api.openai.com/v1/embeddings").respond( + 500, headers={"retry-after-ms": "1"}, json={"error": {"message": "overloaded", "type": "server_error"}} + ) + with pytest.raises(litellm.InternalServerError): + await router.aembedding(model="embed", input=["Say hi"]) + + assert len(route.calls) == provider_hits