fix(router): keep a deployment's own litellm_params over router default_litellm_params

Router defaults now fill only the keys a deployment leaves unset, so a deployment's own value (vertex_location, api_base, drop_params) beats default_litellm_params. The proxy token counters, drop-params checks, and the Entra cooldown check use the same order
This commit is contained in:
mateo-berri 2026-11-13 20:36:58 -08:00
parent 752eb7c5b7
commit 20566c0eed
18 changed files with 416 additions and 95 deletions

View file

@ -193,6 +193,7 @@ from litellm.responses.streaming_iterator import partial_responses_incomplete_ev
from litellm.router import Router
from litellm.router_api import RoutingContext
from litellm.router_strategy.auto_router.auto_router_model_naming import AUTO_ROUTER_MODEL_PREFIX
from litellm.router_utils.default_litellm_params import params_sent_to_deployment
from litellm.types.extraction import ExtractionResponse
from litellm.types.guardrails import GuardrailEventHooks
from litellm.types.router import RouterRateLimitError
@ -869,14 +870,16 @@ def _router_defaults(llm_router: Router | None) -> Mapping[str, object]:
def counted_drop_policies(
data: Mapping[str, object], llm_router: Router | None, team_id: str | None
) -> tuple[DropPolicy, ...]:
requested: Final = {**_router_defaults(llm_router), **data}
router_defaults: Final = _router_defaults(llm_router)
model: Final = data.get("model")
deployments: Final = (
_named_deployments(llm_router, model, team_id) if llm_router is not None and isinstance(model, str) else ()
)
if not deployments:
return (drop_policy(requested), _UNSEEN_DEPLOYMENT)
return tuple(drop_policy({**deployment, **requested}) for deployment in deployments)
return (drop_policy({**router_defaults, **data}), _UNSEEN_DEPLOYMENT)
return tuple(
drop_policy(params_sent_to_deployment(deployment, router_defaults, data)) for deployment in deployments
)
def resolve_counted_tool_fields(

View file

@ -207,6 +207,7 @@ from litellm.router_strategy.auto_router.auto_router_tuning_baseline import (
snapshot_tuning_baselines,
tuning_limit_violation,
)
from litellm.router_utils.default_litellm_params import params_sent_to_deployment
from litellm.rust_bridge.catalog import Target
from litellm.types.utils import (
PRICING_OVERRIDES_KEY,
@ -15969,7 +15970,8 @@ def _count_context_management(
)
)
sent_params: Final = tuple(
{**params, **router_defaults, **request_litellm_params} for params in deployment_params or ({},)
params_sent_to_deployment(params, router_defaults, request_litellm_params)
for params in deployment_params or ({},)
)
if any(context_management_explicitly_dropped(params.get("additional_drop_params")) for params in sent_params):
return None
@ -16072,7 +16074,7 @@ def _responses_routing_kwargs(responses_request: Mapping[str, object] | None) ->
def _inference_params(
litellm_params: Mapping[str, object], responses_request: Mapping[str, object]
) -> Mapping[str, object]:
return {**litellm_params, **_router_default_litellm_params(llm_router), **responses_request}
return params_sent_to_deployment(litellm_params, _router_default_litellm_params(llm_router), responses_request)
def _count_drop_policy(deployment: Mapping[str, object] | None, responses_request: Mapping[str, object]) -> DropPolicy:
@ -16314,12 +16316,17 @@ async def _count_request_tokens(
if _model is not None:
model_to_use = _model
sent_params: Final[Mapping[str, object]] = {**_router_default_litellm_params(llm_router), **request_litellm_params}
router_defaults: Final = _router_default_litellm_params(llm_router)
sent_params: Final = params_sent_to_deployment(
_DEPLOYMENT_LITELLM_PARAMS.validate_python((deployment or {}).get("litellm_params") or {}),
router_defaults,
request_litellm_params,
)
count_deployment: Final = _deployment_with_request_drop_settings(deployment, sent_params)
count_litellm_params: Final = _DEPLOYMENT_LITELLM_PARAMS.validate_python(
(count_deployment or {}).get("litellm_params") or {}
)
deployment_defaults: Final = {**count_litellm_params, **_router_default_litellm_params(llm_router)}
deployment_defaults: Final = params_sent_to_deployment(count_litellm_params, router_defaults, {})
conversation_empty: Final = not messages and not contents
if prompt and not conversation_empty and not deployment_drops_params(count_deployment):
raise ProxyException(

View file

@ -242,6 +242,7 @@ from litellm.router_strategy.least_busy import (
picks_reserve,
release_started_attempt,
)
from litellm.router_utils import default_litellm_params as default_params
from litellm.router_utils.cooldown_handlers import get_cooldown_deployment_info, mark_deployment_selection_failure
from litellm.router_utils.endpoint_eligibility import endpoint_for_call, selected_endpoint
from litellm.router_utils.kubernetes_pod_discovery import (
@ -3203,13 +3204,11 @@ class RustCore(
)
def params_sent_to_deployment(self, request: Mapping[str, object], deployment_id: object) -> Mapping[str, object]:
"""The request as a deployment received it: its params, the router defaults, its tool choice, then
"""The request as a deployment received it: its params, the router defaults it leaves unset, then
the request (``_params_sent_to_deployment``)."""
record: Final = self._handle.deployment(deployment_id) if isinstance(deployment_id, str) else None
params: Final = as_str_mapping((as_str_mapping(record) or {}).get("litellm_params")) or {}
defaults: Final = {k: v for k, v in self._resolved.default_litellm_params.items() if v is not None}
tool_choice: Final = {"tool_choice": params["tool_choice"]} if params.get("tool_choice") is not None else {}
return MappingProxyType({**params, **defaults, **tool_choice, **request})
return default_params.params_sent_to_deployment(params, self._resolved.default_litellm_params, request)
def _drive_sync_steps(self, call: "RouterCall[_FailureReportSteps | _SuccessReportSteps]") -> Mapping[str, object]:
"""Drive a call whose only steps are cache ops (``report_failure``) to its end."""
@ -3434,15 +3433,17 @@ class RustCore(
) -> list[str]: # mutable-ok: the step's answer
"""The candidates that refuse the audio or videos a Decisions call carries."""
group: Final = str(step.get("model_group"))
defaults: Final = {k: v for k, v in self._resolved.default_litellm_params.items() if v is not None}
sent: Final = {**defaults, **request}
audio: Final = bool(request.get("audio"))
videos: Final = bool(request.get("videos"))
return [
str(id)
for id in _as_list(step.get("candidates"))
if refuses_decisions_media(
as_str_mapping(self._handle.deployment(str(id))) or {}, group, sent, audio=audio, videos=videos
as_str_mapping(self._handle.deployment(str(id))) or {},
group,
self.params_sent_to_deployment(request, str(id)),
audio=audio,
videos=videos,
)
]

View file

@ -52,6 +52,7 @@ from litellm.router_strategy.complexity_router.context_compaction import (
initialize_compaction_state,
surface_for_call,
)
from litellm.router_utils.default_litellm_params import router_defaults_for_deployment
from litellm.router_utils.endpoint_eligibility import endpoint_for_call, endpoint_selection
from litellm.vector_stores.search_query import store_search_params
@ -2176,7 +2177,7 @@ class RustEndpoints:
return await self._direct("amoderation", prepared)
def text_completion(self, request: Mapping[str, object]) -> object:
"""The sync `text_completion`: one pick, no retries, the group name as the model."""
"""The sync `text_completion`: one pick, no retries."""
from litellm.litellm_core_utils.routing_common_utils import (
deployment_model_id_pin,
metadata_defaults_under_request,
@ -2197,17 +2198,21 @@ class RustEndpoints:
request_kwargs=kwargs,
)
data: Final = as_dict(deployment.get("litellm_params"))
for key, value in self._core.resolved.default_litellm_params.items():
if key not in kwargs:
kwargs[key] = value
elif key == "metadata":
metadata_bucket(kwargs, key).update(
metadata_defaults_under_request(defaults=as_dict(value), request_metadata=as_dict(kwargs[key]))
defaults: Final = self._core.resolved.default_litellm_params
for key, value in router_defaults_for_deployment(without(defaults, "metadata"), data).items():
kwargs.setdefault(key, value)
if "metadata" in defaults:
metadata_bucket(kwargs, "metadata").update(
metadata_defaults_under_request(
defaults=as_dict(defaults["metadata"]), request_metadata=as_dict(kwargs["metadata"])
)
)
pinned: Final = deployment_model_id_pin(data)
if pinned is not None:
kwargs["model_id"] = pinned
return self._invoke("text_completion")(**{**data, "prompt": prompt, "caching": self._cache_responses, **kwargs})
return self._invoke("text_completion")(
**{**data, "prompt": prompt, "caching": self._cache_responses, **without(kwargs, "model")}
)
async def aadapter_completion(
self,

View file

@ -41,6 +41,11 @@ from litellm.router_strategy.tag_based_routing import (
get_tags_from_request_kwargs,
tags_with_serving_deployment_tags,
)
from litellm.router_utils.default_litellm_params import (
deployment_sets_litellm_param,
params_sent_to_deployment,
router_defaults_for_deployment,
)
from litellm.router_utils.pre_call_checks.io_token_rate_limit_check import (
deployment_has_io_token_limits, # pyright: ignore[reportUnknownVariableType] # litellm's untyped helpers and request values
refund_stale_reservation_before_retry,
@ -119,8 +124,7 @@ def _drops_malformed_request_tools(
) -> bool:
if isinstance(request.get("tools") or [], (list, tuple)) or not deployment_params.get("tools"):
return False
set_defaults: Final = {name: value for name, value in defaults.items() if value is not None}
return drop_policy({**deployment_params, **set_defaults, **request}).drops("tools")
return drop_policy(params_sent_to_deployment(deployment_params, defaults, request)).drops("tools")
def _malformed_request_params_the_deployment_sets(
@ -136,8 +140,7 @@ def _malformed_request_params_the_deployment_sets(
try:
validate_litellm_params_strictly(GenericLiteLLMParams, overrides)
except ValidationError as error:
set_defaults: Final = {name: value for name, value in defaults.items() if value is not None}
policy: Final = drop_policy({**deployment_params, **set_defaults, **request})
policy: Final = drop_policy(params_sent_to_deployment(deployment_params, defaults, request))
malformed: Final = (str(detail["loc"][0]) for detail in error.errors() if detail["loc"])
return frozenset(key for key in malformed if policy.drops(key))
return frozenset()
@ -175,21 +178,13 @@ def merge_tools_from_deployment(
)
def _deployment_sets_litellm_param(deployment_params: Mapping[str, object], key: str) -> bool:
value: Final = deployment_params.get(key)
if value is None:
return False
field: Final = LiteLLM_Params.model_fields.get(key)
return field is None or value != field.default # pyright: ignore[reportAny] # litellm's untyped helpers and request values
def _forwarded_alias_marker_keys_the_deployment_sets(
deployment_params: Mapping[str, object], forwarded_keys: object
) -> tuple[str, ...]:
if not isinstance(forwarded_keys, tuple):
return ()
keys: Final[tuple[object, ...]] = forwarded_keys # pyright: ignore[reportUnknownVariableType] # the router's own marker
return tuple(key for key in keys if isinstance(key, str) and _deployment_sets_litellm_param(deployment_params, key))
return tuple(key for key in keys if isinstance(key, str) and deployment_sets_litellm_param(deployment_params, key))
# --- reasoning effort -------------------------------------------------------------------------
@ -355,19 +350,18 @@ def call_timeout(shaping: Shaping, kwargs: Mapping[str, object], data: Mapping[s
def update_kwargs_with_default_litellm_params(
shaping: Shaping,
deployment_params: Mapping[str, object],
kwargs: MutableMapping[str, object], # mutable-ok: the request kwargs the Python router mutates in place
metadata_variable_name: str,
) -> None:
"""``_update_kwargs_with_default_litellm_params``: unset non-None defaults fill the request;
the default metadata fills the keys the request leaves unset, the router's caching groups excepted."""
defaults: Final = dict(without_retrieve_stream_defaults(shaping.default_litellm_params, kwargs))
metadata_defaults: Final = defaults.pop("metadata", {}) or {} # pyright: ignore[reportUnknownVariableType] # litellm's untyped helpers and request values
for key, value in defaults.items():
if value is None:
continue
"""``_update_kwargs_with_default_litellm_params``: the non-None defaults the request leaves unset and
the deployment does not set itself fill the request; the default metadata fills the keys the request
leaves unset, the router's caching groups excepted."""
defaults: Final = without_retrieve_stream_defaults(shaping.default_litellm_params, kwargs)
for key, value in router_defaults_for_deployment(without(defaults, "metadata"), deployment_params).items():
kwargs.setdefault(key, value)
bucket: Final = metadata_bucket(kwargs, metadata_variable_name)
bucket.update(metadata_defaults_under_request(defaults=as_dict(metadata_defaults), request_metadata=bucket)) # pyright: ignore[reportUnknownArgumentType] # litellm's untyped helpers and request values
bucket.update(metadata_defaults_under_request(defaults=as_dict(defaults.get("metadata")), request_metadata=bucket))
# --- clientside credentials -------------------------------------------------------------------
@ -515,11 +509,11 @@ def update_kwargs_with_deployment(
)
_set_timeouts(shaping, deployment_params, kwargs, function_name, splits_stream_timeout)
if router_config_requests_stream_usage(kwargs, (deployment_params, defaults)):
if router_config_requests_stream_usage(kwargs, (params_sent_to_deployment(deployment_params, defaults, {}),)):
kwargs[ # rebind-ok: the request kwargs are shaped in place, as the Python router shapes them
STREAM_USAGE_INJECTED_FIELD
] = True
update_kwargs_with_default_litellm_params(shaping, kwargs, metadata_variable_name)
update_kwargs_with_default_litellm_params(shaping, deployment_params, kwargs, metadata_variable_name)
pinned_model_id: Final = deployment_model_id_pin(deployment_params)
if pinned_model_id is not None:
kwargs["model_id"] = ( # rebind-ok: the request kwargs are shaped in place, as the Python router shapes them

View file

@ -18,7 +18,6 @@ import traceback
import uuid
from collections.abc import Awaitable, Callable, Iterable, Mapping
from functools import partial
from types import MappingProxyType
from typing import TYPE_CHECKING, Final
from typing_extensions import NotRequired, ReadOnly, TypedDict
@ -46,6 +45,7 @@ from litellm.router_strategy.auto_router.auto_router_model_naming import (
count_capability_routers,
)
from litellm.router_utils.cooldown_handlers import set_cooldown_deployments
from litellm.router_utils.default_litellm_params import params_sent_to_deployment
from litellm.router_utils.endpoint_eligibility import endpoint_selection
from litellm.router_utils.pre_call_checks.io_token_rate_limit_check import async_io_token_refund_failure
from litellm.router_utils.router_callbacks.track_deployment_metrics import (
@ -538,9 +538,7 @@ class RustLegacyMembers(RustRegistryMembers, RustOptionalChecks):
"""``_params_sent_to_deployment`` of a ``Deployment``, which ``RoutingContext.deployment_params`` hands in."""
params: Final = deployment.litellm_params.model_dump(exclude_none=True) if deployment else {}
configured: Final = as_str_mapping(RoutingContext(self).settings().default_litellm_params) # pyright: ignore[reportArgumentType, reportUnknownMemberType, reportUnknownArgumentType] # the core's RoutingContext
defaults: Final = {k: v for k, v in (configured or {}).items() if v is not None}
tool_choice: Final = {"tool_choice": params["tool_choice"]} if params.get("tool_choice") is not None else {}
return MappingProxyType({**params, **defaults, **tool_choice, **request_kwargs})
return params_sent_to_deployment(params, configured or {}, request_kwargs)
__all__ = ["RustLegacyMembers"]

View file

@ -30,6 +30,7 @@ from litellm.litellm_core_utils.litellm_logging import mask_api_base_credentials
from litellm.router_deprecated import renamed_router_param
from litellm.router_utils.cooldown_cache import CooldownCacheValue
from litellm.router_utils.cooldown_callbacks import router_cooldown_event_callback
from litellm.router_utils.default_litellm_params import params_sent_to_deployment
from litellm.types.llms.anthropic import MESSAGE_THREAD_NOT_FOUND_ERROR_CODE
from litellm.types.router import Deployment
from litellm.types.utils import (
@ -888,8 +889,9 @@ def is_caller_entra_credential_failure(
credential_name: Final = deployment.litellm_params.litellm_credential_name
configured_by_precedence: Final[tuple[Mapping[str, object], ...]] = (
router_defaults,
_CREDENTIAL_VALUES_ADAPTER.validate_python(deployment.litellm_params.model_dump(exclude_none=True)),
_CREDENTIAL_VALUES_ADAPTER.validate_python(
params_sent_to_deployment(deployment.litellm_params.model_dump(exclude_none=True), router_defaults, {})
),
_CREDENTIAL_VALUES_ADAPTER.validate_python(
CredentialAccessor.get_credential_values(credential_name) if credential_name is not None else {}
),

View file

@ -0,0 +1,33 @@
from collections.abc import Mapping
from types import MappingProxyType
from typing import Final
from litellm.types.router import LiteLLM_Params
def deployment_sets_litellm_param(deployment_params: Mapping[str, object], key: str) -> bool:
value: Final = deployment_params.get(key)
if value is None:
return False
field: Final = LiteLLM_Params.model_fields.get(key)
return field is None or value != field.default # pyright: ignore[reportAny] # pydantic types FieldInfo.default as Any
def router_defaults_for_deployment(
router_defaults: Mapping[str, object], deployment_params: Mapping[str, object]
) -> Mapping[str, object]:
return MappingProxyType(
{
key: value
for key, value in router_defaults.items()
if value is not None and not deployment_sets_litellm_param(deployment_params, key)
}
)
def params_sent_to_deployment(
deployment_params: Mapping[str, object], router_defaults: Mapping[str, object], request: Mapping[str, object]
) -> Mapping[str, object]:
return MappingProxyType(
{**deployment_params, **router_defaults_for_deployment(router_defaults, deployment_params), **request}
)

View file

@ -6,6 +6,7 @@ from pydantic import TypeAdapter, ValidationError
import litellm
from litellm.litellm_core_utils.core_helpers import normalize_drop_params
from litellm.router_utils.default_litellm_params import params_sent_to_deployment
@dataclass(frozen=True, slots=True)
@ -91,7 +92,8 @@ def request_drops_params(
request_flag: Final = _explicit_drop_params(kwargs)
if request_flag is not None:
return request_flag
router_flag: Final = _explicit_drop_params(router_defaults)
if router_flag is not None:
return router_flag
return any(_explicit_drop_params(params) is True for params in deployments)
if not deployments:
return _explicit_drop_params(router_defaults) is True
return any(
_explicit_drop_params(params_sent_to_deployment(params, router_defaults, {})) is True for params in deployments
)

View file

@ -4174,7 +4174,7 @@ async def test_clips_to_a_group_whose_only_taker_is_cooling_down_are_refused_rat
@pytest.mark.asyncio
@pytest.mark.parametrize("surface", tuple(_MEDIA_SURFACES))
async def test_a_router_default_drop_params_overrides_a_deployment_when_choosing_who_takes_clips(
async def test_a_deployment_drop_params_beats_a_router_default_when_choosing_who_takes_clips(
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, local_model_cost_map: None, surface: str
) -> None:
monkeypatch.setattr(litellm, "drop_params", False)
@ -4194,13 +4194,13 @@ async def test_a_router_default_drop_params_overrides_a_deployment_when_choosing
return await response if asyncio.iscoroutine(response) else response
await send()
assert respx_mock.calls.last.request.headers["authorization"] == "Bearer takes"
assert json.loads(respx_mock.calls.last.request.content)["audio"] == list(_AUDIO_CLIPS)
await send(drop_params=True)
assert respx_mock.calls.last.request.headers["authorization"] == "Bearer drops"
assert "audio" not in json.loads(respx_mock.calls.last.request.content)
await send(drop_params=False)
assert respx_mock.calls.last.request.headers["authorization"] == "Bearer takes"
assert json.loads(respx_mock.calls.last.request.content)["audio"] == list(_AUDIO_CLIPS)
@pytest.mark.asyncio
async def test_a_deployment_that_turns_audio_off_is_refused_or_dropped_even_when_the_cost_map_allows_it(

View file

@ -6000,6 +6000,28 @@ class TestResponsesInputTokens:
assert counter.await_args.kwargs["request"].tools is None
assert "tools" not in counter.await_args.kwargs["responses_request"]
@pytest.mark.parametrize(
("deployment_drop_params", "expected_status"),
[({}, 200), ({"drop_params": False}, 400)],
ids=["router-default-drops", "deployment-keeps-over-router-default"],
)
def test_malformed_tools_follow_the_deployment_drop_params_over_the_router_default(
self, deployment_drop_params: dict[str, bool], expected_status: int
) -> None:
router: Final = litellm.Router(
model_list=[
{
"model_name": "gpt-5.6",
"litellm_params": {"model": "openai/gpt-5.6", "api_key": "sk-test", **deployment_drop_params},
}
],
default_litellm_params={"drop_params": True},
)
response, _ = self._post_input_tokens({"model": "gpt-5.6", "input": "hi", "tools": 5}, llm_router=router)
assert response.status_code == expected_status, response.text
def test_malformed_tools_for_a_model_outside_the_router_get_a_clean_400(self) -> None:
router: Final = litellm.Router(
model_list=[{"model_name": "gpt-5.2", "litellm_params": {"model": "openai/gpt-5.2", "api_key": "sk-test"}}]

View file

@ -1582,7 +1582,10 @@ def _build_log_messages(log_text: JsonValue) -> list[dict[str, JsonValue]]:
async def _count_on_a_model_without_a_count_api(
monkeypatch: pytest.MonkeyPatch, body: Mapping[str, object], router_defaults: dict[str, JsonValue] | None = None
monkeypatch: pytest.MonkeyPatch,
body: Mapping[str, object],
router_defaults: dict[str, JsonValue] | None = None,
deployment_params: Mapping[str, JsonValue] = MappingProxyType({}),
) -> JSONResponse | dict[str, JsonValue]:
import litellm.proxy.proxy_server as proxy_server
@ -1590,7 +1593,7 @@ async def _count_on_a_model_without_a_count_api(
model_list=[
{
"model_name": "mistral-tool-result",
"litellm_params": {"model": "mistral/mistral-small-latest", "api_key": "sk-test"},
"litellm_params": {"model": "mistral/mistral-small-latest", "api_key": "sk-test", **deployment_params},
}
],
default_litellm_params=router_defaults,
@ -1633,14 +1636,22 @@ async def test_count_tokens_without_a_count_api_counts_a_non_string_tool_result_
@pytest.mark.asyncio
@pytest.mark.parametrize(("body_drop_params", "expected_status"), (({}, None), ({"drop_params": False}, 400)))
async def test_count_tokens_honors_router_default_drop_params_unless_the_body_overrides_it(
monkeypatch: pytest.MonkeyPatch, body_drop_params: dict[str, JsonValue], expected_status: int | None
@pytest.mark.parametrize(
("deployment_params", "body_drop_params", "expected_status"),
(({}, {}, None), ({}, {"drop_params": False}, 400), ({"drop_params": False}, {}, 400)),
ids=("router-default", "body-over-router-default", "deployment-over-router-default"),
)
async def test_count_tokens_honors_router_default_drop_params_unless_the_body_or_deployment_overrides_it(
monkeypatch: pytest.MonkeyPatch,
deployment_params: dict[str, JsonValue],
body_drop_params: dict[str, JsonValue],
expected_status: int | None,
) -> None:
response: Final = await _count_on_a_model_without_a_count_api(
monkeypatch,
{"model": "mistral-tool-result", "messages": _build_log_messages(None), **body_drop_params},
router_defaults={"drop_params": True},
deployment_params=deployment_params,
)
match response:
@ -2992,6 +3003,7 @@ _NATIVE_RESPONSES_BASE: Final = {
_FIREWORKS_DEEPSEEK_V4_1: "https://unused.invalid",
_XAI_GROK_4_3: "https://unused.invalid",
}
_BRIDGED_RESPONSES_BASE: Final = "https://unused.cognitiveservices.azure.com/models"
def _responses_function(tool: Mapping[str, object], **strict: bool | None) -> dict[str, object]:
@ -3300,26 +3312,34 @@ async def test_a_responses_count_that_asks_for_the_chat_completions_bridge_frame
@pytest.mark.asyncio
async def test_a_responses_count_frames_for_the_api_base_router_defaults_send_inference_to(
monkeypatch: pytest.MonkeyPatch,
@pytest.mark.parametrize(
("deployment_params", "inference_api_base"),
[
pytest.param({}, _BRIDGED_RESPONSES_BASE, id="router-default-fills-an-unset-api-base"),
pytest.param(
{"api_base": _NATIVE_RESPONSES_BASE[_AZURE_GROK_4_1]},
_NATIVE_RESPONSES_BASE[_AZURE_GROK_4_1],
id="deployment-api-base-beats-the-router-default",
),
],
)
async def test_a_responses_count_frames_for_the_api_base_inference_is_sent_to(
monkeypatch: pytest.MonkeyPatch, deployment_params: dict[str, object], inference_api_base: str
):
router: Final = Router(
model_list=[
{
"model_name": "counted",
"litellm_params": {
"api_key": "unused",
"model": _AZURE_GROK_4_1,
"api_base": _NATIVE_RESPONSES_BASE[_AZURE_GROK_4_1],
},
"litellm_params": {"api_key": "unused", "model": _AZURE_GROK_4_1, **deployment_params},
"model_info": {},
}
],
default_litellm_params={"api_base": "https://unused.cognitiveservices.azure.com/models"},
default_litellm_params={"api_base": _BRIDGED_RESPONSES_BASE},
)
chat_count: Final = await _count_with_router(monkeypatch, router, (_GET_WEATHER,), responses=False)
sent_alone: Final = _single_deployment_router({"model": _AZURE_GROK_4_1, "api_base": inference_api_base}, {})
expected: Final = await _count_with_router(monkeypatch, sent_alone, (_GET_WEATHER,), responses=True)
assert await _count_with_router(monkeypatch, router, (_GET_WEATHER,), responses=True) == chat_count
assert await _count_with_router(monkeypatch, router, (_GET_WEATHER,), responses=True) == expected
@pytest.mark.asyncio
@ -4157,10 +4177,20 @@ async def test_messages_count_tokens_for_a_model_with_no_deployment_drops_an_inv
@pytest.mark.asyncio
async def test_gemini_messages_count_tokens_rejects_an_invalid_edit_without_drop_params() -> None:
@pytest.mark.parametrize(
("deployment_params", "router_defaults"),
(({}, {}), ({"drop_params": False}, {"drop_params": True})),
ids=("no-drop-params", "deployment-over-router-default"),
)
async def test_gemini_messages_count_tokens_rejects_an_invalid_edit_without_drop_params(
deployment_params: dict[str, JsonValue], router_defaults: dict[str, JsonValue]
) -> None:
with patch.object(litellm, "drop_params", False):
response, sent_bodies = await _count_through_the_messages_endpoint(
Router(model_list=[_gemini_deployment("gemini-agent", {})]),
Router(
model_list=[_gemini_deployment("gemini-agent", deployment_params)],
default_litellm_params=router_defaults,
),
{"model": "gemini-agent", "context_management": _INVALID_COMPACT_EDIT},
)
@ -5480,6 +5510,7 @@ async def _claude_count(
monkeypatch: pytest.MonkeyPatch,
deployment_params: Mapping[str, JsonValue],
responses_request: Mapping[str, object],
router_defaults: dict[str, JsonValue] | None = None,
) -> tuple[TokenCountResponse, list[dict[str, JsonValue]]]:
router: Final = Router(
model_list=[
@ -5492,7 +5523,8 @@ async def _claude_count(
**deployment_params,
},
}
]
],
default_litellm_params=router_defaults,
)
monkeypatch.setattr(litellm.proxy.proxy_server, "llm_router", router)
bodies: Final[list[dict[str, JsonValue]]] = [] # mutable-ok: collects what the mock peer receives
@ -5516,14 +5548,15 @@ async def _claude_count(
@pytest.mark.asyncio
@pytest.mark.parametrize(
("deployment_params", "request_params"),
[({"drop_params": True}, {}), ({}, {"drop_params": True})],
ids=["deployment", "request"],
("deployment_params", "request_params", "router_defaults"),
[({"drop_params": True}, {}, None), ({}, {"drop_params": True}, None), ({}, {}, {"drop_params": True})],
ids=["deployment", "request", "router-default"],
)
async def test_responses_count_on_claude_downgrades_forced_tool_choice_under_drop_params(
monkeypatch: pytest.MonkeyPatch,
deployment_params: Mapping[str, JsonValue],
request_params: Mapping[str, JsonValue],
router_defaults: dict[str, JsonValue] | None,
) -> None:
monkeypatch.setitem(litellm.model_cost, "claude-zeta-9", {"supports_forced_tool_use": False})
@ -5531,6 +5564,7 @@ async def test_responses_count_on_claude_downgrades_forced_tool_choice_under_dro
monkeypatch,
deployment_params,
{"input": "Weather?", "tools": [_WEATHER_TOOL], "tool_choice": "required", **request_params},
router_defaults,
)
assert (result.error, result.total_tokens) == (False, 77)
@ -5539,13 +5573,25 @@ async def test_responses_count_on_claude_downgrades_forced_tool_choice_under_dro
@pytest.mark.asyncio
@pytest.mark.parametrize(
("deployment_params", "router_defaults"),
[({}, None), ({"drop_params": False}, {"drop_params": True})],
ids=["no-drop-params", "deployment-over-router-default"],
)
async def test_responses_count_on_claude_rejects_forced_tool_choice_without_drop_params(
monkeypatch: pytest.MonkeyPatch,
deployment_params: Mapping[str, JsonValue],
router_defaults: dict[str, JsonValue] | None,
) -> None:
monkeypatch.setitem(litellm.model_cost, "claude-zeta-9", {"supports_forced_tool_use": False})
with pytest.raises(ProxyException) as rejected:
await _claude_count(monkeypatch, {}, {"input": "Weather?", "tools": [_WEATHER_TOOL], "tool_choice": "required"})
await _claude_count(
monkeypatch,
deployment_params,
{"input": "Weather?", "tools": [_WEATHER_TOOL], "tool_choice": "required"},
router_defaults,
)
assert (rejected.value.code, "forced tool use" in rejected.value.message) == ("400", True)

View file

@ -1138,7 +1138,7 @@ async def test_jev_bounds_complete_input_without_changing_forwarded_messages(
[
("false", None, None, None, "truncated"),
("true", None, None, None, "within_budget"),
("true", "false", None, None, "truncated"),
("true", "false", None, None, "within_budget"),
("true", None, 8192, None, "truncated"),
("true", None, 8192, "default", "truncated"),
("true", None, None, "default", "truncated"),
@ -1199,7 +1199,7 @@ async def test_jev_budget_uses_effective_headers_and_every_deployment(
assert telemetry["classifier_input_tokens"] <= telemetry["classifier_input_budget_tokens"]
sent: Final = JevSystemOneRequest.model_validate_json(route.calls[0].request.content)
if other_member is None:
assert route.calls[0].request.headers[header] == (default_header or mode)
assert route.calls[0].request.headers[header] == mode
if explicit_cap is not None:
assert telemetry["classifier_input_budget_tokens"] < explicit_cap
assert (ask in sent.state) is (expected_status == "within_budget")

View file

@ -0,0 +1,34 @@
from collections.abc import Mapping
from typing import Final
import pytest
from litellm.router_utils.default_litellm_params import params_sent_to_deployment
_ROUTER_DEFAULTS: Final = {"vertex_location": "us-east5", "use_in_pass_through": True, "timeout": None}
@pytest.mark.parametrize(
("deployment", "request_params", "sent"),
[
({"vertex_location": "eu"}, {}, {"vertex_location": "eu", "use_in_pass_through": True}),
({}, {}, {"vertex_location": "us-east5", "use_in_pass_through": True}),
({"vertex_location": None}, {}, {"vertex_location": "us-east5", "use_in_pass_through": True}),
({"vertex_location": "eu"}, {"vertex_location": "asia-northeast1"}, {"vertex_location": "asia-northeast1"}),
({"use_in_pass_through": False}, {}, {"vertex_location": "us-east5", "use_in_pass_through": True}),
],
ids=[
"deployment-over-router-default",
"router-default-fills-unset",
"router-default-fills-none",
"request-over-both",
"deployment-field-default-counts-as-unset",
],
)
def test_params_sent_to_deployment_prefer_the_request_then_the_deployment_then_the_router_default(
deployment: Mapping[str, object], request_params: Mapping[str, object], sent: Mapping[str, object]
) -> None:
merged: Final = params_sent_to_deployment(deployment, _ROUTER_DEFAULTS, request_params)
assert {key: merged[key] for key in sent} == sent
assert "timeout" not in merged

View file

@ -55,7 +55,7 @@ def test_the_drop_params_decision_is_consulted_only_for_a_non_integer_priority()
assert resolve_request_priority("1", 3, drops_params=_never) == 1
def test_the_global_drop_params_wins_then_the_request_then_the_router_defaults_then_the_deployments(
def test_the_global_drop_params_wins_then_the_request_then_each_deployment_over_the_router_defaults(
monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.setattr(litellm, "drop_params", False)
@ -67,7 +67,10 @@ def test_the_global_drop_params_wins_then_the_request_then_the_router_defaults_t
assert request_drops_params({}, {}, ({"drop_params": True},)) is True
assert request_drops_params({}, {}, ({"drop_params": True}, {"model": "openai/gpt-5.4-nano"})) is True
assert request_drops_params({}, {}, ({"drop_params": False}, {"model": "openai/gpt-5.4-nano"})) is False
assert request_drops_params({}, {"drop_params": False}, ({"drop_params": True},)) is False
assert request_drops_params({}, {"drop_params": False}, ({"drop_params": True},)) is True
assert request_drops_params({}, {"drop_params": True}, ({"drop_params": False},)) is False
one_keeps_one_inherits: Final = ({"drop_params": False}, {"model": "openai/gpt-5.4-nano"})
assert request_drops_params({}, {"drop_params": True}, one_keeps_one_inherits) is True
monkeypatch.setattr(litellm, "drop_params", True)
assert request_drops_params({}, {}, ()) is True
assert request_drops_params({}, {}, ({"model": "openai/gpt-5.4-nano"},)) is True

View file

@ -216,18 +216,17 @@ async def test_sync_text_completion_keeps_the_request_metadata_over_the_default_
assert _bucket(_sent(host))["requester_metadata"] == {"env": "dev", "region": "us", "team": "t1"}
async def test_sync_text_completion_sends_the_model_group_name_not_the_deployment_model(
async def test_sync_text_completion_sends_the_deployment_model_not_the_model_group_name(
make_core: MakeCore, host: EndpointHost
):
"""The request's model="g" is spread after the deployment params, so it replaces "openai/d1"."""
core: Final = make_core(model_list=[deployment("g", "d1", api_base="https://d1.example")])
core.text_completion(model="g", prompt="p")
call: Final = host.calls[0]
assert (call.model, call.kwargs["api_base"]) == ("g", "https://d1.example")
assert (call.model, call.kwargs["api_base"]) == ("openai/d1", "https://d1.example")
async def test_sync_text_completion_is_not_retried(make_core: MakeCore, host: EndpointHost):
host.answer("g", litellm.InternalServerError(message="boom", llm_provider="openai", model="d1"))
host.answer("openai/d1", litellm.InternalServerError(message="boom", llm_provider="openai", model="d1"))
core: Final = make_core(model_list=[deployment("g", "d1")], num_retries=3)
with pytest.raises(litellm.InternalServerError):
core.text_completion(model="g", prompt="p")

View file

@ -640,7 +640,7 @@ def test_deployment_params_merge_in_the_order_the_deployment_receives_them(make_
params: Final = RoutingContext(core).deployment_params({"top_p": 0.7}, target) # pyright: ignore[reportArgumentType] # the core a context wraps
assert params["model"] == "openai/a"
assert params["temperature"] == 0.9
assert params["temperature"] == 0.1
assert params["tool_choice"] == "auto"
assert params["top_p"] == 0.7
assert RoutingContext(core).deployment_params({}, None).get("temperature") == 0.9 # pyright: ignore[reportArgumentType] # see above

View file

@ -15107,7 +15107,21 @@ async def test_router_caller_entra_field_that_changes_nothing_still_cools_down_a
@pytest.mark.usefixtures("_vcr_outcome_gate", "isolate_litellm_state", "setup_and_teardown")
async def test_router_caller_repeating_the_secret_an_empty_router_default_selects_still_cools_down(monkeypatch):
@pytest.mark.parametrize(
("deployment_secret", "router_default_secret", "caller_secret"),
[
pytest.param({}, "", "environment-secret", id="empty-router-default-selects-the-environment"),
pytest.param(
{"client_secret": "configured-secret"},
"router-default-secret",
"configured-secret",
id="deployment-secret-beats-the-router-default",
),
],
)
async def test_router_caller_repeating_the_configured_secret_still_cools_down(
monkeypatch, deployment_secret, router_default_secret, caller_secret
):
monkeypatch.setenv("AZURE_CLIENT_SECRET", "environment-secret")
router = Router(
model_list=[
@ -15119,12 +15133,12 @@ async def test_router_caller_repeating_the_secret_an_empty_router_default_select
"api_version": "2025-04-01-preview",
"tenant_id": "configured-tenant",
"client_id": "configured-client",
"client_secret": "configured-secret",
**deployment_secret,
},
"model_info": {"id": "configured-deployment"},
}
],
default_litellm_params={"client_secret": ""},
default_litellm_params={"client_secret": router_default_secret},
)
with pytest.raises(litellm.AuthenticationError):
@ -15138,7 +15152,7 @@ async def test_router_caller_repeating_the_secret_an_empty_router_default_select
model="gpt-4.1",
)
),
client_secret="environment-secret",
client_secret=caller_secret,
)
with pytest.raises(litellm.RouterRateLimitError) as excinfo:
@ -20514,3 +20528,161 @@ def test_registry_finds_a_team_deployment_by_public_and_internal_name_for_its_te
assert [row["model_info"]["id"] for row in registry.deployments(internal_name, team_id="team-a")] == ["team-row"]
assert list(registry.deployments("public-chat", team_id="team-b")) == []
assert list(registry.deployments("public-chat")) == []
_ROUTER_DEFAULT_REGION: Final = "us-east-1"
_BEDROCK_URL: Final = r"https://bedrock-runtime\.[a-z0-9-]+\.amazonaws\.com/model/.+"
_BEDROCK_CONVERSE_REPLY: Final = {
"output": {"message": {"role": "assistant", "content": [{"text": "hi"}]}},
"stopReason": "end_turn",
"usage": {"inputTokens": 1, "outputTokens": 1, "totalTokens": 2},
}
_REGION_PRECEDENCE: Final = pytest.mark.parametrize(
("deployment_region", "request_region", "called_region"),
[
("eu-west-1", None, "eu-west-1"),
(None, None, _ROUTER_DEFAULT_REGION),
("eu-west-1", "ap-northeast-1", "ap-northeast-1"),
],
ids=["deployment-over-router-default", "router-default-fills-unset", "request-over-both"],
)
def _bedrock_router_with_a_default_region(model: str, deployment_region: str | None) -> Router:
region: Final = {} if deployment_region is None else {"aws_region_name": deployment_region}
return Router(
model_list=[
{
"model_name": "regional",
"litellm_params": {
"model": model,
"aws_access_key_id": "unit-access-key",
"aws_secret_access_key": "unit-secret-key",
**region,
},
}
],
default_litellm_params={"aws_region_name": _ROUTER_DEFAULT_REGION},
num_retries=0,
)
def _request_region(request_region: str | None) -> Mapping[str, str]:
return {} if request_region is None else {"aws_region_name": request_region}
def _called_regions(route: respx.Route) -> list[str]:
return [call.request.url.host.split(".")[1] for call in route.calls]
@pytest.mark.asyncio
@pytest.mark.usefixtures("_router_httpx_transport")
@_REGION_PRECEDENCE
async def test_acompletion_calls_the_deployment_region_over_a_conflicting_router_default(
deployment_region: str | None, request_region: str | None, called_region: str
) -> None:
router: Final = _bedrock_router_with_a_default_region(
"bedrock/converse/anthropic.claude-haiku-5-5", deployment_region
)
with respx.mock(assert_all_called=True) as upstream:
route: Final = upstream.post(url__regex=_BEDROCK_URL).respond(json=_BEDROCK_CONVERSE_REPLY)
await router.acompletion(
model="regional", messages=[{"role": "user", "content": "Say hi"}], **_request_region(request_region)
)
assert _called_regions(route) == [called_region]
@_REGION_PRECEDENCE
def test_text_completion_calls_the_deployment_region_over_a_conflicting_router_default(
deployment_region: str | None, request_region: str | None, called_region: str
) -> None:
router: Final = _bedrock_router_with_a_default_region(
"bedrock/converse/anthropic.claude-haiku-5-5", deployment_region
)
with respx.mock(assert_all_called=True) as upstream:
route: Final = upstream.post(url__regex=_BEDROCK_URL).respond(json=_BEDROCK_CONVERSE_REPLY)
router.text_completion(model="regional", prompt="Say hi", **_request_region(request_region))
assert _called_regions(route) == [called_region]
@pytest.mark.asyncio
@pytest.mark.usefixtures("_router_httpx_transport")
@_REGION_PRECEDENCE
async def test_aembedding_calls_the_deployment_region_over_a_conflicting_router_default(
deployment_region: str | None, request_region: str | None, called_region: str
) -> None:
router: Final = _bedrock_router_with_a_default_region("bedrock/amazon.titan-embed-text-v2:0", deployment_region)
with respx.mock(assert_all_called=True) as upstream:
route: Final = upstream.post(url__regex=_BEDROCK_URL).respond(
json={"embedding": [0.1, 0.2], "inputTextTokenCount": 2}
)
await router.aembedding(model="regional", input=["Say hi"], **_request_region(request_region))
assert _called_regions(route) == [called_region]
def test_text_completion_calls_the_provider_with_the_deployment_model_not_the_group_name() -> None:
router: Final = Router(
model_list=[
{
"model_name": "text-group",
"litellm_params": {
"model": "text-completion-openai/unit-text-model",
"api_base": "https://sync-text-group.invalid/v1",
"api_key": "unit-provider-key",
},
}
],
num_retries=0,
)
with respx.mock(assert_all_called=True) as upstream:
provider: Final = upstream.post("https://sync-text-group.invalid/v1/completions").respond(
200,
json={
"id": "cmpl-unit",
"object": "text_completion",
"created": 1,
"model": "unit-text-model",
"choices": [{"text": "provider called", "index": 0, "finish_reason": "stop", "logprobs": None}],
"usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2},
},
)
response: Final = router.text_completion(model="text-group", prompt="Say hi")
assert response.choices[0].text == "provider called"
assert json.loads(provider.calls.last.request.content)["model"] == "unit-text-model"
@pytest.mark.asyncio
@pytest.mark.usefixtures("_router_httpx_transport")
@pytest.mark.parametrize(
("deployment_params", "provider_hits"),
[({"max_retries": 1}, 2), ({"max_retries": 3}, 4)],
ids=["deployment-max-retries-1", "deployment-max-retries-3"],
)
async def test_aembedding_retries_at_the_provider_as_often_as_the_deployment_sets(
deployment_params: Mapping[str, int], provider_hits: int
) -> None:
router: Final = Router(
model_list=[
{
"model_name": "embed",
"litellm_params": {"model": "openai/text-embedding-3-large", "api_key": "sk-test", **deployment_params},
}
],
num_retries=0,
)
with respx.mock(assert_all_called=True) as upstream:
route: Final = upstream.post("https://api.openai.com/v1/embeddings").respond(
500, headers={"retry-after-ms": "1"}, json={"error": {"message": "overloaded", "type": "server_error"}}
)
with pytest.raises(litellm.InternalServerError):
await router.aembedding(model="embed", input=["Say hi"])
assert len(route.calls) == provider_hits