From 07fe19bb15604abe4eae568ed3499e8fa7e8a583 Mon Sep 17 00:00:00 2001 From: Nate Armstrong Date: Thu, 12 Nov 2026 17:24:23 -0800 Subject: [PATCH] fix(router): validate request fields and ignore scalar fallback overrides (#51502) * fix(router): coerce request fallback overrides and reject invalid values Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(router): validate request fields and ignore scalar fallback overrides Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * refactor(router): centralize fallback filtering in request model Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: nate Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/router_rust/core.py | 110 +++++++++++----- tests/unit/test_router/core/test_fallbacks.py | 123 ++++++++++++++++++ .../core/test_fallbacks_characterization.py | 7 +- 3 files changed, 207 insertions(+), 33 deletions(-) diff --git a/litellm/router_rust/core.py b/litellm/router_rust/core.py index 8d5778b8a63..fbb07ea7553 100644 --- a/litellm/router_rust/core.py +++ b/litellm/router_rust/core.py @@ -29,9 +29,9 @@ from functools import partial from time import perf_counter from time import time as unix_time from types import MappingProxyType -from typing import TYPE_CHECKING, Final, Literal, Protocol, TypeAlias, TypeVar, final, overload +from typing import TYPE_CHECKING, Annotated, Final, Literal, Protocol, TypeAlias, TypeVar, final, overload -from pydantic import TypeAdapter +from pydantic import BaseModel, ConfigDict, Field, TypeAdapter, ValidationError, field_validator, model_validator from litellm._internal_context import service_target from litellm.constants import DEFAULT_COOLDOWN_TIME_SECONDS, HEALTH_CHECKS_TARGET @@ -1007,10 +1007,6 @@ def _listing_team_id(request: Mapping[str, object]) -> str | None: return team_id if isinstance(team_id, str) else None -# Request fallback settings, which replace the router's for the call. -_REQUEST_FALLBACK_KEYS: Final = ("fallbacks", "context_window_fallbacks", "content_policy_fallbacks", "max_fallbacks") - - def _strings(value: object) -> "tuple[str, ...]": """Every string in ``value``, a name or a fallback list of names and dict entries.""" if isinstance(value, str): @@ -1018,7 +1014,7 @@ def _strings(value: object) -> "tuple[str, ...]": if isinstance(value, Mapping): entries: Final[Mapping[object, object]] = value # pyright: ignore[reportUnknownVariableType] # read as objects return tuple(itertools.chain.from_iterable(map(_strings, itertools.chain.from_iterable(entries.items())))) - if isinstance(value, list | tuple): + if isinstance(value, Sequence): items: Final[Sequence[object]] = value # pyright: ignore[reportUnknownVariableType] # see above return tuple(itertools.chain.from_iterable(map(_strings, items))) # pyright: ignore[reportUnknownArgumentType] # read as objects return () @@ -1030,31 +1026,82 @@ def _routed_model(request: Mapping[str, object]) -> object: return model if model is None or isinstance(model, str) else str(model) -def _start_fields(request: Mapping[str, object]) -> Mapping[str, object]: - """The request fields the Rust core routes on.""" - fields: Final = { - **request_retry_fields(request.get("num_retries")), - **({"cooldown_time": request["cooldown_time"]} if request.get("cooldown_time") is not None else {}), - } - # `_common_checks_available_deployment`: only True names a litellm model. - specific: Final = {"specific_deployment": True} if request.get("specific_deployment") is True else {} - # `record_disable_fallbacks(kwargs, disable_fallbacks is True)`: only True turns them off. - disabled: Final = {"disable_fallbacks": True} if request.get("disable_fallbacks") is True else {} - lists: Final = {key: _plain(request[key]) for key in _REQUEST_FALLBACK_KEYS if key in request} - weights: Final = request.get("_router_weights") - team_id: Final = get_request_team_id(request) - listing_team_id: Final = _listing_team_id(request) +class RequestStartFields(BaseModel): + model_config = ConfigDict(frozen=True) + + fallbacks: Sequence[str | Mapping[str, object]] | None = None + context_window_fallbacks: Sequence[str | Mapping[str, object]] | None = None + content_policy_fallbacks: Sequence[str | Mapping[str, object]] | None = None + max_fallbacks: Annotated[int, Field(ge=0, le=_U32_MAX)] | None = None + specific_deployment: bool = False + disable_fallbacks: bool = False + cooldown_time: object | None = None + router_weights: object | None = Field(default=None, validation_alias="_router_weights") + team_id: str | None = None + listing_team_id: str | None = None + routing_strategy: str | None = None + + @model_validator(mode="before") + @classmethod + def ignore_standalone_fallback_strings(cls, value: object) -> object: + request_fields: Final = as_str_mapping(value) + if request_fields is None: + return value + return { + key: item + for key, item in request_fields.items() + if key not in ("fallbacks", "context_window_fallbacks", "content_policy_fallbacks") + or not isinstance(item, str) + } + + @field_validator("specific_deployment", "disable_fallbacks", mode="before") + @classmethod + def literal_true_only(cls, value: object) -> bool: + return value is True + + +def _request_start_fields(request: Mapping[str, object]) -> RequestStartFields: strategy: Final = request.get("routing_strategy") override: Final = normalize_strategy(strategy) if isinstance(strategy, str | RoutingStrategy) else None + fields: Final = { + **request, + "team_id": get_request_team_id(request), + "listing_team_id": _listing_team_id(request), + "routing_strategy": override, + } + try: + return RequestStartFields.model_validate(fields) + except ValidationError: + import litellm.exceptions as litellm_exceptions + + model: Final = request.get("model") + raise litellm_exceptions.BadRequestError( + message="Invalid router request fields", + model=model if isinstance(model, str) else None, + llm_provider="", + ) from None + + +def _start_fields(request: Mapping[str, object]) -> Mapping[str, object]: + """The request fields the Rust core routes on.""" + normalized: Final = _request_start_fields(request) + excluded: Final = { + name: True + for name, omit in ( + ("specific_deployment", not normalized.specific_deployment), + ("disable_fallbacks", not normalized.disable_fallbacks), + ("cooldown_time", normalized.cooldown_time is None), + ("router_weights", normalized.router_weights is None), + ("team_id", not normalized.team_id), + ("listing_team_id", normalized.listing_team_id is None), + ("routing_strategy", normalized.routing_strategy is None), + ) + if omit + } + request_fields: Final = normalized.model_dump(exclude_unset=True, exclude=excluded) return { - **fields, - **specific, - **({"routing_strategy": override} if override is not None else {}), - **disabled, - **({"router_weights": weights} if weights is not None else {}), - **lists, - **({"team_id": team_id} if team_id else {}), - **({"listing_team_id": listing_team_id} if listing_team_id is not None else {}), + **request_retry_fields(request.get("num_retries")), + **request_fields, **caller_fields(request), } @@ -2849,11 +2896,14 @@ class RustCore( """The model names a retry policy lookup may run on (each hop looks up the name it calls): the requested one and every name in the fallback lists the walk may take.""" resolved: Final = self._resolved + normalized: Final = _request_start_fields(request) lists: Final = ( resolved.fallbacks, resolved.context_window_fallbacks, resolved.content_policy_fallbacks, - *(request.get(key) for key in _REQUEST_FALLBACK_KEYS), + normalized.fallbacks, + normalized.context_window_fallbacks, + normalized.content_policy_fallbacks, ) return frozenset((*_strings(request.get("model")), *_strings(lists))) diff --git a/tests/unit/test_router/core/test_fallbacks.py b/tests/unit/test_router/core/test_fallbacks.py index ef51634b920..5f7235da27c 100644 --- a/tests/unit/test_router/core/test_fallbacks.py +++ b/tests/unit/test_router/core/test_fallbacks.py @@ -91,6 +91,51 @@ async def test_a_failed_group_falls_back_to_the_chain_from_the_winning_source( assert headers(response)["x-litellm-model-group"] == expected[-1] +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("field", "error"), + [ + ("fallbacks", rate_limit), + ("context_window_fallbacks", context_window), + ("content_policy_fallbacks", content_policy), + ], + ids=["fallbacks", "context-window", "content-policy"], +) +async def test_request_fallback_string_is_ignored_and_router_chain_is_used( + make_core: MakeCore, host: ScriptedHost, field: str, error: Callable[[], Exception] +) -> None: + core = build(make_core, **{field: [{"a": ["b"]}]}) + host.answer("openai/a", error()) + + response = await complete(core, **{field: "unconfigured-scalar-model"}) + + assert host.deployments_called("acompletion") == ["a", "b"] + assert headers(response)["x-litellm-model-group"] == "b" + + +@pytest.mark.parametrize( + ("field", "error"), + [ + ("fallbacks", rate_limit), + ("context_window_fallbacks", context_window), + ("content_policy_fallbacks", content_policy), + ], + ids=["fallbacks", "context-window", "content-policy"], +) +def test_sync_request_fallback_string_is_ignored_and_router_chain_is_used( + make_core: MakeCore, host: ScriptedHost, field: str, error: Callable[[], Exception] +) -> None: + core = build(make_core, **{field: [{"a": ["b"]}]}) + host.answer("openai/a", error()) + + response: Final = core.completion( # pyright: ignore[reportAttributeAccessIssue] # endpoint method + model="a", messages=MESSAGES, **{field: "unconfigured-scalar-model"} + ) + + assert host.deployments_called("completion") == ["a", "b"] + assert headers(response)["x-litellm-model-group"] == "b" + + @pytest.mark.asyncio async def test_an_empty_request_fallback_list_overrides_the_constructor_chain( make_core: MakeCore, host: ScriptedHost @@ -104,6 +149,19 @@ async def test_an_empty_request_fallback_list_overrides_the_constructor_chain( assert host.deployments_called("acompletion") == ["a"] +@pytest.mark.asyncio +async def test_a_null_request_fallback_list_overrides_the_constructor_chain( + make_core: MakeCore, host: ScriptedHost +) -> None: + core = build(make_core, fallbacks=[{"a": ["b"]}]) + host.answer("openai/a", rate_limit()) + + with pytest.raises(litellm.RateLimitError): + await complete(core, fallbacks=None) + + assert host.deployments_called("acompletion") == ["a"] + + @pytest.mark.asyncio @pytest.mark.parametrize("field", ["assign", "settings"]) async def test_fallbacks_applied_by_changeset_take_effect(make_core: MakeCore, host: ScriptedHost, field: str) -> None: @@ -300,6 +358,71 @@ async def test_max_fallbacks_applied_by_changeset_bounds_the_walk(make_core: Mak assert host.deployments_called("acompletion") == ["a", "b"] +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("value", "expected"), + [ + (True, ["a", "b"]), + (False, ["a"]), + (1.0, ["a", "b"]), + ("1", ["a", "b"]), + ("1.0", ["a", "b"]), + (0, ["a"]), + ], + ids=["true", "false", "integral-float", "integer-string", "decimal-string", "zero"], +) +async def test_request_max_fallbacks_uses_bounded_integer_coercion( + make_core: MakeCore, host: ScriptedHost, value: object, expected: list[str] +) -> None: + core = build(make_core, model_list=GROUPS_ABCD, fallbacks=CHAIN_ABCD) + host.answer("*", rate_limit()) + + with pytest.raises(litellm.RateLimitError): + await complete(core, max_fallbacks=value) + + assert host.deployments_called("acompletion") == expected + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("field", "value"), + [ + ("fallbacks", 3), + ("fallbacks", ["c", 1]), + ("context_window_fallbacks", [None]), + ("content_policy_fallbacks", [True]), + ("max_fallbacks", "not-an-integer"), + ("max_fallbacks", -1), + ("max_fallbacks", 2**32), + ("max_fallbacks", 1.5), + ("max_fallbacks", float("nan")), + ("max_fallbacks", float("inf")), + ], + ids=[ + "fallbacks-not-a-list", + "fallbacks-invalid-entry", + "context-window-invalid-entry", + "content-policy-invalid-entry", + "max-fallbacks-invalid-string", + "max-fallbacks-negative", + "max-fallbacks-too-large", + "max-fallbacks-fractional", + "max-fallbacks-nan", + "max-fallbacks-infinite", + ], +) +async def test_invalid_request_fallback_fields_raise_bad_request( + make_core: MakeCore, host: ScriptedHost, field: str, value: object +) -> None: + core = build(make_core) + + with pytest.raises(litellm.BadRequestError) as raised: + await complete(core, **{field: value}) + + assert raised.value.status_code == 400 + assert host.deployments_called("acompletion") == [] + + @pytest.mark.asyncio async def test_a_fallback_group_with_no_deployments_is_passed_over(make_core: MakeCore, host: ScriptedHost) -> None: core = build(make_core, fallbacks=[{"a": ["missing", "c"]}]) diff --git a/tests/unit/test_router/core/test_fallbacks_characterization.py b/tests/unit/test_router/core/test_fallbacks_characterization.py index 41a751358a9..3e6c1f3666a 100644 --- a/tests/unit/test_router/core/test_fallbacks_characterization.py +++ b/tests/unit/test_router/core/test_fallbacks_characterization.py @@ -248,13 +248,14 @@ async def test_the_request_fallbacks_kwarg_is_not_forwarded_to_the_provider( @pytest.mark.asyncio -async def test_disable_fallbacks_given_as_a_string_does_not_disable_them( - make_core: MakeCore, host: ScriptedHost +@pytest.mark.parametrize("value", ["true", 1, False], ids=["string", "integer", "false"]) +async def test_disable_fallbacks_requires_literal_true( + make_core: MakeCore, host: ScriptedHost, value: object ) -> None: core = build(make_core, fallbacks=[{"a": ["b"]}]) host.answer("openai/a", rate_limit()) - await complete(core, disable_fallbacks="true") + await complete(core, disable_fallbacks=value) assert host.deployments_called("acompletion") == ["a", "b"]