Merge pull request #26968 from stuxf/chore/router-override-trust

chore(proxy): tighten router-settings-override and mock-testing trust

litellm/proxy/auth/user_api_key_auth.py

@@ -2284,20 +2284,69 @@ async def _enforce_key_and_fallback_model_access(
                 llm_router=llm_router,
             )
 
-        if fallback_models is not None:
-            for m in fallback_models:
-                await can_key_call_model(
-                    model=m["model"] if isinstance(m, dict) else m,
-                    llm_model_list=llm_model_list,
-                    valid_token=valid_token,
-                    llm_router=llm_router,
-                )
-                await is_valid_fallback_model(
-                    model=m["model"] if isinstance(m, dict) else m,
-                    llm_router=llm_router,
-                    user_model=None,
+        # Validate every fallback model name reachable by this request.
+        # All three fields (``fallbacks``, ``context_window_fallbacks``,
+        # ``content_policy_fallbacks``) are forwarded to the router as
+        # per-request kwargs whether they appear at the top level of
+        # ``request_data`` or nested under ``router_settings_override``.
+        # Both surfaces must be validated against the API key's model
+        # allowlist or a caller can smuggle a restricted model. VERIA-44.
+        fallback_names: List[str] = []
+        override_settings = request_data.get("router_settings_override")
+        for _fb_key in ROUTER_FALLBACK_FIELDS:
+            fallback_names.extend(
+                iter_router_fallback_model_names(request_data.get(_fb_key))
+            )
+            if isinstance(override_settings, dict):
+                fallback_names.extend(
+                    iter_router_fallback_model_names(override_settings.get(_fb_key))
                 )
 
+        for _name in dict.fromkeys(fallback_names):  # dedupe, preserve order
+            await can_key_call_model(
+                model=_name,
+                llm_model_list=llm_model_list,
+                valid_token=valid_token,
+                llm_router=llm_router,
+            )
+            await is_valid_fallback_model(
+                model=_name,
+                llm_router=llm_router,
+                user_model=None,
+            )
+
+
+ROUTER_FALLBACK_FIELDS: Tuple[str, ...] = (
+    "fallbacks",
+    "context_window_fallbacks",
+    "content_policy_fallbacks",
+)
+
+
+def iter_router_fallback_model_names(fallbacks: Any) -> Iterator[str]:
+    """Yield leaf model names from any of the supported fallbacks shapes.
+
+    Handles the simple top-level shape (``str`` or ``{"model": str}``) and
+    the nested router-config shape (``[{primary: [fallback_list]}]``).
+    """
+    if not isinstance(fallbacks, list):
+        return
+    for entry in fallbacks:
+        if isinstance(entry, str):
+            yield entry
+        elif isinstance(entry, dict):
+            if isinstance(entry.get("model"), str):
+                yield entry["model"]
+                continue
+            for fallback_list in entry.values():
+                if not isinstance(fallback_list, list):
+                    continue
+                for m in fallback_list:
+                    if isinstance(m, str):
+                        yield m
+                    elif isinstance(m, dict) and isinstance(m.get("model"), str):
+                        yield m["model"]
+
 
 async def _run_post_custom_auth_checks(
     valid_token: UserAPIKeyAuth,

litellm/proxy/route_llm_request.py

@@ -6,6 +6,18 @@ from fastapi import HTTPException, status
 import litellm
 from litellm.proxy._types import UserAPIKeyAuth
 
+# Router-internal mock_testing_* flag names — kept in sync with
+# ``litellm.types.router.MockRouterTestingParams`` by the test
+# ``test_mock_testing_kwarg_names_matches_dataclass``. Hardcoding (rather
+# than deriving via ``dataclasses.fields(MockRouterTestingParams)`` at
+# import time) avoids a cyclic import: ``litellm.types.router`` imports
+# back into proxy modules before this module finishes loading.
+_MOCK_TESTING_KWARG_NAMES: tuple = (
+    "mock_testing_fallbacks",
+    "mock_testing_context_fallbacks",
+    "mock_testing_content_policy_fallbacks",
+)
+
 if TYPE_CHECKING:
     from litellm.router import Router as _Router
 
@@ -322,6 +334,13 @@ async def route_request(  # noqa: PLR0915 - Complex routing function, refactorin
     """
     await add_shared_session_to_data(data)
 
+    # Strip router-internal mock_testing_* flags. Combined with an
+    # unauthorized fallback in ``router_settings_override`` they let a
+    # caller deterministically execute requests against restricted
+    # models. VERIA-44.
+    for _key in _MOCK_TESTING_KWARG_NAMES:
+        data.pop(_key, None)
+
     team_id = get_team_id_from_data(data)
     router_model_names = llm_router.model_names if llm_router is not None else []
 

tests/test_litellm/proxy/auth/test_router_override_fallback_auth.py

@@ -0,0 +1,233 @@
+"""
+VERIA-44: ``router_settings_override.fallbacks`` must be validated
+against the API key's model allowlist at auth time. Without this, the
+override is promoted to per-request kwargs after auth and lets a caller
+execute requests against models their API key cannot call.
+"""
+
+from typing import List
+from unittest.mock import AsyncMock, patch
+
+import pytest
+
+from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth
+from litellm.proxy.auth.user_api_key_auth import (
+    _enforce_key_and_fallback_model_access,
+    iter_router_fallback_model_names,
+)
+
+
+def _key_with_models(models: List[str]) -> UserAPIKeyAuth:
+    return UserAPIKeyAuth(
+        api_key="hashed",
+        user_id="caller",
+        user_role=LitellmUserRoles.INTERNAL_USER,
+        models=models,
+    )
+
+
+# ── iter_router_fallback_model_names ─────────────────────────────────────────
+
+
+def testiter_router_fallback_model_names_router_config_shape():
+    """Router-config shape: ``[{primary: [fallback_list]}]``."""
+    assert list(
+        iter_router_fallback_model_names(
+            [{"gpt-3.5-turbo": ["gpt-4", "claude-3"]}, {"gpt-4o": ["o1"]}]
+        )
+    ) == ["gpt-4", "claude-3", "o1"]
+
+
+def testiter_router_fallback_model_names_simple_string_shape():
+    """Simple top-level shape: list of strings."""
+    assert list(iter_router_fallback_model_names(["gpt-4", "claude-3"])) == [
+        "gpt-4",
+        "claude-3",
+    ]
+
+
+def testiter_router_fallback_model_names_client_side_shape():
+    """ClientSideFallbackModel shape: ``[{"model": "..."}]``."""
+    assert list(
+        iter_router_fallback_model_names([{"model": "gpt-4"}, {"model": "claude-3"}])
+    ) == ["gpt-4", "claude-3"]
+
+
+def testiter_router_fallback_model_names_empty_or_none():
+    assert list(iter_router_fallback_model_names(None)) == []
+    assert list(iter_router_fallback_model_names([])) == []
+    assert list(iter_router_fallback_model_names("not a list")) == []
+
+
+# ── _enforce_key_and_fallback_model_access ────────────────────────────────────
+
+
+@pytest.mark.asyncio
+async def test_router_override_fallbacks_validated_against_key_allowlist():
+    """A fallback nested inside ``router_settings_override`` is validated
+    against the API key's allowed models — not just the top-level
+    ``fallbacks`` field."""
+    valid_token = _key_with_models(["gpt-3.5-turbo"])
+    request_data = {
+        "model": "gpt-3.5-turbo",
+        "router_settings_override": {
+            "fallbacks": [{"gpt-3.5-turbo": ["unauthorized-model"]}],
+        },
+    }
+
+    seen_models: List[str] = []
+
+    async def fake_can_key_call_model(model, llm_model_list, valid_token, llm_router):
+        seen_models.append(model)
+
+    with (
+        patch(
+            "litellm.proxy.auth.user_api_key_auth.can_key_call_model",
+            side_effect=fake_can_key_call_model,
+        ),
+        patch(
+            "litellm.proxy.auth.user_api_key_auth.is_valid_fallback_model",
+            new=AsyncMock(),
+        ),
+    ):
+        await _enforce_key_and_fallback_model_access(
+            valid_token=valid_token,
+            request_data=request_data,
+            route="/v1/chat/completions",
+            llm_model_list=None,
+            llm_router=None,
+        )
+
+    # Both the primary model and the override-nested fallback must be
+    # checked against the API key's allowlist.
+    assert "gpt-3.5-turbo" in seen_models
+    assert "unauthorized-model" in seen_models
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    "fallback_field",
+    [
+        "fallbacks",
+        "context_window_fallbacks",
+        "content_policy_fallbacks",
+    ],
+)
+async def test_router_override_all_fallback_fields_validated(fallback_field):
+    """All three fallback fields the router accepts as per-request kwargs
+    are validated — context_window_fallbacks and content_policy_fallbacks
+    are promoted in route_llm_request.py too."""
+    valid_token = _key_with_models(["gpt-3.5-turbo"])
+    request_data = {
+        "model": "gpt-3.5-turbo",
+        "router_settings_override": {
+            fallback_field: [{"gpt-3.5-turbo": ["smuggled-model"]}],
+        },
+    }
+
+    seen: List[str] = []
+
+    async def fake_can_key_call_model(model, llm_model_list, valid_token, llm_router):
+        seen.append(model)
+
+    with (
+        patch(
+            "litellm.proxy.auth.user_api_key_auth.can_key_call_model",
+            side_effect=fake_can_key_call_model,
+        ),
+        patch(
+            "litellm.proxy.auth.user_api_key_auth.is_valid_fallback_model",
+            new=AsyncMock(),
+        ),
+    ):
+        await _enforce_key_and_fallback_model_access(
+            valid_token=valid_token,
+            request_data=request_data,
+            route="/v1/chat/completions",
+            llm_model_list=None,
+            llm_router=None,
+        )
+
+    assert "smuggled-model" in seen
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    "fallback_field",
+    [
+        "fallbacks",
+        "context_window_fallbacks",
+        "content_policy_fallbacks",
+    ],
+)
+async def test_top_level_fallback_fields_validated(fallback_field):
+    """All three top-level fallback fields are forwarded to the router as
+    per-request kwargs, so all three must be validated against the API
+    key's allowlist. Greptile P1 follow-up: previously only the
+    ``fallbacks`` field was walked at the top level."""
+    valid_token = _key_with_models(["gpt-3.5-turbo"])
+    request_data = {
+        "model": "gpt-3.5-turbo",
+        fallback_field: [{"gpt-3.5-turbo": ["top-level-smuggled"]}],
+    }
+
+    seen: List[str] = []
+
+    async def fake_can_key_call_model(model, llm_model_list, valid_token, llm_router):
+        seen.append(model)
+
+    with (
+        patch(
+            "litellm.proxy.auth.user_api_key_auth.can_key_call_model",
+            side_effect=fake_can_key_call_model,
+        ),
+        patch(
+            "litellm.proxy.auth.user_api_key_auth.is_valid_fallback_model",
+            new=AsyncMock(),
+        ),
+    ):
+        await _enforce_key_and_fallback_model_access(
+            valid_token=valid_token,
+            request_data=request_data,
+            route="/v1/chat/completions",
+            llm_model_list=None,
+            llm_router=None,
+        )
+
+    assert "top-level-smuggled" in seen
+
+
+@pytest.mark.asyncio
+async def test_router_override_without_fallbacks_does_not_break_auth():
+    """``router_settings_override`` set without any fallback fields is a
+    no-op for the auth check — only the primary model is validated."""
+    valid_token = _key_with_models(["gpt-3.5-turbo"])
+    request_data = {
+        "model": "gpt-3.5-turbo",
+        "router_settings_override": {"num_retries": 3, "timeout": 30},
+    }
+
+    seen: List[str] = []
+
+    async def fake_can_key_call_model(model, llm_model_list, valid_token, llm_router):
+        seen.append(model)
+
+    with (
+        patch(
+            "litellm.proxy.auth.user_api_key_auth.can_key_call_model",
+            side_effect=fake_can_key_call_model,
+        ),
+        patch(
+            "litellm.proxy.auth.user_api_key_auth.is_valid_fallback_model",
+            new=AsyncMock(),
+        ),
+    ):
+        await _enforce_key_and_fallback_model_access(
+            valid_token=valid_token,
+            request_data=request_data,
+            route="/v1/chat/completions",
+            llm_model_list=None,
+            llm_router=None,
+        )
+
+    assert seen == ["gpt-3.5-turbo"]

tests/test_litellm/proxy/test_route_llm_request.py

@@ -241,6 +241,53 @@ async def test_route_request_with_router_settings_override_preserves_existing():
     assert call_kwargs["timeout"] == 30
 
 
+def test_mock_testing_kwarg_names_matches_dataclass():
+    """``_MOCK_TESTING_KWARG_NAMES`` is hardcoded to avoid a cyclic import
+    against ``litellm.types.router``. This test guards against drift —
+    if a new ``mock_testing_*`` field is added to ``MockRouterTestingParams``
+    the strip list must be updated to keep covering it."""
+    from dataclasses import fields
+
+    from litellm.proxy.route_llm_request import _MOCK_TESTING_KWARG_NAMES
+    from litellm.types.router import MockRouterTestingParams
+
+    assert set(_MOCK_TESTING_KWARG_NAMES) == {
+        f.name for f in fields(MockRouterTestingParams)
+    }
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    "mock_flag",
+    [
+        "mock_testing_fallbacks",
+        "mock_testing_context_fallbacks",
+        "mock_testing_content_policy_fallbacks",
+    ],
+)
+async def test_route_request_strips_mock_testing_flags(mock_flag):
+    """VERIA-44: router-internal testing flags must not survive a
+    user-supplied request body. Without this strip, an attacker can
+    combine ``mock_testing_fallbacks=true`` with an unauthorized fallback
+    in ``router_settings_override`` to deterministically execute requests
+    against restricted models."""
+    data = {
+        "model": "gpt-3.5-turbo",
+        "messages": [{"role": "user", "content": "Hello"}],
+        mock_flag: True,
+    }
+    llm_router = MagicMock()
+    llm_router.acompletion.return_value = "ok"
+
+    await route_request(data, llm_router, None, "acompletion")
+
+    call_kwargs = llm_router.acompletion.call_args[1]
+    assert mock_flag not in call_kwargs
+    # The flag is also gone from the original data dict so any subsequent
+    # processing (e.g. logging) doesn't see it either.
+    assert mock_flag not in data
+
+
 @pytest.mark.parametrize(
     "route_type", ["agenerate_content", "agenerate_content_stream"]
 )