Merge pull request #35423 from BerriAI/litellm_/mock-testing-feature-flag-6e30ad

feat(proxy)!: gate all mock testing request params behind a single config flag

litellm/proxy/proxy_server.py

@@ -1041,6 +1041,8 @@ async def proxy_startup_event(app: FastAPI):
         redis_usage_cache=transaction_buffer_redis_cache,
     )
 
+    ProxyStartupEvent._warn_if_mock_testing_params_enabled(general_settings=general_settings)
+
     ## SEMANTIC TOOL FILTER ##
     # Read litellm_settings from config for semantic filter initialization
     try:
@@ -7725,6 +7727,37 @@ class ProxyStartupEvent:
 
         proxy_logging_obj.startup_event(llm_router=llm_router, redis_usage_cache=redis_usage_cache)
 
+    @staticmethod
+    def _warn_if_mock_testing_params_enabled(general_settings: dict) -> None:
+        """Announce, loudly, that any caller may inject synthetic failures."""
+        from litellm.proxy.route_llm_request import (
+            GATED_MOCK_PARAM_NAMES,
+            MOCK_TESTING_CONFIG_KEY,
+        )
+
+        if general_settings.get(MOCK_TESTING_CONFIG_KEY, False) is not True:
+            return
+
+        verbose_proxy_logger.warning(
+            "\n%s\n"
+            " DANGEROUS SETTING ENABLED\n"
+            " general_settings.%s = true\n"
+            "\n"
+            " Any caller with a valid key on this proxy can now inject synthetic\n"
+            " failures and latency into their own requests using these body params:\n"
+            "%s\n"
+            "\n"
+            " A request using them consumes a connection and a concurrency slot\n"
+            " without reaching a provider, and returns an error the caller chose.\n"
+            "\n"
+            " Intended for testing fallback chains. Do not leave enabled.\n"
+            "%s",
+            "=" * 72,
+            MOCK_TESTING_CONFIG_KEY,
+            "\n".join(f"   {name}" for name in GATED_MOCK_PARAM_NAMES),
+            "=" * 72,
+        )
+
     @staticmethod
     def _validate_redis_transaction_buffer_config(
         general_settings: dict,

litellm/proxy/route_llm_request.py

@@ -8,18 +8,24 @@ import litellm
 from litellm.proxy._types import UserAPIKeyAuth
 from litellm.router_utils.common_utils import _is_proxy_admin_request
 
-# Router-internal mock_testing_* flag names — kept in sync with
-# ``litellm.types.router.MockRouterTestingParams`` by the test
-# ``test_mock_testing_kwarg_names_matches_dataclass``. Hardcoding (rather
-# than deriving via ``dataclasses.fields(MockRouterTestingParams)`` at
+# Client-supplied params that make the router or the call path fabricate a
+# failure or a delay instead of calling the provider. The ``mock_testing_*``
+# names are kept in sync with ``litellm.types.router.MockRouterTestingParams``
+# by ``test_gated_mock_params_cover_mock_router_testing_params``. Hardcoding
+# (rather than deriving via ``dataclasses.fields(MockRouterTestingParams)`` at
 # import time) avoids a cyclic import: ``litellm.types.router`` imports
 # back into proxy modules before this module finishes loading.
-_MOCK_TESTING_KWARG_NAMES: tuple = (
+GATED_MOCK_PARAM_NAMES: tuple[str, ...] = (
     "mock_testing_fallbacks",
     "mock_testing_context_fallbacks",
     "mock_testing_content_policy_fallbacks",
+    "mock_testing_rate_limit_error",
+    "mock_timeout",
+    "mock_delay",
 )
 
+MOCK_TESTING_CONFIG_KEY = "dangerously_allow_mock_testing_request_params"
+
 if TYPE_CHECKING:
     from litellm.router import Router as _Router
 
@@ -169,6 +175,41 @@ def raise_if_required_body_param_missing(route_type: str, data: Mapping[str, obj
     )
 
 
+class MockTestingParamsDisabledError(HTTPException):
+    def __init__(self, params: tuple[str, ...]):
+        super().__init__(
+            status_code=status.HTTP_400_BAD_REQUEST,
+            detail={  # mutable-ok: HTTPException.detail has no immutable form; same shape as the sibling errors here
+                "error": (
+                    f"Mock testing request params are disabled on this proxy: {', '.join(params)}. "
+                    f"An admin can enable them by setting `general_settings.{MOCK_TESTING_CONFIG_KEY}: true` "
+                    "in config.yaml. This setting cannot be changed from the Admin UI or the API."
+                )
+            },
+        )
+
+
+def raise_if_mock_testing_params_disallowed(data: Mapping[str, object], *, allowed: bool) -> None:
+    """Reject client-supplied mock testing params unless an admin opted in.
+
+    Rejecting (rather than silently dropping) keeps a request that asked for a
+    synthetic failure from returning a normal success, which reads as a passing
+    fallback test that never ran.
+    """
+    if allowed:
+        return
+    present = tuple(name for name in GATED_MOCK_PARAM_NAMES if name in data)
+    if present:
+        raise MockTestingParamsDisabledError(params=present)
+
+
+def mock_testing_params_allowed() -> bool:
+    """Read the opt-in from the running proxy's ``general_settings``."""
+    import litellm.proxy.proxy_server as proxy_server
+
+    return proxy_server.general_settings.get(MOCK_TESTING_CONFIG_KEY, False) is True
+
+
 def get_team_id_from_data(data: dict) -> Optional[str]:
     """
     Get the team id from the data's metadata or litellm_metadata params.
@@ -381,12 +422,7 @@ async def route_request(
 
     await add_shared_session_to_data(data)
 
-    # Strip router-internal mock_testing_* flags. Combined with an
-    # unauthorized fallback in ``router_settings_override`` they let a
-    # caller deterministically execute requests against restricted
-    # models. VERIA-44.
-    for _key in _MOCK_TESTING_KWARG_NAMES:
-        data.pop(_key, None)
+    raise_if_mock_testing_params_disallowed(data, allowed=mock_testing_params_allowed())
 
     data.pop("enable_tag_filtering", None)
 

tests/test_litellm/proxy/test_proxy_server.py

@@ -10685,3 +10685,83 @@ async def test_async_data_generator_forwards_usage_chunk_without_strip_marker():
     assert len(data_frames) == 4
     assert any('"usage"' in frame and '"completion_tokens":188' in frame.replace(" ", "") for frame in data_frames)
     assert frames[-1] == "data: [DONE]\n\n"
+
+
+@pytest.mark.asyncio
+async def test_config_field_update_rejects_mock_testing_flag():
+    """The mock-testing opt-in is deliberately absent from
+    ``ConfigGeneralSettings`` so that ``/config/field/update`` refuses it. If
+    someone later adds the field for tidiness, this test fails and tells them
+    they have just opened an API write path into a config-file-only setting."""
+    from fastapi import HTTPException
+
+    from litellm.proxy._types import ConfigFieldUpdate
+    from litellm.proxy.proxy_server import update_config_general_settings
+    from litellm.proxy.route_llm_request import MOCK_TESTING_CONFIG_KEY
+
+    admin = UserAPIKeyAuth(
+        user_role=LitellmUserRoles.PROXY_ADMIN,
+        api_key="sk-test",
+    )
+
+    with patch.object(proxy_server_module, "prisma_client", MagicMock()):
+        with pytest.raises(HTTPException) as exc_info:
+            await update_config_general_settings(
+                data=ConfigFieldUpdate(
+                    field_name=MOCK_TESTING_CONFIG_KEY,
+                    field_value=True,
+                    config_type="general_settings",
+                ),
+                user_api_key_dict=admin,
+            )
+
+    assert exc_info.value.status_code == 400
+
+
+def test_config_update_body_drops_mock_testing_flag():
+    """``/config/update`` parses its body as ``ConfigYAML``, whose
+    ``general_settings`` is a ``ConfigGeneralSettings``. Undeclared keys are
+    dropped on parse, so the flag never reaches the DB by that route either."""
+    from litellm.proxy._types import ConfigYAML
+    from litellm.proxy.route_llm_request import MOCK_TESTING_CONFIG_KEY
+
+    parsed = ConfigYAML.model_validate({"general_settings": {MOCK_TESTING_CONFIG_KEY: True}})
+
+    assert parsed.general_settings is not None
+    assert MOCK_TESTING_CONFIG_KEY not in parsed.general_settings.model_dump(exclude_none=True)
+
+
+def test_startup_warns_when_mock_testing_params_enabled(caplog):
+    """Enabling the opt-in must announce itself, naming every param it
+    unlocks — the config key says ``mock_testing`` but the gate also covers
+    ``mock_timeout`` and ``mock_delay``, so coverage cannot be inferred from
+    the name alone."""
+    import logging
+
+    from litellm.proxy.proxy_server import ProxyStartupEvent
+    from litellm.proxy.route_llm_request import (
+        GATED_MOCK_PARAM_NAMES,
+        MOCK_TESTING_CONFIG_KEY,
+    )
+
+    with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"):
+        ProxyStartupEvent._warn_if_mock_testing_params_enabled(
+            general_settings={MOCK_TESTING_CONFIG_KEY: True}
+        )
+
+    assert MOCK_TESTING_CONFIG_KEY in caplog.text
+    for param_name in GATED_MOCK_PARAM_NAMES:
+        assert param_name in caplog.text
+
+
+def test_startup_is_silent_when_mock_testing_params_disabled(caplog):
+    """A proxy that never set the opt-in must not emit the warning."""
+    import logging
+
+    from litellm.proxy.proxy_server import ProxyStartupEvent
+    from litellm.proxy.route_llm_request import MOCK_TESTING_CONFIG_KEY
+
+    with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"):
+        ProxyStartupEvent._warn_if_mock_testing_params_enabled(general_settings={})
+
+    assert MOCK_TESTING_CONFIG_KEY not in caplog.text

tests/test_litellm/proxy/test_route_llm_request.py

@@ -8,6 +8,8 @@ sys.path.insert(0, os.path.abspath("../../.."))  # Adds the parent directory to
 
 from unittest.mock import MagicMock
 
+from fastapi import HTTPException
+
 from litellm.proxy.route_llm_request import ProxyModelNotFoundError, route_request
 
 
@@ -471,38 +473,147 @@ async def test_route_request_with_router_settings_override_preserves_existing():
     assert call_kwargs["timeout"] == 30
 
 
-def test_mock_testing_kwarg_names_matches_dataclass():
-    """``_MOCK_TESTING_KWARG_NAMES`` is hardcoded to avoid a cyclic import
-    against ``litellm.types.router``. This test guards against drift —
-    if a new ``mock_testing_*`` field is added to ``MockRouterTestingParams``
-    the strip list must be updated to keep covering it."""
+def test_gated_mock_params_cover_mock_router_testing_params():
+    """``GATED_MOCK_PARAM_NAMES`` is hardcoded to avoid a cyclic import
+    against ``litellm.types.router``. This test guards against drift — if a
+    new ``mock_testing_*`` field is added to ``MockRouterTestingParams`` the
+    gate must be updated to keep covering it. The gate is a superset: it also
+    covers params consumed outside that dataclass."""
     from dataclasses import fields
 
-    from litellm.proxy.route_llm_request import _MOCK_TESTING_KWARG_NAMES
+    from litellm.proxy.route_llm_request import GATED_MOCK_PARAM_NAMES
     from litellm.types.router import MockRouterTestingParams
 
-    assert set(_MOCK_TESTING_KWARG_NAMES) == {f.name for f in fields(MockRouterTestingParams)}
+    assert {f.name for f in fields(MockRouterTestingParams)} <= set(GATED_MOCK_PARAM_NAMES)
+    assert {"mock_testing_rate_limit_error", "mock_timeout", "mock_delay"} <= set(GATED_MOCK_PARAM_NAMES)
 
 
-@pytest.mark.asyncio
 @pytest.mark.parametrize(
-    "mock_flag",
+    "mock_param",
     [
         "mock_testing_fallbacks",
         "mock_testing_context_fallbacks",
         "mock_testing_content_policy_fallbacks",
+        "mock_testing_rate_limit_error",
+        "mock_timeout",
+        "mock_delay",
     ],
 )
-async def test_route_request_strips_mock_testing_flags(mock_flag):
-    """VERIA-44: router-internal testing flags must not survive a
-    user-supplied request body. Without this strip, an attacker can
-    combine ``mock_testing_fallbacks=true`` with an unauthorized fallback
-    in ``router_settings_override`` to deterministically execute requests
-    against restricted models."""
+def test_mock_params_rejected_when_not_allowed(mock_param):
+    """Every gated param must be rejected by name when the proxy has not
+    opted in, and the error must point the caller at the config key."""
+    from litellm.proxy.route_llm_request import (
+        MOCK_TESTING_CONFIG_KEY,
+        raise_if_mock_testing_params_disallowed,
+    )
+
+    data = {"model": "gpt-3.5-turbo", mock_param: True}
+
+    with pytest.raises(HTTPException) as exc_info:
+        raise_if_mock_testing_params_disallowed(data, allowed=False)
+
+    assert exc_info.value.status_code == 400
+    error_message = exc_info.value.detail["error"]
+    assert mock_param in error_message
+    assert MOCK_TESTING_CONFIG_KEY in error_message
+
+
+@pytest.mark.parametrize(
+    "mock_param",
+    [
+        "mock_testing_fallbacks",
+        "mock_testing_context_fallbacks",
+        "mock_testing_content_policy_fallbacks",
+        "mock_testing_rate_limit_error",
+        "mock_timeout",
+        "mock_delay",
+    ],
+)
+def test_mock_params_pass_through_when_allowed(mock_param):
+    """With the opt-in set, gated params must survive untouched — a gate that
+    rejects correctly but strips anyway would leave the feature unusable."""
+    from litellm.proxy.route_llm_request import raise_if_mock_testing_params_disallowed
+
+    data = {"model": "gpt-3.5-turbo", mock_param: True}
+
+    raise_if_mock_testing_params_disallowed(data, allowed=True)
+
+    assert data[mock_param] is True
+
+
+def test_mock_param_gate_reports_every_param_present():
+    """A request carrying several gated params must name all of them, so a
+    caller fixing one is not surprised by the next."""
+    from litellm.proxy.route_llm_request import raise_if_mock_testing_params_disallowed
+
+    data = {
+        "model": "gpt-3.5-turbo",
+        "mock_testing_fallbacks": True,
+        "mock_delay": 30,
+    }
+
+    with pytest.raises(HTTPException) as exc_info:
+        raise_if_mock_testing_params_disallowed(data, allowed=False)
+
+    error_message = exc_info.value.detail["error"]
+    assert "mock_testing_fallbacks" in error_message
+    assert "mock_delay" in error_message
+
+
+def test_ordinary_request_is_not_rejected_by_the_mock_param_gate():
+    """The gate must not fire on a request that carries no gated param."""
+    from litellm.proxy.route_llm_request import raise_if_mock_testing_params_disallowed
+
     data = {
         "model": "gpt-3.5-turbo",
         "messages": [{"role": "user", "content": "Hello"}],
-        mock_flag: True,
+        "mock_response": "hi",
+    }
+
+    raise_if_mock_testing_params_disallowed(data, allowed=False)
+
+
+@pytest.mark.asyncio
+async def test_route_request_rejects_mock_params_by_default(monkeypatch):
+    """End-to-end through ``route_request``: with no opt-in configured the
+    request is rejected before it ever reaches the router."""
+    import litellm.proxy.proxy_server as proxy_server
+
+    monkeypatch.setattr(proxy_server, "general_settings", {}, raising=False)
+
+    data = {
+        "model": "gpt-3.5-turbo",
+        "messages": [{"role": "user", "content": "Hello"}],
+        "mock_testing_fallbacks": True,
+    }
+    llm_router = MagicMock()
+
+    with pytest.raises(HTTPException) as exc_info:
+        await route_request(data, llm_router, None, "acompletion")
+
+    assert exc_info.value.status_code == 400
+    llm_router.acompletion.assert_not_called()
+
+
+@pytest.mark.asyncio
+async def test_route_request_forwards_mock_params_when_opted_in(monkeypatch):
+    """End-to-end through ``route_request``: with the opt-in set the param
+    reaches the router, which is what makes a fallback drill possible."""
+    import litellm.proxy.proxy_server as proxy_server
+
+    from litellm.proxy.route_llm_request import MOCK_TESTING_CONFIG_KEY
+
+    monkeypatch.setattr(
+        proxy_server,
+        "general_settings",
+        {MOCK_TESTING_CONFIG_KEY: True},
+        raising=False,
+    )
+
+    data = {
+        "model": "gpt-3.5-turbo",
+        "messages": [{"role": "user", "content": "Hello"}],
+        "mock_testing_fallbacks": True,
     }
     llm_router = MagicMock()
     llm_router.acompletion.return_value = "ok"
@@ -510,10 +621,7 @@ async def test_route_request_strips_mock_testing_flags(mock_flag):
     await route_request(data, llm_router, None, "acompletion")
 
     call_kwargs = llm_router.acompletion.call_args[1]
-    assert mock_flag not in call_kwargs
-    # The flag is also gone from the original data dict so any subsequent
-    # processing (e.g. logging) doesn't see it either.
-    assert mock_flag not in data
+    assert call_kwargs["mock_testing_fallbacks"] is True
 
 
 @pytest.mark.parametrize("route_type", ["agenerate_content", "agenerate_content_stream"])