From 594d0d7a0a5513d31e280811dd2eac9603d2f036 Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Fri, 31 Jul 2026 17:03:57 -0700 Subject: [PATCH 1/2] feat(proxy)!: gate all mock testing request params behind a single config flag Handling of the client-supplied mock testing params was split across three places with different behavior for each. Three were dropped from every proxy request, two reached the router untouched, and a request that asked for a synthetic failure came back as an ordinary success with nothing to indicate that no failure had been injected Put all six behind one opt-in, general_settings. dangerously_allow_mock_testing_request_params, and reject rather than drop when it is unset, so a fallback drill cannot report a pass for a test that never ran. The rejection names the params it saw and the config key to set, which is also the answer for anyone following the older docs The flag is config-file only. It is deliberately absent from ConfigGeneralSettings, and that absence is what makes /config/update drop it on parse and /config/field/update reject it; the tests pin both so the field cannot be added back for tidiness without the reason surfacing. Enabling it logs a startup warning naming every param it unlocks BREAKING CHANGE: mock_timeout and mock_testing_rate_limit_error now require general_settings.dangerously_allow_mock_testing_request_params to be set in config.yaml. Previously they were accepted unconditionally --- litellm/proxy/proxy_server.py | 33 ++++ litellm/proxy/route_llm_request.py | 61 ++++++-- tests/test_litellm/proxy/test_proxy_server.py | 80 ++++++++++ .../proxy/test_route_llm_request.py | 148 +++++++++++++++--- 4 files changed, 291 insertions(+), 31 deletions(-) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index a60ea2da019..06e9cc1fbee 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -1041,6 +1041,8 @@ async def proxy_startup_event(app: FastAPI): redis_usage_cache=transaction_buffer_redis_cache, ) + ProxyStartupEvent._warn_if_mock_testing_params_enabled(general_settings=general_settings) + ## SEMANTIC TOOL FILTER ## # Read litellm_settings from config for semantic filter initialization try: @@ -7712,6 +7714,37 @@ class ProxyStartupEvent: proxy_logging_obj.startup_event(llm_router=llm_router, redis_usage_cache=redis_usage_cache) + @staticmethod + def _warn_if_mock_testing_params_enabled(general_settings: dict) -> None: + """Announce, loudly, that any caller may inject synthetic failures.""" + from litellm.proxy.route_llm_request import ( + GATED_MOCK_PARAM_NAMES, + MOCK_TESTING_CONFIG_KEY, + ) + + if general_settings.get(MOCK_TESTING_CONFIG_KEY, False) is not True: + return + + verbose_proxy_logger.warning( + "\n%s\n" + " DANGEROUS SETTING ENABLED\n" + " general_settings.%s = true\n" + "\n" + " Any caller with a valid key on this proxy can now inject synthetic\n" + " failures and latency into their own requests using these body params:\n" + "%s\n" + "\n" + " A request using them consumes a connection and a concurrency slot\n" + " without reaching a provider, and returns an error the caller chose.\n" + "\n" + " Intended for testing fallback chains. Do not leave enabled.\n" + "%s", + "=" * 72, + MOCK_TESTING_CONFIG_KEY, + "\n".join(f" {name}" for name in GATED_MOCK_PARAM_NAMES), + "=" * 72, + ) + @staticmethod def _validate_redis_transaction_buffer_config( general_settings: dict, diff --git a/litellm/proxy/route_llm_request.py b/litellm/proxy/route_llm_request.py index 1f5aacc2115..aebdbca86ad 100644 --- a/litellm/proxy/route_llm_request.py +++ b/litellm/proxy/route_llm_request.py @@ -8,18 +8,24 @@ import litellm from litellm.proxy._types import UserAPIKeyAuth from litellm.router_utils.common_utils import _is_proxy_admin_request -# Router-internal mock_testing_* flag names — kept in sync with -# ``litellm.types.router.MockRouterTestingParams`` by the test -# ``test_mock_testing_kwarg_names_matches_dataclass``. Hardcoding (rather -# than deriving via ``dataclasses.fields(MockRouterTestingParams)`` at +# Client-supplied params that make the router or the call path fabricate a +# failure or a delay instead of calling the provider. The ``mock_testing_*`` +# names are kept in sync with ``litellm.types.router.MockRouterTestingParams`` +# by ``test_gated_mock_params_cover_mock_router_testing_params``. Hardcoding +# (rather than deriving via ``dataclasses.fields(MockRouterTestingParams)`` at # import time) avoids a cyclic import: ``litellm.types.router`` imports # back into proxy modules before this module finishes loading. -_MOCK_TESTING_KWARG_NAMES: tuple = ( +GATED_MOCK_PARAM_NAMES: tuple[str, ...] = ( "mock_testing_fallbacks", "mock_testing_context_fallbacks", "mock_testing_content_policy_fallbacks", + "mock_testing_rate_limit_error", + "mock_timeout", + "mock_delay", ) +MOCK_TESTING_CONFIG_KEY = "dangerously_allow_mock_testing_request_params" + if TYPE_CHECKING: from litellm.router import Router as _Router @@ -169,6 +175,41 @@ def raise_if_required_body_param_missing(route_type: str, data: Mapping[str, obj ) +class MockTestingParamsDisabledError(HTTPException): + def __init__(self, params: tuple[str, ...]): + super().__init__( + status_code=status.HTTP_400_BAD_REQUEST, + detail={ # mutable-ok: HTTPException.detail has no immutable form; same shape as the sibling errors here + "error": ( + f"Mock testing request params are disabled on this proxy: {', '.join(params)}. " + f"An admin can enable them by setting `general_settings.{MOCK_TESTING_CONFIG_KEY}: true` " + "in config.yaml. This setting cannot be changed from the Admin UI or the API." + ) + }, + ) + + +def raise_if_mock_testing_params_disallowed(data: Mapping[str, object], *, allowed: bool) -> None: + """Reject client-supplied mock testing params unless an admin opted in. + + Rejecting (rather than silently dropping) keeps a request that asked for a + synthetic failure from returning a normal success, which reads as a passing + fallback test that never ran. + """ + if allowed: + return + present = tuple(name for name in GATED_MOCK_PARAM_NAMES if name in data) + if present: + raise MockTestingParamsDisabledError(params=present) + + +def mock_testing_params_allowed() -> bool: + """Read the opt-in from the running proxy's ``general_settings``.""" + import litellm.proxy.proxy_server as proxy_server + + return proxy_server.general_settings.get(MOCK_TESTING_CONFIG_KEY, False) is True + + def get_team_id_from_data(data: dict) -> Optional[str]: """ Get the team id from the data's metadata or litellm_metadata params. @@ -381,12 +422,10 @@ async def route_request( await add_shared_session_to_data(data) - # Strip router-internal mock_testing_* flags. Combined with an - # unauthorized fallback in ``router_settings_override`` they let a - # caller deterministically execute requests against restricted - # models. VERIA-44. - for _key in _MOCK_TESTING_KWARG_NAMES: - data.pop(_key, None) + # Gated here rather than alongside the sibling untrusted-param checks in + # ``litellm_pre_call_utils``: the Responses WebSocket route calls + # ``route_request`` directly and never runs ``add_litellm_data_to_request``. + raise_if_mock_testing_params_disallowed(data, allowed=mock_testing_params_allowed()) data.pop("enable_tag_filtering", None) diff --git a/tests/test_litellm/proxy/test_proxy_server.py b/tests/test_litellm/proxy/test_proxy_server.py index b9a33bd2cef..192c674771f 100644 --- a/tests/test_litellm/proxy/test_proxy_server.py +++ b/tests/test_litellm/proxy/test_proxy_server.py @@ -10678,3 +10678,83 @@ async def test_async_data_generator_forwards_usage_chunk_without_strip_marker(): assert len(data_frames) == 4 assert any('"usage"' in frame and '"completion_tokens":188' in frame.replace(" ", "") for frame in data_frames) assert frames[-1] == "data: [DONE]\n\n" + + +@pytest.mark.asyncio +async def test_config_field_update_rejects_mock_testing_flag(): + """The mock-testing opt-in is deliberately absent from + ``ConfigGeneralSettings`` so that ``/config/field/update`` refuses it. If + someone later adds the field for tidiness, this test fails and tells them + they have just opened an API write path into a config-file-only setting.""" + from fastapi import HTTPException + + from litellm.proxy._types import ConfigFieldUpdate + from litellm.proxy.proxy_server import update_config_general_settings + from litellm.proxy.route_llm_request import MOCK_TESTING_CONFIG_KEY + + admin = UserAPIKeyAuth( + user_role=LitellmUserRoles.PROXY_ADMIN, + api_key="sk-test", + ) + + with patch.object(proxy_server_module, "prisma_client", MagicMock()): + with pytest.raises(HTTPException) as exc_info: + await update_config_general_settings( + data=ConfigFieldUpdate( + field_name=MOCK_TESTING_CONFIG_KEY, + field_value=True, + config_type="general_settings", + ), + user_api_key_dict=admin, + ) + + assert exc_info.value.status_code == 400 + + +def test_config_update_body_drops_mock_testing_flag(): + """``/config/update`` parses its body as ``ConfigYAML``, whose + ``general_settings`` is a ``ConfigGeneralSettings``. Undeclared keys are + dropped on parse, so the flag never reaches the DB by that route either.""" + from litellm.proxy._types import ConfigYAML + from litellm.proxy.route_llm_request import MOCK_TESTING_CONFIG_KEY + + parsed = ConfigYAML.model_validate({"general_settings": {MOCK_TESTING_CONFIG_KEY: True}}) + + assert parsed.general_settings is not None + assert MOCK_TESTING_CONFIG_KEY not in parsed.general_settings.model_dump(exclude_none=True) + + +def test_startup_warns_when_mock_testing_params_enabled(caplog): + """Enabling the opt-in must announce itself, naming every param it + unlocks — the config key says ``mock_testing`` but the gate also covers + ``mock_timeout`` and ``mock_delay``, so coverage cannot be inferred from + the name alone.""" + import logging + + from litellm.proxy.proxy_server import ProxyStartupEvent + from litellm.proxy.route_llm_request import ( + GATED_MOCK_PARAM_NAMES, + MOCK_TESTING_CONFIG_KEY, + ) + + with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"): + ProxyStartupEvent._warn_if_mock_testing_params_enabled( + general_settings={MOCK_TESTING_CONFIG_KEY: True} + ) + + assert MOCK_TESTING_CONFIG_KEY in caplog.text + for param_name in GATED_MOCK_PARAM_NAMES: + assert param_name in caplog.text + + +def test_startup_is_silent_when_mock_testing_params_disabled(caplog): + """A proxy that never set the opt-in must not emit the warning.""" + import logging + + from litellm.proxy.proxy_server import ProxyStartupEvent + from litellm.proxy.route_llm_request import MOCK_TESTING_CONFIG_KEY + + with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"): + ProxyStartupEvent._warn_if_mock_testing_params_enabled(general_settings={}) + + assert MOCK_TESTING_CONFIG_KEY not in caplog.text diff --git a/tests/test_litellm/proxy/test_route_llm_request.py b/tests/test_litellm/proxy/test_route_llm_request.py index 93b3ef1cce8..e74693a4f6c 100644 --- a/tests/test_litellm/proxy/test_route_llm_request.py +++ b/tests/test_litellm/proxy/test_route_llm_request.py @@ -8,6 +8,8 @@ sys.path.insert(0, os.path.abspath("../../..")) # Adds the parent directory to from unittest.mock import MagicMock +from fastapi import HTTPException + from litellm.proxy.route_llm_request import ProxyModelNotFoundError, route_request @@ -471,38 +473,147 @@ async def test_route_request_with_router_settings_override_preserves_existing(): assert call_kwargs["timeout"] == 30 -def test_mock_testing_kwarg_names_matches_dataclass(): - """``_MOCK_TESTING_KWARG_NAMES`` is hardcoded to avoid a cyclic import - against ``litellm.types.router``. This test guards against drift — - if a new ``mock_testing_*`` field is added to ``MockRouterTestingParams`` - the strip list must be updated to keep covering it.""" +def test_gated_mock_params_cover_mock_router_testing_params(): + """``GATED_MOCK_PARAM_NAMES`` is hardcoded to avoid a cyclic import + against ``litellm.types.router``. This test guards against drift — if a + new ``mock_testing_*`` field is added to ``MockRouterTestingParams`` the + gate must be updated to keep covering it. The gate is a superset: it also + covers params consumed outside that dataclass.""" from dataclasses import fields - from litellm.proxy.route_llm_request import _MOCK_TESTING_KWARG_NAMES + from litellm.proxy.route_llm_request import GATED_MOCK_PARAM_NAMES from litellm.types.router import MockRouterTestingParams - assert set(_MOCK_TESTING_KWARG_NAMES) == {f.name for f in fields(MockRouterTestingParams)} + assert {f.name for f in fields(MockRouterTestingParams)} <= set(GATED_MOCK_PARAM_NAMES) + assert {"mock_testing_rate_limit_error", "mock_timeout", "mock_delay"} <= set(GATED_MOCK_PARAM_NAMES) -@pytest.mark.asyncio @pytest.mark.parametrize( - "mock_flag", + "mock_param", [ "mock_testing_fallbacks", "mock_testing_context_fallbacks", "mock_testing_content_policy_fallbacks", + "mock_testing_rate_limit_error", + "mock_timeout", + "mock_delay", ], ) -async def test_route_request_strips_mock_testing_flags(mock_flag): - """VERIA-44: router-internal testing flags must not survive a - user-supplied request body. Without this strip, an attacker can - combine ``mock_testing_fallbacks=true`` with an unauthorized fallback - in ``router_settings_override`` to deterministically execute requests - against restricted models.""" +def test_mock_params_rejected_when_not_allowed(mock_param): + """Every gated param must be rejected by name when the proxy has not + opted in, and the error must point the caller at the config key.""" + from litellm.proxy.route_llm_request import ( + MOCK_TESTING_CONFIG_KEY, + raise_if_mock_testing_params_disallowed, + ) + + data = {"model": "gpt-3.5-turbo", mock_param: True} + + with pytest.raises(HTTPException) as exc_info: + raise_if_mock_testing_params_disallowed(data, allowed=False) + + assert exc_info.value.status_code == 400 + error_message = exc_info.value.detail["error"] + assert mock_param in error_message + assert MOCK_TESTING_CONFIG_KEY in error_message + + +@pytest.mark.parametrize( + "mock_param", + [ + "mock_testing_fallbacks", + "mock_testing_context_fallbacks", + "mock_testing_content_policy_fallbacks", + "mock_testing_rate_limit_error", + "mock_timeout", + "mock_delay", + ], +) +def test_mock_params_pass_through_when_allowed(mock_param): + """With the opt-in set, gated params must survive untouched — a gate that + rejects correctly but strips anyway would leave the feature unusable.""" + from litellm.proxy.route_llm_request import raise_if_mock_testing_params_disallowed + + data = {"model": "gpt-3.5-turbo", mock_param: True} + + raise_if_mock_testing_params_disallowed(data, allowed=True) + + assert data[mock_param] is True + + +def test_mock_param_gate_reports_every_param_present(): + """A request carrying several gated params must name all of them, so a + caller fixing one is not surprised by the next.""" + from litellm.proxy.route_llm_request import raise_if_mock_testing_params_disallowed + + data = { + "model": "gpt-3.5-turbo", + "mock_testing_fallbacks": True, + "mock_delay": 30, + } + + with pytest.raises(HTTPException) as exc_info: + raise_if_mock_testing_params_disallowed(data, allowed=False) + + error_message = exc_info.value.detail["error"] + assert "mock_testing_fallbacks" in error_message + assert "mock_delay" in error_message + + +def test_ordinary_request_is_not_rejected_by_the_mock_param_gate(): + """The gate must not fire on a request that carries no gated param.""" + from litellm.proxy.route_llm_request import raise_if_mock_testing_params_disallowed + data = { "model": "gpt-3.5-turbo", "messages": [{"role": "user", "content": "Hello"}], - mock_flag: True, + "mock_response": "hi", + } + + raise_if_mock_testing_params_disallowed(data, allowed=False) + + +@pytest.mark.asyncio +async def test_route_request_rejects_mock_params_by_default(monkeypatch): + """End-to-end through ``route_request``: with no opt-in configured the + request is rejected before it ever reaches the router.""" + import litellm.proxy.proxy_server as proxy_server + + monkeypatch.setattr(proxy_server, "general_settings", {}, raising=False) + + data = { + "model": "gpt-3.5-turbo", + "messages": [{"role": "user", "content": "Hello"}], + "mock_testing_fallbacks": True, + } + llm_router = MagicMock() + + with pytest.raises(HTTPException) as exc_info: + await route_request(data, llm_router, None, "acompletion") + + assert exc_info.value.status_code == 400 + llm_router.acompletion.assert_not_called() + + +@pytest.mark.asyncio +async def test_route_request_forwards_mock_params_when_opted_in(monkeypatch): + """End-to-end through ``route_request``: with the opt-in set the param + reaches the router, which is what makes a fallback drill possible.""" + import litellm.proxy.proxy_server as proxy_server + + from litellm.proxy.route_llm_request import MOCK_TESTING_CONFIG_KEY + + monkeypatch.setattr( + proxy_server, + "general_settings", + {MOCK_TESTING_CONFIG_KEY: True}, + raising=False, + ) + + data = { + "model": "gpt-3.5-turbo", + "messages": [{"role": "user", "content": "Hello"}], + "mock_testing_fallbacks": True, } llm_router = MagicMock() llm_router.acompletion.return_value = "ok" @@ -510,10 +621,7 @@ async def test_route_request_strips_mock_testing_flags(mock_flag): await route_request(data, llm_router, None, "acompletion") call_kwargs = llm_router.acompletion.call_args[1] - assert mock_flag not in call_kwargs - # The flag is also gone from the original data dict so any subsequent - # processing (e.g. logging) doesn't see it either. - assert mock_flag not in data + assert call_kwargs["mock_testing_fallbacks"] is True @pytest.mark.parametrize("route_type", ["agenerate_content", "agenerate_content_stream"]) From 764b2337701f972ecafd172c9d0705ed976189b6 Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Fri, 31 Jul 2026 17:52:01 -0700 Subject: [PATCH 2/2] refactor(proxy): drop an inaccurate comment on the mock testing gate The comment said the Responses WebSocket route never runs add_litellm_data_to_request. It does, via common_processing_pre_call_logic, so the note recorded a request-flow constraint that does not hold The gate stays in route_request, which is the dispatch chokepoint and where the previous handling lived --- litellm/proxy/route_llm_request.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/litellm/proxy/route_llm_request.py b/litellm/proxy/route_llm_request.py index aebdbca86ad..12e9857219e 100644 --- a/litellm/proxy/route_llm_request.py +++ b/litellm/proxy/route_llm_request.py @@ -422,9 +422,6 @@ async def route_request( await add_shared_session_to_data(data) - # Gated here rather than alongside the sibling untrusted-param checks in - # ``litellm_pre_call_utils``: the Responses WebSocket route calls - # ``route_request`` directly and never runs ``add_litellm_data_to_request``. raise_if_mock_testing_params_disallowed(data, allowed=mock_testing_params_allowed()) data.pop("enable_tag_filtering", None)