diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 15d977b75f2..9023f5fb23f 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -1041,6 +1041,8 @@ async def proxy_startup_event(app: FastAPI): redis_usage_cache=transaction_buffer_redis_cache, ) + ProxyStartupEvent._warn_if_mock_testing_params_enabled(general_settings=general_settings) + ## SEMANTIC TOOL FILTER ## # Read litellm_settings from config for semantic filter initialization try: @@ -7725,6 +7727,37 @@ class ProxyStartupEvent: proxy_logging_obj.startup_event(llm_router=llm_router, redis_usage_cache=redis_usage_cache) + @staticmethod + def _warn_if_mock_testing_params_enabled(general_settings: dict) -> None: + """Announce, loudly, that any caller may inject synthetic failures.""" + from litellm.proxy.route_llm_request import ( + GATED_MOCK_PARAM_NAMES, + MOCK_TESTING_CONFIG_KEY, + ) + + if general_settings.get(MOCK_TESTING_CONFIG_KEY, False) is not True: + return + + verbose_proxy_logger.warning( + "\n%s\n" + " DANGEROUS SETTING ENABLED\n" + " general_settings.%s = true\n" + "\n" + " Any caller with a valid key on this proxy can now inject synthetic\n" + " failures and latency into their own requests using these body params:\n" + "%s\n" + "\n" + " A request using them consumes a connection and a concurrency slot\n" + " without reaching a provider, and returns an error the caller chose.\n" + "\n" + " Intended for testing fallback chains. Do not leave enabled.\n" + "%s", + "=" * 72, + MOCK_TESTING_CONFIG_KEY, + "\n".join(f" {name}" for name in GATED_MOCK_PARAM_NAMES), + "=" * 72, + ) + @staticmethod def _validate_redis_transaction_buffer_config( general_settings: dict, diff --git a/litellm/proxy/route_llm_request.py b/litellm/proxy/route_llm_request.py index 1f5aacc2115..12e9857219e 100644 --- a/litellm/proxy/route_llm_request.py +++ b/litellm/proxy/route_llm_request.py @@ -8,18 +8,24 @@ import litellm from litellm.proxy._types import UserAPIKeyAuth from litellm.router_utils.common_utils import _is_proxy_admin_request -# Router-internal mock_testing_* flag names — kept in sync with -# ``litellm.types.router.MockRouterTestingParams`` by the test -# ``test_mock_testing_kwarg_names_matches_dataclass``. Hardcoding (rather -# than deriving via ``dataclasses.fields(MockRouterTestingParams)`` at +# Client-supplied params that make the router or the call path fabricate a +# failure or a delay instead of calling the provider. The ``mock_testing_*`` +# names are kept in sync with ``litellm.types.router.MockRouterTestingParams`` +# by ``test_gated_mock_params_cover_mock_router_testing_params``. Hardcoding +# (rather than deriving via ``dataclasses.fields(MockRouterTestingParams)`` at # import time) avoids a cyclic import: ``litellm.types.router`` imports # back into proxy modules before this module finishes loading. -_MOCK_TESTING_KWARG_NAMES: tuple = ( +GATED_MOCK_PARAM_NAMES: tuple[str, ...] = ( "mock_testing_fallbacks", "mock_testing_context_fallbacks", "mock_testing_content_policy_fallbacks", + "mock_testing_rate_limit_error", + "mock_timeout", + "mock_delay", ) +MOCK_TESTING_CONFIG_KEY = "dangerously_allow_mock_testing_request_params" + if TYPE_CHECKING: from litellm.router import Router as _Router @@ -169,6 +175,41 @@ def raise_if_required_body_param_missing(route_type: str, data: Mapping[str, obj ) +class MockTestingParamsDisabledError(HTTPException): + def __init__(self, params: tuple[str, ...]): + super().__init__( + status_code=status.HTTP_400_BAD_REQUEST, + detail={ # mutable-ok: HTTPException.detail has no immutable form; same shape as the sibling errors here + "error": ( + f"Mock testing request params are disabled on this proxy: {', '.join(params)}. " + f"An admin can enable them by setting `general_settings.{MOCK_TESTING_CONFIG_KEY}: true` " + "in config.yaml. This setting cannot be changed from the Admin UI or the API." + ) + }, + ) + + +def raise_if_mock_testing_params_disallowed(data: Mapping[str, object], *, allowed: bool) -> None: + """Reject client-supplied mock testing params unless an admin opted in. + + Rejecting (rather than silently dropping) keeps a request that asked for a + synthetic failure from returning a normal success, which reads as a passing + fallback test that never ran. + """ + if allowed: + return + present = tuple(name for name in GATED_MOCK_PARAM_NAMES if name in data) + if present: + raise MockTestingParamsDisabledError(params=present) + + +def mock_testing_params_allowed() -> bool: + """Read the opt-in from the running proxy's ``general_settings``.""" + import litellm.proxy.proxy_server as proxy_server + + return proxy_server.general_settings.get(MOCK_TESTING_CONFIG_KEY, False) is True + + def get_team_id_from_data(data: dict) -> Optional[str]: """ Get the team id from the data's metadata or litellm_metadata params. @@ -381,12 +422,7 @@ async def route_request( await add_shared_session_to_data(data) - # Strip router-internal mock_testing_* flags. Combined with an - # unauthorized fallback in ``router_settings_override`` they let a - # caller deterministically execute requests against restricted - # models. VERIA-44. - for _key in _MOCK_TESTING_KWARG_NAMES: - data.pop(_key, None) + raise_if_mock_testing_params_disallowed(data, allowed=mock_testing_params_allowed()) data.pop("enable_tag_filtering", None) diff --git a/tests/test_litellm/proxy/test_proxy_server.py b/tests/test_litellm/proxy/test_proxy_server.py index 1ecf7e8b83a..ab64224fdc3 100644 --- a/tests/test_litellm/proxy/test_proxy_server.py +++ b/tests/test_litellm/proxy/test_proxy_server.py @@ -10685,3 +10685,83 @@ async def test_async_data_generator_forwards_usage_chunk_without_strip_marker(): assert len(data_frames) == 4 assert any('"usage"' in frame and '"completion_tokens":188' in frame.replace(" ", "") for frame in data_frames) assert frames[-1] == "data: [DONE]\n\n" + + +@pytest.mark.asyncio +async def test_config_field_update_rejects_mock_testing_flag(): + """The mock-testing opt-in is deliberately absent from + ``ConfigGeneralSettings`` so that ``/config/field/update`` refuses it. If + someone later adds the field for tidiness, this test fails and tells them + they have just opened an API write path into a config-file-only setting.""" + from fastapi import HTTPException + + from litellm.proxy._types import ConfigFieldUpdate + from litellm.proxy.proxy_server import update_config_general_settings + from litellm.proxy.route_llm_request import MOCK_TESTING_CONFIG_KEY + + admin = UserAPIKeyAuth( + user_role=LitellmUserRoles.PROXY_ADMIN, + api_key="sk-test", + ) + + with patch.object(proxy_server_module, "prisma_client", MagicMock()): + with pytest.raises(HTTPException) as exc_info: + await update_config_general_settings( + data=ConfigFieldUpdate( + field_name=MOCK_TESTING_CONFIG_KEY, + field_value=True, + config_type="general_settings", + ), + user_api_key_dict=admin, + ) + + assert exc_info.value.status_code == 400 + + +def test_config_update_body_drops_mock_testing_flag(): + """``/config/update`` parses its body as ``ConfigYAML``, whose + ``general_settings`` is a ``ConfigGeneralSettings``. Undeclared keys are + dropped on parse, so the flag never reaches the DB by that route either.""" + from litellm.proxy._types import ConfigYAML + from litellm.proxy.route_llm_request import MOCK_TESTING_CONFIG_KEY + + parsed = ConfigYAML.model_validate({"general_settings": {MOCK_TESTING_CONFIG_KEY: True}}) + + assert parsed.general_settings is not None + assert MOCK_TESTING_CONFIG_KEY not in parsed.general_settings.model_dump(exclude_none=True) + + +def test_startup_warns_when_mock_testing_params_enabled(caplog): + """Enabling the opt-in must announce itself, naming every param it + unlocks — the config key says ``mock_testing`` but the gate also covers + ``mock_timeout`` and ``mock_delay``, so coverage cannot be inferred from + the name alone.""" + import logging + + from litellm.proxy.proxy_server import ProxyStartupEvent + from litellm.proxy.route_llm_request import ( + GATED_MOCK_PARAM_NAMES, + MOCK_TESTING_CONFIG_KEY, + ) + + with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"): + ProxyStartupEvent._warn_if_mock_testing_params_enabled( + general_settings={MOCK_TESTING_CONFIG_KEY: True} + ) + + assert MOCK_TESTING_CONFIG_KEY in caplog.text + for param_name in GATED_MOCK_PARAM_NAMES: + assert param_name in caplog.text + + +def test_startup_is_silent_when_mock_testing_params_disabled(caplog): + """A proxy that never set the opt-in must not emit the warning.""" + import logging + + from litellm.proxy.proxy_server import ProxyStartupEvent + from litellm.proxy.route_llm_request import MOCK_TESTING_CONFIG_KEY + + with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"): + ProxyStartupEvent._warn_if_mock_testing_params_enabled(general_settings={}) + + assert MOCK_TESTING_CONFIG_KEY not in caplog.text diff --git a/tests/test_litellm/proxy/test_route_llm_request.py b/tests/test_litellm/proxy/test_route_llm_request.py index 93b3ef1cce8..e74693a4f6c 100644 --- a/tests/test_litellm/proxy/test_route_llm_request.py +++ b/tests/test_litellm/proxy/test_route_llm_request.py @@ -8,6 +8,8 @@ sys.path.insert(0, os.path.abspath("../../..")) # Adds the parent directory to from unittest.mock import MagicMock +from fastapi import HTTPException + from litellm.proxy.route_llm_request import ProxyModelNotFoundError, route_request @@ -471,38 +473,147 @@ async def test_route_request_with_router_settings_override_preserves_existing(): assert call_kwargs["timeout"] == 30 -def test_mock_testing_kwarg_names_matches_dataclass(): - """``_MOCK_TESTING_KWARG_NAMES`` is hardcoded to avoid a cyclic import - against ``litellm.types.router``. This test guards against drift — - if a new ``mock_testing_*`` field is added to ``MockRouterTestingParams`` - the strip list must be updated to keep covering it.""" +def test_gated_mock_params_cover_mock_router_testing_params(): + """``GATED_MOCK_PARAM_NAMES`` is hardcoded to avoid a cyclic import + against ``litellm.types.router``. This test guards against drift — if a + new ``mock_testing_*`` field is added to ``MockRouterTestingParams`` the + gate must be updated to keep covering it. The gate is a superset: it also + covers params consumed outside that dataclass.""" from dataclasses import fields - from litellm.proxy.route_llm_request import _MOCK_TESTING_KWARG_NAMES + from litellm.proxy.route_llm_request import GATED_MOCK_PARAM_NAMES from litellm.types.router import MockRouterTestingParams - assert set(_MOCK_TESTING_KWARG_NAMES) == {f.name for f in fields(MockRouterTestingParams)} + assert {f.name for f in fields(MockRouterTestingParams)} <= set(GATED_MOCK_PARAM_NAMES) + assert {"mock_testing_rate_limit_error", "mock_timeout", "mock_delay"} <= set(GATED_MOCK_PARAM_NAMES) -@pytest.mark.asyncio @pytest.mark.parametrize( - "mock_flag", + "mock_param", [ "mock_testing_fallbacks", "mock_testing_context_fallbacks", "mock_testing_content_policy_fallbacks", + "mock_testing_rate_limit_error", + "mock_timeout", + "mock_delay", ], ) -async def test_route_request_strips_mock_testing_flags(mock_flag): - """VERIA-44: router-internal testing flags must not survive a - user-supplied request body. Without this strip, an attacker can - combine ``mock_testing_fallbacks=true`` with an unauthorized fallback - in ``router_settings_override`` to deterministically execute requests - against restricted models.""" +def test_mock_params_rejected_when_not_allowed(mock_param): + """Every gated param must be rejected by name when the proxy has not + opted in, and the error must point the caller at the config key.""" + from litellm.proxy.route_llm_request import ( + MOCK_TESTING_CONFIG_KEY, + raise_if_mock_testing_params_disallowed, + ) + + data = {"model": "gpt-3.5-turbo", mock_param: True} + + with pytest.raises(HTTPException) as exc_info: + raise_if_mock_testing_params_disallowed(data, allowed=False) + + assert exc_info.value.status_code == 400 + error_message = exc_info.value.detail["error"] + assert mock_param in error_message + assert MOCK_TESTING_CONFIG_KEY in error_message + + +@pytest.mark.parametrize( + "mock_param", + [ + "mock_testing_fallbacks", + "mock_testing_context_fallbacks", + "mock_testing_content_policy_fallbacks", + "mock_testing_rate_limit_error", + "mock_timeout", + "mock_delay", + ], +) +def test_mock_params_pass_through_when_allowed(mock_param): + """With the opt-in set, gated params must survive untouched — a gate that + rejects correctly but strips anyway would leave the feature unusable.""" + from litellm.proxy.route_llm_request import raise_if_mock_testing_params_disallowed + + data = {"model": "gpt-3.5-turbo", mock_param: True} + + raise_if_mock_testing_params_disallowed(data, allowed=True) + + assert data[mock_param] is True + + +def test_mock_param_gate_reports_every_param_present(): + """A request carrying several gated params must name all of them, so a + caller fixing one is not surprised by the next.""" + from litellm.proxy.route_llm_request import raise_if_mock_testing_params_disallowed + + data = { + "model": "gpt-3.5-turbo", + "mock_testing_fallbacks": True, + "mock_delay": 30, + } + + with pytest.raises(HTTPException) as exc_info: + raise_if_mock_testing_params_disallowed(data, allowed=False) + + error_message = exc_info.value.detail["error"] + assert "mock_testing_fallbacks" in error_message + assert "mock_delay" in error_message + + +def test_ordinary_request_is_not_rejected_by_the_mock_param_gate(): + """The gate must not fire on a request that carries no gated param.""" + from litellm.proxy.route_llm_request import raise_if_mock_testing_params_disallowed + data = { "model": "gpt-3.5-turbo", "messages": [{"role": "user", "content": "Hello"}], - mock_flag: True, + "mock_response": "hi", + } + + raise_if_mock_testing_params_disallowed(data, allowed=False) + + +@pytest.mark.asyncio +async def test_route_request_rejects_mock_params_by_default(monkeypatch): + """End-to-end through ``route_request``: with no opt-in configured the + request is rejected before it ever reaches the router.""" + import litellm.proxy.proxy_server as proxy_server + + monkeypatch.setattr(proxy_server, "general_settings", {}, raising=False) + + data = { + "model": "gpt-3.5-turbo", + "messages": [{"role": "user", "content": "Hello"}], + "mock_testing_fallbacks": True, + } + llm_router = MagicMock() + + with pytest.raises(HTTPException) as exc_info: + await route_request(data, llm_router, None, "acompletion") + + assert exc_info.value.status_code == 400 + llm_router.acompletion.assert_not_called() + + +@pytest.mark.asyncio +async def test_route_request_forwards_mock_params_when_opted_in(monkeypatch): + """End-to-end through ``route_request``: with the opt-in set the param + reaches the router, which is what makes a fallback drill possible.""" + import litellm.proxy.proxy_server as proxy_server + + from litellm.proxy.route_llm_request import MOCK_TESTING_CONFIG_KEY + + monkeypatch.setattr( + proxy_server, + "general_settings", + {MOCK_TESTING_CONFIG_KEY: True}, + raising=False, + ) + + data = { + "model": "gpt-3.5-turbo", + "messages": [{"role": "user", "content": "Hello"}], + "mock_testing_fallbacks": True, } llm_router = MagicMock() llm_router.acompletion.return_value = "ok" @@ -510,10 +621,7 @@ async def test_route_request_strips_mock_testing_flags(mock_flag): await route_request(data, llm_router, None, "acompletion") call_kwargs = llm_router.acompletion.call_args[1] - assert mock_flag not in call_kwargs - # The flag is also gone from the original data dict so any subsequent - # processing (e.g. logging) doesn't see it either. - assert mock_flag not in data + assert call_kwargs["mock_testing_fallbacks"] is True @pytest.mark.parametrize("route_type", ["agenerate_content", "agenerate_content_stream"])