diff --git a/litellm/router.py b/litellm/router.py index 60e1004dc1c..0473f848b73 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -2706,6 +2706,9 @@ class Router: **kwargs, ): try: + kwargs.pop("_unrendered_messages", None) + kwargs.pop("_original_prompt_params", None) + kwargs.pop("_in_prompt_factory", None) kwargs["model"] = model kwargs["messages"] = messages kwargs["stream"] = stream diff --git a/litellm/router_utils/fallback_event_handlers.py b/litellm/router_utils/fallback_event_handlers.py index ab6dc29b16f..cab3f6298eb 100644 --- a/litellm/router_utils/fallback_event_handlers.py +++ b/litellm/router_utils/fallback_event_handlers.py @@ -474,11 +474,24 @@ def creates_provider_scoped_resource(kwargs: Mapping[str, object]) -> bool: return getattr(kwargs.get("original_function"), "__name__", None) in PROVIDER_SCOPED_CREATION_FUNCTION_NAMES +ALLOWED_FALLBACK_PROMPT_PARAMS: Final = frozenset({ + "prompt_id", + "prompt_variables", + "prompt_label", + "prompt_version", + "prompt_environment", +}) + + def _restore_fallback_prompt_state(kwargs: dict[str, object]) -> None: - if "_unrendered_messages" in kwargs: - kwargs["messages"] = safe_deep_copy(kwargs["_unrendered_messages"]) - if "_original_prompt_params" in kwargs and isinstance(kwargs["_original_prompt_params"], dict): - kwargs.update(kwargs["_original_prompt_params"]) + unrendered: Final = kwargs.get("_unrendered_messages") + if isinstance(unrendered, list): + kwargs["messages"] = safe_deep_copy(unrendered) + original_prompt_params: Final = kwargs.get("_original_prompt_params") + if isinstance(original_prompt_params, dict): + for key in ALLOWED_FALLBACK_PROMPT_PARAMS: + if key in original_prompt_params: + kwargs[key] = original_prompt_params[key] kwargs.pop("_in_prompt_factory", None) diff --git a/tests/test_litellm/test_router.py b/tests/test_litellm/test_router.py index a598421ab9b..71b6e02440f 100644 --- a/tests/test_litellm/test_router.py +++ b/tests/test_litellm/test_router.py @@ -17077,3 +17077,51 @@ def test_router_dispatch_prompt_completion_direct(monkeypatch: pytest.MonkeyPatc assert response.choices[0].message.content == "dispatched reply" +def test_router_fallback_ignores_injected_original_prompt_params( + prompt_version_clean_registry, monkeypatch: pytest.MonkeyPatch +) -> None: + calls: Final[list[dict[str, object]]] = [] + + async def mock_failover_injected_acompletion(*args, **kwargs): + calls.append(dict(kwargs)) + if kwargs.get("model") == "openai/gpt-4o-primary": + raise litellm.RateLimitError( + message="Rate limit reached", + response=httpx.Response(429, request=httpx.Request("POST", "https://api.openai.com/v1/chat/completions")), + llm_provider="openai", + model="gpt-4o-primary", + ) + return litellm.ModelResponse( + choices=[litellm.utils.Choices(message=litellm.utils.Message(content="safe reply", role="assistant"))] + ) + + monkeypatch.setattr(litellm, "acompletion", mock_failover_injected_acompletion) + + router: Final = Router( + model_list=[ + { + "model_name": "primary-model", + "litellm_params": {"model": "openai/gpt-4o-primary"}, + }, + { + "model_name": "fallback-model", + "litellm_params": {"model": "openai/gpt-4o-fallback"}, + }, + ], + fallbacks=[{"primary-model": ["fallback-model"]}], + num_retries=0, + ) + + response: Final = asyncio.run( + router.acompletion( + model="primary-model", + messages=[{"role": "user", "content": "test query"}], + _original_prompt_params={"api_base": "https://attacker.example"}, + ) + ) + + assert isinstance(response, litellm.ModelResponse) + assert response.choices[0].message.content == "safe reply" + assert len(calls) == 2 + for call in calls: + assert "api_base" not in call