From 61f5973cd3cf3e1c3bebf4cb0c292b8dc888fd22 Mon Sep 17 00:00:00 2001 From: Jaeyeon Kim Date: Mon, 26 Oct 2026 13:23:46 +0000 Subject: [PATCH] fix: preserve generic response contracts for shadow requests Signed-off-by: Jaeyeon Kim --- litellm/router.py | 8 ++++- tests/unit/test_router_silent_experiment.py | 34 +++++++++++++++++++++ 2 files changed, 41 insertions(+), 1 deletion(-) diff --git a/litellm/router.py b/litellm/router.py index e709c7afdd9..2c0a724a79e 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -4497,10 +4497,12 @@ class Router: """ try: verbose_router_logger.info("Starting silent experiment for model %s", silent_model) + remaining_kwargs: Final = {key: value for key, value in silent_kwargs.items() if key != "attempt_function"} await self._ageneric_api_call_with_fallbacks( model=silent_model, original_function=original_function, - **silent_kwargs, + attempt_function=None, + **remaining_kwargs, ) except Exception as e: # noqa: BLE001 - a background mirror must never break the primary request verbose_router_logger.error("Silent experiment failed for model %s: %s", silent_model, e) @@ -6452,6 +6454,8 @@ class Router: carry_over_routed_deployment(live_kwargs=kwargs, snapshot=hop_kwargs) if kwargs.get("stream") and isinstance(response, BaseResponsesAPIStreamingIterator): return await self._aresponses_streaming_iterator(response=response, initial_kwargs=hop_kwargs) + if response is None: + raise TypeError("Responses request returned no response") return response async def _aanthropic_messages_streaming_iterator( @@ -7015,6 +7019,8 @@ class Router: response=cast("AsyncIterator[bytes]", response), # cast-ok: stream=True always returns a byte iterator initial_kwargs=hop_kwargs, ) + if response is None: + raise TypeError("Anthropic Messages request returned no response") return response async def _dispatch_generic_call_type( diff --git a/tests/unit/test_router_silent_experiment.py b/tests/unit/test_router_silent_experiment.py index 9c4c7270e83..4c59332997a 100644 --- a/tests/unit/test_router_silent_experiment.py +++ b/tests/unit/test_router_silent_experiment.py @@ -949,3 +949,37 @@ async def test_generic_mirroring_skips_deployment_mcp(mcp_deployment, mcp_config provider.assert_awaited_once() assert provider.await_args.kwargs["model"] == "openai/gpt-4o-mini" + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "call_type,attempt_method", + [ + ("aresponses", "_ageneric_api_call_with_fallbacks_responses_attempt"), + ("anthropic_messages", "_ageneric_api_call_with_fallbacks_anthropic_messages_attempt"), + ], +) +async def test_generic_primary_attempt_rejects_missing_provider_response(call_type, attempt_method): + provider: Final = AsyncMock(return_value=None) + provider.__name__ = call_type + router: Final = Router(model_list=_generic_silent_model_list()) + with pytest.raises(TypeError, match="request returned no response"): + await getattr(router, attempt_method)(model="silent-model", original_generic_function=provider, input="hi") + + +@pytest.mark.asyncio +async def test_generic_mirror_does_not_reuse_primary_attempt_callback(): + primary_attempt: Final = AsyncMock(return_value="primary-only") + provider: Final = AsyncMock(return_value="mirror-result") + provider.__name__ = "aresponses" + router: Final = Router(model_list=_generic_silent_model_list()) + + await router._silent_experiment_ageneric( + silent_model="silent-model", + original_function=provider, + silent_kwargs={"input": "hi", "attempt_function": primary_attempt}, + ) + + primary_attempt.assert_not_awaited() + provider.assert_awaited_once() + assert provider.await_args.kwargs["model"] == "openai/gpt-4o"