fix: preserve generic response contracts for shadow requests

Signed-off-by: Jaeyeon Kim <anencore94@gmail.com>
This commit is contained in:
Jaeyeon Kim 2026-10-26 13:23:46 +00:00
parent 14799041aa
commit 61f5973cd3
2 changed files with 41 additions and 1 deletions

View file

@ -4497,10 +4497,12 @@ class Router:
"""
try:
verbose_router_logger.info("Starting silent experiment for model %s", silent_model)
remaining_kwargs: Final = {key: value for key, value in silent_kwargs.items() if key != "attempt_function"}
await self._ageneric_api_call_with_fallbacks(
model=silent_model,
original_function=original_function,
**silent_kwargs,
attempt_function=None,
**remaining_kwargs,
)
except Exception as e: # noqa: BLE001 - a background mirror must never break the primary request
verbose_router_logger.error("Silent experiment failed for model %s: %s", silent_model, e)
@ -6452,6 +6454,8 @@ class Router:
carry_over_routed_deployment(live_kwargs=kwargs, snapshot=hop_kwargs)
if kwargs.get("stream") and isinstance(response, BaseResponsesAPIStreamingIterator):
return await self._aresponses_streaming_iterator(response=response, initial_kwargs=hop_kwargs)
if response is None:
raise TypeError("Responses request returned no response")
return response
async def _aanthropic_messages_streaming_iterator(
@ -7015,6 +7019,8 @@ class Router:
response=cast("AsyncIterator[bytes]", response), # cast-ok: stream=True always returns a byte iterator
initial_kwargs=hop_kwargs,
)
if response is None:
raise TypeError("Anthropic Messages request returned no response")
return response
async def _dispatch_generic_call_type(

View file

@ -949,3 +949,37 @@ async def test_generic_mirroring_skips_deployment_mcp(mcp_deployment, mcp_config
provider.assert_awaited_once()
assert provider.await_args.kwargs["model"] == "openai/gpt-4o-mini"
@pytest.mark.asyncio
@pytest.mark.parametrize(
"call_type,attempt_method",
[
("aresponses", "_ageneric_api_call_with_fallbacks_responses_attempt"),
("anthropic_messages", "_ageneric_api_call_with_fallbacks_anthropic_messages_attempt"),
],
)
async def test_generic_primary_attempt_rejects_missing_provider_response(call_type, attempt_method):
provider: Final = AsyncMock(return_value=None)
provider.__name__ = call_type
router: Final = Router(model_list=_generic_silent_model_list())
with pytest.raises(TypeError, match="request returned no response"):
await getattr(router, attempt_method)(model="silent-model", original_generic_function=provider, input="hi")
@pytest.mark.asyncio
async def test_generic_mirror_does_not_reuse_primary_attempt_callback():
primary_attempt: Final = AsyncMock(return_value="primary-only")
provider: Final = AsyncMock(return_value="mirror-result")
provider.__name__ = "aresponses"
router: Final = Router(model_list=_generic_silent_model_list())
await router._silent_experiment_ageneric(
silent_model="silent-model",
original_function=provider,
silent_kwargs={"input": "hi", "attempt_function": primary_attempt},
)
primary_attempt.assert_not_awaited()
provider.assert_awaited_once()
assert provider.await_args.kwargs["model"] == "openai/gpt-4o"