mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-18 09:45:23 +00:00
fix: preserve generic response contracts for shadow requests
Signed-off-by: Jaeyeon Kim <anencore94@gmail.com>
This commit is contained in:
parent
14799041aa
commit
61f5973cd3
2 changed files with 41 additions and 1 deletions
|
|
@ -4497,10 +4497,12 @@ class Router:
|
|||
"""
|
||||
try:
|
||||
verbose_router_logger.info("Starting silent experiment for model %s", silent_model)
|
||||
remaining_kwargs: Final = {key: value for key, value in silent_kwargs.items() if key != "attempt_function"}
|
||||
await self._ageneric_api_call_with_fallbacks(
|
||||
model=silent_model,
|
||||
original_function=original_function,
|
||||
**silent_kwargs,
|
||||
attempt_function=None,
|
||||
**remaining_kwargs,
|
||||
)
|
||||
except Exception as e: # noqa: BLE001 - a background mirror must never break the primary request
|
||||
verbose_router_logger.error("Silent experiment failed for model %s: %s", silent_model, e)
|
||||
|
|
@ -6452,6 +6454,8 @@ class Router:
|
|||
carry_over_routed_deployment(live_kwargs=kwargs, snapshot=hop_kwargs)
|
||||
if kwargs.get("stream") and isinstance(response, BaseResponsesAPIStreamingIterator):
|
||||
return await self._aresponses_streaming_iterator(response=response, initial_kwargs=hop_kwargs)
|
||||
if response is None:
|
||||
raise TypeError("Responses request returned no response")
|
||||
return response
|
||||
|
||||
async def _aanthropic_messages_streaming_iterator(
|
||||
|
|
@ -7015,6 +7019,8 @@ class Router:
|
|||
response=cast("AsyncIterator[bytes]", response), # cast-ok: stream=True always returns a byte iterator
|
||||
initial_kwargs=hop_kwargs,
|
||||
)
|
||||
if response is None:
|
||||
raise TypeError("Anthropic Messages request returned no response")
|
||||
return response
|
||||
|
||||
async def _dispatch_generic_call_type(
|
||||
|
|
|
|||
|
|
@ -949,3 +949,37 @@ async def test_generic_mirroring_skips_deployment_mcp(mcp_deployment, mcp_config
|
|||
|
||||
provider.assert_awaited_once()
|
||||
assert provider.await_args.kwargs["model"] == "openai/gpt-4o-mini"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"call_type,attempt_method",
|
||||
[
|
||||
("aresponses", "_ageneric_api_call_with_fallbacks_responses_attempt"),
|
||||
("anthropic_messages", "_ageneric_api_call_with_fallbacks_anthropic_messages_attempt"),
|
||||
],
|
||||
)
|
||||
async def test_generic_primary_attempt_rejects_missing_provider_response(call_type, attempt_method):
|
||||
provider: Final = AsyncMock(return_value=None)
|
||||
provider.__name__ = call_type
|
||||
router: Final = Router(model_list=_generic_silent_model_list())
|
||||
with pytest.raises(TypeError, match="request returned no response"):
|
||||
await getattr(router, attempt_method)(model="silent-model", original_generic_function=provider, input="hi")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_generic_mirror_does_not_reuse_primary_attempt_callback():
|
||||
primary_attempt: Final = AsyncMock(return_value="primary-only")
|
||||
provider: Final = AsyncMock(return_value="mirror-result")
|
||||
provider.__name__ = "aresponses"
|
||||
router: Final = Router(model_list=_generic_silent_model_list())
|
||||
|
||||
await router._silent_experiment_ageneric(
|
||||
silent_model="silent-model",
|
||||
original_function=provider,
|
||||
silent_kwargs={"input": "hi", "attempt_function": primary_attempt},
|
||||
)
|
||||
|
||||
primary_attempt.assert_not_awaited()
|
||||
provider.assert_awaited_once()
|
||||
assert provider.await_args.kwargs["model"] == "openai/gpt-4o"
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue