This commit is contained in:
Jaeyeon Kim(김재연) 2026-10-27 02:23:21 +00:00 • committed by GitHub
commit ec92ba44d3
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
3 changed files with 462 additions and 35 deletions

View file

@ -2161,6 +2161,11 @@ DEFAULT_COMPETITOR_DISCOVERY_MODEL: Final = "gpt-4o-mini"
ADVISOR_NATIVE_PROVIDERS: Final[frozenset] = frozenset({"anthropic"})
# Hard cap on advisor iterations per request to prevent runaway loops.
ADVISOR_MAX_USES: Final[int] = 5
# Generic-router call types (non chat-completion) that are safe to mirror to a
# `silent_model`. Only side-effect-free inference endpoints belong here: the same
# router helper also serves file/fine-tuning/passthrough calls that must never be
# replayed against a second deployment.
SILENT_MODEL_MIRROR_ALLOWED_CALL_TYPES: Final[frozenset[str]] = frozenset({"aresponses", "anthropic_messages"})
# Description injected into the synthetic advisor tool definition sent to non-native providers.
ADVISOR_TOOL_DESCRIPTION: Final[str] = (
"Consult a highly intelligent advisor model when you need expert guidance, "

View file

@ -79,6 +79,7 @@ from litellm.constants import (
ROUTING_REQUEST_TAGS_METADATA_KEY,
RUNTIME_UPDATABLE_ROUTER_SETTINGS,
SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY,
SILENT_MODEL_MIRROR_ALLOWED_CALL_TYPES,
)
from litellm.integrations.custom_guardrail import CustomGuardrail, is_guardrail_intervention
from litellm.integrations.custom_logger import CustomLogger
@ -1122,6 +1123,23 @@ def as_output_cap(value: object) -> int | None:
return cap if cap >= 0 else None
def _is_silent_experiment_marker(value: object) -> bool:
"""True when a metadata dict carries the silent-experiment marker."""
return isinstance(value, Mapping) and bool(
cast(Mapping[str, object], value).get("is_silent_experiment", False) # cast-ok: narrowed by isinstance above
)
def _has_mcp_tool(tools: object) -> bool:
"""True when a Responses/Messages `tools` list carries an MCP tool definition."""
if not isinstance(tools, Sequence) or isinstance(tools, str):
return False
for tool in cast(Sequence[object], tools): # cast-ok: narrowed by isinstance above
if isinstance(tool, Mapping) and cast(Mapping[str, object], tool).get("type") == "mcp": # cast-ok: narrowed
return True
return False
def _exact_base_model_max_input_tokens(base_model: str) -> int | None:
try:
model_info: Final = litellm.get_model_info(model=base_model)
@ -3217,7 +3235,12 @@ class Router:
self._stamp_failed_deployment_id_with_effective_model_info(e, deployment, kwargs)
raise e
def _get_silent_experiment_kwargs(self, **kwargs) -> dict:
def _get_silent_experiment_kwargs(
self,
metadata_variable_name: str = "metadata",
/,
**kwargs: object, # kwargs-ok: the primary call's kwargs, forwarded verbatim into the silent call
) -> dict:
"""
Prepare kwargs for a silent experiment by ensuring isolation from the primary call.
@ -3226,29 +3249,36 @@ class Router:
parent_otel_span — an OTel Span that is not deepcopy-able). Force a shallow
copy of the metadata dict so mutations (model_group, is_silent_experiment)
never corrupt the main call's metadata.
`metadata_variable_name` selects the kwargs key that carries the router's own
metadata: "metadata" for chat completions, "litellm_metadata" for calls routed
through `_ageneric_api_call_with_fallbacks` (Responses API, Anthropic Messages),
where `metadata` is the provider request-body field and must not be touched.
"""
from litellm.litellm_core_utils.core_helpers import safe_deep_copy
silent_kwargs: Final = safe_deep_copy(kwargs)
silent_kwargs: Final[dict[str, object]] = cast( # cast-ok: safe_deep_copy is untyped # mutable-ok: fresh copy
dict[str, object], safe_deep_copy(kwargs)
)
# safe_deep_copy may fall back to the original metadata reference when
# deepcopy fails (UserAPIKeyAuth.parent_otel_span is not deepcopy-able).
# Detect this via identity check and force a shallow copy so that setting
# model_group / is_silent_experiment on the silent dict doesn't corrupt
# the primary call's metadata.
original_metadata: Final = kwargs.get("metadata")
if original_metadata is not None and silent_kwargs.get("metadata") is original_metadata:
silent_kwargs["metadata"] = dict(original_metadata)
if "metadata" not in silent_kwargs:
silent_kwargs["metadata"] = {}
# Always shallow-copy the metadata dict so that setting model_group /
# is_silent_experiment on the silent dict doesn't corrupt the primary call's.
copied_metadata: Final = silent_kwargs.get(metadata_variable_name)
silent_metadata: Final[dict[str, object]] = ( # mutable-ok: the silent call's own metadata dict
dict(cast(Mapping[str, object], copied_metadata)) # cast-ok: narrowed by the isinstance check
if isinstance(copied_metadata, Mapping)
else {}
)
# OTel spans are not safe to use across event loops. The silent
# experiment runs in a new event loop, so strip the span to prevent
# cross-loop tracing races or span corruption.
silent_kwargs["metadata"].pop("litellm_parent_otel_span", None)
silent_metadata.pop("litellm_parent_otel_span", None)
silent_kwargs["metadata"]["is_silent_experiment"] = True
silent_metadata["is_silent_experiment"] = True
silent_kwargs[metadata_variable_name] = silent_metadata
# Pop logging objects and call IDs to ensure a fresh logging context
# This prevents collisions in the Proxy's database (spend_logs)
@ -3276,13 +3306,18 @@ class Router:
async for _ in response:
pass
def _silent_experiment_completion(self, silent_model: str, messages: Sequence[Mapping[str, str]], **kwargs):
def _silent_experiment_completion(
self,
silent_model: str,
messages: Sequence[Mapping[str, str]],
**kwargs: object, # kwargs-ok: the primary call's kwargs, forwarded verbatim into the silent call
):
"""
Run a silent experiment in the background (thread).
"""
try:
# Prevent infinite recursion if silent model also has a silent model
if kwargs.get("metadata", {}).get("is_silent_experiment", False):
if _is_silent_experiment_marker(kwargs.get("metadata")):
return
messages = copy.deepcopy(messages)
@ -4426,13 +4461,71 @@ class Router:
wrapper_ref: Final[weakref.ref[FallbackAwareStreamWrapper]] = weakref.ref(wrapped_response)
return wrapped_response
async def _silent_experiment_acompletion(self, silent_model: str, messages: Sequence[Mapping[str, str]], **kwargs):
def _silent_experiment_generic_kwargs(
self,
**kwargs: object, # kwargs-ok: the primary call's kwargs, snapshotted before the router mutates them
) -> Mapping[str, object] | None:
"""
Snapshot the primary call's kwargs for a generic-path silent experiment, or
return None when the request must not be mirrored.
Called synchronously before the mirror task is scheduled: `create_task` only
starts running once the caller yields, by which point
`_update_kwargs_with_deployment` has merged the primary deployment's metadata
and tags into `kwargs` in place. Copying here keeps the mirror on the caller's
request. Router metadata for these call types lives in `litellm_metadata`;
`metadata` is the provider request-body field (OpenAI Responses `metadata`,
Anthropic `metadata`) and is left untouched so the marker never reaches the
provider.
"""
# Prevent infinite recursion if the silent model also has a silent model
if _is_silent_experiment_marker(kwargs.get("metadata")) or _is_silent_experiment_marker(
kwargs.get("litellm_metadata")
):
return None
# MCP tools may execute against the provider (Anthropic `mcp_servers`, Responses
# MCP tools with require_approval="never"); replaying those is a side effect.
if kwargs.get("mcp_servers") or _has_mcp_tool(kwargs.get("tools")):
return None
return {**self._get_silent_experiment_kwargs("litellm_metadata", **kwargs), "stream": False}
async def _silent_experiment_ageneric(
self,
silent_model: str,
original_function: Callable[..., Awaitable[object]],
silent_kwargs: Mapping[str, object],
) -> None:
"""
Run a generic-path silent experiment (Responses API, Anthropic Messages) in the
background. Counterpart of `_silent_experiment_acompletion`; `silent_kwargs`
comes from `_silent_experiment_generic_kwargs`. `model_group` is re-stamped to
`silent_model` by `_update_kwargs_before_fallbacks` inside the call, so metrics
attribute the mirror to the silent model.
"""
try:
verbose_router_logger.info("Starting silent experiment for model %s", silent_model)
remaining_kwargs: Final = {key: value for key, value in silent_kwargs.items() if key != "attempt_function"}
await self._ageneric_api_call_with_fallbacks(
model=silent_model,
original_function=original_function,
attempt_function=None,
**remaining_kwargs,
)
except Exception as e: # noqa: BLE001 - a background mirror must never break the primary request
verbose_router_logger.error("Silent experiment failed for model %s: %s", silent_model, e)
async def _silent_experiment_acompletion(
self,
silent_model: str,
messages: Sequence[Mapping[str, str]],
**kwargs: object, # kwargs-ok: the primary call's kwargs, forwarded verbatim into the silent call
):
"""
Run a silent experiment in the background.
"""
try:
# Prevent infinite recursion if silent model also has a silent model
if kwargs.get("metadata", {}).get("is_silent_experiment", False):
if _is_silent_experiment_marker(kwargs.get("metadata")):
return
messages = copy.deepcopy(messages)
@ -6258,6 +6351,31 @@ class Router:
return await original_generic_function(model=model, **kwargs)
raise e
deployment_has_mcp: Final = bool(deployment["litellm_params"].get("mcp_servers")) or _has_mcp_tool(
deployment["litellm_params"].get("tools")
)
if deployment_has_mcp and _is_silent_experiment_marker(kwargs.get("litellm_metadata")):
return None
silent_model: Final[object] = deployment["litellm_params"].get("silent_model")
if (
isinstance(silent_model, str)
and handler_name in SILENT_MODEL_MIRROR_ALLOWED_CALL_TYPES
and not deployment_has_mcp
):
# Snapshot now: `_update_kwargs_with_deployment` below mutates kwargs in place,
# and the task only starts running once this coroutine yields.
silent_kwargs: Final = self._silent_experiment_generic_kwargs(**kwargs)
if silent_kwargs is not None:
asyncio.create_task(
self._silent_experiment_ageneric(
silent_model=silent_model,
original_function=cast( # cast-ok: generic handlers are async callables; the bare Callable annotation predates this
Callable[..., Awaitable[object]], original_generic_function
),
silent_kwargs=silent_kwargs,
)
)
self._update_kwargs_with_deployment(
deployment=deployment,
kwargs=kwargs,
@ -6266,6 +6384,8 @@ class Router:
)
data: Final = deployment["litellm_params"].copy()
# Router-only setting: must never reach the provider handler (#34890)
data.pop("silent_model", None)
model_name: Final = data["model"]
self.total_calls[model_name] += 1
@ -6291,6 +6411,7 @@ class Router:
# Only set custom_llm_provider if it's not None
if custom_llm_provider is not None:
response_kwargs["custom_llm_provider"] = custom_llm_provider
response_kwargs.pop("silent_model", None)
compacted_input: Final = await compact_to_fit(
self,
@ -6370,6 +6491,8 @@ class Router:
carry_over_routed_deployment(live_kwargs=kwargs, snapshot=hop_kwargs)
if kwargs.get("stream") and isinstance(response, BaseResponsesAPIStreamingIterator):
return await self._aresponses_streaming_iterator(response=response, initial_kwargs=hop_kwargs)
if response is None:
raise TypeError("Responses request returned no response")
return response
async def _aanthropic_messages_streaming_iterator(
@ -6933,6 +7056,8 @@ class Router:
response=cast("AsyncIterator[bytes]", response), # cast-ok: stream=True always returns a byte iterator
initial_kwargs=hop_kwargs,
)
if response is None:
raise TypeError("Anthropic Messages request returned no response")
return response
async def _dispatch_generic_call_type(

View file

@ -13,7 +13,7 @@ import respx
import litellm
from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
from litellm.router import Router
from litellm.router import Router, _has_mcp_tool, _is_silent_experiment_marker
from litellm.router import _silent_experiment_kwargs_snapshot
from litellm.router import _silent_experiment_targets
@ -488,11 +488,7 @@ async def test_router_silent_experiment_acompletion():
# Find the silent call
silent_call = next(
(
c
for c in call_args_list
if c[1].get("metadata", {}).get("is_silent_experiment") is True
),
(c for c in call_args_list if c[1].get("metadata", {}).get("is_silent_experiment") is True),
None,
)
assert silent_call is not None
@ -500,11 +496,7 @@ async def test_router_silent_experiment_acompletion():
# Find the primary call
primary_call = next(
(
c
for c in call_args_list
if not c[1].get("metadata", {}).get("is_silent_experiment")
),
(c for c in call_args_list if not c[1].get("metadata", {}).get("is_silent_experiment")),
None,
)
assert primary_call is not None
@ -573,11 +565,7 @@ def test_router_silent_experiment_completion():
# Find the silent call
silent_call = next(
(
c
for c in call_args_list
if c[1].get("metadata", {}).get("is_silent_experiment") is True
),
(c for c in call_args_list if c[1].get("metadata", {}).get("is_silent_experiment") is True),
None,
)
assert silent_call is not None
@ -613,8 +601,6 @@ def test_silent_experiment_sends_shadow_request_attributed_to_the_silent_model(r
assert primary_metadata == {"model_group": "primary-model"}
_EMBEDDING_API_BASE: Final = "https://embeddings.example.test/v1"
@ -674,6 +660,8 @@ async def test_aembedding_with_silent_model_sends_provider_body_without_it(
request_body: Final = json.loads(route.calls.last.request.read())
assert request_body == {"model": "embed-model", "input": ["black dresses"], "input_type": "query"}
assert response.data[0]["embedding"] == [0.1, 0.2]
@pytest.mark.parametrize("run_silent_experiment", SILENT_EXPERIMENT_RUNNERS)
def test_silent_experiment_does_not_launch_from_a_shadow_request(run_silent_experiment):
router = Router(model_list=_streaming_model_list(["shadow-a"]))
@ -686,3 +674,312 @@ def test_silent_experiment_does_not_launch_from_a_shadow_request(run_silent_expe
)
acompletion.assert_not_awaited()
# ---------------------------------------------------------------------------
# Generic router path (Responses API / Anthropic Messages) — issues #31888, #34890
# ---------------------------------------------------------------------------
def _generic_silent_model_list():
return [
{
"model_name": "primary-model",
"litellm_params": {
"model": "openai/gpt-4o-mini",
"api_key": "fake-key",
"silent_model": "silent-model",
},
},
{
"model_name": "silent-model",
"litellm_params": {
"model": "openai/gpt-4o",
"api_key": "fake-key",
},
},
]
def _split_generic_calls(mock):
"""Return (primary_call, silent_call) from a mocked generic handler."""
silent = next(
(c for c in mock.call_args_list if (c.kwargs.get("litellm_metadata") or {}).get("is_silent_experiment")),
None,
)
primary = next(
(c for c in mock.call_args_list if not (c.kwargs.get("litellm_metadata") or {}).get("is_silent_experiment")),
None,
)
return primary, silent
@pytest.mark.asyncio
async def test_router_silent_experiment_aresponses():
"""
Regression for #31888: silent_model on a deployment must fire a background
aresponses call when the primary request goes through /v1/responses.
"""
mock_aresponses = AsyncMock(return_value=MagicMock())
mock_aresponses.__name__ = "aresponses"
router = Router(model_list=_generic_silent_model_list())
router.aresponses = router.factory_function(mock_aresponses, call_type="aresponses")
await router.aresponses(
model="primary-model",
input=[{"role": "user", "content": "hi"}],
metadata={"trace": "user-supplied"},
)
await asyncio.sleep(0.1)
assert mock_aresponses.call_count == 2
primary_call, silent_call = _split_generic_calls(mock_aresponses)
assert primary_call is not None
assert silent_call is not None
assert primary_call.kwargs["model"] == "openai/gpt-4o-mini"
assert primary_call.kwargs["litellm_metadata"]["model_group"] == "primary-model"
assert silent_call.kwargs["model"] == "openai/gpt-4o"
# model_group is overridden so metrics/logging attribute the mirror to the silent model
assert silent_call.kwargs["litellm_metadata"]["model_group"] == "silent-model"
# The router-only setting must never reach the provider handler (#34890)
for call in mock_aresponses.call_args_list:
assert "silent_model" not in call.kwargs
# `metadata` is the OpenAI Responses request-body field: the marker must not be
# written there, and the caller's value must pass through untouched.
assert silent_call.kwargs["metadata"] == {"trace": "user-supplied"}
assert primary_call.kwargs["metadata"] == {"trace": "user-supplied"}
@pytest.mark.asyncio
async def test_router_silent_experiment_anthropic_messages():
"""
Regression for #34890: same contract for /v1/messages. Anthropic validates the
`metadata` body field strictly, so the marker must live in litellm_metadata.
"""
mock_messages = AsyncMock(return_value=MagicMock())
mock_messages.__name__ = "anthropic_messages"
router = Router(model_list=_generic_silent_model_list())
router.aanthropic_messages = router.factory_function(mock_messages, call_type="anthropic_messages")
await router.aanthropic_messages(
model="primary-model",
messages=[{"role": "user", "content": "hi"}],
max_tokens=16,
)
await asyncio.sleep(0.1)
assert mock_messages.call_count == 2
primary_call, silent_call = _split_generic_calls(mock_messages)
assert primary_call is not None
assert silent_call is not None
assert primary_call.kwargs["model"] == "openai/gpt-4o-mini"
assert silent_call.kwargs["model"] == "openai/gpt-4o"
assert silent_call.kwargs["litellm_metadata"]["model_group"] == "silent-model"
for call in mock_messages.call_args_list:
assert "silent_model" not in call.kwargs
assert "is_silent_experiment" not in (call.kwargs.get("metadata") or {})
@pytest.mark.asyncio
async def test_router_silent_experiment_generic_does_not_corrupt_primary_metadata():
"""
The primary call's litellm_metadata must not see the silent marker or the
silent model_group, even when deepcopy falls back to a shared reference.
"""
mock_aresponses = AsyncMock(return_value=MagicMock())
mock_aresponses.__name__ = "aresponses"
router = Router(model_list=_generic_silent_model_list())
router.aresponses = router.factory_function(mock_aresponses, call_type="aresponses")
litellm_metadata = {
"user_api_key_auth": _FakeUserAPIKeyAuth(
key_alias="primary-key",
parent_otel_span=_NonCopyableSpan(),
)
}
await router.aresponses(
model="primary-model",
input=[{"role": "user", "content": "hi"}],
litellm_metadata=litellm_metadata,
)
await asyncio.sleep(0.1)
assert mock_aresponses.call_count == 2
primary_call, silent_call = _split_generic_calls(mock_aresponses)
assert primary_call is not None
assert silent_call is not None
assert primary_call.kwargs["litellm_metadata"]["model_group"] == "primary-model"
assert "is_silent_experiment" not in primary_call.kwargs["litellm_metadata"]
assert silent_call.kwargs["litellm_metadata"]["model_group"] == "silent-model"
@pytest.mark.asyncio
async def test_router_silent_experiment_skips_non_allowlisted_generic_call_types():
"""
Only side-effect-free inference call types are mirrored. The same generic
helper serves file/fine-tuning/passthrough calls that must not be replayed
against the silent deployment.
"""
sentinel_response = MagicMock()
mock_file_content = AsyncMock(return_value=sentinel_response)
mock_file_content.__name__ = "afile_content"
router = Router(model_list=_generic_silent_model_list())
router.afile_content = router.factory_function(mock_file_content, call_type="afile_content")
response = await router.afile_content(model="primary-model", file_id="file-123")
await asyncio.sleep(0.1)
assert response is sentinel_response
assert mock_file_content.call_count == 1
assert "silent_model" not in mock_file_content.call_args.kwargs
def test_silent_experiment_generic_kwargs_skips_recursion_and_mcp():
"""
The generic-path snapshot must return None when the request is already a
silent experiment (marker in either litellm_metadata or metadata) or when it
carries MCP tooling that the provider may execute.
"""
router = Router(model_list=_generic_silent_model_list())
base = {"input": [{"role": "user", "content": "hi"}]}
assert router._silent_experiment_generic_kwargs(**base, litellm_metadata={"is_silent_experiment": True}) is None
assert router._silent_experiment_generic_kwargs(**base, metadata={"is_silent_experiment": True}) is None
assert router._silent_experiment_generic_kwargs(**base, mcp_servers=[{"type": "url", "url": "x"}]) is None
assert (
router._silent_experiment_generic_kwargs(
**base, tools=[{"type": "mcp", "server_label": "x", "require_approval": "never"}]
)
is None
)
snapshot = router._silent_experiment_generic_kwargs(**base, tools=[{"type": "web_search"}])
assert snapshot is not None
assert snapshot["litellm_metadata"]["is_silent_experiment"] is True
assert snapshot["stream"] is False
assert "metadata" not in snapshot
def test_silent_experiment_generic_kwargs_snapshots_before_mutation():
"""
Regression: the snapshot is taken synchronously, so metadata merged into the
caller's kwargs afterwards (deployment tags etc.) never reaches the mirror.
"""
router = Router(model_list=_generic_silent_model_list())
litellm_metadata = {"model_group": "primary-model"}
snapshot = router._silent_experiment_generic_kwargs(input="hi", litellm_metadata=litellm_metadata)
assert snapshot is not None
litellm_metadata["tags"] = ["primary-deployment-tag"]
assert "tags" not in snapshot["litellm_metadata"]
@pytest.mark.asyncio
async def test_silent_experiment_ageneric_error_is_caught():
"""
A failing mirror must never propagate to the caller.
"""
router = Router(model_list=_generic_silent_model_list())
with patch.object(
router,
"_ageneric_api_call_with_fallbacks",
new_callable=AsyncMock,
side_effect=Exception("downstream failure"),
):
result = await router._silent_experiment_ageneric(
silent_model="silent-model",
original_function=litellm.aresponses,
silent_kwargs={"input": [{"role": "user", "content": "hi"}]},
)
assert result is None
def test_is_silent_experiment_marker():
assert _is_silent_experiment_marker({"is_silent_experiment": True}) is True
assert _is_silent_experiment_marker({"is_silent_experiment": False}) is False
assert _is_silent_experiment_marker({}) is False
assert _is_silent_experiment_marker(None) is False
assert _is_silent_experiment_marker("is_silent_experiment") is False
def test_has_mcp_tool():
assert _has_mcp_tool([{"type": "mcp", "server_label": "x"}]) is True
assert _has_mcp_tool([{"type": "web_search"}, {"type": "mcp"}]) is True
assert _has_mcp_tool([{"type": "web_search"}]) is False
assert _has_mcp_tool([]) is False
assert _has_mcp_tool(None) is False
assert _has_mcp_tool("mcp") is False
@pytest.mark.asyncio
@pytest.mark.parametrize("mcp_deployment", ["primary-model", "silent-model"])
@pytest.mark.parametrize(
"mcp_config",
[
{"tools": [{"type": "mcp", "server_label": "test", "server_url": "https://mcp.example.test"}]},
{"mcp_servers": [{"type": "url", "name": "test", "url": "https://mcp.example.test"}]},
],
)
async def test_generic_mirroring_skips_deployment_mcp(mcp_deployment, mcp_config):
provider: Final = AsyncMock(return_value=MagicMock())
provider.__name__ = "aresponses"
router: Final = Router(
model_list=[
{
**deployment,
"litellm_params": {
**deployment["litellm_params"],
**(mcp_config if deployment["model_name"] == mcp_deployment else {}),
},
}
for deployment in _generic_silent_model_list()
]
)
endpoint: Final = router.factory_function(provider, call_type="aresponses")
await endpoint(model="primary-model", input="hi")
mirrors: Final = tuple(
task for task in asyncio.all_tasks() if task.get_coro().__qualname__.endswith("._silent_experiment_ageneric")
)
await asyncio.gather(*mirrors)
provider.assert_awaited_once()
assert provider.await_args.kwargs["model"] == "openai/gpt-4o-mini"
@pytest.mark.asyncio
@pytest.mark.parametrize(
"call_type,attempt_method",
[
("aresponses", "_ageneric_api_call_with_fallbacks_responses_attempt"),
("anthropic_messages", "_ageneric_api_call_with_fallbacks_anthropic_messages_attempt"),
],
)
async def test_generic_primary_attempt_rejects_missing_provider_response(call_type, attempt_method):
provider: Final = AsyncMock(return_value=None)
provider.__name__ = call_type
router: Final = Router(model_list=_generic_silent_model_list())
with pytest.raises(TypeError, match="request returned no response"):
await getattr(router, attempt_method)(model="silent-model", original_generic_function=provider, input="hi")
@pytest.mark.asyncio
async def test_generic_mirror_does_not_reuse_primary_attempt_callback():
primary_attempt: Final = AsyncMock(return_value="primary-only")
provider: Final = AsyncMock(return_value="mirror-result")
provider.__name__ = "aresponses"
router: Final = Router(model_list=_generic_silent_model_list())
await router._silent_experiment_ageneric(
silent_model="silent-model",
original_function=provider,
silent_kwargs={"input": "hi", "attempt_function": primary_attempt},
)
primary_attempt.assert_not_awaited()
provider.assert_awaited_once()
assert provider.await_args.kwargs["model"] == "openai/gpt-4o"