mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-13 23:11:40 +00:00
Merge 089e316b63 into 1c61c2606e
This commit is contained in:
Jaeyeon Kim(김재연)
GitHub
commit
94bfc93115
3 changed files with 379 additions and 33 deletions
|
|
@ -1909,6 +1909,11 @@ DEFAULT_COMPETITOR_DISCOVERY_MODEL: Final = "gpt-4o-mini"
|
|||
ADVISOR_NATIVE_PROVIDERS: Final[frozenset] = frozenset({"anthropic"})
|
||||
# Hard cap on advisor iterations per request to prevent runaway loops.
|
||||
ADVISOR_MAX_USES: Final[int] = 5
|
||||
# Generic-router call types (non chat-completion) that are safe to mirror to a
|
||||
# `silent_model`. Only side-effect-free inference endpoints belong here: the same
|
||||
# router helper also serves file/fine-tuning/passthrough calls that must never be
|
||||
# replayed against a second deployment.
|
||||
SILENT_MODEL_MIRROR_ALLOWED_CALL_TYPES: Final[frozenset[str]] = frozenset({"aresponses", "anthropic_messages"})
|
||||
# Description injected into the synthetic advisor tool definition sent to non-native providers.
|
||||
ADVISOR_TOOL_DESCRIPTION: Final[str] = (
|
||||
"Consult a highly intelligent advisor model when you need expert guidance, "
|
||||
|
|
|
|||
|
|
@ -24,6 +24,7 @@ from collections import defaultdict
|
|||
from collections.abc import (
|
||||
AsyncGenerator,
|
||||
AsyncIterator,
|
||||
Awaitable,
|
||||
Callable,
|
||||
Generator,
|
||||
Iterator,
|
||||
|
|
@ -66,6 +67,7 @@ from litellm.constants import (
|
|||
ROUTING_REQUEST_TAGS_METADATA_KEY,
|
||||
RUNTIME_UPDATABLE_ROUTER_SETTINGS,
|
||||
SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY,
|
||||
SILENT_MODEL_MIRROR_ALLOWED_CALL_TYPES,
|
||||
)
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.litellm_core_utils.asyncify import run_async_function
|
||||
|
|
@ -684,6 +686,23 @@ def as_output_cap(value: object) -> int | None:
|
|||
return cap if cap >= 0 else None
|
||||
|
||||
|
||||
def _is_silent_experiment_marker(value: object) -> bool:
|
||||
"""True when a metadata dict carries the silent-experiment marker."""
|
||||
return isinstance(value, Mapping) and bool(
|
||||
cast(Mapping[str, object], value).get("is_silent_experiment", False) # cast-ok: narrowed by isinstance above
|
||||
)
|
||||
|
||||
|
||||
def _has_mcp_tool(tools: object) -> bool:
|
||||
"""True when a Responses/Messages `tools` list carries an MCP tool definition."""
|
||||
if not isinstance(tools, Sequence) or isinstance(tools, str):
|
||||
return False
|
||||
for tool in cast(Sequence[object], tools): # cast-ok: narrowed by isinstance above
|
||||
if isinstance(tool, Mapping) and cast(Mapping[str, object], tool).get("type") == "mcp": # cast-ok: narrowed
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
class Router:
|
||||
model_names: set = set()
|
||||
cache_responses: bool | None = False
|
||||
|
|
@ -2512,7 +2531,12 @@ class Router:
|
|||
self._stamp_failed_deployment_id_with_effective_model_info(e, deployment, kwargs)
|
||||
raise e
|
||||
|
||||
def _get_silent_experiment_kwargs(self, **kwargs) -> dict:
|
||||
def _get_silent_experiment_kwargs(
|
||||
self,
|
||||
metadata_variable_name: str = "metadata",
|
||||
/,
|
||||
**kwargs: object, # kwargs-ok: the primary call's kwargs, forwarded verbatim into the silent call
|
||||
) -> dict:
|
||||
"""
|
||||
Prepare kwargs for a silent experiment by ensuring isolation from the primary call.
|
||||
|
||||
|
|
@ -2521,29 +2545,36 @@ class Router:
|
|||
parent_otel_span — an OTel Span that is not deepcopy-able). Force a shallow
|
||||
copy of the metadata dict so mutations (model_group, is_silent_experiment)
|
||||
never corrupt the main call's metadata.
|
||||
|
||||
`metadata_variable_name` selects the kwargs key that carries the router's own
|
||||
metadata: "metadata" for chat completions, "litellm_metadata" for calls routed
|
||||
through `_ageneric_api_call_with_fallbacks` (Responses API, Anthropic Messages),
|
||||
where `metadata` is the provider request-body field and must not be touched.
|
||||
"""
|
||||
from litellm.litellm_core_utils.core_helpers import safe_deep_copy
|
||||
|
||||
silent_kwargs: Final = safe_deep_copy(kwargs)
|
||||
silent_kwargs: Final[dict[str, object]] = cast( # cast-ok: safe_deep_copy is untyped # mutable-ok: fresh copy
|
||||
dict[str, object], safe_deep_copy(kwargs)
|
||||
)
|
||||
|
||||
# safe_deep_copy may fall back to the original metadata reference when
|
||||
# deepcopy fails (UserAPIKeyAuth.parent_otel_span is not deepcopy-able).
|
||||
# Detect this via identity check and force a shallow copy so that setting
|
||||
# model_group / is_silent_experiment on the silent dict doesn't corrupt
|
||||
# the primary call's metadata.
|
||||
original_metadata: Final = kwargs.get("metadata")
|
||||
if original_metadata is not None and silent_kwargs.get("metadata") is original_metadata:
|
||||
silent_kwargs["metadata"] = dict(original_metadata)
|
||||
|
||||
if "metadata" not in silent_kwargs:
|
||||
silent_kwargs["metadata"] = {}
|
||||
# Always shallow-copy the metadata dict so that setting model_group /
|
||||
# is_silent_experiment on the silent dict doesn't corrupt the primary call's.
|
||||
copied_metadata: Final = silent_kwargs.get(metadata_variable_name)
|
||||
silent_metadata: Final[dict[str, object]] = ( # mutable-ok: the silent call's own metadata dict
|
||||
dict(cast(Mapping[str, object], copied_metadata)) # cast-ok: narrowed by the isinstance check
|
||||
if isinstance(copied_metadata, Mapping)
|
||||
else {}
|
||||
)
|
||||
|
||||
# OTel spans are not safe to use across event loops. The silent
|
||||
# experiment runs in a new event loop, so strip the span to prevent
|
||||
# cross-loop tracing races or span corruption.
|
||||
silent_kwargs["metadata"].pop("litellm_parent_otel_span", None)
|
||||
silent_metadata.pop("litellm_parent_otel_span", None)
|
||||
|
||||
silent_kwargs["metadata"]["is_silent_experiment"] = True
|
||||
silent_metadata["is_silent_experiment"] = True
|
||||
silent_kwargs[metadata_variable_name] = silent_metadata
|
||||
|
||||
# Force stream=False so the response is fully consumed and callbacks fire
|
||||
silent_kwargs["stream"] = False
|
||||
|
|
@ -2557,13 +2588,18 @@ class Router:
|
|||
|
||||
return silent_kwargs
|
||||
|
||||
def _silent_experiment_completion(self, silent_model: str, messages: Sequence[Mapping[str, str]], **kwargs):
|
||||
def _silent_experiment_completion(
|
||||
self,
|
||||
silent_model: str,
|
||||
messages: Sequence[Mapping[str, str]],
|
||||
**kwargs: object, # kwargs-ok: the primary call's kwargs, forwarded verbatim into the silent call
|
||||
):
|
||||
"""
|
||||
Run a silent experiment in the background (thread).
|
||||
"""
|
||||
try:
|
||||
# Prevent infinite recursion if silent model also has a silent model
|
||||
if kwargs.get("metadata", {}).get("is_silent_experiment", False):
|
||||
if _is_silent_experiment_marker(kwargs.get("metadata")):
|
||||
return
|
||||
|
||||
messages = copy.deepcopy(messages)
|
||||
|
|
@ -3453,13 +3489,69 @@ class Router:
|
|||
wrapper_ref: Final = weakref.ref(wrapped_response)
|
||||
return wrapped_response
|
||||
|
||||
async def _silent_experiment_acompletion(self, silent_model: str, messages: Sequence[Mapping[str, str]], **kwargs):
|
||||
def _silent_experiment_generic_kwargs(
|
||||
self,
|
||||
**kwargs: object, # kwargs-ok: the primary call's kwargs, snapshotted before the router mutates them
|
||||
) -> Mapping[str, object] | None:
|
||||
"""
|
||||
Snapshot the primary call's kwargs for a generic-path silent experiment, or
|
||||
return None when the request must not be mirrored.
|
||||
|
||||
Called synchronously before the mirror task is scheduled: `create_task` only
|
||||
starts running once the caller yields, by which point
|
||||
`_update_kwargs_with_deployment` has merged the primary deployment's metadata
|
||||
and tags into `kwargs` in place. Copying here keeps the mirror on the caller's
|
||||
request. Router metadata for these call types lives in `litellm_metadata`;
|
||||
`metadata` is the provider request-body field (OpenAI Responses `metadata`,
|
||||
Anthropic `metadata`) and is left untouched so the marker never reaches the
|
||||
provider.
|
||||
"""
|
||||
# Prevent infinite recursion if the silent model also has a silent model
|
||||
if _is_silent_experiment_marker(kwargs.get("metadata")) or _is_silent_experiment_marker(
|
||||
kwargs.get("litellm_metadata")
|
||||
):
|
||||
return None
|
||||
# MCP tools may execute against the provider (Anthropic `mcp_servers`, Responses
|
||||
# MCP tools with require_approval="never"); replaying those is a side effect.
|
||||
if kwargs.get("mcp_servers") or _has_mcp_tool(kwargs.get("tools")):
|
||||
return None
|
||||
return self._get_silent_experiment_kwargs("litellm_metadata", **kwargs)
|
||||
|
||||
async def _silent_experiment_ageneric(
|
||||
self,
|
||||
silent_model: str,
|
||||
original_function: Callable[..., Awaitable[object]],
|
||||
silent_kwargs: Mapping[str, object],
|
||||
) -> None:
|
||||
"""
|
||||
Run a generic-path silent experiment (Responses API, Anthropic Messages) in the
|
||||
background. Counterpart of `_silent_experiment_acompletion`; `silent_kwargs`
|
||||
comes from `_silent_experiment_generic_kwargs`. `model_group` is re-stamped to
|
||||
`silent_model` by `_update_kwargs_before_fallbacks` inside the call, so metrics
|
||||
attribute the mirror to the silent model.
|
||||
"""
|
||||
try:
|
||||
verbose_router_logger.info("Starting silent experiment for model %s", silent_model)
|
||||
await self._ageneric_api_call_with_fallbacks(
|
||||
model=silent_model,
|
||||
original_function=original_function,
|
||||
**silent_kwargs,
|
||||
)
|
||||
except Exception as e: # noqa: BLE001 - a background mirror must never break the primary request
|
||||
verbose_router_logger.error("Silent experiment failed for model %s: %s", silent_model, e)
|
||||
|
||||
async def _silent_experiment_acompletion(
|
||||
self,
|
||||
silent_model: str,
|
||||
messages: Sequence[Mapping[str, str]],
|
||||
**kwargs: object, # kwargs-ok: the primary call's kwargs, forwarded verbatim into the silent call
|
||||
):
|
||||
"""
|
||||
Run a silent experiment in the background.
|
||||
"""
|
||||
try:
|
||||
# Prevent infinite recursion if silent model also has a silent model
|
||||
if kwargs.get("metadata", {}).get("is_silent_experiment", False):
|
||||
if _is_silent_experiment_marker(kwargs.get("metadata")):
|
||||
return
|
||||
|
||||
messages = copy.deepcopy(messages)
|
||||
|
|
@ -5247,9 +5339,29 @@ class Router:
|
|||
return await original_generic_function(model=model, **kwargs)
|
||||
raise e
|
||||
|
||||
silent_model: Final[object] = deployment["litellm_params"].get("silent_model")
|
||||
generic_handler: Final[object] = cast(object, original_generic_function) # cast-ok: name only
|
||||
handler_name: Final[object] = getattr(generic_handler, "__name__", None)
|
||||
if isinstance(silent_model, str) and handler_name in SILENT_MODEL_MIRROR_ALLOWED_CALL_TYPES:
|
||||
# Snapshot now: `_update_kwargs_with_deployment` below mutates kwargs in place,
|
||||
# and the task only starts running once this coroutine yields.
|
||||
silent_kwargs: Final = self._silent_experiment_generic_kwargs(**kwargs)
|
||||
if silent_kwargs is not None:
|
||||
asyncio.create_task(
|
||||
self._silent_experiment_ageneric(
|
||||
silent_model=silent_model,
|
||||
original_function=cast( # cast-ok: generic handlers are async callables; the bare Callable annotation predates this
|
||||
Callable[..., Awaitable[object]], original_generic_function
|
||||
),
|
||||
silent_kwargs=silent_kwargs,
|
||||
)
|
||||
)
|
||||
|
||||
self._update_kwargs_with_deployment(deployment=deployment, kwargs=kwargs, function_name=function_name)
|
||||
|
||||
data: Final = deployment["litellm_params"].copy()
|
||||
# Router-only setting: must never reach the provider handler (#34890)
|
||||
data.pop("silent_model", None)
|
||||
model_name: Final = data["model"]
|
||||
self.total_calls[model_name] += 1
|
||||
|
||||
|
|
@ -5269,6 +5381,7 @@ class Router:
|
|||
# Only set custom_llm_provider if it's not None
|
||||
if custom_llm_provider is not None:
|
||||
response_kwargs["custom_llm_provider"] = custom_llm_provider
|
||||
response_kwargs.pop("silent_model", None)
|
||||
|
||||
response = original_generic_function(**response_kwargs)
|
||||
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@ from unittest.mock import AsyncMock, MagicMock, patch
|
|||
import pytest
|
||||
|
||||
import litellm
|
||||
from litellm.router import Router
|
||||
from litellm.router import Router, _has_mcp_tool, _is_silent_experiment_marker
|
||||
|
||||
|
||||
class _NonCopyableSpan:
|
||||
|
|
@ -185,11 +185,7 @@ async def test_router_silent_experiment_acompletion():
|
|||
|
||||
# Find the silent call
|
||||
silent_call = next(
|
||||
(
|
||||
c
|
||||
for c in call_args_list
|
||||
if c[1].get("metadata", {}).get("is_silent_experiment") is True
|
||||
),
|
||||
(c for c in call_args_list if c[1].get("metadata", {}).get("is_silent_experiment") is True),
|
||||
None,
|
||||
)
|
||||
assert silent_call is not None
|
||||
|
|
@ -197,11 +193,7 @@ async def test_router_silent_experiment_acompletion():
|
|||
|
||||
# Find the primary call
|
||||
primary_call = next(
|
||||
(
|
||||
c
|
||||
for c in call_args_list
|
||||
if not c[1].get("metadata", {}).get("is_silent_experiment")
|
||||
),
|
||||
(c for c in call_args_list if not c[1].get("metadata", {}).get("is_silent_experiment")),
|
||||
None,
|
||||
)
|
||||
assert primary_call is not None
|
||||
|
|
@ -270,14 +262,250 @@ def test_router_silent_experiment_completion():
|
|||
|
||||
# Find the silent call
|
||||
silent_call = next(
|
||||
(
|
||||
c
|
||||
for c in call_args_list
|
||||
if c[1].get("metadata", {}).get("is_silent_experiment") is True
|
||||
),
|
||||
(c for c in call_args_list if c[1].get("metadata", {}).get("is_silent_experiment") is True),
|
||||
None,
|
||||
)
|
||||
assert silent_call is not None
|
||||
assert silent_call[1]["model"] == "openai/gpt-4"
|
||||
# Verify model_group is set to the silent model name for correct metric attribution
|
||||
assert silent_call[1]["metadata"]["model_group"] == "silent-model"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Generic router path (Responses API / Anthropic Messages) — issues #31888, #34890
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _generic_silent_model_list():
|
||||
return [
|
||||
{
|
||||
"model_name": "primary-model",
|
||||
"litellm_params": {
|
||||
"model": "openai/gpt-4o-mini",
|
||||
"api_key": "fake-key",
|
||||
"silent_model": "silent-model",
|
||||
},
|
||||
},
|
||||
{
|
||||
"model_name": "silent-model",
|
||||
"litellm_params": {
|
||||
"model": "openai/gpt-4o",
|
||||
"api_key": "fake-key",
|
||||
},
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
def _split_generic_calls(mock):
|
||||
"""Return (primary_call, silent_call) from a mocked generic handler."""
|
||||
silent = next(
|
||||
(c for c in mock.call_args_list if (c.kwargs.get("litellm_metadata") or {}).get("is_silent_experiment")),
|
||||
None,
|
||||
)
|
||||
primary = next(
|
||||
(c for c in mock.call_args_list if not (c.kwargs.get("litellm_metadata") or {}).get("is_silent_experiment")),
|
||||
None,
|
||||
)
|
||||
return primary, silent
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_router_silent_experiment_aresponses():
|
||||
"""
|
||||
Regression for #31888: silent_model on a deployment must fire a background
|
||||
aresponses call when the primary request goes through /v1/responses.
|
||||
"""
|
||||
mock_aresponses = AsyncMock(return_value=MagicMock())
|
||||
mock_aresponses.__name__ = "aresponses"
|
||||
|
||||
router = Router(model_list=_generic_silent_model_list())
|
||||
router.aresponses = router.factory_function(mock_aresponses, call_type="aresponses")
|
||||
await router.aresponses(
|
||||
model="primary-model",
|
||||
input=[{"role": "user", "content": "hi"}],
|
||||
metadata={"trace": "user-supplied"},
|
||||
)
|
||||
await asyncio.sleep(0.1)
|
||||
|
||||
assert mock_aresponses.call_count == 2
|
||||
primary_call, silent_call = _split_generic_calls(mock_aresponses)
|
||||
assert primary_call is not None
|
||||
assert silent_call is not None
|
||||
|
||||
assert primary_call.kwargs["model"] == "openai/gpt-4o-mini"
|
||||
assert primary_call.kwargs["litellm_metadata"]["model_group"] == "primary-model"
|
||||
|
||||
assert silent_call.kwargs["model"] == "openai/gpt-4o"
|
||||
# model_group is overridden so metrics/logging attribute the mirror to the silent model
|
||||
assert silent_call.kwargs["litellm_metadata"]["model_group"] == "silent-model"
|
||||
|
||||
# The router-only setting must never reach the provider handler (#34890)
|
||||
for call in mock_aresponses.call_args_list:
|
||||
assert "silent_model" not in call.kwargs
|
||||
|
||||
# `metadata` is the OpenAI Responses request-body field: the marker must not be
|
||||
# written there, and the caller's value must pass through untouched.
|
||||
assert silent_call.kwargs["metadata"] == {"trace": "user-supplied"}
|
||||
assert primary_call.kwargs["metadata"] == {"trace": "user-supplied"}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_router_silent_experiment_anthropic_messages():
|
||||
"""
|
||||
Regression for #34890: same contract for /v1/messages. Anthropic validates the
|
||||
`metadata` body field strictly, so the marker must live in litellm_metadata.
|
||||
"""
|
||||
mock_messages = AsyncMock(return_value=MagicMock())
|
||||
mock_messages.__name__ = "anthropic_messages"
|
||||
|
||||
router = Router(model_list=_generic_silent_model_list())
|
||||
router.aanthropic_messages = router.factory_function(mock_messages, call_type="anthropic_messages")
|
||||
await router.aanthropic_messages(
|
||||
model="primary-model",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
max_tokens=16,
|
||||
)
|
||||
await asyncio.sleep(0.1)
|
||||
|
||||
assert mock_messages.call_count == 2
|
||||
primary_call, silent_call = _split_generic_calls(mock_messages)
|
||||
assert primary_call is not None
|
||||
assert silent_call is not None
|
||||
assert primary_call.kwargs["model"] == "openai/gpt-4o-mini"
|
||||
assert silent_call.kwargs["model"] == "openai/gpt-4o"
|
||||
assert silent_call.kwargs["litellm_metadata"]["model_group"] == "silent-model"
|
||||
for call in mock_messages.call_args_list:
|
||||
assert "silent_model" not in call.kwargs
|
||||
assert "is_silent_experiment" not in (call.kwargs.get("metadata") or {})
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_router_silent_experiment_generic_does_not_corrupt_primary_metadata():
|
||||
"""
|
||||
The primary call's litellm_metadata must not see the silent marker or the
|
||||
silent model_group, even when deepcopy falls back to a shared reference.
|
||||
"""
|
||||
mock_aresponses = AsyncMock(return_value=MagicMock())
|
||||
mock_aresponses.__name__ = "aresponses"
|
||||
|
||||
router = Router(model_list=_generic_silent_model_list())
|
||||
router.aresponses = router.factory_function(mock_aresponses, call_type="aresponses")
|
||||
litellm_metadata = {
|
||||
"user_api_key_auth": _FakeUserAPIKeyAuth(
|
||||
key_alias="primary-key",
|
||||
parent_otel_span=_NonCopyableSpan(),
|
||||
)
|
||||
}
|
||||
await router.aresponses(
|
||||
model="primary-model",
|
||||
input=[{"role": "user", "content": "hi"}],
|
||||
litellm_metadata=litellm_metadata,
|
||||
)
|
||||
await asyncio.sleep(0.1)
|
||||
|
||||
assert mock_aresponses.call_count == 2
|
||||
primary_call, silent_call = _split_generic_calls(mock_aresponses)
|
||||
assert primary_call is not None
|
||||
assert silent_call is not None
|
||||
assert primary_call.kwargs["litellm_metadata"]["model_group"] == "primary-model"
|
||||
assert "is_silent_experiment" not in primary_call.kwargs["litellm_metadata"]
|
||||
assert silent_call.kwargs["litellm_metadata"]["model_group"] == "silent-model"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_router_silent_experiment_skips_non_allowlisted_generic_call_types():
|
||||
"""
|
||||
Only side-effect-free inference call types are mirrored. The same generic
|
||||
helper serves file/fine-tuning/passthrough calls that must not be replayed
|
||||
against the silent deployment.
|
||||
"""
|
||||
sentinel_response = MagicMock()
|
||||
mock_file_content = AsyncMock(return_value=sentinel_response)
|
||||
mock_file_content.__name__ = "afile_content"
|
||||
|
||||
router = Router(model_list=_generic_silent_model_list())
|
||||
router.afile_content = router.factory_function(mock_file_content, call_type="afile_content")
|
||||
response = await router.afile_content(model="primary-model", file_id="file-123")
|
||||
await asyncio.sleep(0.1)
|
||||
|
||||
assert response is sentinel_response
|
||||
assert mock_file_content.call_count == 1
|
||||
assert "silent_model" not in mock_file_content.call_args.kwargs
|
||||
|
||||
|
||||
def test_silent_experiment_generic_kwargs_skips_recursion_and_mcp():
|
||||
"""
|
||||
The generic-path snapshot must return None when the request is already a
|
||||
silent experiment (marker in either litellm_metadata or metadata) or when it
|
||||
carries MCP tooling that the provider may execute.
|
||||
"""
|
||||
router = Router(model_list=_generic_silent_model_list())
|
||||
base = {"input": [{"role": "user", "content": "hi"}]}
|
||||
|
||||
assert router._silent_experiment_generic_kwargs(**base, litellm_metadata={"is_silent_experiment": True}) is None
|
||||
assert router._silent_experiment_generic_kwargs(**base, metadata={"is_silent_experiment": True}) is None
|
||||
assert router._silent_experiment_generic_kwargs(**base, mcp_servers=[{"type": "url", "url": "x"}]) is None
|
||||
assert (
|
||||
router._silent_experiment_generic_kwargs(
|
||||
**base, tools=[{"type": "mcp", "server_label": "x", "require_approval": "never"}]
|
||||
)
|
||||
is None
|
||||
)
|
||||
|
||||
snapshot = router._silent_experiment_generic_kwargs(**base, tools=[{"type": "web_search"}])
|
||||
assert snapshot is not None
|
||||
assert snapshot["litellm_metadata"]["is_silent_experiment"] is True
|
||||
assert snapshot["stream"] is False
|
||||
assert "metadata" not in snapshot
|
||||
|
||||
|
||||
def test_silent_experiment_generic_kwargs_snapshots_before_mutation():
|
||||
"""
|
||||
Regression: the snapshot is taken synchronously, so metadata merged into the
|
||||
caller's kwargs afterwards (deployment tags etc.) never reaches the mirror.
|
||||
"""
|
||||
router = Router(model_list=_generic_silent_model_list())
|
||||
litellm_metadata = {"model_group": "primary-model"}
|
||||
snapshot = router._silent_experiment_generic_kwargs(input="hi", litellm_metadata=litellm_metadata)
|
||||
assert snapshot is not None
|
||||
litellm_metadata["tags"] = ["primary-deployment-tag"]
|
||||
assert "tags" not in snapshot["litellm_metadata"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_silent_experiment_ageneric_error_is_caught():
|
||||
"""
|
||||
A failing mirror must never propagate to the caller.
|
||||
"""
|
||||
router = Router(model_list=_generic_silent_model_list())
|
||||
|
||||
with patch.object(
|
||||
router,
|
||||
"_ageneric_api_call_with_fallbacks",
|
||||
new_callable=AsyncMock,
|
||||
side_effect=Exception("downstream failure"),
|
||||
):
|
||||
result = await router._silent_experiment_ageneric(
|
||||
silent_model="silent-model",
|
||||
original_function=litellm.aresponses,
|
||||
silent_kwargs={"input": [{"role": "user", "content": "hi"}]},
|
||||
)
|
||||
|
||||
assert result is None
|
||||
|
||||
|
||||
def test_is_silent_experiment_marker():
|
||||
assert _is_silent_experiment_marker({"is_silent_experiment": True}) is True
|
||||
assert _is_silent_experiment_marker({"is_silent_experiment": False}) is False
|
||||
assert _is_silent_experiment_marker({}) is False
|
||||
assert _is_silent_experiment_marker(None) is False
|
||||
assert _is_silent_experiment_marker("is_silent_experiment") is False
|
||||
|
||||
|
||||
def test_has_mcp_tool():
|
||||
assert _has_mcp_tool([{"type": "mcp", "server_label": "x"}]) is True
|
||||
assert _has_mcp_tool([{"type": "web_search"}, {"type": "mcp"}]) is True
|
||||
assert _has_mcp_tool([{"type": "web_search"}]) is False
|
||||
assert _has_mcp_tool([]) is False
|
||||
assert _has_mcp_tool(None) is False
|
||||
assert _has_mcp_tool("mcp") is False
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue