diff --git a/litellm/constants.py b/litellm/constants.py index 6b984c2673c..d57217bb6dc 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -1909,6 +1909,11 @@ DEFAULT_COMPETITOR_DISCOVERY_MODEL: Final = "gpt-4o-mini" ADVISOR_NATIVE_PROVIDERS: Final[frozenset] = frozenset({"anthropic"}) # Hard cap on advisor iterations per request to prevent runaway loops. ADVISOR_MAX_USES: Final[int] = 5 +# Generic-router call types (non chat-completion) that are safe to mirror to a +# `silent_model`. Only side-effect-free inference endpoints belong here: the same +# router helper also serves file/fine-tuning/passthrough calls that must never be +# replayed against a second deployment. +SILENT_MODEL_MIRROR_ALLOWED_CALL_TYPES: Final[frozenset[str]] = frozenset({"aresponses", "anthropic_messages"}) # Description injected into the synthetic advisor tool definition sent to non-native providers. ADVISOR_TOOL_DESCRIPTION: Final[str] = ( "Consult a highly intelligent advisor model when you need expert guidance, " diff --git a/litellm/router.py b/litellm/router.py index 8865543badd..f52ab37a745 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -24,6 +24,7 @@ from collections import defaultdict from collections.abc import ( AsyncGenerator, AsyncIterator, + Awaitable, Callable, Generator, Iterator, @@ -66,6 +67,7 @@ from litellm.constants import ( ROUTING_REQUEST_TAGS_METADATA_KEY, RUNTIME_UPDATABLE_ROUTER_SETTINGS, SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY, + SILENT_MODEL_MIRROR_ALLOWED_CALL_TYPES, ) from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.asyncify import run_async_function @@ -684,6 +686,23 @@ def as_output_cap(value: object) -> int | None: return cap if cap >= 0 else None +def _is_silent_experiment_marker(value: object) -> bool: + """True when a metadata dict carries the silent-experiment marker.""" + return isinstance(value, Mapping) and bool( + cast(Mapping[str, object], value).get("is_silent_experiment", False) # cast-ok: narrowed by isinstance above + ) + + +def _has_mcp_tool(tools: object) -> bool: + """True when a Responses/Messages `tools` list carries an MCP tool definition.""" + if not isinstance(tools, Sequence) or isinstance(tools, str): + return False + for tool in cast(Sequence[object], tools): # cast-ok: narrowed by isinstance above + if isinstance(tool, Mapping) and cast(Mapping[str, object], tool).get("type") == "mcp": # cast-ok: narrowed + return True + return False + + class Router: model_names: set = set() cache_responses: bool | None = False @@ -2512,7 +2531,12 @@ class Router: self._stamp_failed_deployment_id_with_effective_model_info(e, deployment, kwargs) raise e - def _get_silent_experiment_kwargs(self, **kwargs) -> dict: + def _get_silent_experiment_kwargs( + self, + metadata_variable_name: str = "metadata", + /, + **kwargs: object, # kwargs-ok: the primary call's kwargs, forwarded verbatim into the silent call + ) -> dict: """ Prepare kwargs for a silent experiment by ensuring isolation from the primary call. @@ -2521,29 +2545,36 @@ class Router: parent_otel_span — an OTel Span that is not deepcopy-able). Force a shallow copy of the metadata dict so mutations (model_group, is_silent_experiment) never corrupt the main call's metadata. + + `metadata_variable_name` selects the kwargs key that carries the router's own + metadata: "metadata" for chat completions, "litellm_metadata" for calls routed + through `_ageneric_api_call_with_fallbacks` (Responses API, Anthropic Messages), + where `metadata` is the provider request-body field and must not be touched. """ from litellm.litellm_core_utils.core_helpers import safe_deep_copy - silent_kwargs: Final = safe_deep_copy(kwargs) + silent_kwargs: Final[dict[str, object]] = cast( # cast-ok: safe_deep_copy is untyped # mutable-ok: fresh copy + dict[str, object], safe_deep_copy(kwargs) + ) # safe_deep_copy may fall back to the original metadata reference when # deepcopy fails (UserAPIKeyAuth.parent_otel_span is not deepcopy-able). - # Detect this via identity check and force a shallow copy so that setting - # model_group / is_silent_experiment on the silent dict doesn't corrupt - # the primary call's metadata. - original_metadata: Final = kwargs.get("metadata") - if original_metadata is not None and silent_kwargs.get("metadata") is original_metadata: - silent_kwargs["metadata"] = dict(original_metadata) - - if "metadata" not in silent_kwargs: - silent_kwargs["metadata"] = {} + # Always shallow-copy the metadata dict so that setting model_group / + # is_silent_experiment on the silent dict doesn't corrupt the primary call's. + copied_metadata: Final = silent_kwargs.get(metadata_variable_name) + silent_metadata: Final[dict[str, object]] = ( # mutable-ok: the silent call's own metadata dict + dict(cast(Mapping[str, object], copied_metadata)) # cast-ok: narrowed by the isinstance check + if isinstance(copied_metadata, Mapping) + else {} + ) # OTel spans are not safe to use across event loops. The silent # experiment runs in a new event loop, so strip the span to prevent # cross-loop tracing races or span corruption. - silent_kwargs["metadata"].pop("litellm_parent_otel_span", None) + silent_metadata.pop("litellm_parent_otel_span", None) - silent_kwargs["metadata"]["is_silent_experiment"] = True + silent_metadata["is_silent_experiment"] = True + silent_kwargs[metadata_variable_name] = silent_metadata # Force stream=False so the response is fully consumed and callbacks fire silent_kwargs["stream"] = False @@ -2557,13 +2588,18 @@ class Router: return silent_kwargs - def _silent_experiment_completion(self, silent_model: str, messages: Sequence[Mapping[str, str]], **kwargs): + def _silent_experiment_completion( + self, + silent_model: str, + messages: Sequence[Mapping[str, str]], + **kwargs: object, # kwargs-ok: the primary call's kwargs, forwarded verbatim into the silent call + ): """ Run a silent experiment in the background (thread). """ try: # Prevent infinite recursion if silent model also has a silent model - if kwargs.get("metadata", {}).get("is_silent_experiment", False): + if _is_silent_experiment_marker(kwargs.get("metadata")): return messages = copy.deepcopy(messages) @@ -3453,13 +3489,69 @@ class Router: wrapper_ref: Final = weakref.ref(wrapped_response) return wrapped_response - async def _silent_experiment_acompletion(self, silent_model: str, messages: Sequence[Mapping[str, str]], **kwargs): + def _silent_experiment_generic_kwargs( + self, + **kwargs: object, # kwargs-ok: the primary call's kwargs, snapshotted before the router mutates them + ) -> Mapping[str, object] | None: + """ + Snapshot the primary call's kwargs for a generic-path silent experiment, or + return None when the request must not be mirrored. + + Called synchronously before the mirror task is scheduled: `create_task` only + starts running once the caller yields, by which point + `_update_kwargs_with_deployment` has merged the primary deployment's metadata + and tags into `kwargs` in place. Copying here keeps the mirror on the caller's + request. Router metadata for these call types lives in `litellm_metadata`; + `metadata` is the provider request-body field (OpenAI Responses `metadata`, + Anthropic `metadata`) and is left untouched so the marker never reaches the + provider. + """ + # Prevent infinite recursion if the silent model also has a silent model + if _is_silent_experiment_marker(kwargs.get("metadata")) or _is_silent_experiment_marker( + kwargs.get("litellm_metadata") + ): + return None + # MCP tools may execute against the provider (Anthropic `mcp_servers`, Responses + # MCP tools with require_approval="never"); replaying those is a side effect. + if kwargs.get("mcp_servers") or _has_mcp_tool(kwargs.get("tools")): + return None + return self._get_silent_experiment_kwargs("litellm_metadata", **kwargs) + + async def _silent_experiment_ageneric( + self, + silent_model: str, + original_function: Callable[..., Awaitable[object]], + silent_kwargs: Mapping[str, object], + ) -> None: + """ + Run a generic-path silent experiment (Responses API, Anthropic Messages) in the + background. Counterpart of `_silent_experiment_acompletion`; `silent_kwargs` + comes from `_silent_experiment_generic_kwargs`. `model_group` is re-stamped to + `silent_model` by `_update_kwargs_before_fallbacks` inside the call, so metrics + attribute the mirror to the silent model. + """ + try: + verbose_router_logger.info("Starting silent experiment for model %s", silent_model) + await self._ageneric_api_call_with_fallbacks( + model=silent_model, + original_function=original_function, + **silent_kwargs, + ) + except Exception as e: # noqa: BLE001 - a background mirror must never break the primary request + verbose_router_logger.error("Silent experiment failed for model %s: %s", silent_model, e) + + async def _silent_experiment_acompletion( + self, + silent_model: str, + messages: Sequence[Mapping[str, str]], + **kwargs: object, # kwargs-ok: the primary call's kwargs, forwarded verbatim into the silent call + ): """ Run a silent experiment in the background. """ try: # Prevent infinite recursion if silent model also has a silent model - if kwargs.get("metadata", {}).get("is_silent_experiment", False): + if _is_silent_experiment_marker(kwargs.get("metadata")): return messages = copy.deepcopy(messages) @@ -5247,9 +5339,29 @@ class Router: return await original_generic_function(model=model, **kwargs) raise e + silent_model: Final[object] = deployment["litellm_params"].get("silent_model") + generic_handler: Final[object] = cast(object, original_generic_function) # cast-ok: name only + handler_name: Final[object] = getattr(generic_handler, "__name__", None) + if isinstance(silent_model, str) and handler_name in SILENT_MODEL_MIRROR_ALLOWED_CALL_TYPES: + # Snapshot now: `_update_kwargs_with_deployment` below mutates kwargs in place, + # and the task only starts running once this coroutine yields. + silent_kwargs: Final = self._silent_experiment_generic_kwargs(**kwargs) + if silent_kwargs is not None: + asyncio.create_task( + self._silent_experiment_ageneric( + silent_model=silent_model, + original_function=cast( # cast-ok: generic handlers are async callables; the bare Callable annotation predates this + Callable[..., Awaitable[object]], original_generic_function + ), + silent_kwargs=silent_kwargs, + ) + ) + self._update_kwargs_with_deployment(deployment=deployment, kwargs=kwargs, function_name=function_name) data: Final = deployment["litellm_params"].copy() + # Router-only setting: must never reach the provider handler (#34890) + data.pop("silent_model", None) model_name: Final = data["model"] self.total_calls[model_name] += 1 @@ -5269,6 +5381,7 @@ class Router: # Only set custom_llm_provider if it's not None if custom_llm_provider is not None: response_kwargs["custom_llm_provider"] = custom_llm_provider + response_kwargs.pop("silent_model", None) response = original_generic_function(**response_kwargs) diff --git a/tests/test_litellm/test_router_silent_experiment.py b/tests/test_litellm/test_router_silent_experiment.py index bfdf39bad71..4bb29fa3e7f 100644 --- a/tests/test_litellm/test_router_silent_experiment.py +++ b/tests/test_litellm/test_router_silent_experiment.py @@ -5,7 +5,7 @@ from unittest.mock import AsyncMock, MagicMock, patch import pytest import litellm -from litellm.router import Router +from litellm.router import Router, _has_mcp_tool, _is_silent_experiment_marker class _NonCopyableSpan: @@ -185,11 +185,7 @@ async def test_router_silent_experiment_acompletion(): # Find the silent call silent_call = next( - ( - c - for c in call_args_list - if c[1].get("metadata", {}).get("is_silent_experiment") is True - ), + (c for c in call_args_list if c[1].get("metadata", {}).get("is_silent_experiment") is True), None, ) assert silent_call is not None @@ -197,11 +193,7 @@ async def test_router_silent_experiment_acompletion(): # Find the primary call primary_call = next( - ( - c - for c in call_args_list - if not c[1].get("metadata", {}).get("is_silent_experiment") - ), + (c for c in call_args_list if not c[1].get("metadata", {}).get("is_silent_experiment")), None, ) assert primary_call is not None @@ -270,14 +262,250 @@ def test_router_silent_experiment_completion(): # Find the silent call silent_call = next( - ( - c - for c in call_args_list - if c[1].get("metadata", {}).get("is_silent_experiment") is True - ), + (c for c in call_args_list if c[1].get("metadata", {}).get("is_silent_experiment") is True), None, ) assert silent_call is not None assert silent_call[1]["model"] == "openai/gpt-4" # Verify model_group is set to the silent model name for correct metric attribution assert silent_call[1]["metadata"]["model_group"] == "silent-model" + + +# --------------------------------------------------------------------------- +# Generic router path (Responses API / Anthropic Messages) — issues #31888, #34890 +# --------------------------------------------------------------------------- + + +def _generic_silent_model_list(): + return [ + { + "model_name": "primary-model", + "litellm_params": { + "model": "openai/gpt-4o-mini", + "api_key": "fake-key", + "silent_model": "silent-model", + }, + }, + { + "model_name": "silent-model", + "litellm_params": { + "model": "openai/gpt-4o", + "api_key": "fake-key", + }, + }, + ] + + +def _split_generic_calls(mock): + """Return (primary_call, silent_call) from a mocked generic handler.""" + silent = next( + (c for c in mock.call_args_list if (c.kwargs.get("litellm_metadata") or {}).get("is_silent_experiment")), + None, + ) + primary = next( + (c for c in mock.call_args_list if not (c.kwargs.get("litellm_metadata") or {}).get("is_silent_experiment")), + None, + ) + return primary, silent + + +@pytest.mark.asyncio +async def test_router_silent_experiment_aresponses(): + """ + Regression for #31888: silent_model on a deployment must fire a background + aresponses call when the primary request goes through /v1/responses. + """ + mock_aresponses = AsyncMock(return_value=MagicMock()) + mock_aresponses.__name__ = "aresponses" + + router = Router(model_list=_generic_silent_model_list()) + router.aresponses = router.factory_function(mock_aresponses, call_type="aresponses") + await router.aresponses( + model="primary-model", + input=[{"role": "user", "content": "hi"}], + metadata={"trace": "user-supplied"}, + ) + await asyncio.sleep(0.1) + + assert mock_aresponses.call_count == 2 + primary_call, silent_call = _split_generic_calls(mock_aresponses) + assert primary_call is not None + assert silent_call is not None + + assert primary_call.kwargs["model"] == "openai/gpt-4o-mini" + assert primary_call.kwargs["litellm_metadata"]["model_group"] == "primary-model" + + assert silent_call.kwargs["model"] == "openai/gpt-4o" + # model_group is overridden so metrics/logging attribute the mirror to the silent model + assert silent_call.kwargs["litellm_metadata"]["model_group"] == "silent-model" + + # The router-only setting must never reach the provider handler (#34890) + for call in mock_aresponses.call_args_list: + assert "silent_model" not in call.kwargs + + # `metadata` is the OpenAI Responses request-body field: the marker must not be + # written there, and the caller's value must pass through untouched. + assert silent_call.kwargs["metadata"] == {"trace": "user-supplied"} + assert primary_call.kwargs["metadata"] == {"trace": "user-supplied"} + + +@pytest.mark.asyncio +async def test_router_silent_experiment_anthropic_messages(): + """ + Regression for #34890: same contract for /v1/messages. Anthropic validates the + `metadata` body field strictly, so the marker must live in litellm_metadata. + """ + mock_messages = AsyncMock(return_value=MagicMock()) + mock_messages.__name__ = "anthropic_messages" + + router = Router(model_list=_generic_silent_model_list()) + router.aanthropic_messages = router.factory_function(mock_messages, call_type="anthropic_messages") + await router.aanthropic_messages( + model="primary-model", + messages=[{"role": "user", "content": "hi"}], + max_tokens=16, + ) + await asyncio.sleep(0.1) + + assert mock_messages.call_count == 2 + primary_call, silent_call = _split_generic_calls(mock_messages) + assert primary_call is not None + assert silent_call is not None + assert primary_call.kwargs["model"] == "openai/gpt-4o-mini" + assert silent_call.kwargs["model"] == "openai/gpt-4o" + assert silent_call.kwargs["litellm_metadata"]["model_group"] == "silent-model" + for call in mock_messages.call_args_list: + assert "silent_model" not in call.kwargs + assert "is_silent_experiment" not in (call.kwargs.get("metadata") or {}) + + +@pytest.mark.asyncio +async def test_router_silent_experiment_generic_does_not_corrupt_primary_metadata(): + """ + The primary call's litellm_metadata must not see the silent marker or the + silent model_group, even when deepcopy falls back to a shared reference. + """ + mock_aresponses = AsyncMock(return_value=MagicMock()) + mock_aresponses.__name__ = "aresponses" + + router = Router(model_list=_generic_silent_model_list()) + router.aresponses = router.factory_function(mock_aresponses, call_type="aresponses") + litellm_metadata = { + "user_api_key_auth": _FakeUserAPIKeyAuth( + key_alias="primary-key", + parent_otel_span=_NonCopyableSpan(), + ) + } + await router.aresponses( + model="primary-model", + input=[{"role": "user", "content": "hi"}], + litellm_metadata=litellm_metadata, + ) + await asyncio.sleep(0.1) + + assert mock_aresponses.call_count == 2 + primary_call, silent_call = _split_generic_calls(mock_aresponses) + assert primary_call is not None + assert silent_call is not None + assert primary_call.kwargs["litellm_metadata"]["model_group"] == "primary-model" + assert "is_silent_experiment" not in primary_call.kwargs["litellm_metadata"] + assert silent_call.kwargs["litellm_metadata"]["model_group"] == "silent-model" + + +@pytest.mark.asyncio +async def test_router_silent_experiment_skips_non_allowlisted_generic_call_types(): + """ + Only side-effect-free inference call types are mirrored. The same generic + helper serves file/fine-tuning/passthrough calls that must not be replayed + against the silent deployment. + """ + sentinel_response = MagicMock() + mock_file_content = AsyncMock(return_value=sentinel_response) + mock_file_content.__name__ = "afile_content" + + router = Router(model_list=_generic_silent_model_list()) + router.afile_content = router.factory_function(mock_file_content, call_type="afile_content") + response = await router.afile_content(model="primary-model", file_id="file-123") + await asyncio.sleep(0.1) + + assert response is sentinel_response + assert mock_file_content.call_count == 1 + assert "silent_model" not in mock_file_content.call_args.kwargs + + +def test_silent_experiment_generic_kwargs_skips_recursion_and_mcp(): + """ + The generic-path snapshot must return None when the request is already a + silent experiment (marker in either litellm_metadata or metadata) or when it + carries MCP tooling that the provider may execute. + """ + router = Router(model_list=_generic_silent_model_list()) + base = {"input": [{"role": "user", "content": "hi"}]} + + assert router._silent_experiment_generic_kwargs(**base, litellm_metadata={"is_silent_experiment": True}) is None + assert router._silent_experiment_generic_kwargs(**base, metadata={"is_silent_experiment": True}) is None + assert router._silent_experiment_generic_kwargs(**base, mcp_servers=[{"type": "url", "url": "x"}]) is None + assert ( + router._silent_experiment_generic_kwargs( + **base, tools=[{"type": "mcp", "server_label": "x", "require_approval": "never"}] + ) + is None + ) + + snapshot = router._silent_experiment_generic_kwargs(**base, tools=[{"type": "web_search"}]) + assert snapshot is not None + assert snapshot["litellm_metadata"]["is_silent_experiment"] is True + assert snapshot["stream"] is False + assert "metadata" not in snapshot + + +def test_silent_experiment_generic_kwargs_snapshots_before_mutation(): + """ + Regression: the snapshot is taken synchronously, so metadata merged into the + caller's kwargs afterwards (deployment tags etc.) never reaches the mirror. + """ + router = Router(model_list=_generic_silent_model_list()) + litellm_metadata = {"model_group": "primary-model"} + snapshot = router._silent_experiment_generic_kwargs(input="hi", litellm_metadata=litellm_metadata) + assert snapshot is not None + litellm_metadata["tags"] = ["primary-deployment-tag"] + assert "tags" not in snapshot["litellm_metadata"] + + +@pytest.mark.asyncio +async def test_silent_experiment_ageneric_error_is_caught(): + """ + A failing mirror must never propagate to the caller. + """ + router = Router(model_list=_generic_silent_model_list()) + + with patch.object( + router, + "_ageneric_api_call_with_fallbacks", + new_callable=AsyncMock, + side_effect=Exception("downstream failure"), + ): + result = await router._silent_experiment_ageneric( + silent_model="silent-model", + original_function=litellm.aresponses, + silent_kwargs={"input": [{"role": "user", "content": "hi"}]}, + ) + + assert result is None + + +def test_is_silent_experiment_marker(): + assert _is_silent_experiment_marker({"is_silent_experiment": True}) is True + assert _is_silent_experiment_marker({"is_silent_experiment": False}) is False + assert _is_silent_experiment_marker({}) is False + assert _is_silent_experiment_marker(None) is False + assert _is_silent_experiment_marker("is_silent_experiment") is False + + +def test_has_mcp_tool(): + assert _has_mcp_tool([{"type": "mcp", "server_label": "x"}]) is True + assert _has_mcp_tool([{"type": "web_search"}, {"type": "mcp"}]) is True + assert _has_mcp_tool([{"type": "web_search"}]) is False + assert _has_mcp_tool([]) is False + assert _has_mcp_tool(None) is False + assert _has_mcp_tool("mcp") is False