fix(router): keep silent_model out of embedding provider requests (#44064)

Co-authored-by: yassin <yassin@berri.ai>
Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

litellm/types/litellm_params.py

@@ -151,6 +151,7 @@ class DeploymentOptions:
     order: int | None = None
     tag_regex: Sequence[str] | None = None
     max_file_size_mb: float | None = None
+    silent_model: str | Sequence[str] | None = None
 
 
 @dataclass(frozen=True, slots=True, kw_only=True)

tests/unit/test_router_silent_experiment.py

@@ -1,11 +1,14 @@
 import asyncio
+import json
 import time
 from collections.abc import Callable, Mapping
 from types import SimpleNamespace
 from typing import Final
 from unittest.mock import AsyncMock, MagicMock, patch
 
+import httpx
 import pytest
+import respx
 
 import litellm
 from litellm.integrations.custom_logger import CustomLogger
@@ -603,6 +606,67 @@ def test_silent_experiment_sends_shadow_request_attributed_to_the_silent_model(r
     assert primary_metadata == {"model_group": "primary-model"}
 
 
+
+
+_EMBEDDING_API_BASE: Final = "https://embeddings.example.test/v1"
+
+
+def _strict_embedding_route(respx_mock: respx.MockRouter) -> respx.Route:
+    return respx_mock.post(f"{_EMBEDDING_API_BASE}/embeddings").mock(
+        return_value=httpx.Response(
+            200,
+            json={
+                "object": "list",
+                "data": [{"object": "embedding", "index": 0, "embedding": [0.1, 0.2]}],
+                "model": "embed-model",
+                "usage": {"prompt_tokens": 2, "total_tokens": 2},
+            },
+        )
+    )
+
+
+def _embedding_router_with_silent_model() -> Router:
+    return Router(
+        model_list=[
+            {
+                "model_name": "embed-primary",
+                "litellm_params": {
+                    "model": "openai/embed-model",
+                    "api_base": _EMBEDDING_API_BASE,
+                    "api_key": "fake-key",
+                    "silent_model": "embed-shadow",
+                },
+            }
+        ]
+    )
+
+
+def test_embedding_with_silent_model_sends_provider_body_without_it(respx_mock: respx.MockRouter) -> None:
+    route: Final = _strict_embedding_route(respx_mock)
+
+    response: Final = _embedding_router_with_silent_model().embedding(
+        model="embed-primary", input=["black dresses"], input_type="query"
+    )
+
+    request_body: Final = json.loads(route.calls.last.request.read())
+    assert request_body == {"model": "embed-model", "input": ["black dresses"], "input_type": "query"}
+    assert response.data[0]["embedding"] == [0.1, 0.2]
+
+
+@pytest.mark.asyncio
+async def test_aembedding_with_silent_model_sends_provider_body_without_it(
+    respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
+    route: Final = _strict_embedding_route(respx_mock)
+
+    response: Final = await _embedding_router_with_silent_model().aembedding(
+        model="embed-primary", input=["black dresses"], input_type="query"
+    )
+
+    request_body: Final = json.loads(route.calls.last.request.read())
+    assert request_body == {"model": "embed-model", "input": ["black dresses"], "input_type": "query"}
+    assert response.data[0]["embedding"] == [0.1, 0.2]
 @pytest.mark.parametrize("run_silent_experiment", SILENT_EXPERIMENT_RUNNERS)
 def test_silent_experiment_does_not_launch_from_a_shadow_request(run_silent_experiment):
     router = Router(model_list=_streaming_model_list(["shadow-a"]))

tests/unit/types/test_litellm_params.py

@@ -129,6 +129,7 @@ OPTION_NAMES: Final = (
     "order",
     "tag_regex",
     "max_file_size_mb",
+    "silent_model",
     "auto_router_config_path",
     "auto_router_config",
     "auto_router_default_model",