diff --git a/litellm/router.py b/litellm/router.py index e3c17bfb288..789fc81d8d3 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -2477,7 +2477,6 @@ class Router: ) silent_model: Final = litellm_params.pop("silent_model", None) - shadow_kwargs: Final = _silent_experiment_kwargs_snapshot(kwargs) for silent_target in _silent_experiment_targets(silent_model): # Mirroring traffic to a secondary model # Use threading.Thread (not ThreadPoolExecutor) - executor.submit() @@ -2486,7 +2485,7 @@ class Router: threading.Thread( target=self._silent_experiment_completion, args=(silent_target, messages), - kwargs=shadow_kwargs, + kwargs=_silent_experiment_kwargs_snapshot(kwargs), daemon=True, ).start() @@ -3591,7 +3590,6 @@ class Router: ) silent_model: Final = litellm_params.pop("silent_model", None) - shadow_kwargs: Final = _silent_experiment_kwargs_snapshot(kwargs) for silent_target in _silent_experiment_targets(silent_model): # Mirroring traffic to a secondary model # This is a silent experiment, so we don't want to block the primary request @@ -3599,7 +3597,7 @@ class Router: self._silent_experiment_acompletion( silent_model=silent_target, messages=messages, # Use messages instead of *args - **shadow_kwargs, + **_silent_experiment_kwargs_snapshot(kwargs), ) ) diff --git a/tests/test_litellm/test_router_silent_experiment.py b/tests/test_litellm/test_router_silent_experiment.py index 64d9400960d..d62962da275 100644 --- a/tests/test_litellm/test_router_silent_experiment.py +++ b/tests/test_litellm/test_router_silent_experiment.py @@ -1,6 +1,6 @@ import asyncio import time -from collections.abc import Callable +from collections.abc import Callable, Mapping from types import SimpleNamespace from typing import Final from unittest.mock import AsyncMock, MagicMock, patch @@ -321,6 +321,41 @@ def test_sync_shadow_gets_kwargs_snapshot_taken_before_primary_mutates_them(reco assert "primary-only" not in shadow_metadata.get("tags", []) +def test_sync_shadow_workers_do_not_share_metadata_with_each_other(recording_logger): + workers: list[tuple[Mapping[str, object], Callable[[], None]]] = [] + + class _DeferredThread: + def __init__(self, target, args, kwargs, daemon) -> None: + workers.append((kwargs, lambda: target(*args, **kwargs))) + + def start(self) -> None: + return None + + router = Router(model_list=_streaming_model_list(["shadow-a", "shadow-b"])) + with patch( # test-quality-ok: Router has no thread factory to inject; deferring start is the only deterministic way to expose the race + "litellm.router.threading", SimpleNamespace(Thread=_DeferredThread) + ): + router.completion( + model="primary-model", + messages=[{"role": "user", "content": "hi"}], + mock_response="pong", + metadata={"foo": "bar"}, + ) + assert len(workers) == 2 + (first_kwargs, run_first), (_, run_second) = workers + first_kwargs["metadata"].pop("foo") + run_second() + run_first() + _wait_for_shadow_successes_sync(recording_logger, expected=2) + + metadata_by_group = { + call["litellm_params"]["metadata"]["model_group"]: call["litellm_params"]["metadata"] + for call in recording_logger.shadow_successes() + } + assert metadata_by_group["shadow-b"]["foo"] == "bar" + assert "foo" not in metadata_by_group["shadow-a"] + + @pytest.mark.asyncio async def test_async_shadow_does_not_inherit_primary_deployment_tags(recording_logger): router = Router(model_list=_tagged_primary_model_list())