fix(router): snapshot shadow kwargs per target so concurrent shadows never share metadata

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

litellm/router.py

@@ -2477,7 +2477,6 @@ class Router:
             )
             silent_model: Final = litellm_params.pop("silent_model", None)
 
-            shadow_kwargs: Final = _silent_experiment_kwargs_snapshot(kwargs)
             for silent_target in _silent_experiment_targets(silent_model):
                 # Mirroring traffic to a secondary model
                 # Use threading.Thread (not ThreadPoolExecutor) - executor.submit()
@@ -2486,7 +2485,7 @@ class Router:
                 threading.Thread(
                     target=self._silent_experiment_completion,
                     args=(silent_target, messages),
-                    kwargs=shadow_kwargs,
+                    kwargs=_silent_experiment_kwargs_snapshot(kwargs),
                     daemon=True,
                 ).start()
 
@@ -3591,7 +3590,6 @@ class Router:
             )
             silent_model: Final = litellm_params.pop("silent_model", None)
 
-            shadow_kwargs: Final = _silent_experiment_kwargs_snapshot(kwargs)
             for silent_target in _silent_experiment_targets(silent_model):
                 # Mirroring traffic to a secondary model
                 # This is a silent experiment, so we don't want to block the primary request
@@ -3599,7 +3597,7 @@ class Router:
                     self._silent_experiment_acompletion(
                         silent_model=silent_target,
                         messages=messages,  # Use messages instead of *args
-                        **shadow_kwargs,
+                        **_silent_experiment_kwargs_snapshot(kwargs),
                     )
                 )
 

tests/test_litellm/test_router_silent_experiment.py

@@ -1,6 +1,6 @@
 import asyncio
 import time
-from collections.abc import Callable
+from collections.abc import Callable, Mapping
 from types import SimpleNamespace
 from typing import Final
 from unittest.mock import AsyncMock, MagicMock, patch
@@ -321,6 +321,41 @@ def test_sync_shadow_gets_kwargs_snapshot_taken_before_primary_mutates_them(reco
     assert "primary-only" not in shadow_metadata.get("tags", [])
 
 
+def test_sync_shadow_workers_do_not_share_metadata_with_each_other(recording_logger):
+    workers: list[tuple[Mapping[str, object], Callable[[], None]]] = []
+
+    class _DeferredThread:
+        def __init__(self, target, args, kwargs, daemon) -> None:
+            workers.append((kwargs, lambda: target(*args, **kwargs)))
+
+        def start(self) -> None:
+            return None
+
+    router = Router(model_list=_streaming_model_list(["shadow-a", "shadow-b"]))
+    with patch(  # test-quality-ok: Router has no thread factory to inject; deferring start is the only deterministic way to expose the race
+        "litellm.router.threading", SimpleNamespace(Thread=_DeferredThread)
+    ):
+        router.completion(
+            model="primary-model",
+            messages=[{"role": "user", "content": "hi"}],
+            mock_response="pong",
+            metadata={"foo": "bar"},
+        )
+    assert len(workers) == 2
+    (first_kwargs, run_first), (_, run_second) = workers
+    first_kwargs["metadata"].pop("foo")
+    run_second()
+    run_first()
+    _wait_for_shadow_successes_sync(recording_logger, expected=2)
+
+    metadata_by_group = {
+        call["litellm_params"]["metadata"]["model_group"]: call["litellm_params"]["metadata"]
+        for call in recording_logger.shadow_successes()
+    }
+    assert metadata_by_group["shadow-b"]["foo"] == "bar"
+    assert "foo" not in metadata_by_group["shadow-a"]
+
+
 @pytest.mark.asyncio
 async def test_async_shadow_does_not_inherit_primary_deployment_tags(recording_logger):
     router = Router(model_list=_tagged_primary_model_list())