feat: per-incumbent-model shadow eval results; skip self-shadowing (#36321)

A shadowed key's real traffic can be a mix of models, and per-tier win
rates blend those incumbents together: if the router beats gpt-4o but
loses to a fine-tune within the same tier, the tier rate hides both
facts. The verdict rollup now groups by (tier, real_model) and the
results carry a second stratification, by_current_model, listing win
rates against each model the key actually uses. The UI renders it as a
second table, only when the traffic really was mixed — a single
incumbent would just repeat the tier table's totals. Tier confidence is
now turn-weighted across the merged rows rather than averaged per row.

Requests already served by the router being shadowed are now skipped
before sampling: duplicating them compares the router to itself —
guaranteed ties, judge spend for zero information. Traffic routed by a
*different* auto-router still samples, which is a meaningful
router-vs-router comparison. Skipped requests still count toward the
job's request_count so the seen/judged ratio stays honest.

Co-authored-by: Abhimanyu Kapur <38531241+akapur99@users.noreply.github.com>
Co-authored-by: Claude <noreply@anthropic.com>

litellm/integrations/shadow_eval_logger.py

@@ -238,6 +238,19 @@ async def _key_or_team_is_over_budget(metadata: Mapping[str, object]) -> bool:
     return False
 
 
+def _request_was_routed_by(request_metadata: Mapping[str, object], router_name: str) -> bool:
+    """Whether this request was already served by the router being shadowed.
+
+    Duplicating such a request compares the router to itself: guaranteed ties, judge
+    spend for zero information. Requests routed by a *different* auto-router still
+    sample, which is a meaningful router-vs-router comparison.
+    """
+    decision: Final = request_metadata.get("routing_decision")
+    if not isinstance(decision, Mapping):
+        return False
+    return decision.get("router_model_name") == router_name
+
+
 def _job_is_over_spend_cap(job: ActiveShadowEvalJob) -> bool:
     """Whether the job has spent past what its start-time estimate justifies.
 
@@ -328,6 +341,8 @@ class ShadowEvalLogger(CustomLogger):
                 return
             if payload.get("call_type") not in (None, "completion", "acompletion", "chat_completion"):
                 return  # only chat-shaped traffic is comparable
+            if _request_was_routed_by(request_metadata, job.router_name):
+                return
             if self._inflight_shadow_tasks >= _MAX_CONCURRENT_SHADOW_TASKS:
                 return
             raw_messages: Final = kwargs.get("messages")

tests/test_litellm/integrations/test_shadow_eval_logger.py

@@ -179,6 +179,47 @@ class TestSuccessHookSkipPaths:
         # 0% sampling: request seen but never shadowed.
         assert logger._pending_seen == {"j1": 1}
 
+    @staticmethod
+    def _routed_kwargs(router_model_name: str):
+        return {
+            "standard_logging_object": {
+                "id": "req-1",
+                "model": "gpt-4o-mini",
+                "call_type": "acompletion",
+                "metadata": {"user_api_key_hash": "key-hash"},
+            },
+            "litellm_params": {"metadata": {"routing_decision": {"router_model_name": router_model_name}}},
+            "messages": [{"role": "user", "content": "hi"}],
+        }
+
+    async def test_skips_requests_already_served_by_the_shadowed_router(self):
+        """Duplicating the router's own traffic compares it to itself: paid ties, no signal."""
+        job = ActiveShadowEvalJob(
+            id="j1", router_name="claude-auto", shadow_percentage=100.0, judge_model="m", status="running"
+        )
+        logger, _, router = _logger_with_mocks(job)
+        logger._run_shadow_eval = AsyncMock()
+
+        await logger.async_log_success_event(self._routed_kwargs("claude-auto"), MagicMock(), None, None)
+        seen_before_flush = dict(logger._pending_seen)
+        await asyncio.sleep(0)
+
+        logger._run_shadow_eval.assert_not_awaited()
+        router.acompletion.assert_not_called()
+        assert seen_before_flush == {"j1": 1}, "skipped for judging, but still counted toward requests seen"
+
+    async def test_traffic_from_a_different_router_still_samples(self):
+        job = ActiveShadowEvalJob(
+            id="j1", router_name="claude-auto", shadow_percentage=100.0, judge_model="m", status="running"
+        )
+        logger, _, _ = _logger_with_mocks(job)
+        logger._run_shadow_eval = AsyncMock()
+
+        await logger.async_log_success_event(self._routed_kwargs("other-router"), MagicMock(), None, None)
+        await asyncio.sleep(0.01)
+
+        logger._run_shadow_eval.assert_awaited_once()
+
 
 @pytest.mark.asyncio
 class TestCallRouterShadowForwardsParameters: