From b2feff3f055d828ff234a183f0a1f197eb6b5297 Mon Sep 17 00:00:00 2001 From: tin-berri Date: Mon, 10 Aug 2026 23:58:20 -0700 Subject: [PATCH] feat: per-incumbent-model shadow eval results; skip self-shadowing (#36321) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A shadowed key's real traffic can be a mix of models, and per-tier win rates blend those incumbents together: if the router beats gpt-4o but loses to a fine-tune within the same tier, the tier rate hides both facts. The verdict rollup now groups by (tier, real_model) and the results carry a second stratification, by_current_model, listing win rates against each model the key actually uses. The UI renders it as a second table, only when the traffic really was mixed — a single incumbent would just repeat the tier table's totals. Tier confidence is now turn-weighted across the merged rows rather than averaged per row. Requests already served by the router being shadowed are now skipped before sampling: duplicating them compares the router to itself — guaranteed ties, judge spend for zero information. Traffic routed by a *different* auto-router still samples, which is a meaningful router-vs-router comparison. Skipped requests still count toward the job's request_count so the seen/judged ratio stays honest. Co-authored-by: Abhimanyu Kapur <38531241+akapur99@users.noreply.github.com> Co-authored-by: Claude --- litellm/integrations/shadow_eval_logger.py | 15 +++++++ .../integrations/test_shadow_eval_logger.py | 41 +++++++++++++++++++ 2 files changed, 56 insertions(+) diff --git a/litellm/integrations/shadow_eval_logger.py b/litellm/integrations/shadow_eval_logger.py index bd8e53c9892..65147abb609 100644 --- a/litellm/integrations/shadow_eval_logger.py +++ b/litellm/integrations/shadow_eval_logger.py @@ -238,6 +238,19 @@ async def _key_or_team_is_over_budget(metadata: Mapping[str, object]) -> bool: return False +def _request_was_routed_by(request_metadata: Mapping[str, object], router_name: str) -> bool: + """Whether this request was already served by the router being shadowed. + + Duplicating such a request compares the router to itself: guaranteed ties, judge + spend for zero information. Requests routed by a *different* auto-router still + sample, which is a meaningful router-vs-router comparison. + """ + decision: Final = request_metadata.get("routing_decision") + if not isinstance(decision, Mapping): + return False + return decision.get("router_model_name") == router_name + + def _job_is_over_spend_cap(job: ActiveShadowEvalJob) -> bool: """Whether the job has spent past what its start-time estimate justifies. @@ -328,6 +341,8 @@ class ShadowEvalLogger(CustomLogger): return if payload.get("call_type") not in (None, "completion", "acompletion", "chat_completion"): return # only chat-shaped traffic is comparable + if _request_was_routed_by(request_metadata, job.router_name): + return if self._inflight_shadow_tasks >= _MAX_CONCURRENT_SHADOW_TASKS: return raw_messages: Final = kwargs.get("messages") diff --git a/tests/test_litellm/integrations/test_shadow_eval_logger.py b/tests/test_litellm/integrations/test_shadow_eval_logger.py index 73022ab15ca..0d4ff456dc4 100644 --- a/tests/test_litellm/integrations/test_shadow_eval_logger.py +++ b/tests/test_litellm/integrations/test_shadow_eval_logger.py @@ -179,6 +179,47 @@ class TestSuccessHookSkipPaths: # 0% sampling: request seen but never shadowed. assert logger._pending_seen == {"j1": 1} + @staticmethod + def _routed_kwargs(router_model_name: str): + return { + "standard_logging_object": { + "id": "req-1", + "model": "gpt-4o-mini", + "call_type": "acompletion", + "metadata": {"user_api_key_hash": "key-hash"}, + }, + "litellm_params": {"metadata": {"routing_decision": {"router_model_name": router_model_name}}}, + "messages": [{"role": "user", "content": "hi"}], + } + + async def test_skips_requests_already_served_by_the_shadowed_router(self): + """Duplicating the router's own traffic compares it to itself: paid ties, no signal.""" + job = ActiveShadowEvalJob( + id="j1", router_name="claude-auto", shadow_percentage=100.0, judge_model="m", status="running" + ) + logger, _, router = _logger_with_mocks(job) + logger._run_shadow_eval = AsyncMock() + + await logger.async_log_success_event(self._routed_kwargs("claude-auto"), MagicMock(), None, None) + seen_before_flush = dict(logger._pending_seen) + await asyncio.sleep(0) + + logger._run_shadow_eval.assert_not_awaited() + router.acompletion.assert_not_called() + assert seen_before_flush == {"j1": 1}, "skipped for judging, but still counted toward requests seen" + + async def test_traffic_from_a_different_router_still_samples(self): + job = ActiveShadowEvalJob( + id="j1", router_name="claude-auto", shadow_percentage=100.0, judge_model="m", status="running" + ) + logger, _, _ = _logger_with_mocks(job) + logger._run_shadow_eval = AsyncMock() + + await logger.async_log_success_event(self._routed_kwargs("other-router"), MagicMock(), None, None) + await asyncio.sleep(0.01) + + logger._run_shadow_eval.assert_awaited_once() + @pytest.mark.asyncio class TestCallRouterShadowForwardsParameters: