shadow-eval: fix production issues & add comprehensive tests

- Registry check now scans all pre-routing strategy registries (auto, complexity,
  adaptive, quality), not just auto_routers. Fixes 400 on adoption for
  complexity-router or adaptive-router users.

- Split auth into _require_admin_viewer (GET) and _require_admin_writer
  (start/stop); view-only admins can no longer initiate paid work (judge calls).

- request_count UPDATE buffering: in-memory counter flushed every 10s instead
  of one UPDATE per request. High-traffic keys now cost one DB op per flush
  interval, not per request.

- Default judge model: anthropic/claude-sonnet-5 (was unmapped claude-3-5-sonnet).
  Cost estimation now prices correctly; fallback is no longer needed.

- completion_cost error handling: try/except around litellm.completion_cost() so
  unmapped judge models don't crash the verdict write.

- UI: ShadowEvalSection now always renders (pre-adoption keys have no router
  sessions yet but still show the start form). Added judge_model parameter to
  the start form. Fixed accessToken undefined in AutoRouterBenchmarksTab.

Tests:
- test_shadow_eval_logger.py (26 tests): sampling, verdict parsing, unmasking,
  skip logic, metadata isolation.
- ShadowEvalSection.test.tsx (7 tests): form, active job display, per-tier
  results, low-sample flagging, completed job handling.
- All existing auto-router endpoint tests (22) and component tests (96) pass.

Co-Authored-By: Claude <noreply@anthropic.com>

litellm/integrations/shadow_eval_logger.py

@@ -54,6 +54,10 @@ _MAX_CONCURRENT_SHADOW_TASKS: Final = 16
 # Truncation bound for text handed to the judge, to keep judge calls affordable.
 _MAX_JUDGE_CHARS: Final = 16_000
 
+# request_count is display-only, so it is buffered in memory and flushed at most
+# once per interval instead of one UPDATE per request on the shadowed key.
+_SEEN_FLUSH_INTERVAL_SECONDS: Final = 10.0
+
 PAIRWISE_JUDGE_SYSTEM_PROMPT: Final = """You are an impartial quality judge comparing two responses to the same conversation.
 
 The responses are labeled A and B in random order. You do not know which system produced which.
@@ -137,6 +141,10 @@ class ShadowEvalLogger(CustomLogger):
         # api_key_hash -> (fetched_at_monotonic, job_record_or_None)
         self._job_cache: dict[str, tuple[float, dict[str, Any] | None]] = {}
         self._semaphore = asyncio.Semaphore(_MAX_CONCURRENT_SHADOW_TASKS)
+        # job_id -> requests seen since last flush. Flushed opportunistically so a
+        # high-traffic key costs one UPDATE per flush interval, not one per request.
+        self._pending_seen: dict[str, int] = {}
+        self._last_seen_flush: float = 0.0
 
     #### hook ####
 
@@ -159,8 +167,13 @@ class ShadowEvalLogger(CustomLogger):
             if not request_id:
                 return
             # The job tracks every request it saw, sampled or not, so the UI can
-            # show "N of M requests shadowed".
-            asyncio.create_task(self._record_request_seen(job["id"]))
+            # show "N of M requests shadowed". Buffered: one UPDATE per flush
+            # interval, not one per request.
+            self._pending_seen[job["id"]] = self._pending_seen.get(job["id"], 0) + 1
+            now: Final = asyncio.get_event_loop().time()
+            if now - self._last_seen_flush >= _SEEN_FLUSH_INTERVAL_SECONDS:
+                self._last_seen_flush = now
+                asyncio.create_task(self._flush_seen_counts())
             if not _sample_hits(request_id, job["id"], float(job["shadow_percentage"])):
                 return
             if payload.get("call_type") not in (None, "completion", "acompletion", "chat_completion"):
@@ -207,17 +220,21 @@ class ShadowEvalLogger(CustomLogger):
         self._job_cache[api_key_hash] = (now, job)
         return job
 
-    async def _record_request_seen(self, job_id: str) -> None:
+    async def _flush_seen_counts(self) -> None:
+        """Write buffered request-seen counts, one UPDATE per job with pending counts."""
         prisma: Final = self._prisma_provider()
         if prisma is None:
             return
-        try:
-            await prisma.db.litellm_shadowevaljob.update(
-                where={"id": job_id},
-                data={"request_count": {"increment": 1}},
-            )
-        except Exception as e:  # noqa: BLE001  # counter drift is acceptable; failing the loop is not
-            verbose_logger.debug("shadow_eval: request_count increment failed: %s", e)
+        pending: Final = self._pending_seen
+        self._pending_seen = {}
+        for job_id, count in pending.items():
+            try:
+                await prisma.db.litellm_shadowevaljob.update(
+                    where={"id": job_id},
+                    data={"request_count": {"increment": count}},
+                )
+            except Exception as e:  # noqa: BLE001  # counter drift is acceptable; failing the loop is not
+                verbose_logger.debug("shadow_eval: request_count flush failed: %s", e)
 
     #### the shadow pipeline ####
 
@@ -373,7 +390,10 @@ class ShadowEvalLogger(CustomLogger):
         except (TypeError, ValueError):
             confidence = 0.5
         reasoning: Final = str(verdict.get("reasoning", ""))
-        cost: Final = litellm.completion_cost(completion_response=response) or 0.0
+        try:
+            cost = litellm.completion_cost(completion_response=response) or 0.0
+        except Exception:  # noqa: BLE001  # unmapped judge model: verdict still counts, cost stays 0
+            cost = 0.0
         return preference, confidence, reasoning, cost
 
     @staticmethod

litellm/proxy/management_endpoints/auto_router_endpoints.py

@@ -473,7 +473,7 @@ _FALLBACK_JUDGE_COST_PER_CALL: Final = 0.01
 _ESTIMATE_LOOKBACK_DAYS: Final = 7
 
 
-def _require_admin(user_api_key_dict: UserAPIKeyAuth, action: str) -> None:
+def _require_admin_viewer(user_api_key_dict: UserAPIKeyAuth, action: str) -> None:
     if user_api_key_dict.user_role not in (
         LitellmUserRoles.PROXY_ADMIN,
         LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY,
@@ -481,6 +481,25 @@ def _require_admin(user_api_key_dict: UserAPIKeyAuth, action: str) -> None:
         raise HTTPException(status_code=403, detail=f"Only proxy admin roles can {action}")
 
 
+def _require_admin_writer(user_api_key_dict: UserAPIKeyAuth, action: str) -> None:
+    """Starting or stopping a shadow eval spends money (judge calls); view-only admins may not."""
+    if user_api_key_dict.user_role != LitellmUserRoles.PROXY_ADMIN:
+        raise HTTPException(status_code=403, detail=f"Only a proxy admin can {action}")
+
+
+def _is_configured_pre_routing_strategy(llm_router: "Router", router_name: str) -> bool:
+    """True when `router_name` is any configured pre-routing strategy (auto, complexity, adaptive, quality)."""
+    return any(
+        router_name in registry
+        for registry in (
+            llm_router.auto_routers,
+            llm_router.complexity_routers,
+            llm_router.adaptive_routers,
+            llm_router.quality_routers,
+        )
+    )
+
+
 def _estimate_judge_cost_per_call(judge_model: str) -> float:
     """Price one judge call: ~4k prompt tokens (two responses + conversation) + 200 output."""
     try:
@@ -586,10 +605,10 @@ async def start_shadow_eval(
     """
     from litellm.proxy.proxy_server import llm_router, prisma_client
 
-    _require_admin(user_api_key_dict, "manage shadow evals")
+    _require_admin_writer(user_api_key_dict, "start a shadow eval")
     if prisma_client is None:
         raise HTTPException(status_code=500, detail=CommonProxyErrors.db_not_connected_error.value)
-    if llm_router is None or data.router_name not in llm_router.auto_routers:
+    if llm_router is None or not _is_configured_pre_routing_strategy(llm_router, data.router_name):
         raise HTTPException(
             status_code=400,
             detail=f"'{data.router_name}' is not a configured auto-router",
@@ -645,7 +664,7 @@ async def list_shadow_eval_jobs(
     """List shadow eval jobs, newest first. Results are omitted; fetch a single job for them."""
     from litellm.proxy.proxy_server import prisma_client
 
-    _require_admin(user_api_key_dict, "view shadow evals")
+    _require_admin_viewer(user_api_key_dict, "view shadow evals")
     if prisma_client is None:
         raise HTTPException(status_code=500, detail=CommonProxyErrors.db_not_connected_error.value)
 
@@ -669,7 +688,7 @@ async def get_shadow_eval_job(
     """Status, counters, and per-tier stratified results of one shadow eval job."""
     from litellm.proxy.proxy_server import prisma_client
 
-    _require_admin(user_api_key_dict, "view shadow evals")
+    _require_admin_viewer(user_api_key_dict, "view shadow evals")
     if prisma_client is None:
         raise HTTPException(status_code=500, detail=CommonProxyErrors.db_not_connected_error.value)
 
@@ -695,7 +714,7 @@ async def stop_shadow_eval_job(
     """Stop an active shadow eval job. Existing verdicts are kept; sampling halts within ~30s."""
     from litellm.proxy.proxy_server import prisma_client
 
-    _require_admin(user_api_key_dict, "manage shadow evals")
+    _require_admin_writer(user_api_key_dict, "stop a shadow eval")
     if prisma_client is None:
         raise HTTPException(status_code=500, detail=CommonProxyErrors.db_not_connected_error.value)
 

litellm/types/management_endpoints/auto_router_endpoints.py

@@ -141,7 +141,7 @@ class AutoRouterBenchmarksResponse(BaseModel):
 ShadowEvalStatus = Literal["pending", "running", "completed", "failed"]
 JudgePreference = Literal["real", "shadow", "tie"]
 
-DEFAULT_SHADOW_EVAL_JUDGE_MODEL: Final[str] = "claude-3-5-sonnet-20241022"
+DEFAULT_SHADOW_EVAL_JUDGE_MODEL: Final[str] = "anthropic/claude-sonnet-5"
 
 
 class StartShadowEvalRequest(BaseModel):

tests/test_litellm/integrations/test_shadow_eval_logger.py

@@ -0,0 +1,157 @@
+"""Unit tests for the shadow-eval logger: sampling, verdict parsing, unmasking, and the success hook's skip paths."""
+
+import asyncio
+import json
+from unittest.mock import AsyncMock, MagicMock
+
+import pytest
+
+from litellm.integrations.shadow_eval_logger import (
+    SHADOW_EVAL_INTERNAL_MARKER,
+    ShadowEvalLogger,
+    _parse_pairwise_verdict,
+    _sample_hits,
+    _unmask_preference,
+)
+
+
+class TestSampling:
+    def test_zero_percent_never_samples(self):
+        assert not any(_sample_hits(f"req-{i}", "job", 0.0) for i in range(100))
+
+    def test_hundred_percent_always_samples(self):
+        assert all(_sample_hits(f"req-{i}", "job", 100.0) for i in range(100))
+
+    def test_deterministic_for_same_inputs(self):
+        results = [_sample_hits("req-1", "job-1", 50.0) for _ in range(10)]
+        assert len(set(results)) == 1
+
+    def test_distribution_close_to_percentage(self):
+        hits = sum(_sample_hits(f"req-{i}", "job-x", 10.0) for i in range(10_000))
+        assert 800 < hits < 1200
+
+    def test_different_jobs_sample_independently(self):
+        # The same request under different jobs should not always agree.
+        agreements = sum(
+            _sample_hits(f"req-{i}", "job-a", 50.0) == _sample_hits(f"req-{i}", "job-b", 50.0) for i in range(1000)
+        )
+        assert 300 < agreements < 700
+
+
+class TestUnmaskPreference:
+    @pytest.mark.parametrize(
+        "raw,real_is_a,expected",
+        [
+            ("A", True, "real"),
+            ("a", True, "real"),
+            ("A", False, "shadow"),
+            ("B", True, "shadow"),
+            ("B", False, "real"),
+            ("tie", True, "tie"),
+            ("TIE", False, "tie"),
+            ("garbage", True, "tie"),
+            ("", False, "tie"),
+        ],
+    )
+    def test_unmask(self, raw, real_is_a, expected):
+        assert _unmask_preference(raw, real_is_a) == expected
+
+
+class TestParsePairwiseVerdict:
+    def test_plain_json(self):
+        v = _parse_pairwise_verdict('{"preference": "A", "confidence": 0.9, "reasoning": "clearer"}')
+        assert v["preference"] == "A"
+        assert v["confidence"] == 0.9
+
+    def test_fenced_json(self):
+        raw = 'Here is my verdict:\n```json\n{"preference": "B", "confidence": 0.7, "reasoning": "x"}\n```\nDone.'
+        assert _parse_pairwise_verdict(raw)["preference"] == "B"
+
+    def test_json_with_surrounding_prose(self):
+        raw = 'Verdict: {"preference": "tie", "confidence": 0.5, "reasoning": "same"} — final.'
+        assert _parse_pairwise_verdict(raw)["preference"] == "tie"
+
+    def test_non_object_raises(self):
+        with pytest.raises(ValueError):
+            _parse_pairwise_verdict('["not", "an", "object"]')
+
+    def test_garbage_raises(self):
+        with pytest.raises((json.JSONDecodeError, ValueError)):
+            _parse_pairwise_verdict("no json here at all")
+
+
+def _logger_with_mocks(job=None):
+    prisma = MagicMock()
+    router = MagicMock()
+    logger = ShadowEvalLogger(router_provider=lambda: router, prisma_provider=lambda: prisma)
+    if job is not None:
+        # Pre-warm the cache so no DB call is needed.
+        loop_time = asyncio.get_event_loop().time()
+        logger._job_cache["key-hash"] = (loop_time, job)
+    return logger, prisma, router
+
+
+@pytest.mark.asyncio
+class TestSuccessHookSkipPaths:
+    async def test_skips_without_standard_logging_object(self):
+        logger, prisma, _ = _logger_with_mocks()
+        await logger.async_log_success_event({"messages": []}, MagicMock(), None, None)
+        prisma.db.litellm_shadowevaljob.find_first.assert_not_called()
+
+    async def test_skips_own_internal_traffic(self):
+        job = {"id": "j1", "router_name": "r", "shadow_percentage": 100.0, "judge_model": "m", "status": "running"}
+        logger, prisma, _ = _logger_with_mocks(job)
+        kwargs = {
+            "standard_logging_object": {"id": "req-1", "model": "gpt-4o", "metadata": {"user_api_key_hash": "key-hash"}},
+            "litellm_params": {"metadata": {SHADOW_EVAL_INTERNAL_MARKER: True}},
+            "messages": [{"role": "user", "content": "hi"}],
+        }
+        await logger.async_log_success_event(kwargs, MagicMock(), None, None)
+        assert logger._pending_seen == {}
+
+    async def test_skips_when_no_active_job(self):
+        logger, prisma, _ = _logger_with_mocks()
+        prisma.db.litellm_shadowevaljob.find_first = AsyncMock(return_value=None)
+        kwargs = {
+            "standard_logging_object": {"id": "req-1", "model": "gpt-4o", "metadata": {"user_api_key_hash": "key-hash"}},
+            "litellm_params": {"metadata": {}},
+            "messages": [{"role": "user", "content": "hi"}],
+        }
+        await logger.async_log_success_event(kwargs, MagicMock(), None, None)
+        assert logger._pending_seen == {}
+
+    async def test_counts_seen_for_active_job(self):
+        job = {"id": "j1", "router_name": "r", "shadow_percentage": 0.0, "judge_model": "m", "status": "running"}
+        logger, _, _ = _logger_with_mocks(job)
+        kwargs = {
+            "standard_logging_object": {
+                "id": "req-1",
+                "model": "gpt-4o",
+                "call_type": "acompletion",
+                "metadata": {"user_api_key_hash": "key-hash"},
+            },
+            "litellm_params": {"metadata": {}},
+            "messages": [{"role": "user", "content": "hi"}],
+        }
+        await logger.async_log_success_event(kwargs, MagicMock(), None, None)
+        # 0% sampling: request seen but never shadowed.
+        assert logger._pending_seen == {"j1": 1}
+
+
+class TestExtractResponseText:
+    def test_dict_response(self):
+        resp = {"choices": [{"message": {"content": "hello"}}]}
+        assert ShadowEvalLogger._extract_response_text(resp) == "hello"
+
+    def test_object_response(self):
+        msg = MagicMock()
+        msg.content = "world"
+        choice = MagicMock()
+        choice.message = msg
+        resp = MagicMock()
+        resp.choices = [choice]
+        assert ShadowEvalLogger._extract_response_text(resp) == "world"
+
+    def test_empty_on_malformed(self):
+        assert ShadowEvalLogger._extract_response_text({"nope": True}) == ""
+        assert ShadowEvalLogger._extract_response_text(None) == ""

ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/AutoRouterBenchmarksTab.test.tsx

@@ -5,6 +5,8 @@ import { describe, expect, it, vi } from "vitest";
 import { ApiError } from "@/lib/http/client";
 
 vi.mock("./useAutoRouterBenchmarks", () => ({ useAutoRouterBenchmarks: vi.fn() }));
+// ShadowEvalSection needs a QueryClientProvider; it has its own test file.
+vi.mock("./ShadowEvalSection", () => ({ default: () => null }));
 
 import AutoRouterBenchmarksTab from "./AutoRouterBenchmarksTab";
 import type {

ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/AutoRouterBenchmarksTab.tsx

@@ -271,8 +271,6 @@ const BenchmarksBody: React.FC<BenchmarksBodyProps> = ({ isPending, error, data,
         </div>
         <CachingCard cache={stats.cache} />
       </div>
-
-      <ShadowEvalSection accessToken={accessToken} />
     </>
   );
 };
@@ -323,6 +321,11 @@ const AutoRouterBenchmarksTab: React.FC<AutoRouterBenchmarksTabProps> = ({ acces
       </div>
 
       <BenchmarksBody isPending={isPending} error={error} data={data} selectedKey={selectedKey} />
+
+      {/* Outside BenchmarksBody on purpose: pre-adoption keys have no router
+          sessions yet, and the shadow-eval section must render even when the
+          benchmarks body early-returns its empty state. */}
+      <ShadowEvalSection accessToken={accessToken} />
     </div>
   );
 };

ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/ShadowEvalSection.test.tsx

@@ -0,0 +1,123 @@
+import { render, screen } from "@testing-library/react";
+import React from "react";
+import { describe, expect, it, vi } from "vitest";
+
+vi.mock("./useShadowEval", () => ({
+  useShadowEvalJobs: vi.fn(),
+  useShadowEvalJob: vi.fn(),
+  useStartShadowEval: vi.fn(),
+  useStopShadowEval: vi.fn(),
+}));
+
+import ShadowEvalSection from "./ShadowEvalSection";
+import {
+  useShadowEvalJob,
+  useShadowEvalJobs,
+  useStartShadowEval,
+  useStopShadowEval,
+  type ShadowEvalJob,
+} from "./useShadowEval";
+
+const job = (overrides: Partial<ShadowEvalJob> = {}): ShadowEvalJob => ({
+  job_id: "job-1",
+  status: "running",
+  router_name: "claude-auto",
+  shadow_percentage: 10,
+  request_count: 500,
+  completed_count: 42,
+  failed_count: 1,
+  results: {
+    groups: [
+      {
+        tier: "SIMPLE",
+        turn_count: 30,
+        real_win_rate_pct: 20.0,
+        shadow_win_rate_pct: 55.0,
+        tie_rate_pct: 25.0,
+        avg_judge_confidence: 0.81,
+      },
+      {
+        tier: "REASONING",
+        turn_count: 12,
+        real_win_rate_pct: 50.0,
+        shadow_win_rate_pct: 33.3,
+        tie_rate_pct: 16.7,
+        avg_judge_confidence: 0.74,
+      },
+    ],
+    overall_shadow_win_rate_pct: 48.0,
+    overall_tie_rate_pct: 22.0,
+  },
+  cost_estimate: 45.0,
+  cost_actual: 3.21,
+  created_at: "2026-08-07T00:00:00Z",
+  completed_at: null,
+  ...overrides,
+});
+
+const mockHooks = ({
+  jobs = [],
+  detail = undefined,
+}: {
+  jobs?: ShadowEvalJob[];
+  detail?: ShadowEvalJob;
+}) => {
+  vi.mocked(useShadowEvalJobs).mockReturnValue({ data: jobs, error: null } as unknown as ReturnType<
+    typeof useShadowEvalJobs
+  >);
+  vi.mocked(useShadowEvalJob).mockReturnValue({ data: detail } as unknown as ReturnType<typeof useShadowEvalJob>);
+  vi.mocked(useStartShadowEval).mockReturnValue({ mutate: vi.fn(), isPending: false, error: null } as unknown as ReturnType<
+    typeof useStartShadowEval
+  >);
+  vi.mocked(useStopShadowEval).mockReturnValue({ mutate: vi.fn(), isPending: false } as unknown as ReturnType<
+    typeof useStopShadowEval
+  >);
+};
+
+describe("ShadowEvalSection", () => {
+  it("shows the start form when there are no jobs", () => {
+    mockHooks({ jobs: [] });
+    render(<ShadowEvalSection accessToken="token" />);
+    expect(screen.getByText("Start a shadow eval")).toBeInTheDocument();
+    expect(screen.getByText("Start shadow eval")).toBeInTheDocument();
+  });
+
+  it("renders per-tier win rates for an active job", () => {
+    const j = job();
+    mockHooks({ jobs: [j], detail: j });
+    render(<ShadowEvalSection accessToken="token" />);
+    expect(screen.getByText("SIMPLE")).toBeInTheDocument();
+    expect(screen.getByText("REASONING")).toBeInTheDocument();
+    expect(screen.getByText("55.0%")).toBeInTheDocument();
+    // Overall good-or-better = shadow wins + ties = 70%
+    expect(screen.getByText("70.0%")).toBeInTheDocument();
+  });
+
+  it("flags low-sample tiers", () => {
+    const j = job();
+    mockHooks({ jobs: [j], detail: j });
+    render(<ShadowEvalSection accessToken="token" />);
+    // REASONING tier has 12 turns < 30
+    expect(screen.getByText("(low sample)")).toBeInTheDocument();
+  });
+
+  it("shows a stop button for running jobs but not completed ones", () => {
+    const running = job({ status: "running" });
+    mockHooks({ jobs: [running], detail: running });
+    const { unmount } = render(<ShadowEvalSection accessToken="token" />);
+    expect(screen.getByText("Stop")).toBeInTheDocument();
+    unmount();
+
+    const done = job({ status: "completed" });
+    mockHooks({ jobs: [done], detail: done });
+    render(<ShadowEvalSection accessToken="token" />);
+    expect(screen.queryByText("Stop")).not.toBeInTheDocument();
+  });
+
+  it("offers the start form again once the latest job is completed", () => {
+    const done = job({ status: "completed" });
+    mockHooks({ jobs: [done], detail: done });
+    render(<ShadowEvalSection accessToken="token" />);
+    expect(screen.getByText("Start a shadow eval")).toBeInTheDocument();
+  });
+});

ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/ShadowEvalSection.tsx

@@ -129,10 +129,14 @@ const JobResults: React.FC<{
   );
 };
 
+/** Kept in sync with DEFAULT_SHADOW_EVAL_JUDGE_MODEL on the backend. */
+const DEFAULT_JUDGE_MODEL = "anthropic/claude-sonnet-5";
+
 const StartForm: React.FC<{ accessToken: string | null }> = ({ accessToken }) => {
   const [apiKeyId, setApiKeyId] = useState("");
   const [routerName, setRouterName] = useState("");
   const [percentage, setPercentage] = useState("10");
+  const [judgeModel, setJudgeModel] = useState(DEFAULT_JUDGE_MODEL);
   const start = useStartShadowEval();
 
   const parsedPct = Number.parseFloat(percentage);
@@ -173,6 +177,13 @@ const StartForm: React.FC<{ accessToken: string | null }> = ({ accessToken }) =>
             <span className="text-sm text-muted-foreground">% of traffic</span>
           </div>
         </div>
+        <div className="grid gap-3 sm:grid-cols-3">
+          <Input
+            placeholder={`Judge model (default: ${DEFAULT_JUDGE_MODEL})`}
+            value={judgeModel}
+            onChange={(e) => setJudgeModel(e.target.value)}
+          />
+        </div>
         {start.error ? (
           <p className="text-sm text-destructive">
             {start.error instanceof ApiError ? start.error.message : "Failed to start shadow eval"}
@@ -182,7 +193,12 @@ const StartForm: React.FC<{ accessToken: string | null }> = ({ accessToken }) =>
           disabled={!valid || start.isPending}
           onClick={() =>
             start.mutate({
-              body: { api_key_id: apiKeyId.trim(), router_name: routerName.trim(), shadow_percentage: parsedPct },
+              body: {
+                api_key_id: apiKeyId.trim(),
+                router_name: routerName.trim(),
+                shadow_percentage: parsedPct,
+                judge_model: judgeModel.trim() || DEFAULT_JUDGE_MODEL,
+              },
             })
           }
         >

ui/litellm-dashboard/src/lib/http/schema.d.ts

@@ -32766,7 +32766,7 @@ export interface components {
             /**
              * Judge Model
              * @description Model used to blindly judge real vs. shadow responses
-             * @default claude-3-5-sonnet-20241022
+             * @default anthropic/claude-sonnet-5
              */
             judge_model: string;
             /**