feat(ui): explain the shadow eval mechanism and clarify its metrics

The section assumed the reader already knew what a shadow eval was: the
only explanation was one line of muted text, the hero metrics were
unlabeled jargon with no denominator, and the fact that a tie counts in
the router's favor — the key to reading every number on the card — was
stated nowhere.

- Header: plain-language framing question plus a collapsible 'How this
  works' walking through sampling, shadow duplication, blind A/B
  judging, and billing.
- Hero metrics renamed to 'Router matched or beat your current model'
  (with 'of N judged responses' under the number) and 'Router strictly
  won'; both carry tooltips, the first spelling out why a tie favors
  the router.
- New win/tie/loss stacked bar with a three-part legend, so the verdict
  split is visible instead of derivable by subtraction.
- Tier table: 'Ties' and 'Judge confidence' get explanatory tooltips,
  '(low sample)' explains its threshold on hover.
- Status line says what failed ('errored (shadow or judge call)') and
  what the estimate is ('$3.21 judge spend of ~$45.00 estimated').

Co-Authored-By: Claude <noreply@anthropic.com>

ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/ShadowEvalSection.test.tsx

@@ -139,10 +139,39 @@ describe("ShadowEvalSection", () => {
     expect(screen.getByText("SIMPLE")).toBeInTheDocument();
     expect(screen.getByText("REASONING")).toBeInTheDocument();
     expect(screen.getByText("55.0%")).toBeInTheDocument();
-    // Overall good-or-better = shadow wins + ties = 70%
+    // Overall matched-or-beat = shadow wins + ties = 70%
     expect(screen.getByText("70.0%")).toBeInTheDocument();
   });
 
+  it("states the metric's denominator next to the headline number", () => {
+    const j = job();
+    mockHooks({ jobs: [j], detail: j });
+    render(<ShadowEvalSection accessToken="token" />);
+    expect(screen.getByText("Router matched or beat your current model")).toBeInTheDocument();
+    expect(screen.getByText("of 42 judged responses")).toBeInTheDocument();
+  });
+
+  it("shows the win/tie/loss split as a bar whose segments match the rates", () => {
+    const j = job();
+    mockHooks({ jobs: [j], detail: j });
+    render(<ShadowEvalSection accessToken="token" />);
+    // 48% router wins, 22% ties, 30% current-model wins
+    expect(screen.getByText(/Router won/)).toBeInTheDocument();
+    expect(screen.getByText(/Tie 22.0%/)).toBeInTheDocument();
+    expect(screen.getByText(/Current model won 30.0%/)).toBeInTheDocument();
+    expect(screen.getByTestId("verdict-segment-Router won")).toHaveStyle({ width: "48%" });
+  });
+
+  it("explains the mechanism behind a 'How this works' toggle", async () => {
+    const user = userEvent.setup();
+    mockHooks({ jobs: [] });
+    render(<ShadowEvalSection accessToken="token" />);
+
+    expect(screen.queryByText(/LLM judge compares the two answers blind/)).not.toBeInTheDocument();
+    await user.click(screen.getByText("How this works"));
+    expect(screen.getByText(/LLM judge compares the two answers blind/)).toBeInTheDocument();
+  });
+
   it("flags low-sample tiers", () => {
     const j = job();
     mockHooks({ jobs: [j], detail: j });

ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/ShadowEvalSection.tsx

@@ -10,10 +10,12 @@ import { SearchSelect, type SearchSelectOption } from "@/components/shared/Searc
 import { Badge } from "@/components/ui/badge";
 import { Button } from "@/components/ui/button";
 import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card";
+import { Collapsible, CollapsibleContent, CollapsibleTrigger } from "@/components/ui/collapsible";
 import { Input } from "@/components/ui/input";
 import { Label } from "@/components/ui/label";
 import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select";
 import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from "@/components/ui/table";
+import { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from "@/components/ui/tooltip";
 import { ApiError } from "@/lib/http/client";
 
 import { usd } from "./costOptimizationUtils";
@@ -55,16 +57,79 @@ const StatusBadge: React.FC<{ status: string }> = ({ status }) => (
 /** Verdict counts are meaningless below this; the table warns instead of misleading. */
 const MIN_TURNS_FOR_CONFIDENCE = 30;
 
+const HOW_IT_WORKS_STEPS = [
+  "A sampled slice of the key's live traffic is picked at random. Users are never affected — they keep getting answers from your current models.",
+  "Each sampled request is quietly duplicated through the auto-router, which classifies the prompt and picks whatever model it would have picked.",
+  "An LLM judge compares the two answers blind: it sees them only as “A” and “B” in random order, never which system produced which.",
+  "Verdicts accumulate below, broken down by the router's difficulty tier. Shadow and judge calls bill to the shadowed key, capped per job.",
+] as const;
+
+const HowThisWorks: React.FC = () => {
+  const [open, setOpen] = useState(false);
+  return (
+    <Collapsible open={open} onOpenChange={setOpen}>
+      <CollapsibleTrigger className="text-xs text-muted-foreground underline decoration-dotted underline-offset-2 hover:text-foreground">
+        {open ? "Hide how this works" : "How this works"}
+      </CollapsibleTrigger>
+      <CollapsibleContent>
+        <ol className="mt-2 max-w-3xl list-decimal space-y-1 pl-5 text-xs text-muted-foreground">
+          {HOW_IT_WORKS_STEPS.map((step) => (
+            <li key={step}>{step}</li>
+          ))}
+        </ol>
+      </CollapsibleContent>
+    </Collapsible>
+  );
+};
+
+const HeadTooltip: React.FC<{ label: string; tooltip: string; className?: string }> = ({
+  label,
+  tooltip,
+  className,
+}) => (
+  <TableHead className={className}>
+    <TooltipProvider delay={200}>
+      <Tooltip>
+        <TooltipTrigger render={<span />}>
+          <span className="underline decoration-dotted underline-offset-2">{label}</span>
+        </TooltipTrigger>
+        <TooltipContent className="max-w-xs">{tooltip}</TooltipContent>
+      </Tooltip>
+    </TooltipProvider>
+  </TableHead>
+);
+
+const LOW_SAMPLE_TOOLTIP = `Fewer than ${MIN_TURNS_FOR_CONFIDENCE} judged turns — treat as directional only.`;
+
+const LowSampleFlag: React.FC = () => (
+  <TooltipProvider delay={200}>
+    <Tooltip>
+      <TooltipTrigger render={<span className="ml-2 text-xs text-muted-foreground" />}>
+        <span className="underline decoration-dotted underline-offset-2">(low sample)</span>
+      </TooltipTrigger>
+      <TooltipContent className="max-w-xs">{LOW_SAMPLE_TOOLTIP}</TooltipContent>
+    </Tooltip>
+  </TooltipProvider>
+);
+
 const TierResultsTable: React.FC<{ groups: readonly ShadowEvalTierResult[] }> = ({ groups }) => (
   <Table>
     <TableHeader>
       <TableRow>
         <TableHead>Router tier</TableHead>
         <TableHead className="text-right">Judged turns</TableHead>
-        <TableHead className="text-right">Router pick wins</TableHead>
+        <TableHead className="text-right">Router wins</TableHead>
         <TableHead className="text-right">Current model wins</TableHead>
-        <TableHead className="text-right">Ties</TableHead>
-        <TableHead className="text-right">Judge confidence</TableHead>
+        <HeadTooltip
+          className="text-right"
+          label="Ties"
+          tooltip="The judge saw the same quality from both. Ties count in the router's favor — same answer quality, usually at lower cost."
+        />
+        <HeadTooltip
+          className="text-right"
+          label="Judge confidence"
+          tooltip="The judge's self-reported certainty in its verdicts (0 to 1), averaged over this tier's turns."
+        />
       </TableRow>
     </TableHeader>
     <TableBody>
@@ -72,9 +137,7 @@ const TierResultsTable: React.FC<{ groups: readonly ShadowEvalTierResult[] }> =
         <TableRow key={g.tier}>
           <TableCell className="font-medium text-foreground">
             {g.tier}
-            {g.turn_count < MIN_TURNS_FOR_CONFIDENCE ? (
-              <span className="ml-2 text-xs text-muted-foreground">(low sample)</span>
-            ) : null}
+            {g.turn_count < MIN_TURNS_FOR_CONFIDENCE ? <LowSampleFlag /> : null}
           </TableCell>
           <TableCell className="text-right tabular-nums">{g.turn_count.toLocaleString()}</TableCell>
           <TableCell className="text-right font-medium tabular-nums text-foreground">
@@ -89,6 +152,52 @@ const TierResultsTable: React.FC<{ groups: readonly ShadowEvalTierResult[] }> =
   </Table>
 );
 
+const MetricLabel: React.FC<{ label: string; tooltip: string }> = ({ label, tooltip }) => (
+  <TooltipProvider delay={200}>
+    <Tooltip>
+      <TooltipTrigger render={<span className="w-fit text-[11px] uppercase tracking-wide text-muted-foreground" />}>
+        <span className="underline decoration-dotted underline-offset-2">{label}</span>
+      </TooltipTrigger>
+      <TooltipContent className="max-w-xs">{tooltip}</TooltipContent>
+    </Tooltip>
+  </TooltipProvider>
+);
+
+const VerdictBar: React.FC<{ results: NonNullable<ShadowEvalJob["results"]> }> = ({ results }) => {
+  const routerWins = results.overall_shadow_win_rate_pct;
+  const ties = results.overall_tie_rate_pct;
+  const currentWins = Math.max(0, 100 - routerWins - ties);
+  const segments = [
+    { label: "Router won", value: routerWins, className: "bg-emerald-500" },
+    { label: "Tie", value: ties, className: "bg-emerald-200" },
+    { label: "Current model won", value: currentWins, className: "bg-muted-foreground/30" },
+  ];
+  return (
+    <div className="space-y-2 border-b px-6 py-4">
+      <div className="flex h-2 w-full overflow-hidden rounded-full" role="img" aria-label="Verdict breakdown">
+        {segments.map((segment) =>
+          segment.value > 0 ? (
+            <div
+              key={segment.label}
+              data-testid={`verdict-segment-${segment.label}`}
+              className={segment.className}
+              style={{ width: `${segment.value}%` }}
+            />
+          ) : null,
+        )}
+      </div>
+      <div className="flex flex-wrap gap-x-4 gap-y-1 text-xs text-muted-foreground">
+        {segments.map((segment) => (
+          <span key={segment.label} className="flex items-center gap-1.5">
+            <span className={`size-2 rounded-full ${segment.className}`} />
+            {segment.label} {pct(segment.value)}
+          </span>
+        ))}
+      </div>
+    </div>
+  );
+};
+
 const JobResults: React.FC<{
   job: ShadowEvalJob;
   onStop: () => void;
@@ -107,9 +216,10 @@ const JobResults: React.FC<{
               Shadowing {job.shadow_percentage}% via <span className="font-mono text-xs">{job.router_name}</span>
             </p>
             <p className="text-xs text-muted-foreground">
-              {job.completed_count.toLocaleString()} judged · {job.failed_count.toLocaleString()} failed ·{" "}
+              {job.completed_count.toLocaleString()} responses judged · {job.failed_count.toLocaleString()} errored
+              (shadow or judge call) ·{" "}
               {job.cost_actual != null ? `${usd(job.cost_actual)} judge spend` : "no judge spend yet"}
-              {job.cost_estimate != null ? ` (est. ${usd(job.cost_estimate)})` : ""}
+              {job.cost_estimate != null ? ` of ~${usd(job.cost_estimate)} estimated` : ""}
               {active && endsIn(job.ends_at) ? ` · ${endsIn(job.ends_at)}` : ""}
             </p>
           </div>
@@ -125,16 +235,24 @@ const JobResults: React.FC<{
         <>
           <div className="grid gap-0 border-b sm:grid-cols-2">
             <div className="flex flex-col justify-center gap-1 px-6 py-4">
-              <p className="text-[11px] uppercase tracking-wide text-muted-foreground">
-                Router pick judged as good or better
-              </p>
+              <MetricLabel
+                label="Router matched or beat your current model"
+                tooltip="Router wins plus ties. A tie counts in the router's favor: the judge saw the same quality, and the router usually picked a cheaper model."
+              />
               <p className="text-3xl font-semibold text-foreground">{okOrBetter != null ? pct(okOrBetter) : "—"}</p>
+              <p className="text-xs text-muted-foreground">
+                of {job.completed_count.toLocaleString()} judged responses
+              </p>
             </div>
             <div className="flex flex-col justify-center gap-1 border-t px-6 py-4 sm:border-l sm:border-t-0">
-              <p className="text-[11px] uppercase tracking-wide text-muted-foreground">Router pick strictly better</p>
+              <MetricLabel
+                label="Router strictly won"
+                tooltip="The judge preferred the router's answer outright, ties excluded."
+              />
               <p className="text-3xl font-semibold text-foreground">{pct(results.overall_shadow_win_rate_pct)}</p>
             </div>
           </div>
+          <VerdictBar results={results} />
           <TierResultsTable groups={results.groups} />
         </>
       ) : (
@@ -453,11 +571,15 @@ const ShadowEvalSection: React.FC<ShadowEvalSectionProps> = ({ accessToken }) =>
 
   return (
     <div id="shadow-eval-section" className="space-y-4 scroll-mt-6">
-      <div className="flex flex-wrap items-baseline gap-2">
-        <h3 className="text-lg font-semibold text-foreground">Shadow eval</h3>
-        <p className="text-xs text-muted-foreground">
-          pre-adoption quality check: your current model vs. what the router would have picked
-        </p>
+      <div className="space-y-1">
+        <div className="flex flex-wrap items-baseline gap-2">
+          <h3 className="text-lg font-semibold text-foreground">Shadow eval</h3>
+          <p className="text-sm text-muted-foreground">
+            Would the auto-router have answered as well as the models you use today? Find out on your real traffic,
+            before switching anything.
+          </p>
+        </div>
+        <HowThisWorks />
       </div>
 
       {latest && latestDetail ? (