fix(ui): judge model field is a real selector, not a pre-filled default

The judge model box was a free-text input pre-filled with
anthropic/claude-sonnet-5, implying it was sent as-is when nothing was
actually defaulted client-side (the backend's own default only applies
if the field is omitted entirely). Replace it with a searchable combobox
backed by the model cost map (litellm_provider + mode === "chat"),
starting empty and requiring an explicit pick.

Recommends three models spanning different providers — anthropic/
claude-sonnet-5, openai/gpt-4o, gemini/gemini-2.5-pro — pinned to the
top of the list with a 'Recommended' badge, all backend-verified to
resolve correctly via litellm.get_llm_provider/cost_per_token.

Co-Authored-By: Claude <noreply@anthropic.com>

ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/ShadowEvalSection.test.tsx

@@ -34,6 +34,18 @@ vi.mock("@/app/(dashboard)/hooks/models/useModels", () => ({
   })),
 }));
 
+vi.mock("@/app/(dashboard)/hooks/models/useModelCostMap", () => ({
+  useModelCostMap: vi.fn(() => ({
+    data: {
+      "claude-sonnet-5": { litellm_provider: "anthropic", mode: "chat" },
+      "gpt-4o": { litellm_provider: "openai", mode: "chat" },
+      "gemini/gemini-2.5-pro": { litellm_provider: "gemini", mode: "chat" },
+      "gpt-4o-mini": { litellm_provider: "openai", mode: "chat" },
+      "text-embedding-3-large": { litellm_provider: "openai", mode: "embedding" },
+    },
+  })),
+}));
+
 import ShadowEvalSection from "./ShadowEvalSection";
 import {
   useShadowEvalJob,
@@ -172,6 +184,8 @@ describe("ShadowEvalSection", () => {
     await user.click(await screen.findByText("prod-alpha"));
     await user.click(screen.getByPlaceholderText("Select an auto-router"));
     await user.click(await screen.findByText("gpt-auto"));
+    await user.click(screen.getByPlaceholderText("Select a judge model"));
+    await user.click(await screen.findByRole("option", { name: /anthropic\/claude-sonnet-5/ }));
     await user.click(screen.getByText("Start shadow eval"));
 
     const expectedBody = {
@@ -198,11 +212,42 @@ describe("ShadowEvalSection", () => {
     });
   });
 
-  it("explains what the judge model is for and recommends a tier", () => {
+  it("explains what the judge model is for and recommends models across providers", () => {
     mockHooks({ jobs: [] });
     render(<ShadowEvalSection accessToken="token" />);
     expect(screen.getByText("Judge model")).toBeInTheDocument();
-    expect(screen.getByText(/mid-tier model \(Claude Sonnet or GPT-4o class\)/)).toBeInTheDocument();
+    expect(screen.getByText("anthropic/claude-sonnet-5")).toBeInTheDocument();
+    expect(screen.getByText("openai/gpt-4o")).toBeInTheDocument();
+    expect(screen.getByText("gemini/gemini-2.5-pro")).toBeInTheDocument();
+  });
+
+  it("does not send a default judge model without an explicit pick", async () => {
+    const user = userEvent.setup();
+    const mutate = vi.fn();
+    mockHooks({ jobs: [] });
+    vi.mocked(useStartShadowEval).mockReturnValue({ mutate, isPending: false, error: null } as unknown as ReturnType<
+      typeof useStartShadowEval
+    >);
+    render(<ShadowEvalSection accessToken="token" />);
+
+    await user.click(screen.getByPlaceholderText("Search keys by alias"));
+    await user.click(await screen.findByText("prod-alpha"));
+    await user.click(screen.getByPlaceholderText("Select an auto-router"));
+    await user.click(await screen.findByText("gpt-auto"));
+
+    expect(screen.getByText("Start shadow eval")).toBeDisabled();
+    expect(mutate).not.toHaveBeenCalled();
+  });
+
+  it("offers the recommended judge models ahead of the rest of the catalog", async () => {
+    const user = userEvent.setup();
+    mockHooks({ jobs: [] });
+    render(<ShadowEvalSection accessToken="token" />);
+
+    await user.click(screen.getByPlaceholderText("Select a judge model"));
+
+    const recommended = await screen.findAllByText("Recommended");
+    expect(recommended).toHaveLength(3);
   });
 
   it("shows when an active job will end", () => {

ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/ShadowEvalSection.tsx

@@ -4,6 +4,7 @@ import React, { useMemo, useState } from "react";
 
 import { useKeys } from "@/app/(dashboard)/hooks/keys/useKeys";
 import { useAutoRouters } from "@/app/(dashboard)/hooks/models/useModels";
+import { useModelCostMap } from "@/app/(dashboard)/hooks/models/useModelCostMap";
 import { PaginatedSearchSelect } from "@/components/shared/PaginatedSearchSelect";
 import { SearchSelect, type SearchSelectOption } from "@/components/shared/SearchSelect";
 import { Badge } from "@/components/ui/badge";
@@ -147,8 +148,38 @@ const JobResults: React.FC<{
   );
 };
 
-/** Kept in sync with DEFAULT_SHADOW_EVAL_JUDGE_MODEL on the backend. */
-const DEFAULT_JUDGE_MODEL = "anthropic/claude-sonnet-5";
+/** No default is sent — these are suggestions shown ahead of the rest of the catalog,
+ * one per provider, so a pick doesn't depend on having one provider configured. */
+const RECOMMENDED_JUDGE_MODELS = ["anthropic/claude-sonnet-5", "openai/gpt-4o", "gemini/gemini-2.5-pro"] as const;
+
+interface CostMapEntry {
+  litellm_provider?: string;
+  mode?: string;
+}
+
+const useJudgeModelOptions = (): SearchSelectOption[] => {
+  const { data: costMap } = useModelCostMap();
+  return useMemo(() => {
+    if (!costMap) return [];
+    const seen = new Set<string>();
+    const options: SearchSelectOption[] = [];
+    for (const recommended of RECOMMENDED_JUDGE_MODELS) {
+      seen.add(recommended);
+      options.push({ label: recommended, value: recommended, sublabel: "Recommended" });
+    }
+    const rest: SearchSelectOption[] = [];
+    for (const [key, value] of Object.entries(costMap as Record<string, CostMapEntry>)) {
+      const provider = value?.litellm_provider;
+      if (value?.mode !== "chat" || !provider) continue;
+      const model = key.startsWith(`${provider}/`) ? key : `${provider}/${key}`;
+      if (seen.has(model)) continue;
+      seen.add(model);
+      rest.push({ label: model, value: model });
+    }
+    rest.sort((a, b) => a.label.localeCompare(b.label));
+    return [...options, ...rest];
+  }, [costMap]);
+};
 
 const DURATION_OPTIONS = [
   { value: "1", label: "1 day" },
@@ -193,8 +224,9 @@ const StartForm: React.FC<{ accessToken: string | null }> = ({ accessToken }) =>
   const [routerName, setRouterName] = useState("");
   const [percentage, setPercentage] = useState("10");
   const [durationDays, setDurationDays] = useState(DEFAULT_DURATION_DAYS);
-  const [judgeModel, setJudgeModel] = useState(DEFAULT_JUDGE_MODEL);
+  const [judgeModel, setJudgeModel] = useState("");
   const { data: autoRouters } = useAutoRouters();
+  const judgeModelOptions = useJudgeModelOptions();
   const start = useStartShadowEval();
 
   const routerOptions = useMemo<SearchSelectOption[]>(() => {
@@ -206,8 +238,9 @@ const StartForm: React.FC<{ accessToken: string | null }> = ({ accessToken }) =>
   }, [autoRouters]);
 
   const parsedPct = Number.parseFloat(percentage);
-  const valid =
-    Boolean(accessToken) && apiKeyId.trim() !== "" && routerName.trim() !== "" && parsedPct > 0 && parsedPct <= 100;
+  const percentageValid = parsedPct > 0 && parsedPct <= 100;
+  const requiredFieldsPicked = apiKeyId.trim() !== "" && routerName.trim() !== "" && judgeModel.trim() !== "";
+  const valid = Boolean(accessToken) && requiredFieldsPicked && percentageValid;
 
   return (
     <Card size="sm">
@@ -281,16 +314,19 @@ const StartForm: React.FC<{ accessToken: string | null }> = ({ accessToken }) =>
             <Label htmlFor="shadow-eval-judge" className="text-xs">
               Judge model
             </Label>
-            <Input
-              id="shadow-eval-judge"
-              placeholder={`Default: ${DEFAULT_JUDGE_MODEL}`}
+            <SearchSelect
+              options={judgeModelOptions}
               value={judgeModel}
-              onChange={(e) => setJudgeModel(e.target.value)}
+              onValueChange={setJudgeModel}
+              placeholder="Select a judge model"
+              emptyText="No chat models available"
             />
             <p className="text-xs text-muted-foreground">
-              The judge only compares two answers blind — a mid-tier model (Claude Sonnet or GPT-4o class) is the sweet
-              spot. Small &quot;nano/mini&quot; models give unreliable verdicts; frontier reasoning models cost more
-              without changing outcomes.
+              The judge only compares two answers blind — a mid-tier model is the sweet spot. Recommended:{" "}
+              <span className="font-mono">anthropic/claude-sonnet-5</span>,{" "}
+              <span className="font-mono">openai/gpt-4o</span>, or{" "}
+              <span className="font-mono">gemini/gemini-2.5-pro</span>. Small &quot;nano/mini&quot; models give
+              unreliable verdicts; frontier reasoning models cost more without changing outcomes.
             </p>
           </div>
         </div>
@@ -308,7 +344,7 @@ const StartForm: React.FC<{ accessToken: string | null }> = ({ accessToken }) =>
                 router_name: routerName.trim(),
                 shadow_percentage: parsedPct,
                 duration_days: Number.parseInt(durationDays, 10),
-                judge_model: judgeModel.trim() || DEFAULT_JUDGE_MODEL,
+                judge_model: judgeModel,
               },
             })
           }