mirror of
https://github.com/BerriAI/litellm.git
synced 2026-08-28 05:25:59 +00:00
feat(ui): direction picker and reverse-mode display for shadow evals (#36994)
* feat(ui): direction picker and reverse-mode display for shadow evals * fix(ui): include configured model groups in the shadow eval baseline picker
This commit is contained in:
tin-berri
GitHub
parent
4eadf92ade
commit
540caa6574
4 changed files with 253 additions and 44 deletions
|
|
@ -46,6 +46,7 @@ vi.mock("@/app/(dashboard)/hooks/models/useModels", () => ({
|
|||
{ model_name: "gpt-auto", litellm_params: { model: "auto_router/gpt-auto" } },
|
||||
],
|
||||
})),
|
||||
usePlainModelGroups: vi.fn(() => new Set(["prod-claude"])),
|
||||
}));
|
||||
|
||||
vi.mock("@/app/(dashboard)/hooks/models/useModelCostMap", () => ({
|
||||
|
|
@ -72,6 +73,8 @@ const job = (overrides: Partial<ShadowEvalJob> = {}): ShadowEvalJob => ({
|
|||
job_id: "job-1",
|
||||
status: "running",
|
||||
router_name: "claude-auto",
|
||||
direction: "forward",
|
||||
baseline_model: null,
|
||||
judge_model: "anthropic/claude-sonnet-5",
|
||||
shadow_percentage: 10,
|
||||
max_turns: 200,
|
||||
|
|
@ -367,6 +370,58 @@ describe("ShadowEvalSection", () => {
|
|||
expect(start.mutate).toHaveBeenCalledWith(expectedBody);
|
||||
});
|
||||
|
||||
it("requires a baseline model in reverse mode and submits it, while forward mode never shows the picker", async () => {
|
||||
const user = userEvent.setup();
|
||||
const { start } = mockHooks({});
|
||||
render(<ShadowEvalSection />);
|
||||
|
||||
expect(screen.queryByPlaceholderText("Select a baseline model")).not.toBeInTheDocument();
|
||||
|
||||
await user.click(screen.getByText("Adoption check: key's traffic vs the router"));
|
||||
await user.click(await screen.findByText("Regression check: router's picks vs a baseline"));
|
||||
await user.click(screen.getByPlaceholderText("Search keys by alias"));
|
||||
await user.click(await screen.findByText("prod-alpha"));
|
||||
await user.click(screen.getByPlaceholderText("Select an auto-router"));
|
||||
await user.click(await screen.findByText("gpt-auto"));
|
||||
await user.click(screen.getByPlaceholderText("Select a judge model"));
|
||||
await user.click(await screen.findByRole("option", { name: /anthropic\/claude-sonnet-5/ }));
|
||||
|
||||
expect(screen.getByText("Start shadow eval")).toBeDisabled();
|
||||
|
||||
await user.click(screen.getByPlaceholderText("Select a baseline model"));
|
||||
expect(await screen.findByRole("option", { name: /openai\/gpt-4o/ })).toBeInTheDocument();
|
||||
await user.click(screen.getByRole("option", { name: /prod-claude/ }));
|
||||
await user.click(screen.getByText("Start shadow eval"));
|
||||
|
||||
const expectedBody = {
|
||||
api_key_id: "hash-alpha",
|
||||
router_name: "gpt-auto",
|
||||
direction: "reverse",
|
||||
baseline_model: "prod-claude",
|
||||
shadow_percentage: 10,
|
||||
duration_days: 7,
|
||||
max_turns: 200,
|
||||
judge_model: "anthropic/claude-sonnet-5",
|
||||
};
|
||||
expect(start.mutate).toHaveBeenCalledWith(expectedBody);
|
||||
});
|
||||
|
||||
it("flips the arm labels and headline for a reverse job's results", () => {
|
||||
const j = job({ direction: "reverse", baseline_model: "openai/gpt-4o" });
|
||||
mockHooks({ jobs: [j], detailsById: { "job-1": j } });
|
||||
render(<ShadowEvalSection />);
|
||||
|
||||
expect(screen.getByText(/on 10% of its traffic/)).toBeInTheDocument();
|
||||
expect(screen.getByText("Router matched or beat the baseline")).toBeInTheDocument();
|
||||
expect(screen.getByText("52.0%")).toBeInTheDocument();
|
||||
expect(screen.getByText(/Router won 30.0%/)).toBeInTheDocument();
|
||||
expect(screen.getByText(/Baseline won 48.0%/)).toBeInTheDocument();
|
||||
expect(screen.getAllByText("Baseline wins")).toHaveLength(2);
|
||||
expect(screen.getByText("Router pick")).toBeInTheDocument();
|
||||
expect(screen.queryByText(/Current model/)).not.toBeInTheDocument();
|
||||
expect(screen.queryByText("Compared against")).not.toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("keeps an older job's verdicts reachable through the previous evaluations list", async () => {
|
||||
const user = userEvent.setup();
|
||||
const emptyOverrides: Partial<ShadowEvalJob> = {
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@ import React, { useMemo, useState } from "react";
|
|||
import { useInfiniteKeys } from "@/app/(dashboard)/hooks/keys/useKeys";
|
||||
import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized";
|
||||
import { useModelCostMap } from "@/app/(dashboard)/hooks/models/useModelCostMap";
|
||||
import { useAutoRouters } from "@/app/(dashboard)/hooks/models/useModels";
|
||||
import { useAutoRouters, usePlainModelGroups } from "@/app/(dashboard)/hooks/models/useModels";
|
||||
import { PaginatedSearchSelect } from "@/components/shared/PaginatedSearchSelect";
|
||||
import { SearchSelect, type SearchSelectOption } from "@/components/shared/SearchSelect";
|
||||
import { Badge } from "@/components/ui/badge";
|
||||
|
|
@ -31,6 +31,37 @@ const pct = (value: number): string => `${value.toFixed(1)}%`;
|
|||
|
||||
const MIN_TURNS_FOR_CONFIDENCE = 30;
|
||||
|
||||
type ShadowEvalDirection = ShadowEvalJob["direction"];
|
||||
|
||||
const otherArmLabel = (direction: ShadowEvalDirection): string =>
|
||||
direction === "reverse" ? "Baseline" : "Current model";
|
||||
|
||||
const routerWinRate = (direction: ShadowEvalDirection, slice: ShadowEvalSlice): number =>
|
||||
direction === "reverse" ? slice.real_win_rate_pct : slice.shadow_win_rate_pct;
|
||||
|
||||
const otherArmWinRate = (direction: ShadowEvalDirection, slice: ShadowEvalSlice): number =>
|
||||
direction === "reverse" ? slice.shadow_win_rate_pct : slice.real_win_rate_pct;
|
||||
|
||||
const routerMatchedOrBeatPct = (
|
||||
direction: ShadowEvalDirection,
|
||||
results: NonNullable<ShadowEvalJob["results"]>,
|
||||
): number =>
|
||||
direction === "reverse"
|
||||
? 100 - results.overall_shadow_win_rate_pct
|
||||
: results.overall_shadow_win_rate_pct + results.overall_tie_rate_pct;
|
||||
|
||||
const jobHeadline = (job: ShadowEvalJob): React.ReactNode =>
|
||||
job.direction === "reverse" ? (
|
||||
<>
|
||||
Comparing <span className="font-mono text-xs">{job.router_name}</span> to{" "}
|
||||
<span className="font-mono text-xs">{job.baseline_model}</span> on {job.shadow_percentage}% of its traffic
|
||||
</>
|
||||
) : (
|
||||
<>
|
||||
Shadowing {job.shadow_percentage}% via <span className="font-mono text-xs">{job.router_name}</span>
|
||||
</>
|
||||
);
|
||||
|
||||
const isActive = (job: ShadowEvalJob): boolean => job.status === "running";
|
||||
|
||||
const endsIn = (endsAt: string | null | undefined): string | null => {
|
||||
|
|
@ -54,16 +85,22 @@ const StatusBadge: React.FC<{ status: string }> = ({ status }) => (
|
|||
</Badge>
|
||||
);
|
||||
|
||||
const SliceTable: React.FC<{ groupHeader: string; slices: readonly ShadowEvalSlice[] }> = ({ groupHeader, slices }) => (
|
||||
const SliceTable: React.FC<{
|
||||
groupHeader: string;
|
||||
direction: ShadowEvalDirection;
|
||||
slices: readonly ShadowEvalSlice[];
|
||||
}> = ({ groupHeader, direction, slices }) => (
|
||||
<Table>
|
||||
<TableHeader>
|
||||
<TableRow>
|
||||
<TableHead>{groupHeader}</TableHead>
|
||||
{["Judged turns", "Router wins", "Current model wins", "Ties", "Judge confidence"].map((label) => (
|
||||
<TableHead key={label} className="text-right">
|
||||
{label}
|
||||
</TableHead>
|
||||
))}
|
||||
{["Judged turns", "Router wins", `${otherArmLabel(direction)} wins`, "Ties", "Judge confidence"].map(
|
||||
(label) => (
|
||||
<TableHead key={label} className="text-right">
|
||||
{label}
|
||||
</TableHead>
|
||||
),
|
||||
)}
|
||||
</TableRow>
|
||||
</TableHeader>
|
||||
<TableBody>
|
||||
|
|
@ -77,9 +114,9 @@ const SliceTable: React.FC<{ groupHeader: string; slices: readonly ShadowEvalSli
|
|||
</TableCell>
|
||||
<TableCell className="text-right tabular-nums">{slice.turn_count.toLocaleString()}</TableCell>
|
||||
<TableCell className="text-right font-medium tabular-nums text-foreground">
|
||||
{pct(slice.shadow_win_rate_pct)}
|
||||
{pct(routerWinRate(direction, slice))}
|
||||
</TableCell>
|
||||
<TableCell className="text-right tabular-nums">{pct(slice.real_win_rate_pct)}</TableCell>
|
||||
<TableCell className="text-right tabular-nums">{pct(otherArmWinRate(direction, slice))}</TableCell>
|
||||
<TableCell className="text-right tabular-nums">{pct(slice.tie_rate_pct)}</TableCell>
|
||||
<TableCell className="text-right tabular-nums">{slice.avg_judge_confidence.toFixed(2)}</TableCell>
|
||||
</TableRow>
|
||||
|
|
@ -88,13 +125,23 @@ const SliceTable: React.FC<{ groupHeader: string; slices: readonly ShadowEvalSli
|
|||
</Table>
|
||||
);
|
||||
|
||||
const VerdictBar: React.FC<{ results: NonNullable<ShadowEvalJob["results"]> }> = ({ results }) => {
|
||||
const routerWins = results.overall_shadow_win_rate_pct;
|
||||
const VerdictBar: React.FC<{ direction: ShadowEvalDirection; results: NonNullable<ShadowEvalJob["results"]> }> = ({
|
||||
direction,
|
||||
results,
|
||||
}) => {
|
||||
const ties = results.overall_tie_rate_pct;
|
||||
const routerWins =
|
||||
direction === "reverse"
|
||||
? Math.max(0, 100 - results.overall_shadow_win_rate_pct - ties)
|
||||
: results.overall_shadow_win_rate_pct;
|
||||
const segments = [
|
||||
{ label: "Router won", value: routerWins, fill: "bg-emerald-500" },
|
||||
{ label: "Tie", value: ties, fill: "bg-emerald-200" },
|
||||
{ label: "Current model won", value: Math.max(0, 100 - routerWins - ties), fill: "bg-muted-foreground/30" },
|
||||
{
|
||||
label: `${otherArmLabel(direction)} won`,
|
||||
value: Math.max(0, 100 - routerWins - ties),
|
||||
fill: "bg-muted-foreground/30",
|
||||
},
|
||||
];
|
||||
return (
|
||||
<div className="space-y-2 border-b px-6 py-4">
|
||||
|
|
@ -133,20 +180,22 @@ const ResultsBody: React.FC<{ job: ShadowEvalJob; resultsError?: boolean }> = ({
|
|||
<>
|
||||
<div className="flex flex-col gap-1 border-b px-6 py-4">
|
||||
<p className="text-[11px] uppercase tracking-wide text-muted-foreground">
|
||||
Router matched or beat your current model
|
||||
</p>
|
||||
<p className="text-3xl font-semibold text-foreground">
|
||||
{pct(results.overall_shadow_win_rate_pct + results.overall_tie_rate_pct)}
|
||||
Router matched or beat {job.direction === "reverse" ? "the baseline" : "your current model"}
|
||||
</p>
|
||||
<p className="text-3xl font-semibold text-foreground">{pct(routerMatchedOrBeatPct(job.direction, results))}</p>
|
||||
<p className="text-xs text-muted-foreground">of {(job.judged_count ?? 0).toLocaleString()} judged responses</p>
|
||||
</div>
|
||||
<VerdictBar results={results} />
|
||||
<VerdictBar direction={job.direction} results={results} />
|
||||
{results.by_current_model.length > 0 && (
|
||||
<SliceTable groupHeader="Compared against" slices={results.by_current_model} />
|
||||
<SliceTable
|
||||
groupHeader={job.direction === "reverse" ? "Router pick" : "Compared against"}
|
||||
direction={job.direction}
|
||||
slices={results.by_current_model}
|
||||
/>
|
||||
)}
|
||||
{results.by_tier.length > 0 && (
|
||||
<div className={results.by_current_model.length > 0 ? "border-t" : ""}>
|
||||
<SliceTable groupHeader="Prompt difficulty" slices={results.by_tier} />
|
||||
<SliceTable groupHeader="Prompt difficulty" direction={job.direction} slices={results.by_tier} />
|
||||
</div>
|
||||
)}
|
||||
</>
|
||||
|
|
@ -168,9 +217,7 @@ const JobResults: React.FC<{
|
|||
<div className="flex items-center gap-3">
|
||||
<StatusBadge status={job.status} />
|
||||
<div>
|
||||
<p className="text-sm font-medium text-foreground">
|
||||
Shadowing {job.shadow_percentage}% via <span className="font-mono text-xs">{job.router_name}</span>
|
||||
</p>
|
||||
<p className="text-sm font-medium text-foreground">{jobHeadline(job)}</p>
|
||||
<p className="text-xs text-muted-foreground">
|
||||
{(job.judged_count ?? 0).toLocaleString()} of {job.max_turns.toLocaleString()} turns judged ·{" "}
|
||||
{(job.error_count ?? 0).toLocaleString()} errored · {usd(job.judge_spend ?? 0)} judge spend
|
||||
|
|
@ -201,25 +248,55 @@ interface CostMapEntry {
|
|||
mode?: string;
|
||||
}
|
||||
|
||||
const useJudgeModelOptions = (): SearchSelectOption[] => {
|
||||
const useChatModelNames = (): string[] => {
|
||||
const { data: costMap } = useModelCostMap();
|
||||
return useMemo(() => {
|
||||
if (!costMap) return [];
|
||||
const chatModels = Object.entries(costMap as Record<string, CostMapEntry>)
|
||||
.filter(([, value]) => value?.mode === "chat" && value?.litellm_provider)
|
||||
.map(([key, value]) => (key.startsWith(`${value.litellm_provider}/`) ? key : `${value.litellm_provider}/${key}`));
|
||||
return [...new Set(chatModels)].toSorted((a, b) => a.localeCompare(b));
|
||||
}, [costMap]);
|
||||
};
|
||||
|
||||
const useJudgeModelOptions = (): SearchSelectOption[] => {
|
||||
const chatModels = useChatModelNames();
|
||||
return useMemo(() => {
|
||||
const pinned: SearchSelectOption[] = RECOMMENDED_JUDGE_MODELS.map((model) => ({
|
||||
label: model,
|
||||
value: model,
|
||||
sublabel: "Recommended",
|
||||
}));
|
||||
if (!costMap) return pinned;
|
||||
const pinnedNames = new Set<string>(RECOMMENDED_JUDGE_MODELS);
|
||||
const chatModels = Object.entries(costMap as Record<string, CostMapEntry>)
|
||||
.filter(([, value]) => value?.mode === "chat" && value?.litellm_provider)
|
||||
.map(([key, value]) => (key.startsWith(`${value.litellm_provider}/`) ? key : `${value.litellm_provider}/${key}`));
|
||||
const rest = [...new Set(chatModels)]
|
||||
.filter((model) => !pinnedNames.has(model))
|
||||
.toSorted((a, b) => a.localeCompare(b))
|
||||
.map((model) => ({ label: model, value: model }));
|
||||
const rest = chatModels.filter((model) => !pinnedNames.has(model)).map((model) => ({ label: model, value: model }));
|
||||
return [...pinned, ...rest];
|
||||
}, [costMap]);
|
||||
}, [chatModels]);
|
||||
};
|
||||
|
||||
const useBaselineModelOptions = (): SearchSelectOption[] => {
|
||||
const configuredGroups = usePlainModelGroups();
|
||||
const chatModels = useChatModelNames();
|
||||
return useMemo(() => {
|
||||
const configured = [...configuredGroups]
|
||||
.toSorted((a, b) => a.localeCompare(b))
|
||||
.map((model) => ({ label: model, value: model, sublabel: "Configured on this gateway" }));
|
||||
const rest = chatModels
|
||||
.filter((model) => !configuredGroups.has(model))
|
||||
.map((model) => ({ label: model, value: model }));
|
||||
return [...configured, ...rest];
|
||||
}, [configuredGroups, chatModels]);
|
||||
};
|
||||
|
||||
const DIRECTION_OPTIONS: readonly { value: ShadowEvalDirection; label: string }[] = [
|
||||
{ value: "forward", label: "Adoption check: key's traffic vs the router" },
|
||||
{ value: "reverse", label: "Regression check: router's picks vs a baseline" },
|
||||
] as const;
|
||||
|
||||
const START_FORM_DESCRIPTION: Record<ShadowEvalDirection, string> = {
|
||||
forward:
|
||||
"Duplicates a sampled slice of the key's traffic through the auto-router and has an LLM judge compare both answers blind. The router's answers are never served to users; judge calls bill to the shadowed key.",
|
||||
reverse:
|
||||
"Duplicates a sampled slice of the traffic the auto-router already serves against a fixed baseline model and has an LLM judge compare both answers blind. The baseline's answers are never served to users; judge calls bill to the shadowed key.",
|
||||
};
|
||||
|
||||
const DURATION_OPTIONS = [
|
||||
|
|
@ -282,12 +359,15 @@ const StartForm: React.FC = () => {
|
|||
const { accessToken } = useAuthorized();
|
||||
const [apiKeyId, setApiKeyId] = useState("");
|
||||
const [routerName, setRouterName] = useState("");
|
||||
const [direction, setDirection] = useState<ShadowEvalDirection>("forward");
|
||||
const [baselineModel, setBaselineModel] = useState("");
|
||||
const [percentage, setPercentage] = useState("10");
|
||||
const [durationDays, setDurationDays] = useState("7");
|
||||
const [judgeModel, setJudgeModel] = useState("");
|
||||
const [maxTurns, setMaxTurns] = useState("200");
|
||||
const { data: autoRouters } = useAutoRouters();
|
||||
const judgeModelOptions = useJudgeModelOptions();
|
||||
const baselineModelOptions = useBaselineModelOptions();
|
||||
const start = useStartShadowEval();
|
||||
|
||||
const routerOptions = useMemo<SearchSelectOption[]>(() => {
|
||||
|
|
@ -301,14 +381,17 @@ const StartForm: React.FC = () => {
|
|||
const percentageValid = parsedPct >= 0.1 && parsedPct <= 100;
|
||||
const parsedMaxTurns = Number.parseInt(maxTurns, 10);
|
||||
const maxTurnsValid = parsedMaxTurns >= 1 && parsedMaxTurns <= 2000;
|
||||
const filled = [apiKeyId, routerName, judgeModel].every((field) => field !== "");
|
||||
const filled =
|
||||
[apiKeyId, routerName, judgeModel].every((field) => field !== "") &&
|
||||
(direction === "forward" || baselineModel !== "");
|
||||
const boundsValid = percentageValid && maxTurnsValid;
|
||||
const valid = Boolean(accessToken) && filled && boundsValid;
|
||||
const handleStart = () => {
|
||||
const startBody = {
|
||||
api_key_id: apiKeyId,
|
||||
router_name: routerName,
|
||||
direction: "forward" as const,
|
||||
direction,
|
||||
...(direction === "reverse" ? { baseline_model: baselineModel } : {}),
|
||||
shadow_percentage: parsedPct,
|
||||
duration_days: Number.parseInt(durationDays, 10),
|
||||
max_turns: parsedMaxTurns,
|
||||
|
|
@ -321,13 +404,27 @@ const StartForm: React.FC = () => {
|
|||
<Card size="sm">
|
||||
<CardHeader>
|
||||
<CardTitle className="text-sm font-medium text-foreground">Start a shadow eval</CardTitle>
|
||||
<p className="text-xs text-muted-foreground">
|
||||
Duplicates a sampled slice of the key's traffic through the auto-router and has an LLM judge compare both
|
||||
answers blind. The router's answers are never served to users; judge calls bill to the shadowed key.
|
||||
</p>
|
||||
<p className="text-xs text-muted-foreground">{START_FORM_DESCRIPTION[direction]}</p>
|
||||
</CardHeader>
|
||||
<CardContent className="space-y-3">
|
||||
<div className="grid gap-3 sm:grid-cols-3">
|
||||
<Field label="Direction">
|
||||
<Select
|
||||
value={direction}
|
||||
onValueChange={(v: string | null) => setDirection(v === "reverse" ? "reverse" : "forward")}
|
||||
>
|
||||
<SelectTrigger className="w-full">
|
||||
<SelectValue>{DIRECTION_OPTIONS.find((o) => o.value === direction)?.label}</SelectValue>
|
||||
</SelectTrigger>
|
||||
<SelectContent>
|
||||
{DIRECTION_OPTIONS.map((option) => (
|
||||
<SelectItem key={option.value} value={option.value}>
|
||||
{option.label}
|
||||
</SelectItem>
|
||||
))}
|
||||
</SelectContent>
|
||||
</Select>
|
||||
</Field>
|
||||
<Field label="Key to shadow" htmlFor="shadow-eval-key">
|
||||
<KeySelect value={apiKeyId} onChange={setApiKeyId} />
|
||||
</Field>
|
||||
|
|
@ -390,6 +487,17 @@ const StartForm: React.FC = () => {
|
|||
<p className="text-xs text-destructive">Enter a value from 1 to 2000</p>
|
||||
)}
|
||||
</Field>
|
||||
{direction === "reverse" && (
|
||||
<Field label="Baseline model">
|
||||
<SearchSelect
|
||||
options={baselineModelOptions}
|
||||
value={baselineModel}
|
||||
onValueChange={setBaselineModel}
|
||||
placeholder="Select a baseline model"
|
||||
emptyText="No chat models available"
|
||||
/>
|
||||
</Field>
|
||||
)}
|
||||
<Field label="Judge model" className="sm:col-span-2">
|
||||
<SearchSelect
|
||||
options={judgeModelOptions}
|
||||
|
|
@ -410,7 +518,7 @@ const StartForm: React.FC = () => {
|
|||
|
||||
const previousSummary = (job: ShadowEvalJob): string => {
|
||||
const results = job.results;
|
||||
if (results) return pct(results.overall_shadow_win_rate_pct + results.overall_tie_rate_pct);
|
||||
if (results) return pct(routerMatchedOrBeatPct(job.direction, results));
|
||||
return job.judged_count === 0 ? "no verdicts" : "view results";
|
||||
};
|
||||
|
||||
|
|
@ -429,9 +537,7 @@ const PreviousJob: React.FC<{ job: ShadowEvalJob }> = ({ job }) => {
|
|||
<div className="flex items-center gap-3">
|
||||
<StatusBadge status={shown.status} />
|
||||
<div>
|
||||
<p className="text-sm font-medium text-foreground">
|
||||
{shown.shadow_percentage}% via <span className="font-mono text-xs">{shown.router_name}</span>
|
||||
</p>
|
||||
<p className="text-sm font-medium text-foreground">{jobHeadline(shown)}</p>
|
||||
<p className="text-xs text-muted-foreground">
|
||||
{shown.judged_count != null &&
|
||||
`${shown.judged_count.toLocaleString()} judged · ${(shown.error_count ?? 0).toLocaleString()} errored · ${usd(shown.judge_spend ?? 0)} judge spend · `}
|
||||
|
|
@ -507,8 +613,8 @@ const ShadowEvalSection: React.FC = () => {
|
|||
<div className="flex flex-wrap items-baseline gap-2">
|
||||
<h2 className="text-xl font-semibold text-foreground">Shadow eval</h2>
|
||||
<p className="text-sm text-muted-foreground">
|
||||
Would the auto-router have answered as well as the models you use today? Find out on your real traffic, before
|
||||
switching anything.
|
||||
Blind-judge the auto-router on your real traffic: against the models a key uses today before switching, or
|
||||
against a fixed baseline after it has switched.
|
||||
</p>
|
||||
</div>
|
||||
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ import { beforeEach, describe, expect, it, vi } from "vitest";
|
|||
import {
|
||||
isAutoRouterDeployment,
|
||||
selectAutoRouterModelGroups,
|
||||
selectPlainModelGroups,
|
||||
useAllProxyModels,
|
||||
useAutoRouterModelGroups,
|
||||
useAutoRouters,
|
||||
|
|
@ -977,6 +978,32 @@ describe("selectAutoRouterModelGroups", () => {
|
|||
});
|
||||
});
|
||||
|
||||
describe("selectPlainModelGroups", () => {
|
||||
it("keeps only non-auto-router model groups", () => {
|
||||
const deployments: AutoRouterCandidateDeployment[] = [
|
||||
{ model_name: "smart-router", litellm_params: { model: "auto_router/complexity_router" } },
|
||||
{ model_name: "claude-haiku", litellm_params: { model: "anthropic/claude-haiku-4-5" } },
|
||||
{ model_name: "claude-sonnet", litellm_params: { model: "anthropic/claude-sonnet-4-5" } },
|
||||
{ model_name: "cheap-router", litellm_params: { model: "auto_router/adaptive_router" } },
|
||||
];
|
||||
|
||||
expect(selectPlainModelGroups(deployments)).toEqual(new Set(["claude-haiku", "claude-sonnet"]));
|
||||
});
|
||||
|
||||
it("drops a group name that also fronts an auto-router deployment", () => {
|
||||
const deployments: AutoRouterCandidateDeployment[] = [
|
||||
{ model_name: "shared-name", litellm_params: { model: "auto_router/complexity_router" } },
|
||||
{ model_name: "shared-name", litellm_params: { model: "anthropic/claude-sonnet-4-5" } },
|
||||
];
|
||||
|
||||
expect(selectPlainModelGroups(deployments)).toEqual(new Set());
|
||||
});
|
||||
|
||||
it("drops deployments that have no public model_name", () => {
|
||||
expect(selectPlainModelGroups([{ model_name: "", litellm_params: { model: "openai/gpt-4o" } }])).toEqual(new Set());
|
||||
});
|
||||
});
|
||||
|
||||
describe("useAutoRouterModelGroups", () => {
|
||||
let queryClient: QueryClient;
|
||||
|
||||
|
|
|
|||
|
|
@ -123,6 +123,16 @@ export const selectAutoRouterModelGroups = (deployments: AutoRouterCandidateDepl
|
|||
export const selectAutoRouterDeployments = (deployments: AutoRouterDeployment[]): AutoRouterDeployment[] =>
|
||||
deployments.filter(isAutoRouterDeployment);
|
||||
|
||||
export const selectPlainModelGroups = (deployments: AutoRouterCandidateDeployment[]): ReadonlySet<string> => {
|
||||
const autoRouterGroups = selectAutoRouterModelGroups(deployments);
|
||||
return new Set(
|
||||
deployments
|
||||
.map((deployment) => deployment.model_name)
|
||||
.filter((modelName): modelName is string => Boolean(modelName))
|
||||
.filter((modelName) => !autoRouterGroups.has(modelName)),
|
||||
);
|
||||
};
|
||||
|
||||
export const fetchAllModelDeployments = async (
|
||||
accessToken: string,
|
||||
userId: string,
|
||||
|
|
@ -172,6 +182,17 @@ export const useAutoRouterModelGroups = (): ReadonlySet<string> => {
|
|||
return data ?? NO_AUTO_ROUTERS;
|
||||
};
|
||||
|
||||
export const usePlainModelGroups = (): ReadonlySet<string> => {
|
||||
const { accessToken, userId, userRole } = useAuthorized();
|
||||
const { data } = useQuery<AutoRouterDeployment[], Error, ReadonlySet<string>>({
|
||||
queryKey: autoRouterListKey(userId, userRole),
|
||||
queryFn: async () => await fetchAllModelDeployments(accessToken!, userId!, userRole!),
|
||||
enabled: Boolean(accessToken && userId && userRole),
|
||||
select: selectPlainModelGroups,
|
||||
});
|
||||
return data ?? NO_AUTO_ROUTERS;
|
||||
};
|
||||
|
||||
export const useAutoRouters = (): UseQueryResult<AutoRouterDeployment[], Error> => {
|
||||
const { accessToken, userId, userRole } = useAuthorized();
|
||||
return useQuery<AutoRouterDeployment[], Error, AutoRouterDeployment[]>({
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue