mirror of
https://github.com/BerriAI/litellm.git
synced 2026-08-28 05:25:59 +00:00
feat(ui): shadow evals tab beside auto-router usage (#36588)
This commit is contained in:
tin-berri
GitHub
parent
72ee0bb1c4
commit
83e890bdde
8 changed files with 1123 additions and 4 deletions
|
|
@ -8,6 +8,7 @@ import { ApiError } from "@/lib/http/client";
|
|||
|
||||
vi.mock("./useAutoRouterBenchmarks", () => ({ useAutoRouterBenchmarks: vi.fn() }));
|
||||
vi.mock("@/app/(dashboard)/hooks/models/useModels", () => ({ useAutoRouters: vi.fn() }));
|
||||
vi.mock("./ShadowEvalSection", () => ({ default: () => <div data-testid="shadow-eval-section" /> }));
|
||||
|
||||
import { useAutoRouters } from "@/app/(dashboard)/hooks/models/useModels";
|
||||
|
||||
|
|
@ -274,6 +275,33 @@ describe("AutoRouterBenchmarksTab", () => {
|
|||
expect(screen.getByText("Last 24 hours")).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("shows usage by default and mounts shadow evals only when its sub-tab is selected", () => {
|
||||
mockHook({ data: response([group()]) });
|
||||
renderTab();
|
||||
|
||||
expect(screen.getByRole("tab", { name: "Usage" })).toHaveAttribute("aria-selected", "true");
|
||||
expect(screen.getByText("Total estimated savings")).toBeInTheDocument();
|
||||
expect(screen.queryByTestId("shadow-eval-section")).not.toBeInTheDocument();
|
||||
|
||||
fireEvent.click(screen.getByRole("tab", { name: "Shadow Evals" }));
|
||||
expect(screen.getByRole("tab", { name: "Shadow Evals" })).toHaveAttribute("aria-selected", "true");
|
||||
expect(screen.getByTestId("shadow-eval-section")).toBeInTheDocument();
|
||||
|
||||
fireEvent.click(screen.getByRole("tab", { name: "Usage" }));
|
||||
expect(screen.getByText("Total estimated savings")).toBeInTheDocument();
|
||||
expect(screen.getByTestId("shadow-eval-section")).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("keeps the shadow evals sub-tab reachable while the usage body is in its error state", () => {
|
||||
mockHook({ error: new ApiError("boom", 500, {}) });
|
||||
renderTab();
|
||||
|
||||
expect(screen.getByText("Auto-router usage is unavailable right now")).toBeInTheDocument();
|
||||
|
||||
fireEvent.click(screen.getByRole("tab", { name: "Shadow Evals" }));
|
||||
expect(screen.getByTestId("shadow-eval-section")).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("keeps the window picker reachable while a window has no sessions", () => {
|
||||
mockHook({ data: response([]) });
|
||||
renderTab();
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@ import { Badge } from "@/components/ui/badge";
|
|||
import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card";
|
||||
import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select";
|
||||
import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from "@/components/ui/table";
|
||||
import { Tabs, TabsList, TabsTrigger } from "@/components/ui/tabs";
|
||||
import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs";
|
||||
import { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from "@/components/ui/tooltip";
|
||||
import { ApiError } from "@/lib/http/client";
|
||||
import { formatNumberWithCommas } from "@/utils/dataUtils";
|
||||
|
|
@ -31,6 +31,7 @@ import {
|
|||
type BucketRow,
|
||||
} from "./autoRouterBenchmarks";
|
||||
import { usd } from "./costOptimizationUtils";
|
||||
import ShadowEvalSection from "./ShadowEvalSection";
|
||||
import TierTurnsChart from "./TierTurnsChart";
|
||||
import { useAutoRouterBenchmarks } from "./useAutoRouterBenchmarks";
|
||||
|
||||
|
|
@ -268,7 +269,7 @@ interface AutoRouterBenchmarksTabProps {
|
|||
accessToken: string | null;
|
||||
}
|
||||
|
||||
const AutoRouterBenchmarksTab: React.FC<AutoRouterBenchmarksTabProps> = ({ accessToken }) => {
|
||||
const UsageView: React.FC<AutoRouterBenchmarksTabProps> = ({ accessToken }) => {
|
||||
const [range, setRange] = useState<BenchmarkWindow>("30d");
|
||||
const { data, isPending, error } = useAutoRouterBenchmarks(accessToken, range);
|
||||
const [selectedKey, setSelectedKey] = useState<string>(ALL_ROUTERS);
|
||||
|
|
@ -321,4 +322,36 @@ const AutoRouterBenchmarksTab: React.FC<AutoRouterBenchmarksTabProps> = ({ acces
|
|||
);
|
||||
};
|
||||
|
||||
const AutoRouterBenchmarksTab: React.FC<AutoRouterBenchmarksTabProps> = ({ accessToken }) => {
|
||||
const [visitedTabs, setVisitedTabs] = useState<readonly string[]>(["usage"]);
|
||||
|
||||
const handleTabChange = (value: unknown) => {
|
||||
if (typeof value !== "string") {
|
||||
return;
|
||||
}
|
||||
|
||||
setVisitedTabs((currentTabs) => (currentTabs.includes(value) ? currentTabs : [...currentTabs, value]));
|
||||
};
|
||||
|
||||
return (
|
||||
<Tabs defaultValue="usage" onValueChange={handleTabChange} className="w-full gap-4">
|
||||
<TabsList>
|
||||
<TabsTrigger value="usage" className="px-3">
|
||||
Usage
|
||||
</TabsTrigger>
|
||||
<TabsTrigger value="shadow-evals" className="px-3">
|
||||
Shadow Evals
|
||||
</TabsTrigger>
|
||||
</TabsList>
|
||||
|
||||
<TabsContent value="usage" keepMounted={visitedTabs.includes("usage")}>
|
||||
<UsageView accessToken={accessToken} />
|
||||
</TabsContent>
|
||||
<TabsContent value="shadow-evals" keepMounted={visitedTabs.includes("shadow-evals")}>
|
||||
<ShadowEvalSection />
|
||||
</TabsContent>
|
||||
</Tabs>
|
||||
);
|
||||
};
|
||||
|
||||
export default AutoRouterBenchmarksTab;
|
||||
|
|
|
|||
|
|
@ -0,0 +1,392 @@
|
|||
import { fireEvent, render, screen } from "@testing-library/react";
|
||||
import userEvent from "@testing-library/user-event";
|
||||
import React from "react";
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
import { useInfiniteKeys } from "@/app/(dashboard)/hooks/keys/useKeys";
|
||||
import { ApiError } from "@/lib/http/client";
|
||||
|
||||
vi.mock("./useShadowEval", () => ({
|
||||
useShadowEvalJobs: vi.fn(),
|
||||
useShadowEvalJob: vi.fn(),
|
||||
useStartShadowEval: vi.fn(),
|
||||
useStopShadowEval: vi.fn(),
|
||||
}));
|
||||
|
||||
const authorizedRoleMock = vi.fn(() => ({ accessToken: "token", isViewOnly: false }));
|
||||
vi.mock("@/app/(dashboard)/hooks/useAuthorized", () => ({ default: () => authorizedRoleMock() }));
|
||||
|
||||
vi.mock("@/app/(dashboard)/hooks/keys/useKeys", () => ({
|
||||
useInfiniteKeys: vi.fn(() => ({
|
||||
data: {
|
||||
pages: [
|
||||
{
|
||||
keys: [
|
||||
{ token: "hash-alpha", token_id: "id-1", key_name: "sk-...alpha", key_alias: "prod-alpha" },
|
||||
{ token: "hash-beta", token_id: "id-2", key_name: "sk-...beta", key_alias: "staging-beta" },
|
||||
],
|
||||
total_count: 2,
|
||||
current_page: 1,
|
||||
total_pages: 1,
|
||||
},
|
||||
],
|
||||
},
|
||||
isPending: false,
|
||||
isError: false,
|
||||
fetchNextPage: vi.fn(),
|
||||
hasNextPage: false,
|
||||
isFetchingNextPage: false,
|
||||
})),
|
||||
}));
|
||||
|
||||
vi.mock("@/app/(dashboard)/hooks/models/useModels", () => ({
|
||||
useAutoRouters: vi.fn(() => ({
|
||||
data: [
|
||||
{ model_name: "claude-auto", litellm_params: { model: "auto_router/claude-auto" } },
|
||||
{ model_name: "gpt-auto", litellm_params: { model: "auto_router/gpt-auto" } },
|
||||
],
|
||||
})),
|
||||
}));
|
||||
|
||||
vi.mock("@/app/(dashboard)/hooks/models/useModelCostMap", () => ({
|
||||
useModelCostMap: vi.fn(() => ({
|
||||
data: {
|
||||
"claude-sonnet-5": { litellm_provider: "anthropic", mode: "chat" },
|
||||
"gpt-4o": { litellm_provider: "openai", mode: "chat" },
|
||||
"gemini/gemini-2.5-pro": { litellm_provider: "gemini", mode: "chat" },
|
||||
"text-embedding-3-large": { litellm_provider: "openai", mode: "embedding" },
|
||||
},
|
||||
})),
|
||||
}));
|
||||
|
||||
import ShadowEvalSection from "./ShadowEvalSection";
|
||||
import {
|
||||
useShadowEvalJob,
|
||||
useShadowEvalJobs,
|
||||
useStartShadowEval,
|
||||
useStopShadowEval,
|
||||
type ShadowEvalJob,
|
||||
} from "./useShadowEval";
|
||||
|
||||
const job = (overrides: Partial<ShadowEvalJob> = {}): ShadowEvalJob => ({
|
||||
job_id: "job-1",
|
||||
status: "running",
|
||||
router_name: "claude-auto",
|
||||
judge_model: "anthropic/claude-sonnet-5",
|
||||
shadow_percentage: 10,
|
||||
max_turns: 200,
|
||||
judged_count: 42,
|
||||
error_count: 1,
|
||||
judge_spend: 3.21,
|
||||
results: {
|
||||
by_tier: [
|
||||
{
|
||||
group: "SIMPLE",
|
||||
turn_count: 30,
|
||||
real_win_rate_pct: 20.0,
|
||||
shadow_win_rate_pct: 55.0,
|
||||
tie_rate_pct: 25.0,
|
||||
avg_judge_confidence: 0.81,
|
||||
},
|
||||
{
|
||||
group: "REASONING",
|
||||
turn_count: 12,
|
||||
real_win_rate_pct: 50.0,
|
||||
shadow_win_rate_pct: 33.3,
|
||||
tie_rate_pct: 16.7,
|
||||
avg_judge_confidence: 0.74,
|
||||
},
|
||||
],
|
||||
by_current_model: [
|
||||
{
|
||||
group: "gpt-4o",
|
||||
turn_count: 42,
|
||||
real_win_rate_pct: 30.0,
|
||||
shadow_win_rate_pct: 45.0,
|
||||
tie_rate_pct: 25.0,
|
||||
avg_judge_confidence: 0.8,
|
||||
},
|
||||
],
|
||||
overall_shadow_win_rate_pct: 48.0,
|
||||
overall_tie_rate_pct: 22.0,
|
||||
},
|
||||
created_at: "2026-08-07T00:00:00Z",
|
||||
ends_at: "2026-09-07T00:00:00Z",
|
||||
stopped_at: null,
|
||||
api_key_id: "hashed-key-abc",
|
||||
last_error: null,
|
||||
...overrides,
|
||||
});
|
||||
|
||||
const mockHooks = ({
|
||||
jobs = [],
|
||||
detailsById = {},
|
||||
error = null,
|
||||
detailError = false,
|
||||
isPending = false,
|
||||
}: {
|
||||
jobs?: ShadowEvalJob[];
|
||||
detailsById?: Record<string, ShadowEvalJob>;
|
||||
error?: Error | null;
|
||||
detailError?: boolean;
|
||||
isPending?: boolean;
|
||||
}) => {
|
||||
vi.mocked(useShadowEvalJobs).mockReturnValue({
|
||||
data: error || isPending ? undefined : jobs,
|
||||
error,
|
||||
isPending,
|
||||
} as unknown as ReturnType<typeof useShadowEvalJobs>);
|
||||
vi.mocked(useShadowEvalJob).mockImplementation(
|
||||
(jobId) =>
|
||||
({
|
||||
data: jobId ? detailsById[jobId] : undefined,
|
||||
isError: detailError ?? false,
|
||||
}) as unknown as ReturnType<typeof useShadowEvalJob>,
|
||||
);
|
||||
const start = { mutate: vi.fn(), isPending: false };
|
||||
const stop = { mutate: vi.fn(), isPending: false };
|
||||
vi.mocked(useStartShadowEval).mockReturnValue(start as unknown as ReturnType<typeof useStartShadowEval>);
|
||||
vi.mocked(useStopShadowEval).mockReturnValue(stop as unknown as ReturnType<typeof useStopShadowEval>);
|
||||
return { start, stop };
|
||||
};
|
||||
|
||||
describe("ShadowEvalSection", () => {
|
||||
beforeEach(() => {
|
||||
authorizedRoleMock.mockReturnValue({ accessToken: "token", isViewOnly: false });
|
||||
});
|
||||
|
||||
it("shows a key picker load failure instead of posing as no matching keys", async () => {
|
||||
const user = userEvent.setup();
|
||||
const defaultKeysImpl = vi.mocked(useInfiniteKeys).getMockImplementation();
|
||||
vi.mocked(useInfiniteKeys).mockReturnValue({
|
||||
data: undefined,
|
||||
isPending: false,
|
||||
isError: true,
|
||||
fetchNextPage: vi.fn(),
|
||||
hasNextPage: false,
|
||||
isFetchingNextPage: false,
|
||||
} as unknown as ReturnType<typeof useInfiniteKeys>);
|
||||
mockHooks({});
|
||||
render(<ShadowEvalSection />);
|
||||
|
||||
await user.click(screen.getByPlaceholderText("Search keys by alias"));
|
||||
expect(await screen.findByText("Keys could not be loaded. Refresh the page to retry.")).toBeInTheDocument();
|
||||
expect(screen.queryByText("No matching keys")).not.toBeInTheDocument();
|
||||
if (defaultKeysImpl) vi.mocked(useInfiniteKeys).mockImplementation(defaultKeysImpl);
|
||||
});
|
||||
|
||||
it("offers the start form while the list is still loading", () => {
|
||||
mockHooks({ isPending: true });
|
||||
render(<ShadowEvalSection />);
|
||||
expect(screen.getByText("Loading evaluations...")).toBeInTheDocument();
|
||||
expect(screen.getByText("Start a shadow eval")).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("re-offers the start form when the polled detail sees the job finish before the list does", () => {
|
||||
mockHooks({
|
||||
jobs: [job({ status: "running" })],
|
||||
detailsById: { "job-1": job({ status: "completed" }) },
|
||||
});
|
||||
render(<ShadowEvalSection />);
|
||||
expect(screen.getByText("Start a shadow eval")).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("gives every active job its own card with a stop button, with the form still offered", () => {
|
||||
mockHooks({
|
||||
jobs: [
|
||||
job({ job_id: "job-a", status: "running", api_key_id: "key-a" }),
|
||||
job({ job_id: "job-b", status: "running", api_key_id: "key-b" }),
|
||||
],
|
||||
});
|
||||
render(<ShadowEvalSection />);
|
||||
expect(screen.getAllByRole("button", { name: "Stop" })).toHaveLength(2);
|
||||
expect(screen.getByText("Start a shadow eval")).toBeInTheDocument();
|
||||
expect(screen.queryByText(/Previous evaluations/)).not.toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("renders the active card from the list row while its detail is still loading", () => {
|
||||
mockHooks({ jobs: [job({ status: "running" })], detailsById: {} });
|
||||
render(<ShadowEvalSection />);
|
||||
expect(screen.getByRole("button", { name: "Stop" })).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("hides the start form and stop button from view-only admins", () => {
|
||||
authorizedRoleMock.mockReturnValue({ accessToken: "token", isViewOnly: true });
|
||||
mockHooks({ jobs: [job({ status: "running" })] });
|
||||
render(<ShadowEvalSection />);
|
||||
expect(screen.queryByText("Start a shadow eval")).not.toBeInTheDocument();
|
||||
expect(screen.queryByRole("button", { name: "Stop" })).not.toBeInTheDocument();
|
||||
expect(screen.getByText("running")).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("never labels a collapsed previous eval as empty from a countless list row", () => {
|
||||
const countlessListRow: Partial<ShadowEvalJob> = {
|
||||
job_id: "job-old",
|
||||
status: "stopped",
|
||||
judged_count: null,
|
||||
error_count: null,
|
||||
judge_spend: null,
|
||||
results: null,
|
||||
};
|
||||
mockHooks({ jobs: [job({ status: "running" }), job(countlessListRow)] });
|
||||
render(<ShadowEvalSection />);
|
||||
fireEvent.click(screen.getByRole("button", { name: /Previous evaluations/ }));
|
||||
expect(screen.getByText("view results")).toBeInTheDocument();
|
||||
expect(screen.queryByText("no verdicts")).not.toBeInTheDocument();
|
||||
expect(screen.queryByText(/0 judged/)).not.toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("surfaces a non-403 list failure instead of posing as an empty state", () => {
|
||||
mockHooks({ error: new Error("boom") });
|
||||
render(<ShadowEvalSection />);
|
||||
expect(screen.getByText(/Existing evaluations could not be loaded/)).toBeInTheDocument();
|
||||
expect(screen.getByText("Start a shadow eval")).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("shows a failure line instead of loading forever when the detail fetch errors", () => {
|
||||
mockHooks({
|
||||
jobs: [job({ status: "completed", judged_count: 12, results: null })],
|
||||
detailsById: {},
|
||||
detailError: true,
|
||||
});
|
||||
render(<ShadowEvalSection />);
|
||||
expect(screen.getByText(/Results could not be loaded/)).toBeInTheDocument();
|
||||
expect(screen.queryByText("Loading results...")).not.toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("shows the failure line over the collecting copy when an active job's detail errors", () => {
|
||||
mockHooks({ jobs: [job({ status: "running", results: null })], detailsById: {}, detailError: true });
|
||||
render(<ShadowEvalSection />);
|
||||
expect(screen.getByText(/Results could not be loaded/)).toBeInTheDocument();
|
||||
expect(screen.queryByText(/Collecting verdicts/)).not.toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("never claims no verdicts for a judged job whose results have not loaded yet", () => {
|
||||
mockHooks({ jobs: [job({ status: "completed", judged_count: 12, results: null })], detailsById: {} });
|
||||
render(<ShadowEvalSection />);
|
||||
expect(screen.getByText("Loading results...")).toBeInTheDocument();
|
||||
expect(screen.queryByText(/No verdicts were recorded/)).not.toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("shows the start form when there are no jobs", () => {
|
||||
mockHooks({});
|
||||
render(<ShadowEvalSection />);
|
||||
expect(screen.getByText("Start a shadow eval")).toBeInTheDocument();
|
||||
expect(screen.getByText("Start shadow eval")).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("renders the latest job's results with the headline stat, verdict split, and both stratifications", () => {
|
||||
const j = job();
|
||||
mockHooks({ jobs: [j], detailsById: { "job-1": j } });
|
||||
render(<ShadowEvalSection />);
|
||||
|
||||
expect(screen.getByText("Router matched or beat your current model")).toBeInTheDocument();
|
||||
expect(screen.getByText("70.0%")).toBeInTheDocument();
|
||||
expect(screen.getByText("of 42 judged responses")).toBeInTheDocument();
|
||||
expect(screen.getByText(/Tie 22.0%/)).toBeInTheDocument();
|
||||
expect(screen.getByText(/Current model won 30.0%/)).toBeInTheDocument();
|
||||
expect(screen.getByText("gpt-4o")).toBeInTheDocument();
|
||||
expect(screen.getByText("SIMPLE")).toBeInTheDocument();
|
||||
expect(screen.getByText("REASONING")).toBeInTheDocument();
|
||||
expect(screen.getByText("55.0%")).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("shows the ends-in text while a job is still sampling", () => {
|
||||
const j = job({ ends_at: new Date(Date.now() + 3 * 86_400_000).toISOString() });
|
||||
mockHooks({ jobs: [j], detailsById: { "job-1": j } });
|
||||
render(<ShadowEvalSection />);
|
||||
expect(screen.getByText(/ends in 3 days/)).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("flags rows with fewer than 30 judged turns as low sample", () => {
|
||||
const j = job();
|
||||
mockHooks({ jobs: [j], detailsById: { "job-1": j } });
|
||||
render(<ShadowEvalSection />);
|
||||
expect(screen.getAllByText("(low sample)")).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("surfaces the last failure so a growing error_count is diagnosable", () => {
|
||||
const j = job({ error_count: 7, last_error: "judge call failed: LLM Provider NOT provided" });
|
||||
mockHooks({ jobs: [j], detailsById: { "job-1": j } });
|
||||
render(<ShadowEvalSection />);
|
||||
expect(screen.getByText(/LLM Provider NOT provided/)).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("stops the running job from the stop button", async () => {
|
||||
const user = userEvent.setup();
|
||||
const j = job();
|
||||
const { stop } = mockHooks({ jobs: [j], detailsById: { "job-1": j } });
|
||||
render(<ShadowEvalSection />);
|
||||
|
||||
await user.click(screen.getByText("Stop"));
|
||||
|
||||
expect(stop.mutate).toHaveBeenCalledWith("job-1");
|
||||
});
|
||||
|
||||
it("hides the stop button and offers the start form once the latest job completed", () => {
|
||||
const done = job({ status: "completed" });
|
||||
mockHooks({ jobs: [done], detailsById: { "job-1": done } });
|
||||
render(<ShadowEvalSection />);
|
||||
expect(screen.queryByText("Stop")).not.toBeInTheDocument();
|
||||
expect(screen.getByText("Start a shadow eval")).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("renders nothing for non-admins when the proxy answers 403", () => {
|
||||
mockHooks({ error: new ApiError("forbidden", 403, {}) });
|
||||
const { container } = render(<ShadowEvalSection />);
|
||||
expect(container).toBeEmptyDOMElement();
|
||||
});
|
||||
|
||||
it("keeps the start button disabled until key, router, and judge model are picked, then submits them", async () => {
|
||||
const user = userEvent.setup();
|
||||
const { start } = mockHooks({});
|
||||
render(<ShadowEvalSection />);
|
||||
|
||||
expect(screen.getByText("Start shadow eval")).toBeDisabled();
|
||||
|
||||
await user.click(screen.getByPlaceholderText("Search keys by alias"));
|
||||
await user.click(await screen.findByText("prod-alpha"));
|
||||
await user.click(screen.getByPlaceholderText("Select an auto-router"));
|
||||
await user.click(await screen.findByText("gpt-auto"));
|
||||
|
||||
expect(screen.getByText("Start shadow eval")).toBeDisabled();
|
||||
|
||||
await user.click(screen.getByPlaceholderText("Select a judge model"));
|
||||
await user.click(await screen.findByRole("option", { name: /anthropic\/claude-sonnet-5/ }));
|
||||
await user.click(screen.getByText("Start shadow eval"));
|
||||
|
||||
const expectedBody = {
|
||||
api_key_id: "hash-alpha",
|
||||
router_name: "gpt-auto",
|
||||
shadow_percentage: 10,
|
||||
duration_days: 7,
|
||||
max_turns: 200,
|
||||
judge_model: "anthropic/claude-sonnet-5",
|
||||
};
|
||||
expect(start.mutate).toHaveBeenCalledWith(expectedBody);
|
||||
});
|
||||
|
||||
it("keeps an older job's verdicts reachable through the previous evaluations list", async () => {
|
||||
const user = userEvent.setup();
|
||||
const emptyOverrides: Partial<ShadowEvalJob> = {
|
||||
job_id: "job-new",
|
||||
status: "running",
|
||||
judged_count: 0,
|
||||
error_count: 0,
|
||||
results: null,
|
||||
};
|
||||
const current = job(emptyOverrides);
|
||||
const older = job({ job_id: "job-old", status: "completed", results: null });
|
||||
mockHooks({ jobs: [current, older], detailsById: { "job-new": current, "job-old": job({ job_id: "job-old" }) } });
|
||||
render(<ShadowEvalSection />);
|
||||
|
||||
expect(screen.queryByText("SIMPLE")).not.toBeInTheDocument();
|
||||
|
||||
await user.click(screen.getByRole("button", { name: /Previous evaluations \(1\)/ }));
|
||||
expect(screen.getByText("view results")).toBeInTheDocument();
|
||||
await user.click(screen.getByRole("button", { name: /10% via claude-auto/ }));
|
||||
|
||||
expect(await screen.findByText("SIMPLE")).toBeInTheDocument();
|
||||
expect(screen.getByText("REASONING")).toBeInTheDocument();
|
||||
});
|
||||
});
|
||||
|
|
@ -0,0 +1,531 @@
|
|||
"use client";
|
||||
|
||||
import React, { useMemo, useState } from "react";
|
||||
|
||||
import { useInfiniteKeys } from "@/app/(dashboard)/hooks/keys/useKeys";
|
||||
import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized";
|
||||
import { useModelCostMap } from "@/app/(dashboard)/hooks/models/useModelCostMap";
|
||||
import { useAutoRouters } from "@/app/(dashboard)/hooks/models/useModels";
|
||||
import { PaginatedSearchSelect } from "@/components/shared/PaginatedSearchSelect";
|
||||
import { SearchSelect, type SearchSelectOption } from "@/components/shared/SearchSelect";
|
||||
import { Badge } from "@/components/ui/badge";
|
||||
import { Button } from "@/components/ui/button";
|
||||
import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card";
|
||||
import { Input } from "@/components/ui/input";
|
||||
import { Label } from "@/components/ui/label";
|
||||
import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select";
|
||||
import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from "@/components/ui/table";
|
||||
import { ApiError } from "@/lib/http/client";
|
||||
|
||||
import { usd } from "./costOptimizationUtils";
|
||||
import {
|
||||
useShadowEvalJob,
|
||||
useShadowEvalJobs,
|
||||
useStartShadowEval,
|
||||
useStopShadowEval,
|
||||
type ShadowEvalJob,
|
||||
type ShadowEvalSlice,
|
||||
} from "./useShadowEval";
|
||||
|
||||
const pct = (value: number): string => `${value.toFixed(1)}%`;
|
||||
|
||||
const MIN_TURNS_FOR_CONFIDENCE = 30;
|
||||
|
||||
const isActive = (job: ShadowEvalJob): boolean => job.status === "running";
|
||||
|
||||
const endsIn = (endsAt: string | null | undefined): string | null => {
|
||||
if (!endsAt) return null;
|
||||
const remainingMs = new Date(endsAt).getTime() - Date.now();
|
||||
if (!Number.isFinite(remainingMs)) return null;
|
||||
if (remainingMs <= 0) return "ending now";
|
||||
const days = Math.round(remainingMs / 86_400_000);
|
||||
return days >= 2 ? `ends in ${days} days` : "ends within a day";
|
||||
};
|
||||
|
||||
const STATUS_STYLES: Record<string, string> = {
|
||||
running: "bg-blue-50 text-blue-700",
|
||||
completed: "bg-emerald-50 text-emerald-700",
|
||||
stopped: "bg-secondary text-muted-foreground",
|
||||
};
|
||||
|
||||
const StatusBadge: React.FC<{ status: string }> = ({ status }) => (
|
||||
<Badge variant="secondary" className={STATUS_STYLES[status] ?? STATUS_STYLES.stopped}>
|
||||
{status}
|
||||
</Badge>
|
||||
);
|
||||
|
||||
const SliceTable: React.FC<{ groupHeader: string; slices: readonly ShadowEvalSlice[] }> = ({ groupHeader, slices }) => (
|
||||
<Table>
|
||||
<TableHeader>
|
||||
<TableRow>
|
||||
<TableHead>{groupHeader}</TableHead>
|
||||
{["Judged turns", "Router wins", "Current model wins", "Ties", "Judge confidence"].map((label) => (
|
||||
<TableHead key={label} className="text-right">
|
||||
{label}
|
||||
</TableHead>
|
||||
))}
|
||||
</TableRow>
|
||||
</TableHeader>
|
||||
<TableBody>
|
||||
{slices.map((slice) => (
|
||||
<TableRow key={slice.group}>
|
||||
<TableCell className="font-medium text-foreground">
|
||||
{slice.group}
|
||||
{slice.turn_count < MIN_TURNS_FOR_CONFIDENCE && (
|
||||
<span className="ml-2 text-xs font-normal text-muted-foreground">(low sample)</span>
|
||||
)}
|
||||
</TableCell>
|
||||
<TableCell className="text-right tabular-nums">{slice.turn_count.toLocaleString()}</TableCell>
|
||||
<TableCell className="text-right font-medium tabular-nums text-foreground">
|
||||
{pct(slice.shadow_win_rate_pct)}
|
||||
</TableCell>
|
||||
<TableCell className="text-right tabular-nums">{pct(slice.real_win_rate_pct)}</TableCell>
|
||||
<TableCell className="text-right tabular-nums">{pct(slice.tie_rate_pct)}</TableCell>
|
||||
<TableCell className="text-right tabular-nums">{slice.avg_judge_confidence.toFixed(2)}</TableCell>
|
||||
</TableRow>
|
||||
))}
|
||||
</TableBody>
|
||||
</Table>
|
||||
);
|
||||
|
||||
const VerdictBar: React.FC<{ results: NonNullable<ShadowEvalJob["results"]> }> = ({ results }) => {
|
||||
const routerWins = results.overall_shadow_win_rate_pct;
|
||||
const ties = results.overall_tie_rate_pct;
|
||||
const segments = [
|
||||
{ label: "Router won", value: routerWins, fill: "bg-emerald-500" },
|
||||
{ label: "Tie", value: ties, fill: "bg-emerald-200" },
|
||||
{ label: "Current model won", value: Math.max(0, 100 - routerWins - ties), fill: "bg-muted-foreground/30" },
|
||||
];
|
||||
return (
|
||||
<div className="space-y-2 border-b px-6 py-4">
|
||||
<div className="flex h-2 w-full overflow-hidden rounded-full" role="img" aria-label="Verdict breakdown">
|
||||
{segments
|
||||
.filter((segment) => segment.value > 0)
|
||||
.map((segment) => (
|
||||
<div key={segment.label} className={segment.fill} style={{ width: `${segment.value}%` }} />
|
||||
))}
|
||||
</div>
|
||||
<div className="flex flex-wrap gap-x-4 gap-y-1 text-xs text-muted-foreground">
|
||||
{segments.map((segment) => (
|
||||
<span key={segment.label} className="flex items-center gap-1.5">
|
||||
<span className={`size-2 rounded-full ${segment.fill}`} />
|
||||
{segment.label} {pct(segment.value)}
|
||||
</span>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
};
|
||||
|
||||
const emptyResultsText = (job: ShadowEvalJob, resultsError: boolean): string => {
|
||||
if (resultsError) return "Results could not be loaded. Retrying.";
|
||||
if (isActive(job)) return "Collecting verdicts. Results appear as sampled requests are judged.";
|
||||
if (job.judged_count === 0) return "No verdicts were recorded for this job.";
|
||||
return "Loading results...";
|
||||
};
|
||||
|
||||
const ResultsBody: React.FC<{ job: ShadowEvalJob; resultsError?: boolean }> = ({ job, resultsError = false }) => {
|
||||
const results = job.results;
|
||||
if (!results || (results.by_tier.length === 0 && results.by_current_model.length === 0)) {
|
||||
return <p className="px-6 py-8 text-center text-sm text-muted-foreground">{emptyResultsText(job, resultsError)}</p>;
|
||||
}
|
||||
return (
|
||||
<>
|
||||
<div className="flex flex-col gap-1 border-b px-6 py-4">
|
||||
<p className="text-[11px] uppercase tracking-wide text-muted-foreground">
|
||||
Router matched or beat your current model
|
||||
</p>
|
||||
<p className="text-3xl font-semibold text-foreground">
|
||||
{pct(results.overall_shadow_win_rate_pct + results.overall_tie_rate_pct)}
|
||||
</p>
|
||||
<p className="text-xs text-muted-foreground">of {(job.judged_count ?? 0).toLocaleString()} judged responses</p>
|
||||
</div>
|
||||
<VerdictBar results={results} />
|
||||
{results.by_current_model.length > 0 && (
|
||||
<SliceTable groupHeader="Compared against" slices={results.by_current_model} />
|
||||
)}
|
||||
{results.by_tier.length > 0 && (
|
||||
<div className={results.by_current_model.length > 0 ? "border-t" : ""}>
|
||||
<SliceTable groupHeader="Prompt difficulty" slices={results.by_tier} />
|
||||
</div>
|
||||
)}
|
||||
</>
|
||||
);
|
||||
};
|
||||
|
||||
const JobResults: React.FC<{
|
||||
job: ShadowEvalJob;
|
||||
onStop: () => void;
|
||||
stopPending: boolean;
|
||||
resultsError?: boolean;
|
||||
readOnly?: boolean;
|
||||
}> = ({ job, onStop, stopPending, resultsError = false, readOnly = false }) => {
|
||||
const active = isActive(job);
|
||||
const remaining = endsIn(job.ends_at);
|
||||
return (
|
||||
<Card className="overflow-hidden py-0">
|
||||
<div className="flex flex-wrap items-center justify-between gap-3 border-b px-6 py-4">
|
||||
<div className="flex items-center gap-3">
|
||||
<StatusBadge status={job.status} />
|
||||
<div>
|
||||
<p className="text-sm font-medium text-foreground">
|
||||
Shadowing {job.shadow_percentage}% via <span className="font-mono text-xs">{job.router_name}</span>
|
||||
</p>
|
||||
<p className="text-xs text-muted-foreground">
|
||||
{(job.judged_count ?? 0).toLocaleString()} of {job.max_turns.toLocaleString()} turns judged ·{" "}
|
||||
{(job.error_count ?? 0).toLocaleString()} errored · {usd(job.judge_spend ?? 0)} judge spend
|
||||
{active && remaining ? ` · ${remaining}` : ""}
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
{active && !readOnly && (
|
||||
<Button variant="outline" size="sm" onClick={onStop} disabled={stopPending}>
|
||||
{stopPending ? "Stopping..." : "Stop"}
|
||||
</Button>
|
||||
)}
|
||||
</div>
|
||||
{(job.error_count ?? 0) > 0 && job.last_error != null && (
|
||||
<p className="border-b bg-red-50 px-6 py-2 text-xs text-destructive">
|
||||
Last failure: <span className="font-mono">{job.last_error}</span>
|
||||
</p>
|
||||
)}
|
||||
<ResultsBody job={job} resultsError={resultsError} />
|
||||
</Card>
|
||||
);
|
||||
};
|
||||
|
||||
const RECOMMENDED_JUDGE_MODELS = ["anthropic/claude-sonnet-5", "openai/gpt-4o", "gemini/gemini-2.5-pro"] as const;
|
||||
|
||||
interface CostMapEntry {
|
||||
litellm_provider?: string;
|
||||
mode?: string;
|
||||
}
|
||||
|
||||
const useJudgeModelOptions = (): SearchSelectOption[] => {
|
||||
const { data: costMap } = useModelCostMap();
|
||||
return useMemo(() => {
|
||||
const pinned: SearchSelectOption[] = RECOMMENDED_JUDGE_MODELS.map((model) => ({
|
||||
label: model,
|
||||
value: model,
|
||||
sublabel: "Recommended",
|
||||
}));
|
||||
if (!costMap) return pinned;
|
||||
const pinnedNames = new Set<string>(RECOMMENDED_JUDGE_MODELS);
|
||||
const chatModels = Object.entries(costMap as Record<string, CostMapEntry>)
|
||||
.filter(([, value]) => value?.mode === "chat" && value?.litellm_provider)
|
||||
.map(([key, value]) => (key.startsWith(`${value.litellm_provider}/`) ? key : `${value.litellm_provider}/${key}`));
|
||||
const rest = [...new Set(chatModels)]
|
||||
.filter((model) => !pinnedNames.has(model))
|
||||
.toSorted((a, b) => a.localeCompare(b))
|
||||
.map((model) => ({ label: model, value: model }));
|
||||
return [...pinned, ...rest];
|
||||
}, [costMap]);
|
||||
};
|
||||
|
||||
const DURATION_OPTIONS = [
|
||||
{ value: "1", label: "1 day" },
|
||||
{ value: "3", label: "3 days" },
|
||||
{ value: "7", label: "7 days" },
|
||||
{ value: "14", label: "14 days" },
|
||||
{ value: "30", label: "30 days" },
|
||||
] as const;
|
||||
|
||||
const Field: React.FC<{ label: string; htmlFor?: string; className?: string; children: React.ReactNode }> = ({
|
||||
label,
|
||||
htmlFor,
|
||||
className,
|
||||
children,
|
||||
}) => (
|
||||
<div className={`space-y-1.5 ${className ?? ""}`}>
|
||||
<Label htmlFor={htmlFor} className="text-xs">
|
||||
{label}
|
||||
</Label>
|
||||
{children}
|
||||
</div>
|
||||
);
|
||||
|
||||
const KeySelect: React.FC<{ value: string; onChange: (token: string) => void }> = ({ value, onChange }) => {
|
||||
const [search, setSearch] = useState("");
|
||||
const { data, isPending, isError, fetchNextPage, hasNextPage, isFetchingNextPage } = useInfiniteKeys(50, {
|
||||
selectedKeyAlias: search || null,
|
||||
});
|
||||
const options = useMemo<SearchSelectOption[]>(
|
||||
() =>
|
||||
(data?.pages ?? [])
|
||||
.flatMap((page) => page.keys)
|
||||
.map((key) => ({
|
||||
label: key.key_alias || key.key_name || key.token,
|
||||
value: key.token,
|
||||
sublabel: key.token,
|
||||
})),
|
||||
[data],
|
||||
);
|
||||
return (
|
||||
<PaginatedSearchSelect
|
||||
inputId="shadow-eval-key"
|
||||
options={options}
|
||||
value={value}
|
||||
onValueChange={onChange}
|
||||
onSearchChange={setSearch}
|
||||
onLoadMore={() => void fetchNextPage()}
|
||||
hasNextPage={hasNextPage}
|
||||
isFetchingNextPage={isFetchingNextPage}
|
||||
isLoading={isPending}
|
||||
placeholder="Search keys by alias"
|
||||
emptyText="No matching keys"
|
||||
errorText={isError ? "Keys could not be loaded. Refresh the page to retry." : undefined}
|
||||
/>
|
||||
);
|
||||
};
|
||||
|
||||
const StartForm: React.FC = () => {
|
||||
const { accessToken } = useAuthorized();
|
||||
const [apiKeyId, setApiKeyId] = useState("");
|
||||
const [routerName, setRouterName] = useState("");
|
||||
const [percentage, setPercentage] = useState("10");
|
||||
const [durationDays, setDurationDays] = useState("7");
|
||||
const [judgeModel, setJudgeModel] = useState("");
|
||||
const [maxTurns, setMaxTurns] = useState("200");
|
||||
const { data: autoRouters } = useAutoRouters();
|
||||
const judgeModelOptions = useJudgeModelOptions();
|
||||
const start = useStartShadowEval();
|
||||
|
||||
const routerOptions = useMemo<SearchSelectOption[]>(() => {
|
||||
const names = new Set(
|
||||
(autoRouters ?? []).map((deployment) => deployment.model_name).filter((name): name is string => Boolean(name)),
|
||||
);
|
||||
return [...names].toSorted().map((name) => ({ label: name, value: name }));
|
||||
}, [autoRouters]);
|
||||
|
||||
const parsedPct = Number.parseFloat(percentage);
|
||||
const percentageValid = parsedPct >= 0.1 && parsedPct <= 100;
|
||||
const parsedMaxTurns = Number.parseInt(maxTurns, 10);
|
||||
const maxTurnsValid = parsedMaxTurns >= 1 && parsedMaxTurns <= 2000;
|
||||
const filled = [apiKeyId, routerName, judgeModel].every((field) => field !== "");
|
||||
const boundsValid = percentageValid && maxTurnsValid;
|
||||
const valid = Boolean(accessToken) && filled && boundsValid;
|
||||
const handleStart = () => {
|
||||
const startBody = {
|
||||
api_key_id: apiKeyId,
|
||||
router_name: routerName,
|
||||
shadow_percentage: parsedPct,
|
||||
duration_days: Number.parseInt(durationDays, 10),
|
||||
max_turns: parsedMaxTurns,
|
||||
judge_model: judgeModel,
|
||||
};
|
||||
start.mutate(startBody);
|
||||
};
|
||||
|
||||
return (
|
||||
<Card size="sm">
|
||||
<CardHeader>
|
||||
<CardTitle className="text-sm font-medium text-foreground">Start a shadow eval</CardTitle>
|
||||
<p className="text-xs text-muted-foreground">
|
||||
Duplicates a sampled slice of the key's traffic through the auto-router and has an LLM judge compare both
|
||||
answers blind. The router's answers are never served to users; judge calls bill to the shadowed key.
|
||||
</p>
|
||||
</CardHeader>
|
||||
<CardContent className="space-y-3">
|
||||
<div className="grid gap-3 sm:grid-cols-3">
|
||||
<Field label="Key to shadow" htmlFor="shadow-eval-key">
|
||||
<KeySelect value={apiKeyId} onChange={setApiKeyId} />
|
||||
</Field>
|
||||
<Field label="Auto-router">
|
||||
<SearchSelect
|
||||
options={routerOptions}
|
||||
value={routerName}
|
||||
onValueChange={setRouterName}
|
||||
placeholder="Select an auto-router"
|
||||
emptyText="No auto-routers configured"
|
||||
/>
|
||||
</Field>
|
||||
<Field label="Traffic sampled" htmlFor="shadow-eval-pct">
|
||||
<div className="flex items-center gap-2">
|
||||
<Input
|
||||
id="shadow-eval-pct"
|
||||
type="number"
|
||||
min={0.1}
|
||||
max={100}
|
||||
step={0.1}
|
||||
className="w-24"
|
||||
value={percentage}
|
||||
onChange={(e) => setPercentage(e.target.value)}
|
||||
/>
|
||||
<span className="text-sm text-muted-foreground">% of traffic</span>
|
||||
</div>
|
||||
<div>
|
||||
{percentage.trim() !== "" && !percentageValid && (
|
||||
<p className="text-xs text-destructive">Enter a value from 0.1 to 100</p>
|
||||
)}
|
||||
</div>
|
||||
</Field>
|
||||
<Field label="Duration">
|
||||
<Select value={durationDays} onValueChange={(v: string | null) => setDurationDays(v ?? "7")}>
|
||||
<SelectTrigger className="w-full">
|
||||
<SelectValue>{DURATION_OPTIONS.find((o) => o.value === durationDays)?.label}</SelectValue>
|
||||
</SelectTrigger>
|
||||
<SelectContent>
|
||||
{DURATION_OPTIONS.map((option) => (
|
||||
<SelectItem key={option.value} value={option.value}>
|
||||
{option.label}
|
||||
</SelectItem>
|
||||
))}
|
||||
</SelectContent>
|
||||
</Select>
|
||||
</Field>
|
||||
<Field label="Turn budget">
|
||||
<div className="flex items-center gap-2">
|
||||
<Input
|
||||
type="number"
|
||||
min={1}
|
||||
max={2000}
|
||||
className="w-24"
|
||||
value={maxTurns}
|
||||
onChange={(e) => setMaxTurns(e.target.value)}
|
||||
/>
|
||||
<span className="text-sm text-muted-foreground">turns judged, max</span>
|
||||
</div>
|
||||
{maxTurns.trim() !== "" && !maxTurnsValid && (
|
||||
<p className="text-xs text-destructive">Enter a value from 1 to 2000</p>
|
||||
)}
|
||||
</Field>
|
||||
<Field label="Judge model" className="sm:col-span-2">
|
||||
<SearchSelect
|
||||
options={judgeModelOptions}
|
||||
value={judgeModel}
|
||||
onValueChange={setJudgeModel}
|
||||
placeholder="Select a judge model"
|
||||
emptyText="No chat models available"
|
||||
/>
|
||||
</Field>
|
||||
</div>
|
||||
<Button disabled={!valid || start.isPending} onClick={handleStart}>
|
||||
{start.isPending ? "Starting..." : "Start shadow eval"}
|
||||
</Button>
|
||||
</CardContent>
|
||||
</Card>
|
||||
);
|
||||
};
|
||||
|
||||
const previousSummary = (job: ShadowEvalJob): string => {
|
||||
const results = job.results;
|
||||
if (results) return pct(results.overall_shadow_win_rate_pct + results.overall_tie_rate_pct);
|
||||
return job.judged_count === 0 ? "no verdicts" : "view results";
|
||||
};
|
||||
|
||||
const PreviousJob: React.FC<{ job: ShadowEvalJob }> = ({ job }) => {
|
||||
const [expanded, setExpanded] = useState(false);
|
||||
const { data: detail, isError } = useShadowEvalJob(expanded ? job.job_id : null);
|
||||
const shown = detail ?? job;
|
||||
return (
|
||||
<div className="border-b last:border-b-0">
|
||||
<button
|
||||
type="button"
|
||||
aria-expanded={expanded}
|
||||
onClick={() => setExpanded((open) => !open)}
|
||||
className="flex w-full flex-wrap items-center justify-between gap-3 px-6 py-3 text-left hover:bg-muted/50"
|
||||
>
|
||||
<div className="flex items-center gap-3">
|
||||
<StatusBadge status={shown.status} />
|
||||
<div>
|
||||
<p className="text-sm font-medium text-foreground">
|
||||
{shown.shadow_percentage}% via <span className="font-mono text-xs">{shown.router_name}</span>
|
||||
</p>
|
||||
<p className="text-xs text-muted-foreground">
|
||||
{shown.judged_count != null &&
|
||||
`${shown.judged_count.toLocaleString()} judged · ${(shown.error_count ?? 0).toLocaleString()} errored · ${usd(shown.judge_spend ?? 0)} judge spend · `}
|
||||
{new Date(shown.created_at).toLocaleDateString()}
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
<span className="text-sm font-medium text-foreground">{previousSummary(shown)}</span>
|
||||
</button>
|
||||
{expanded && (
|
||||
<div className="border-t">
|
||||
<ResultsBody job={shown} resultsError={isError} />
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
};
|
||||
|
||||
const PreviousJobs: React.FC<{ jobs: readonly ShadowEvalJob[] }> = ({ jobs }) => {
|
||||
const [open, setOpen] = useState(false);
|
||||
if (jobs.length === 0) return null;
|
||||
return (
|
||||
<Card className="overflow-hidden py-0">
|
||||
<button
|
||||
type="button"
|
||||
aria-expanded={open}
|
||||
onClick={() => setOpen((prev) => !prev)}
|
||||
className="flex w-full items-center justify-between gap-3 px-6 py-3 text-left hover:bg-muted/50"
|
||||
>
|
||||
<span className="text-sm font-medium text-foreground">Previous evaluations ({jobs.length})</span>
|
||||
<span className="text-xs text-muted-foreground">{open ? "Hide" : "Show"}</span>
|
||||
</button>
|
||||
{open && (
|
||||
<div className="border-t">
|
||||
{jobs.map((job) => (
|
||||
<PreviousJob key={job.job_id} job={job} />
|
||||
))}
|
||||
</div>
|
||||
)}
|
||||
</Card>
|
||||
);
|
||||
};
|
||||
|
||||
const JobCard: React.FC<{ job: ShadowEvalJob; readOnly: boolean }> = ({ job, readOnly }) => {
|
||||
const { data: detail, isError } = useShadowEvalJob(job.job_id);
|
||||
const stop = useStopShadowEval();
|
||||
const shown = detail ?? job;
|
||||
return (
|
||||
<JobResults
|
||||
job={shown}
|
||||
onStop={() => stop.mutate(shown.job_id)}
|
||||
stopPending={stop.isPending}
|
||||
resultsError={isError}
|
||||
readOnly={readOnly}
|
||||
/>
|
||||
);
|
||||
};
|
||||
|
||||
const ShadowEvalSection: React.FC = () => {
|
||||
const { data: jobs, error, isPending } = useShadowEvalJobs();
|
||||
const { isViewOnly } = useAuthorized();
|
||||
const { showcased, listed } = useMemo(() => {
|
||||
const active = (jobs ?? []).filter(isActive);
|
||||
const finished = (jobs ?? []).filter((job) => !isActive(job));
|
||||
const shown = active.length > 0 ? active : finished.slice(0, 1);
|
||||
return { showcased: shown, listed: finished.filter((job) => !shown.includes(job)) };
|
||||
}, [jobs]);
|
||||
|
||||
if (error instanceof ApiError && error.status === 403) return null;
|
||||
|
||||
return (
|
||||
<div className="space-y-4">
|
||||
<div className="flex flex-wrap items-baseline gap-2">
|
||||
<h2 className="text-xl font-semibold text-foreground">Shadow eval</h2>
|
||||
<p className="text-sm text-muted-foreground">
|
||||
Would the auto-router have answered as well as the models you use today? Find out on your real traffic, before
|
||||
switching anything.
|
||||
</p>
|
||||
</div>
|
||||
|
||||
{error != null && (
|
||||
<p className="text-sm text-destructive">Existing evaluations could not be loaded. Refresh the page to retry.</p>
|
||||
)}
|
||||
|
||||
{isPending && error == null && <p className="text-sm text-muted-foreground">Loading evaluations...</p>}
|
||||
|
||||
{showcased.map((job) => (
|
||||
<JobCard key={job.job_id} job={job} readOnly={isViewOnly} />
|
||||
))}
|
||||
|
||||
{!isViewOnly && <StartForm />}
|
||||
|
||||
<PreviousJobs jobs={listed} />
|
||||
</div>
|
||||
);
|
||||
};
|
||||
|
||||
export default ShadowEvalSection;
|
||||
|
|
@ -0,0 +1,24 @@
|
|||
import { describe, expect, it, vi } from "vitest";
|
||||
|
||||
vi.mock("@/lib/http/api", () => ({ $api: { useQuery: vi.fn() }, fetchClient: { POST: vi.fn() } }));
|
||||
vi.mock("@/components/molecules/notifications_manager", () => ({ default: { fromBackend: vi.fn() } }));
|
||||
|
||||
import { shadowEvalListPollMs, shadowEvalPollMs } from "./useShadowEval";
|
||||
|
||||
describe("shadowEvalPollMs", () => {
|
||||
it("keeps polling while the job is active or its status is not yet known", () => {
|
||||
expect(shadowEvalPollMs("running")).toBe(15_000);
|
||||
expect(shadowEvalPollMs(undefined)).toBe(15_000);
|
||||
expect(shadowEvalPollMs("completed")).toBe(false);
|
||||
expect(shadowEvalPollMs("stopped")).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe("shadowEvalListPollMs", () => {
|
||||
it("polls the list while any job is running, so finished jobs migrate to previous", () => {
|
||||
expect(shadowEvalListPollMs([{ status: "running" } as never, { status: "stopped" } as never])).toBe(15_000);
|
||||
expect(shadowEvalListPollMs([{ status: "completed" } as never])).toBe(false);
|
||||
expect(shadowEvalListPollMs([])).toBe(false);
|
||||
expect(shadowEvalListPollMs(undefined)).toBe(false);
|
||||
});
|
||||
});
|
||||
|
|
@ -0,0 +1,79 @@
|
|||
import { useMutation, useQueryClient, type QueryClient } from "@tanstack/react-query";
|
||||
|
||||
import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized";
|
||||
import NotificationsManager from "@/components/molecules/notifications_manager";
|
||||
import { $api, fetchClient } from "@/lib/http/api";
|
||||
|
||||
import type { components } from "@/lib/http/schema";
|
||||
|
||||
export type ShadowEvalJob = components["schemas"]["ShadowEvalJobResponse"];
|
||||
export type ShadowEvalSlice = components["schemas"]["ShadowEvalSlice"];
|
||||
export type StartShadowEvalRequest = components["schemas"]["StartShadowEvalRequest"];
|
||||
|
||||
const LIST_PATH = "/auto_router/shadow_eval" as const;
|
||||
const DETAIL_PATH = "/auto_router/shadow_eval/{job_id}" as const;
|
||||
|
||||
const ACTIVE_POLL_MS = 15_000;
|
||||
|
||||
export const shadowEvalPollMs = (status: ShadowEvalJob["status"] | undefined): number | false =>
|
||||
status === "running" || status === undefined ? ACTIVE_POLL_MS : false;
|
||||
|
||||
export const shadowEvalListPollMs = (jobs: ShadowEvalJob[] | undefined): number | false =>
|
||||
jobs?.some((job) => job.status === "running") ? ACTIVE_POLL_MS : false;
|
||||
|
||||
const invalidateShadowEval = (queryClient: QueryClient) =>
|
||||
Promise.all([
|
||||
queryClient.invalidateQueries({ queryKey: ["get", LIST_PATH] }),
|
||||
queryClient.invalidateQueries({ queryKey: ["get", DETAIL_PATH] }),
|
||||
]);
|
||||
|
||||
export const useShadowEvalJobs = () => {
|
||||
const { accessToken } = useAuthorized();
|
||||
return $api.useQuery(
|
||||
"get",
|
||||
LIST_PATH,
|
||||
{},
|
||||
{
|
||||
enabled: Boolean(accessToken),
|
||||
retry: 1,
|
||||
refetchInterval: (query) => shadowEvalListPollMs(query.state.data),
|
||||
},
|
||||
);
|
||||
};
|
||||
|
||||
export const useShadowEvalJob = (jobId: string | null) => {
|
||||
const { accessToken } = useAuthorized();
|
||||
return $api.useQuery(
|
||||
"get",
|
||||
DETAIL_PATH,
|
||||
{ params: { path: { job_id: jobId ?? "" } } },
|
||||
{
|
||||
enabled: Boolean(accessToken) && Boolean(jobId),
|
||||
retry: 1,
|
||||
refetchInterval: (query) => shadowEvalPollMs(query.state.data?.status),
|
||||
},
|
||||
);
|
||||
};
|
||||
|
||||
const useShadowEvalMutation = <TVariables>(mutationFn: (variables: TVariables) => Promise<unknown>) => {
|
||||
const queryClient = useQueryClient();
|
||||
return useMutation({
|
||||
mutationFn,
|
||||
onSuccess: () => invalidateShadowEval(queryClient),
|
||||
onError: (error: unknown) => NotificationsManager.fromBackend(error),
|
||||
});
|
||||
};
|
||||
|
||||
export const useStartShadowEval = () =>
|
||||
useShadowEvalMutation(async (body: StartShadowEvalRequest) => {
|
||||
const { data } = await fetchClient.POST("/auto_router/shadow_eval/start", { body });
|
||||
return data;
|
||||
});
|
||||
|
||||
export const useStopShadowEval = () =>
|
||||
useShadowEvalMutation(async (jobId: string) => {
|
||||
const { data } = await fetchClient.POST("/auto_router/shadow_eval/{job_id}/stop", {
|
||||
params: { path: { job_id: jobId } },
|
||||
});
|
||||
return data;
|
||||
});
|
||||
|
|
@ -1,4 +1,11 @@
|
|||
import { keepPreviousData, useQuery, UseQueryResult } from "@tanstack/react-query";
|
||||
import {
|
||||
keepPreviousData,
|
||||
useInfiniteQuery,
|
||||
useQuery,
|
||||
UseQueryResult,
|
||||
type InfiniteData,
|
||||
type QueryKey,
|
||||
} from "@tanstack/react-query";
|
||||
import { createQueryKeys } from "../common/queryKeysFactory";
|
||||
import { getProxyBaseUrl, getGlobalLitellmHeaderName, deriveErrorMessage, handleError } from "@/components/networking";
|
||||
import { KeyResponse } from "@/components/key_team_helpers/key_list";
|
||||
|
|
@ -113,6 +120,27 @@ export const useKeys = (
|
|||
});
|
||||
};
|
||||
|
||||
const infiniteKeyKeys = createQueryKeys("infiniteKeys");
|
||||
|
||||
export const useInfiniteKeys = (pageSize: number, options: KeyListCallOptions = {}) => {
|
||||
const { accessToken } = useAuthorized();
|
||||
|
||||
const infiniteKeyListOptions = {
|
||||
queryKey: infiniteKeyKeys.list({ limit: pageSize, ...options }),
|
||||
queryFn: async ({ pageParam }: { pageParam: number }) => {
|
||||
if (!accessToken) throw new Error("Access token required");
|
||||
return await keyListCall(accessToken, pageParam, pageSize, options);
|
||||
},
|
||||
initialPageParam: 1,
|
||||
getNextPageParam: (lastPage: KeysResponse) =>
|
||||
lastPage.current_page < lastPage.total_pages ? lastPage.current_page + 1 : undefined,
|
||||
enabled: Boolean(accessToken),
|
||||
staleTime: 30_000,
|
||||
};
|
||||
|
||||
return useInfiniteQuery<KeysResponse, Error, InfiniteData<KeysResponse>, QueryKey, number>(infiniteKeyListOptions);
|
||||
};
|
||||
|
||||
export const deletedKeyKeys = createQueryKeys("deletedKeys");
|
||||
export const useDeletedKeys = (
|
||||
page: number,
|
||||
|
|
|
|||
|
|
@ -31,6 +31,7 @@ interface PaginatedSearchSelectProps {
|
|||
isFetchingNextPage?: boolean;
|
||||
placeholder?: string;
|
||||
emptyText?: string;
|
||||
errorText?: string;
|
||||
loadingText?: string;
|
||||
disabled?: boolean;
|
||||
className?: string;
|
||||
|
|
@ -50,6 +51,7 @@ export function PaginatedSearchSelect({
|
|||
isFetchingNextPage = false,
|
||||
placeholder = "Search…",
|
||||
emptyText = "No results",
|
||||
errorText,
|
||||
loadingText = "Loading…",
|
||||
disabled = false,
|
||||
className,
|
||||
|
|
@ -104,7 +106,9 @@ export function PaginatedSearchSelect({
|
|||
className={`w-full ${className ?? ""}`}
|
||||
/>
|
||||
<ComboboxContent>
|
||||
<ComboboxEmpty>{isLoading ? loadingText : emptyText}</ComboboxEmpty>
|
||||
<ComboboxEmpty className={errorText == null ? undefined : "text-destructive"}>
|
||||
{errorText ?? (isLoading ? loadingText : emptyText)}
|
||||
</ComboboxEmpty>
|
||||
<ComboboxList onScroll={handleScroll} data-testid="paginated-search-select-list">
|
||||
{(item: SearchSelectOption) => (
|
||||
<ComboboxItem key={item.value} value={item}>
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue