scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-06 02:48:13 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 103

Merge cf88e85995 into 3930c5bab6

Browse source
This commit is contained in:
tin-berri 2026-10-01 02:22:33 +08:00 • committed by GitHub
commit 8f722ce41f
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
22 changed files with 670 additions and 67 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

14
ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/AutoRouters/autoRouterRows.test.ts
View file

@ -84,15 +84,21 @@ describe("autoRouterRows", () => {
});
it.each([
["llm", "LLM Classifier"],
["jev", "JEV Classifier"],
])("labels a router using the %s classifier", (classifierType, label) => {
["llm", "LLM Classifier", undefined],
["jev", "Decision Model", undefined],
["jev", "Decision Model", "laya"],
])("labels a router using the %s classifier", (classifierType, label, provider) => {
const row = toAutoRouterRow(
{
...complexityDeployment,
litellm_params: {
...complexityDeployment.litellm_params,
complexity_router_config: { tiers: {}, classifier_type: classifierType, adaptive: true },
complexity_router_config: {
tiers: {},
classifier_type: classifierType,
adaptive: true,
...(provider && { jev_classifier_config: { provider, model: "english" } }),
},
},
},
0,

2
ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/AutoRouters/autoRouterRows.ts
View file

@ -57,7 +57,7 @@ const dedupe = (models: string[]): string[] => Array.from(new Set(models));
const COMPLEXITY_TYPE_LABELS: Record<string, string> = {
llm: "LLM Classifier",
jev: "JEV Classifier",
jev: "Decision Model",
capability: "Capability",
llm_v2: "Fuse v2",
heuristic_first: "Heuristic first",

2
ui/litellm-dashboard/src/components/add_model/AutoRouterClassifierTabs.integration.test.tsx
View file

@ -68,7 +68,7 @@ describe("Auto-router classifier selection", () => {
llm: "LLM",
heuristic_first: "LLM",
hybrid: "LLM",
jev: "Jev",
jev: "Decision Model",
}[classifier_type];
expect(screen.getByRole("radio", { name: new RegExp(`^${family}$`) })).toBeChecked();
fireEvent.click(screen.getByRole("radio", { name: new RegExp(`^${family}$`) }));

2
ui/litellm-dashboard/src/components/add_model/AutoRouterClassifierTabs.tsx
View file

@ -164,7 +164,7 @@ const AutoRouterClassifierTabs: React.FC<AutoRouterClassifierTabsProps> = ({ val
{[
{ value: "heuristics", label: "Heuristics", description: "Classify locally, with no API call" },
{ value: "llm", label: "LLM", description: "Use a judge model to choose a solver" },
{ value: "jev", label: "Jev", description: "Use TypeSafe System One Choice to choose a tier" },
{ value: "jev", label: "Decision Model", description: "Use Jev or open-source Laya to choose a tier" },
].map((option) => (
<Label
key={option.value}

4
ui/litellm-dashboard/src/components/add_model/ClassificationMethodConfig.tsx
View file

@ -642,8 +642,8 @@ const ClassificationMethodConfig: React.FC<ClassificationMethodConfigProps> = ({
/>
<span className="text-xs text-muted-foreground">
Number of prior user turns sent to the classifier provider, excluding tool output and harness reminders.
LLM and Jev default to 3 turns; Jev sends them to the configured TypeSafe endpoint. Set to 0 to omit
conversation history. The current message and selected system text are still sent.
LLM and decision models default to 3 turns and send them to the configured classifier endpoint. Set to 0
to omit conversation history. The current message and selected system text are still sent.
</span>
</div>
<div>

4
ui/litellm-dashboard/src/components/add_model/ClassifierTypeRadios.tsx
View file

@ -54,8 +54,8 @@ const ClassifierTypeRadios: React.FC<ClassifierTypeRadiosProps> = ({ value, clas
<Label className="items-start font-normal leading-normal">
<RadioGroupItem value="jev" className="mt-0.5" />
<span>
<strong className="font-semibold">Jev Classifier</strong>{" "}
<span className="text-muted-foreground">uses TypeSafe System One Choice to decide the tier</span>
<strong className="font-semibold">Decision Model</strong>{" "}
<span className="text-muted-foreground">uses Jev or open-source Laya to decide the tier</span>
</span>
</Label>
<SimpleTooltip content={scorerLockedReason}>

2
ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx
View file

@ -237,7 +237,7 @@ const TierSetToolbar: React.FC<{
{editing && (
<span className="block mt-1 text-xs text-muted-foreground">
Add or remove tiers to define your own set. Every custom tier needs a definition the classifier routes on, and
an edited set requires the LLM or Jev classification method
an edited set requires the LLM or Decision Model classification method
</span>
)}
{editing && keywordRulesError && (

92
ui/litellm-dashboard/src/components/add_model/JevClassifierConfig.integration.test.tsx
View file

@ -12,6 +12,7 @@ import {
} from "../edit_auto_router/edit_auto_router_modal";
import { applyTierSetAction } from "./tier_set_actions";
import { testAutoRouterRouting } from "../networking";
import { selectAutoRouterOption } from "../../../tests/autoRouterSetup";
import { JEV_CONNECTION_TEST_PROMPT } from "./build_auto_router_routing_test_request";
vi.mock("@/app/(dashboard)/hooks/useAuthorized", () => ({
@ -102,8 +103,8 @@ describe("JEV classifier editor", () => {
expect(screen.getByText("Reasoning Effort")).toBeInTheDocument();
expect(screen.getByText("Classifier Prompt")).toBeInTheDocument();
expect(screen.getByRole("switch", { name: "Use images for classification" })).toBeInTheDocument();
fireEvent.click(screen.getByRole("radio", { name: /Jev Classifier/ }));
expect(screen.getByRole("radio", { name: /^Jev Classifier/ })).toBeChecked();
fireEvent.click(screen.getByRole("radio", { name: /Decision Model uses/ }));
expect(screen.getByRole("radio", { name: /^Decision Model uses/ })).toBeChecked();
expect(screen.getByLabelText("Jev Model")).toHaveValue("jev-latest");
expect(screen.getByLabelText("Jev Instructions")).toBeEnabled();
expect(screen.queryByLabelText("Judge model")).not.toBeInTheDocument();
@ -117,7 +118,7 @@ describe("JEV classifier editor", () => {
fireEvent.click(screen.getByRole("switch", { name: "Classifier circuit breaker" }));
fireEvent.click(screen.getByRole("button", { name: "Customize tiers" }));
fireEvent.click(screen.getByRole("button", { name: "Save and reload" }));
expect(screen.getByRole("radio", { name: /Jev Classifier/ })).toBeChecked();
expect(screen.getByRole("radio", { name: /Decision Model uses/ })).toBeChecked();
expect(screen.getByLabelText("Jev Model")).toHaveValue("jev-test");
expect(screen.getByLabelText("Jev Timeout (ms)")).toHaveValue(4200);
expect(screen.getByLabelText("Context Window Size")).toHaveValue("6");
@ -140,6 +141,91 @@ describe("JEV classifier editor", () => {
);
});
it("selects Laya, clears the previous connection and probes the saved checkpoint", async () => {
renderWithProviders(<Form />);
fireEvent.click(screen.getByRole("radio", { name: /Decision Model uses/ }));
fireEvent.click(screen.getByText("Connection settings"));
fireEvent.change(screen.getByLabelText("API Base"), { target: { value: "https://jev.test" } });
fireEvent.change(screen.getByLabelText("API Key"), { target: { value: "old-key" } });
fireEvent.click(screen.getByRole("radio", { name: "Laya (open source)" }));
expect(screen.getByRole("combobox", { name: "Laya Model" })).toHaveTextContent("english");
expect(screen.getByLabelText("API Base")).toHaveValue("");
expect(screen.getByLabelText("API Key")).toHaveValue("");
await selectAutoRouterOption("Laya Model", "multilingual");
fireEvent.change(screen.getByLabelText("API Base"), { target: { value: "http://laya.test:8000" } });
fireEvent.click(screen.getByRole("button", { name: "Probe current config" }));
expect(testAutoRouterRouting).toHaveBeenLastCalledWith(
"token",
expect.objectContaining({
complexity_router_config: expect.objectContaining({
classifier_type: "jev",
jev_classifier_config: {
provider: "laya",
model: "multilingual",
timeout_ms: 3000,
api_base: "http://laya.test:8000",
},
}),
}),
);
fireEvent.click(screen.getByRole("button", { name: "Save and reload" }));
expect(screen.getByRole("radio", { name: "Laya (open source)" })).toBeChecked();
expect(screen.getByRole("combobox", { name: "Laya Model" })).toHaveTextContent("multilingual");
expect(screen.getByLabelText("API Base")).toHaveValue("");
fireEvent.click(screen.getByRole("radio", { name: "TypeSafe Jev" }));
expect(screen.getByLabelText("Jev Model")).toHaveValue("jev-latest");
});
it("shows clear intent after discarded replacements and resets it with the provider", () => {
renderWithProviders(<Form />);
fireEvent.click(screen.getByRole("radio", { name: /Decision Model uses/ }));
fireEvent.click(screen.getByRole("radio", { name: "Laya (open source)" }));
fireEvent.click(screen.getByText("Connection settings"));
fireEvent.click(screen.getByRole("button", { name: "Clear saved API key" }));
fireEvent.change(screen.getByLabelText("API Key"), { target: { value: "replacement" } });
expect(screen.queryByRole("status")).not.toBeInTheDocument();
fireEvent.change(screen.getByLabelText("API Key"), { target: { value: " " } });
expect(screen.getByRole("status")).toHaveTextContent("Saved API key will be cleared on the next save");
fireEvent.click(screen.getByRole("button", { name: "Use gateway connection" }));
fireEvent.change(screen.getByLabelText("API Base"), { target: { value: "https://temporary.test" } });
fireEvent.change(screen.getByLabelText("API Base"), { target: { value: "" } });
expect(screen.getByText("Gateway endpoint selected for the next save")).toBeVisible();
fireEvent.click(screen.getByRole("radio", { name: "TypeSafe Jev" }));
expect(screen.queryByRole("status")).not.toBeInTheDocument();
expect(screen.getByLabelText("API Base")).toHaveValue("");
expect(screen.getByLabelText("API Key")).toHaveValue("");
});
it("keeps an unsupported saved checkpoint visible until replaced", async () => {
const InvalidForm = () => {
const [value, setValue] = useState(
hydrateComplexityRouterConfig(
{
...initial,
classifier_type: "jev",
jev_classifier_config: { provider: "laya", model: "unsupported-checkpoint" },
},
undefined,
),
);
return <JevEditor value={value} onChange={setValue} />;
};
renderWithProviders(<InvalidForm />);
expect(screen.getByRole("combobox", { name: "Laya Model" })).toHaveTextContent("unsupported-checkpoint");
expect(screen.getByRole("combobox", { name: "Laya Model" })).toHaveAttribute("aria-invalid", "true");
await selectAutoRouterOption("Laya Model", "typed-decisions");
expect(screen.getByRole("combobox", { name: "Laya Model" })).toHaveTextContent("typed-decisions");
expect(screen.queryByText("Choose a supported Laya checkpoint before saving or testing")).not.toBeInTheDocument();
});
it("hides connection overrides from team members", () => {
const authorized = useAuthorized();
vi.mocked(useAuthorized).mockReturnValue({ ...authorized, userRole: "Internal User" });
renderWithProviders(<JevEditor value={{ ...initial, classifier_type: "jev" }} onChange={vi.fn()} />);
expect(screen.queryByText("Connection settings")).not.toBeInTheDocument();
expect(screen.getByRole("radio", { name: "Laya (open source)" })).toBeEnabled();
});
it("allows licensed instructions and can restore built-in instructions", () => {
const authorized = useAuthorized();
vi.mocked(useAuthorized).mockReturnValue({ ...authorized, premiumUser: true });

149
ui/litellm-dashboard/src/components/add_model/JevClassifierConfig.tsx
View file

@ -6,7 +6,17 @@ import { Label } from "@/components/ui/label";
import { Textarea } from "@/components/ui/textarea";
import ClassifierCircuitBreakerConfig from "./ClassifierCircuitBreakerConfig";
import type { ComplexityRouterConfigValue } from "./ComplexityRouterConfig";
import { defaultJevClassifierConfig } from "./jev_classifier_config";
import {
defaultJevClassifierConfig,
normalizeJevClassifierConfig,
transitionDecisionModelProvider,
LAYA_CHECKPOINTS,
isLayaCheckpoint,
} from "./jev_classifier_config";
import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized";
import { isProxyAdminRole } from "@/utils/roles";
import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select";
import { RadioGroup, RadioGroupItem } from "@/components/ui/radio-group";
export default function JevClassifierConfig({
value,
@ -16,21 +26,77 @@ export default function JevClassifierConfig({
onChange: (value: ComplexityRouterConfigValue) => void;
}) {
const id = useId();
const { userRole, isViewOnly } = useAuthorized();
const config = value.jev_classifier_config ?? defaultJevClassifierConfig();
const connection = normalizeJevClassifierConfig(config);
const provider = config.provider ?? "typesafe";
const providerLabel = provider === "laya" ? "Laya" : "Jev";
const update = (patch: Partial<typeof config>) =>
onChange({ ...value, jev_classifier_config: { ...config, ...patch } });
return (
<div className="mt-4 space-y-3">
<fieldset className="space-y-2">
<legend className="text-sm font-medium">Decision model provider</legend>
<RadioGroup
value={provider}
onValueChange={(next) => {
if (next === "typesafe" || next === "laya") {
onChange({ ...value, jev_classifier_config: transitionDecisionModelProvider(config, next) });
}
}}
className="flex flex-wrap gap-4"
>
<Label>
<RadioGroupItem value="typesafe" />
TypeSafe Jev
</Label>
<Label>
<RadioGroupItem value="laya" />
Laya (open source)
</Label>
</RadioGroup>
</fieldset>
<p className="text-sm text-muted-foreground">
Uses TypeSafe System One Choice evaluation with your configured tiers
{providerLabel} uses /v1/systemone structured choices to select one of your configured tiers
</p>
<div>
<Label htmlFor={`${id}-model`}>Jev Model</Label>
<Input id={`${id}-model`} value={config.model} onChange={(event) => update({ model: event.target.value })} />
<Label htmlFor={`${id}-model`}>{providerLabel} Model</Label>
{provider === "laya" ? (
<>
<Select
value={config.model}
items={[
...LAYA_CHECKPOINTS.map((model) => ({ value: model, label: model })),
...(!isLayaCheckpoint(config.model)
? [{ value: config.model, label: config.model || "Missing checkpoint" }]
: []),
]}
onValueChange={(model) => model && isLayaCheckpoint(model) && update({ model })}
>
<SelectTrigger id={`${id}-model`} className="w-full" aria-invalid={!isLayaCheckpoint(config.model)}>
<SelectValue />
</SelectTrigger>
<SelectContent>
{LAYA_CHECKPOINTS.map((model) => (
<SelectItem key={model} value={model}>
{model}
</SelectItem>
))}
</SelectContent>
</Select>
{!isLayaCheckpoint(config.model) && (
<p className="mt-1 text-xs text-destructive">
Choose a supported Laya checkpoint before saving or testing
</p>
)}
</>
) : (
<Input id={`${id}-model`} value={config.model} onChange={(event) => update({ model: event.target.value })} />
)}
</div>
<div>
<Label htmlFor={`${id}-timeout`}>Jev Timeout (ms)</Label>
<Label htmlFor={`${id}-timeout`}>{providerLabel} Timeout (ms)</Label>
<Input
id={`${id}-timeout`}
type="number"
@ -40,6 +106,73 @@ export default function JevClassifierConfig({
onChange={(event) => update({ timeout_ms: Number(event.target.value) })}
/>
</div>
{isProxyAdminRole(userRole ?? "") && !isViewOnly && (
<details className="space-y-3 rounded-lg border p-3">
<summary className="cursor-pointer text-sm font-medium">Connection settings</summary>
<p className="text-xs text-muted-foreground">
Saved connection values are hidden. Untouched fields keep the saved connection. New routers use the gateway
connection when these fields are blank
</p>
<div>
<Label htmlFor={`${id}-api-base`}>API Base</Label>
<Input
id={`${id}-api-base`}
value={config.api_base ?? ""}
placeholder={provider === "laya" ? "http://localhost:8000" : "https://api.typesafe.ai"}
onChange={(event) => update({ api_base: event.target.value })}
/>
</div>
<div>
<Label htmlFor={`${id}-api-key`}>API Key</Label>
<Input
id={`${id}-api-key`}
type="password"
autoComplete="new-password"
value={config.api_key ?? ""}
placeholder={provider === "laya" ? "Optional for a keyless Laya server" : "Enter the endpoint's key"}
onChange={(event) => update({ api_key: event.target.value })}
/>
</div>
<div className="flex flex-wrap gap-2">
<Button
variant="outline"
type="button"
onClick={() =>
update({ api_base: null, api_key: null, connection_reset: { api_base: true, api_key: true } })
}
>
Use gateway connection
</Button>
{provider === "laya" && (
<>
<Button
variant="outline"
type="button"
onClick={() =>
update({ api_key: null, connection_reset: { ...config.connection_reset, api_key: true } })
}
>
Clear saved API key
</Button>
<p className="w-full text-xs text-muted-foreground">
Clearing the key keeps a saved custom endpoint and connects without authentication. A gateway
connection still uses its configured key. Use gateway connection clears both router overrides
</p>
</>
)}
</div>
{connection.api_base === null && (
<p role="status" className="text-xs text-muted-foreground">
Gateway endpoint selected for the next save
</p>
)}
{connection.api_key === null && (
<p role="status" className="text-xs text-muted-foreground">
Saved API key will be cleared on the next save
</p>
)}
</details>
)}
<ClassifierCircuitBreakerConfig
value={config}
onChange={(next) =>
@ -50,7 +183,7 @@ export default function JevClassifierConfig({
}
/>
<div>
<Label htmlFor={`${id}-instructions`}>Jev Instructions</Label>
<Label htmlFor={`${id}-instructions`}>{providerLabel} Instructions</Label>
<AutoRouterAllowanceNote
feature="tier_or_classifier_prompt"
label="Custom instructions share the custom-tier allowance"
@ -63,11 +196,11 @@ export default function JevClassifierConfig({
/>
{config.instructions && (
<Button variant="outline" type="button" onClick={() => update({ instructions: undefined })}>
Restore built-in Jev instructions
Restore built-in {providerLabel} instructions
</Button>
)}
<p className="text-xs text-muted-foreground">
Built-in Jev is available without a license and uses the shipped tier criteria
Built-in decision model classification is available without a license and uses the shipped tier criteria
</p>
</div>
</div>

34
ui/litellm-dashboard/src/components/add_model/JevConnectionTest.integration.test.tsx
View file

@ -107,14 +107,40 @@ describe("JEV network probes", () => {
expect(JSON.parse(String(routingCall?.[1]?.body))).toEqual(expectedRequest);
expect(fetchMock).toHaveBeenCalledTimes(5);
expect(screen.getAllByTestId("test-status-success")).toHaveLength(4);
expect(screen.getByRole("status", { name: "Jev connection" })).toHaveTextContent(
expect(screen.getByRole("status", { name: "Decision model connection" })).toHaveTextContent(
cause === "jev_classifier"
? "Jev classification succeeded"
: `Jev was not reached successfully (routing cause: ${cause})`,
? "Decision model classification succeeded"
: `Decision model was not reached successfully (routing cause: ${cause})`,
);
},
);
it("probes a saved Laya classifier using its provider and checkpoint", async () => {
const layaRequest = buildSavedJevConnectionTestRequest(
{ ...config, jev_classifier_config: { provider: "laya", model: "english", api_key: "masked-key" } },
"saved-laya",
);
const fetchMock = vi.fn<typeof fetch>(
async () =>
new Response(
JSON.stringify({
...response("jev_classifier"),
routing_decision: { ...response("jev_classifier").routing_decision, classifier_model: "laya/english" },
}),
),
);
vi.stubGlobal("fetch", fetchMock);
renderWithProviders(<AutoRouterConnectionTest accessToken="token" targets={[]} jevRequest={layaRequest} />);
expect(await screen.findByText("Decision model classification succeeded")).toBeInTheDocument();
expect(JSON.parse(String(fetchMock.mock.calls[0][1]?.body))).toMatchObject({
saved_model_id: "saved-laya",
complexity_router_config: { jev_classifier_config: { provider: "laya", model: "english", timeout_ms: 3000 } },
});
expect(
JSON.parse(String(fetchMock.mock.calls[0][1]?.body)).complexity_router_config.jev_classifier_config,
).not.toHaveProperty("api_key");
});
it("shows routing diagnostics from the real networking response", async () => {
vi.stubGlobal(
"fetch",
@ -131,7 +157,7 @@ describe("JEV network probes", () => {
);
fireEvent.change(screen.getByTestId("auto-router-routing-test-prompt"), { target: { value: "Hello" } });
fireEvent.click(screen.getByTestId("auto-router-routing-test-send"));
expect(await screen.findByText("JEV classifier")).toBeInTheDocument();
expect(await screen.findByText("Decision model classifier")).toBeInTheDocument();
expect(screen.getByText("jev-latest")).toBeInTheDocument();
expect(screen.getByText("80.0%")).toBeInTheDocument();
expect(screen.getByText("SIMPLE: 80.0%")).toBeInTheDocument();

33
ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.integration.test.tsx
View file

@ -274,7 +274,7 @@ describe("AddAutoRouterTab", () => {
mockFetchAvailableModels.mockResolvedValue(ALL_FAMILY_MODELS);
renderWithProviders(<Harness />);
await user.click(await screen.findByRole("button", { name: "Choose models for me" }));
await user.click(screen.getByRole("radio", { name: "Jev" }));
await user.click(screen.getByRole("radio", { name: "Decision Model" }));
await waitFor(() =>
expect(apiClient.post).toHaveBeenLastCalledWith(
"/auto_router/availability",
@ -301,7 +301,7 @@ describe("AddAutoRouterTab", () => {
expect(within(screen.getByRole("alert")).getByRole("link", { name: "Talk to our team" })).toBeVisible();
await user.click(screen.getByRole("button", { name: "Restore defaults" }));
await waitFor(() => expect(screen.queryByRole("alert")).not.toBeInTheDocument());
expect(screen.getByRole("radio", { name: "Jev" })).toBeChecked();
expect(screen.getByRole("radio", { name: "Decision Model" })).toBeChecked();
await waitFor(() => expect(screen.getByRole("button", { name: "Add Auto Router" })).toBeEnabled());
await user.click(screen.getByRole("button", { name: "Add Auto Router" }));
await waitFor(() => expect(handleAddAutoRouterSubmit).toHaveBeenCalled());
@ -317,7 +317,7 @@ describe("AddAutoRouterTab", () => {
mockFetchAvailableModels.mockResolvedValue(ALL_FAMILY_MODELS);
renderWithProviders(<Harness />);
await user.click(await screen.findByRole("button", { name: "Choose models for me" }));
await user.click(screen.getByRole("radio", { name: "Jev" }));
await user.click(screen.getByRole("radio", { name: "Decision Model" }));
fireEvent.change(screen.getByLabelText("Auto Router Name"), { target: { value: "checked-router" } });
await waitFor(() => expect(screen.getByRole("button", { name: "Add Auto Router" })).toBeEnabled());
let complete: ((result: unknown) => void) | undefined;
@ -357,17 +357,22 @@ describe("AddAutoRouterTab", () => {
expect(screen.getByRole("button", { name: "Routing approach" })).toHaveTextContent("Complexity");
});
it.each(["LLM", "Jev"])("keeps %s and the frequency when choosing models automatically", async (family) => {
mockFetchAvailableModels.mockResolvedValue(ALL_FAMILY_MODELS);
renderWithProviders(<Harness />);
const automatic = await screen.findByRole("button", { name: "Choose models for me" });
await userEvent.click(screen.getByRole("radio", { name: family }));
await selectAutoRouterOption("How often to classify", "Every new user message");
await userEvent.click(automatic);
expect(screen.getByRole("radio", { name: family })).toBeChecked();
expect(screen.getByRole("combobox", { name: "How often to classify" })).toHaveTextContent("Every new user message");
expect(screen.getByRole("button", { name: "Advanced settings" })).toHaveAttribute("aria-expanded", "false");
});
it.each(["LLM", "Decision Model"])(
"keeps %s and the frequency when choosing models automatically",
async (family) => {
mockFetchAvailableModels.mockResolvedValue(ALL_FAMILY_MODELS);
renderWithProviders(<Harness />);
const automatic = await screen.findByRole("button", { name: "Choose models for me" });
await userEvent.click(screen.getByRole("radio", { name: family }));
await selectAutoRouterOption("How often to classify", "Every new user message");
await userEvent.click(automatic);
expect(screen.getByRole("radio", { name: family })).toBeChecked();
expect(screen.getByRole("combobox", { name: "How often to classify" })).toHaveTextContent(
"Every new user message",
);
expect(screen.getByRole("button", { name: "Advanced settings" })).toHaveAttribute("aria-expanded", "false");
},
);
it.each(["Capability", "Fuse v2"])(
"creates %s from its dedicated tab without complexity templates",

10
ui/litellm-dashboard/src/components/add_model/auto_router_connection_test.tsx
View file

@ -50,7 +50,7 @@ const AutoRouterConnectionTest: React.FC<AutoRouterConnectionTestProps> = ({
? { status: "success" }
: {
status: "error",
error: `Jev was not reached successfully (routing cause: ${decision.cause ?? "unknown"})`,
error: `Decision model was not reached successfully (routing cause: ${decision.cause ?? "unknown"})`,
},
);
};
@ -91,11 +91,11 @@ const AutoRouterConnectionTest: React.FC<AutoRouterConnectionTestProps> = ({
classifier probe includes its reasoning effort override.
</p>
{jevRequest && (
<div role="status" aria-label="Jev connection" className="rounded-lg border p-3 text-sm">
<strong>Jev Classifier</strong>
<div role="status" aria-label="Decision model connection" className="rounded-lg border p-3 text-sm">
<strong>Decision Model</strong>
<p>
{jevResult.status === "pending" && "Testing Jev classification"}
{jevResult.status === "success" && "Jev classification succeeded"}
{jevResult.status === "pending" && "Testing decision model classification"}
{jevResult.status === "success" && "Decision model classification succeeded"}
{jevResult.status === "error" && jevResult.error}
</p>
</div>

22
ui/litellm-dashboard/src/components/add_model/build_auto_router_routing_test_request.test.ts
View file

@ -68,6 +68,28 @@ describe("buildAutoRouterRoutingTestRequest", () => {
expect(buildSavedJevConnectionTestRequest(config, "saved-id")).toBeUndefined();
},
);
it("sends explicit connection resets in new probes and resolves saved probes on the server", () => {
const config = {
...CONFIG,
classifier_type: "jev" as const,
jev_classifier_config: {
provider: "laya" as const,
model: "english",
timeout_ms: 3000,
api_base: null,
api_key: null,
},
};
const request = buildAutoRouterRoutingTestRequest({ ...params, config });
expect(JSON.parse(JSON.stringify(request)).complexity_router_config.jev_classifier_config).toEqual(
config.jev_classifier_config,
);
const savedRequest = buildSavedJevConnectionTestRequest(config, "saved-laya");
expect(savedRequest?.saved_model_id).toBe("saved-laya");
expect(savedRequest?.complexity_router_config.jev_classifier_config).not.toHaveProperty("api_base");
expect(savedRequest?.complexity_router_config.jev_classifier_config).not.toHaveProperty("api_key");
});
it("sends the prompt with the config being edited", () => {
const request = buildAutoRouterRoutingTestRequest(params);

14
ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.test.ts
View file

@ -75,9 +75,21 @@ describe("buildComplexityRouterConfig", () => {
classifier_type: "jev",
jev_classifier_config: { model: "jev-latest", timeout_ms: 3000, ...patch },
}),
).toBe("Enter a JEV model, a positive whole-number timeout and a positive cooldown");
).toBe("Enter a decision model, a positive whole-number timeout and a positive cooldown");
});
it.each(["", "auto", "unrecognized-checkpoint"])(
"blocks unsupported Laya checkpoint %s before saving or testing",
(model) => {
expect(
getClassifierModelError({
classifier_type: "jev",
jev_classifier_config: { provider: "laya", model, timeout_ms: 3000 },
}),
).not.toBeNull();
},
);
it.each([false, true])("serializes JEV with shared context and no LLM config, custom tiers: %s", (custom) => {
const params: BuildComplexityRouterConfigParams = {
...baseParams,

6
ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.ts
View file

@ -8,7 +8,7 @@ import type { ModelGroup } from "../llm_calls/fetch_models";
import { KeywordTierRule } from "./KeywordTierRules";
import {
type JevClassifierConfig,
jevClassifierConfigSchema,
jevClassifierFormConfigSchema,
normalizeJevClassifierConfig,
} from "./jev_classifier_config";
import {
@ -437,8 +437,8 @@ export const getClassifierModelError = (
>,
): string | null => {
if (effectiveClassifierType(config) === "jev") {
const parsed = jevClassifierConfigSchema.safeParse(config.jev_classifier_config ?? {});
return parsed.success ? null : "Enter a JEV model, a positive whole-number timeout and a positive cooldown";
const parsed = jevClassifierFormConfigSchema.safeParse(config.jev_classifier_config ?? {});
return parsed.success ? null : "Enter a decision model, a positive whole-number timeout and a positive cooldown";
}
if (!usesLlmClassifier(effectiveClassifierType(config)) || config.classifier_llm_config?.model) return null;
return config.custom_tier_set

111
ui/litellm-dashboard/src/components/add_model/jev_classifier_config.test.ts Normal file
View file

@ -0,0 +1,111 @@
import { describe, expect, it } from "vitest";
import {
defaultJevClassifierConfig,
jevClassifierConfigSchema,
jevClassifierFormConfigSchema,
normalizeJevClassifierConfig,
storedJevClassifierConfigSchema,
transitionDecisionModelProvider,
} from "./jev_classifier_config";
describe("decision model configuration", () => {
it("keeps legacy Jev defaults but requires an explicit Laya checkpoint", () => {
expect(jevClassifierConfigSchema.parse({})).toEqual(defaultJevClassifierConfig());
expect(jevClassifierConfigSchema.safeParse({ provider: "laya" }).success).toBe(false);
expect(jevClassifierConfigSchema.safeParse({ provider: "unknown", model: "english" }).success).toBe(false);
expect(jevClassifierConfigSchema.safeParse({ provider: "laya", model: " " }).success).toBe(false);
expect(jevClassifierConfigSchema.safeParse({ provider: "laya", model: "automatic" }).success).toBe(false);
});
it("preserves entered connection overrides when saving and strips hidden connections when loading", () => {
const input = {
provider: "laya",
model: " multilingual ",
timeout_ms: 4500,
api_base: " http://laya.test:8000 ",
api_key: " own-key ",
connection_reset: { api_base: true, api_key: true },
};
const edited = normalizeJevClassifierConfig(jevClassifierFormConfigSchema.parse(input));
const expected = {
provider: "laya",
model: "multilingual",
timeout_ms: 4500,
api_base: "http://laya.test:8000",
api_key: "own-key",
};
expect(edited).toEqual(expected);
expect(storedJevClassifierConfigSchema.parse(input)).toEqual({
provider: "laya",
model: "multilingual",
timeout_ms: 4500,
});
expect(jevClassifierConfigSchema.parse(edited)).toEqual({
provider: "laya",
model: "multilingual",
timeout_ms: 4500,
});
});
it("preserves explicit null clears while leaving untouched or blank transports omitted", () => {
const config = { ...defaultJevClassifierConfig(), provider: "laya" as const, model: "english" };
expect(
normalizeJevClassifierConfig(jevClassifierFormConfigSchema.parse({ ...config, api_base: null, api_key: null })),
).toEqual({ ...config, api_base: null, api_key: null });
expect(normalizeJevClassifierConfig({ ...config, api_key: null })).toEqual({ ...config, api_key: null });
expect(normalizeJevClassifierConfig(config)).toEqual(config);
expect(normalizeJevClassifierConfig({ ...config, api_base: " ", api_key: " " })).toEqual(config);
});
it.each([
{ api_base: " ", api_key: "", expected: { api_base: null, api_key: null } },
{ api_base: " https://new.test ", api_key: " ", expected: { api_base: "https://new.test", api_key: null } },
{ api_base: "", api_key: " new-key ", expected: { api_base: null, api_key: "new-key" } },
])(
"restores each reset independently after replacements are erased: $api_base / $api_key",
({ api_base, api_key, expected }) => {
const config = defaultJevClassifierConfig();
const input = {
...config,
api_base,
api_key,
connection_reset: { api_base: true, api_key: true },
};
const edited = jevClassifierFormConfigSchema.parse(input);
expect(normalizeJevClassifierConfig(edited)).toEqual({ ...config, ...expected });
},
);
it("clears both transports when switching providers while retaining shared settings", () => {
const configured = {
...defaultJevClassifierConfig(),
model: "jev-custom",
api_base: "https://jev.test",
api_key: "secret",
connection_reset: { api_base: true as const, api_key: true as const },
instructions: "Classify into tiers",
circuit_breaker_enabled: false,
};
expect(transitionDecisionModelProvider(configured, "typesafe")).toBe(configured);
const laya = transitionDecisionModelProvider(configured, "laya");
const expectedLaya = {
provider: "laya",
model: "english",
timeout_ms: 3000,
instructions: "Classify into tiers",
circuit_breaker_enabled: false,
};
expect(laya).toEqual(expectedLaya);
const configuredLaya = {
...laya,
api_base: "http://laya.test",
api_key: "laya-secret",
connection_reset: { api_base: true as const, api_key: true as const },
};
expect(transitionDecisionModelProvider(configuredLaya, "typesafe")).toEqual({
...laya,
provider: "typesafe",
model: "jev-latest",
});
});
});

80
ui/litellm-dashboard/src/components/add_model/jev_classifier_config.ts
View file

@ -1,7 +1,13 @@
import { z } from "zod";
export const LAYA_CHECKPOINTS = ["english", "multilingual", "typed-decisions"] as const;
const layaCheckpointSchema = z.enum(LAYA_CHECKPOINTS);
export const isLayaCheckpoint = (model: string): boolean => layaCheckpointSchema.safeParse(model).success;
const jevClassifierConfigFields = {
model: z.string().trim().min(1).default("jev-latest"),
provider: z.enum(["typesafe", "laya"]).optional(),
model: z.string().trim().optional(),
timeout_ms: z.number().int().positive().default(3000),
instructions: z
.string()
@ -11,20 +17,70 @@ const jevClassifierConfigFields = {
circuit_breaker_cooldown_seconds: z.number().finite().positive().optional(),
};
export const jevClassifierConfigSchema = z.object(jevClassifierConfigFields);
const validateDecisionModel = (config: { provider?: string; model?: string }, context: z.RefinementCtx) => {
if (config.provider === "laya" && !layaCheckpointSchema.safeParse(config.model).success) {
context.addIssue({ code: z.ZodIssueCode.custom, path: ["model"], message: "Choose a supported Laya checkpoint" });
} else if (config.model === "") {
context.addIssue({ code: z.ZodIssueCode.custom, path: ["model"], message: "Enter a model" });
}
};
export type JevClassifierConfig = z.infer<typeof jevClassifierConfigSchema>;
const withModelDefault = <T extends { provider?: string; model?: string }>(config: T) => ({
...config,
model: config.model ?? (config.provider === "laya" ? "" : "jev-latest"),
});
export const storedJevClassifierConfigSchema = z.object(jevClassifierConfigFields).transform(withModelDefault);
export const jevClassifierConfigSchema = z
.object(jevClassifierConfigFields)
.superRefine(validateDecisionModel)
.transform(withModelDefault);
const jevClassifierFormConfigFields = {
...jevClassifierConfigFields,
api_base: z.string().trim().nullish(),
api_key: z.string().trim().nullish(),
connection_reset: z.object({ api_base: z.literal(true).optional(), api_key: z.literal(true).optional() }).optional(),
};
export const jevClassifierFormConfigSchema = z
.object(jevClassifierFormConfigFields)
.superRefine(validateDecisionModel)
.transform(withModelDefault);
export type JevClassifierConfig = z.infer<typeof jevClassifierFormConfigSchema>;
export type DecisionModelProvider = NonNullable<JevClassifierConfig["provider"]>;
export const defaultJevClassifierConfig = (): JevClassifierConfig => jevClassifierConfigSchema.parse({});
export const transitionDecisionModelProvider = (
config: JevClassifierConfig,
provider: DecisionModelProvider,
): JevClassifierConfig => {
if ((config.provider ?? "typesafe") === provider) return config;
const { api_base, api_key, connection_reset, ...settings } = config;
return { ...settings, provider, model: provider === "laya" ? "english" : "jev-latest" };
};
const normalizeConnectionValue = (value: string | null | undefined, reset: boolean | undefined) =>
value?.trim() || (value === null || reset ? null : undefined);
export const normalizeJevClassifierConfig = (
config: JevClassifierConfig = defaultJevClassifierConfig(),
): JevClassifierConfig => ({
model: config.model.trim(),
timeout_ms: config.timeout_ms,
...(config.instructions?.trim() && { instructions: config.instructions.trim() }),
...(config.circuit_breaker_enabled !== undefined && { circuit_breaker_enabled: config.circuit_breaker_enabled }),
...(config.circuit_breaker_cooldown_seconds !== undefined && {
circuit_breaker_cooldown_seconds: config.circuit_breaker_cooldown_seconds,
}),
});
): Omit<JevClassifierConfig, "connection_reset"> => {
const apiBase = normalizeConnectionValue(config.api_base, config.connection_reset?.api_base);
const apiKey = normalizeConnectionValue(config.api_key, config.connection_reset?.api_key);
return {
...(config.provider && { provider: config.provider }),
model: config.model.trim(),
timeout_ms: config.timeout_ms,
...(apiBase !== undefined && { api_base: apiBase }),
...(apiKey !== undefined && { api_key: apiKey }),
...(config.instructions?.trim() && { instructions: config.instructions.trim() }),
...(config.circuit_breaker_enabled !== undefined && { circuit_breaker_enabled: config.circuit_breaker_enabled }),
...(config.circuit_breaker_cooldown_seconds !== undefined && {
circuit_breaker_cooldown_seconds: config.circuit_breaker_cooldown_seconds,
}),
};
};

2
ui/litellm-dashboard/src/components/add_model/tier_rows.ts
View file

@ -145,7 +145,7 @@ export const CUSTOM_TIER_RESTRICTIONS = {
heuristicClassifier: {
omit: ["heuristic_first_max_tier", "hybrid_boundary_margin"],
reason:
"The heuristic scorer only produces the built-in tiers, so an edited set needs the LLM or JEV classifier. " +
"The heuristic scorer only produces the built-in tiers, so an edited set needs the LLM or Decision Model classifier. " +
"Heuristic first and hybrid are out for the same reason: their local scorer decides the traffic it is sure of",
},
heuristicScoring: {

20
ui/litellm-dashboard/src/components/edit_auto_router/build_updated_complexity_router_config.test.ts
View file

@ -82,6 +82,26 @@ describe("buildUpdatedComplexityRouterConfig keyword matching", () => {
}
});
it("round trips the Laya provider and checkpoint without writing stored connection fields", () => {
const stored = {
classifier_type: "jev" as const,
tiers: FORM_VALUE.tiers,
jev_classifier_config: {
provider: "laya",
model: "typed-decisions",
timeout_ms: 6000,
api_base: "https://laya.test",
api_key: "sk-masked****",
},
};
const hydrated = hydrateComplexityRouterConfig(stored, undefined);
const saved = buildUpdatedComplexityRouterConfig(stored, hydrated);
expect(saved.jev_classifier_config).toEqual({ provider: "laya", model: "typed-decisions", timeout_ms: 6000 });
expect(hydrateComplexityRouterConfig(saved, undefined).jev_classifier_config).toEqual(
hydrated.jev_classifier_config,
);
});
it("hydrates nullable JEV instructions without resetting the server configuration", () => {
const stored = {
classifier_type: "jev" as const,

128
ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.integration.test.tsx
View file

@ -35,7 +35,9 @@ vi.mock("../networking", () => ({
validateAutoRouterConfig,
}));
vi.mock("@/app/(dashboard)/hooks/useAuthorized", () => ({ default: () => ({ accessToken: "sk-test" }) }));
vi.mock("@/app/(dashboard)/hooks/useAuthorized", () => ({
default: () => ({ accessToken: "sk-test", userRole: "Admin", isViewOnly: false }),
}));
vi.mock("@/components/llm_calls/fetch_models", () => ({
fetchAvailableModels: vi.fn().mockResolvedValue([{ model_group: "gpt-4o-mini" }]),
@ -61,6 +63,24 @@ const MODEL_DATA = {
model_info: { id: "auto-1", access_groups: [] },
};
const LAYA_MODEL_DATA = {
...MODEL_DATA,
litellm_params: {
...MODEL_DATA.litellm_params,
complexity_router_config: {
...STORED_CONFIG,
classifier_type: "jev",
jev_classifier_config: {
provider: "laya",
model: "english",
timeout_ms: 3000,
api_base: "https://laya.test",
api_key: "masked-key",
},
},
},
};
const renderModal = (props: Partial<React.ComponentProps<typeof EditAutoRouterModal>> = {}) =>
renderWithProviders(
<EditAutoRouterModal
@ -129,6 +149,112 @@ describe("EditAutoRouterModal keyword matching", () => {
);
});
it.each([
{ intent: "untouched", action: undefined, transport: {} },
{ intent: "discarded replacement", action: undefined, discardReplacement: true, transport: {} },
{ intent: "gateway reset", action: "Use gateway connection", transport: { api_base: null, api_key: null } },
{ intent: "key reset", action: "Clear saved API key", transport: { api_key: null } },
{
intent: "key reset after discarded replacement",
action: "Clear saved API key",
discardReplacement: true,
transport: { api_key: null },
},
{
intent: "gateway reset after discarded replacements",
action: "Use gateway connection",
discardReplacement: true,
transport: { api_base: null, api_key: null },
},
])("saves Laya connection intent: $intent", async ({ action, transport, discardReplacement }) => {
const user = userEvent.setup();
renderModal({ modelData: LAYA_MODEL_DATA });
openAutoRouterAdvanced("Classification Method");
await user.click(screen.getByText("Connection settings"));
expect(screen.getByLabelText("API Base")).toHaveValue("");
expect(screen.getByLabelText("API Key")).toHaveValue("");
if (action) await user.click(screen.getByRole("button", { name: action }));
if (discardReplacement) {
fireEvent.change(screen.getByLabelText("API Base"), { target: { value: "https://temporary.test" } });
fireEvent.change(screen.getByLabelText("API Key"), { target: { value: "discarded-new-key" } });
fireEvent.change(screen.getByLabelText("API Base"), { target: { value: "" } });
fireEvent.change(screen.getByLabelText("API Key"), { target: { value: " " } });
}
await waitFor(() => expect(screen.getByRole("button", { name: "Save Changes" })).toBeEnabled());
await user.click(screen.getByRole("button", { name: "Save Changes" }));
await waitFor(() => expect(modelPatchUpdateCall).toHaveBeenCalledOnce());
const expectedConfig = { provider: "laya", model: "english", timeout_ms: 3000, ...transport };
expect(savedConfig().jev_classifier_config).toEqual(expectedConfig);
});
it.each(["reopen", "different router"])("discards pending connection resets on %s", async (mode) => {
const user = userEvent.setup();
const props = {
onCancel: vi.fn(),
onSuccess: vi.fn(),
modelData: LAYA_MODEL_DATA,
accessToken: "token",
userRole: "Admin",
};
const { rerender } = renderWithProviders(<EditAutoRouterModal {...props} isVisible />);
openAutoRouterAdvanced("Classification Method");
await user.click(screen.getByText("Connection settings"));
await user.click(screen.getByRole("button", { name: "Use gateway connection" }));
expect(screen.getByText("Saved API key will be cleared on the next save")).toBeVisible();
const reopenedModel =
mode === "reopen"
? LAYA_MODEL_DATA
: { ...LAYA_MODEL_DATA, model_info: { ...LAYA_MODEL_DATA.model_info, id: "another-router" } };
if (mode === "reopen") rerender(<EditAutoRouterModal {...props} isVisible={false} />);
rerender(<EditAutoRouterModal {...props} modelData={reopenedModel} isVisible />);
expect(screen.queryByText("Saved API key will be cleared on the next save")).not.toBeInTheDocument();
await waitFor(() => expect(screen.getByRole("button", { name: "Save Changes" })).toBeEnabled());
await user.click(screen.getByRole("button", { name: "Save Changes" }));
await waitFor(() => expect(modelPatchUpdateCall).toHaveBeenCalledOnce());
expect(savedConfig().jev_classifier_config).toEqual({ provider: "laya", model: "english", timeout_ms: 3000 });
});
it("keeps a newly entered classifier key when correcting the API base before saving", async () => {
const user = userEvent.setup();
renderModal({
modelData: {
...MODEL_DATA,
litellm_params: {
...MODEL_DATA.litellm_params,
complexity_router_config: {
...STORED_CONFIG,
classifier_type: "jev",
jev_classifier_config: {
provider: "laya",
model: "english",
timeout_ms: 3000,
api_base: "https://old-laya.test",
api_key: "masked-saved-key",
},
},
},
},
});
openAutoRouterAdvanced("Classification Method");
await user.click(screen.getByText("Connection settings"));
expect(screen.getByLabelText("API Key")).toHaveValue("");
fireEvent.change(screen.getByLabelText("API Key"), { target: { value: "newly-entered-key" } });
fireEvent.change(screen.getByLabelText("API Base"), { target: { value: "https://new-laya.typo" } });
fireEvent.change(screen.getByLabelText("API Base"), { target: { value: "https://new-laya.test" } });
expect(screen.getByLabelText("API Key")).toHaveValue("newly-entered-key");
await waitFor(() => expect(screen.getByRole("button", { name: "Save Changes" })).toBeEnabled());
await user.click(screen.getByRole("button", { name: "Save Changes" }));
await waitFor(() => expect(modelPatchUpdateCall).toHaveBeenCalledOnce());
const expectedConfig = {
provider: "laya",
model: "english",
timeout_ms: 3000,
api_base: "https://new-laya.test",
api_key: "newly-entered-key",
};
expect(savedConfig().jev_classifier_config).toEqual(expectedConfig);
});
it("renders the advanced sections the create form offers", async () => {
renderModal();

4
ui/litellm-dashboard/src/components/edit_auto_router/hydrate_complexity_router_config.ts
View file

@ -1,4 +1,4 @@
import { defaultJevClassifierConfig, jevClassifierConfigSchema } from "../add_model/jev_classifier_config";
import { defaultJevClassifierConfig, storedJevClassifierConfigSchema } from "../add_model/jev_classifier_config";
import { capabilitySettingsSchema, fuseSettingsSchema } from "../add_model/forecast_classifier_config";
import type { StoredComplexityRouterConfig } from "../add_model/build_complexity_router_config";
import {
@ -80,7 +80,7 @@ export const hydrateComplexityRouterConfig = (
classifier_llm_config: parsedConfig.classifier_type === "jev" ? undefined : parsedConfig.classifier_llm_config,
jev_classifier_config:
parsedConfig.classifier_type === "jev"
? jevClassifierConfigSchema.safeParse(parsedConfig.jev_classifier_config ?? {}).data ??
? storedJevClassifierConfigSchema.safeParse(parsedConfig.jev_classifier_config ?? {}).data ??
defaultJevClassifierConfig()
: undefined,
classifier_context_window_size:

2
ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/RoutingDecisionCard.tsx
View file

@ -143,7 +143,7 @@ function describeCause(decision: RoutingDecision): string {
case "llm_classifier":
return classifierModel ? `LLM classifier (${classifierModel})` : "LLM classifier";
case "jev_classifier":
return "JEV classifier";
return "Decision model classifier";
case "literal_keyword_match":
case "keyword":
return matchedKeyword ? `Keyword match: "${matchedKeyword}"` : "Keyword match";