mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-25 01:02:15 +00:00
feat(ui): complete JEV auto router configuration and connection probes
Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
Moe Khalil
parent
86e079d7a8
commit
969cde4f0c
29 changed files with 978 additions and 113 deletions
|
|
@ -83,13 +83,16 @@ describe("autoRouterRows", () => {
|
|||
expect(row.targets).toEqual(["gpt-4o-mini", "anthropic-sonnet-4-6"]);
|
||||
});
|
||||
|
||||
it("labels a router using the LLM classifier", () => {
|
||||
it.each([
|
||||
["llm", "LLM Classifier"],
|
||||
["jev", "JEV Classifier"],
|
||||
])("labels a router using the %s classifier", (classifierType, label) => {
|
||||
const row = toAutoRouterRow(
|
||||
{
|
||||
...complexityDeployment,
|
||||
litellm_params: {
|
||||
...complexityDeployment.litellm_params,
|
||||
complexity_router_config: { tiers: {}, classifier_type: "llm", adaptive: true },
|
||||
complexity_router_config: { tiers: {}, classifier_type: classifierType, adaptive: true },
|
||||
},
|
||||
},
|
||||
0,
|
||||
|
|
@ -97,7 +100,7 @@ describe("autoRouterRows", () => {
|
|||
null,
|
||||
);
|
||||
|
||||
expect(row.typeLabel).toBe("LLM Classifier");
|
||||
expect(row.typeLabel).toBe(label);
|
||||
});
|
||||
|
||||
it("treats a deployment carrying complexity_router_config as complexity even off the canonical model string", () => {
|
||||
|
|
|
|||
|
|
@ -57,6 +57,7 @@ const dedupe = (models: string[]): string[] => Array.from(new Set(models));
|
|||
|
||||
const COMPLEXITY_TYPE_LABELS: Record<string, string> = {
|
||||
llm: "LLM Classifier",
|
||||
jev: "JEV Classifier",
|
||||
capability: "Capability",
|
||||
llm_v2: "Fuse v2",
|
||||
heuristic_first: "Heuristic first",
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
import { transitionClassifierType } from "./classifier_type_transition";
|
||||
import JevClassifierConfig from "./JevClassifierConfig";
|
||||
import { Info } from "lucide-react";
|
||||
import { SimpleTooltip } from "@/components/ui/tooltip";
|
||||
import { MultiSelect } from "@/components/shared/MultiSelect";
|
||||
|
|
@ -37,6 +38,7 @@ import {
|
|||
effectiveTierLabel,
|
||||
heuristicScoringRole,
|
||||
usesLlmClassifier,
|
||||
usesClassifierContext,
|
||||
DEFAULT_HYBRID_BOUNDARY_MARGIN,
|
||||
HEURISTIC_FIRST_MAX_TIER_KEYS,
|
||||
effectiveClassifierType,
|
||||
|
|
@ -208,6 +210,13 @@ const ClassifierTypeRadios: React.FC<{
|
|||
<span className="text-muted-foreground">calls a model to decide the tier (e.g. a small/fast model)</span>
|
||||
</span>
|
||||
</Label>
|
||||
<Label className="items-start font-normal leading-normal">
|
||||
<RadioGroupItem value="jev" className="mt-0.5" />
|
||||
<span>
|
||||
<strong className="font-semibold">JEV Classifier</strong>{" "}
|
||||
<span className="text-muted-foreground">uses TypeSafe System One Choice to decide the tier</span>
|
||||
</span>
|
||||
</Label>
|
||||
<SimpleTooltip content={scorerLockedReason}>
|
||||
<Label className="items-start font-normal leading-normal has-data-disabled:cursor-not-allowed has-data-disabled:opacity-50">
|
||||
<RadioGroupItem value="heuristic_first" className="mt-0.5" disabled={scorerLocked} />
|
||||
|
|
@ -499,6 +508,7 @@ const ClassificationMethodConfig: React.FC<ClassificationMethodConfigProps> = ({
|
|||
</p>
|
||||
</div>
|
||||
|
||||
{classifierType === "jev" && <JevClassifierConfig value={value} onChange={onChange} />}
|
||||
{usesLlmClassifier(classifierType) && (
|
||||
<div className="mt-4 space-y-3">
|
||||
<div>
|
||||
|
|
@ -591,6 +601,10 @@ const ClassificationMethodConfig: React.FC<ClassificationMethodConfigProps> = ({
|
|||
/>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
{usesClassifierContext(classifierType) && (
|
||||
<div className="mt-4 space-y-3">
|
||||
<RestrictedSection heading="If the classifier fails" by={restrictedBy(value, "classifierFallback")}>
|
||||
<RadioGroup
|
||||
value={value.classifier_fallback ?? DEFAULT_CLASSIFIER_FALLBACK}
|
||||
|
|
|
|||
|
|
@ -1,4 +1,7 @@
|
|||
import RoutingOptions from "./RoutingOptions";
|
||||
import type { JevClassifierConfig } from "./jev_classifier_config";
|
||||
import { type ClassifierType } from "./classifier_types";
|
||||
export { type ClassifierType, usesLlmClassifier, usesClassifierContext } from "./classifier_types";
|
||||
import PlanModeOverrideControls from "./PlanModeOverrideControls";
|
||||
import ForecastClassifierConfig, { ForecastSolverModels } from "./ForecastClassifierConfig";
|
||||
import { isForecastClassifier, type CapabilitySettings, type FuseSettings } from "./forecast_classifier_config";
|
||||
|
|
@ -147,23 +150,6 @@ export interface ClassifierLLMConfig {
|
|||
system_prompt?: string;
|
||||
}
|
||||
|
||||
export type ClassifierType =
|
||||
| "heuristic"
|
||||
| "heuristic_v2"
|
||||
| "llm"
|
||||
| "heuristic_first"
|
||||
| "hybrid"
|
||||
| "capability"
|
||||
| "llm_v2";
|
||||
|
||||
/**
|
||||
* Whether this router can call classifier_llm_config.model. Mirrors the backend's
|
||||
* ComplexityRouterConfig.uses_llm_classifier, and is the single gate for every classifier-only
|
||||
* control and payload key, so a new chaining type cannot strip knobs the operator set.
|
||||
*/
|
||||
export const usesLlmClassifier = (classifierType: ClassifierType): boolean =>
|
||||
(["llm", "heuristic_first", "hybrid", "capability", "llm_v2"] as const).some((type) => type === classifierType);
|
||||
|
||||
export type ClassifierFallback = "heuristic" | "default_model";
|
||||
|
||||
export const DEFAULT_CLASSIFIER_FALLBACK: ClassifierFallback = "heuristic";
|
||||
|
|
@ -200,7 +186,7 @@ export const heuristicScoringRole = (value: ComplexityRouterConfigValue): Heuris
|
|||
// Derived, never written into the value, so undoing a tier edit reverts the form with nothing left behind.
|
||||
export const effectiveClassifierType = (
|
||||
value: Pick<ComplexityRouterConfigValue, "custom_tier_set" | "classifier_type">,
|
||||
): ClassifierType => (value.custom_tier_set ? "llm" : value.classifier_type);
|
||||
): ClassifierType => (value.custom_tier_set && value.classifier_type !== "jev" ? "llm" : value.classifier_type);
|
||||
|
||||
const rowOrigin = (row: TierRow, editing: boolean): string => {
|
||||
if (!editing) return row.id;
|
||||
|
|
@ -251,8 +237,8 @@ const TierSetToolbar: React.FC<{
|
|||
</div>
|
||||
{editing && (
|
||||
<span className="block mt-1 text-xs text-muted-foreground">
|
||||
Add or remove tiers to define your own set. Every custom tier needs a definition the LLM classifier routes on,
|
||||
and an edited set requires the LLM classification method
|
||||
Add or remove tiers to define your own set. Every custom tier needs a definition the classifier routes on, and
|
||||
an edited set requires the LLM or JEV classification method
|
||||
</span>
|
||||
)}
|
||||
{editing && keywordRulesError && (
|
||||
|
|
@ -271,7 +257,7 @@ const FallbackTierField: React.FC<{
|
|||
<div className="mt-4">
|
||||
<div className="flex items-center gap-2 mb-2">
|
||||
<strong className="text-base font-semibold">Fallback Tier</strong>
|
||||
<SimpleTooltip content="Where requests route when the LLM classifier errors, times out, or returns an unparseable reply. Required for an edited tier set: the heuristic scorer cannot produce your tiers.">
|
||||
<SimpleTooltip content="Where requests route when the classifier errors, times out, or returns an unparseable reply. Required for an edited tier set: the heuristic scorer cannot produce your tiers">
|
||||
<Info className="size-4 text-muted-foreground" />
|
||||
</SimpleTooltip>
|
||||
</div>
|
||||
|
|
@ -377,6 +363,7 @@ export interface ComplexityRouterConfigValue {
|
|||
capability_classifier_config?: CapabilitySettings;
|
||||
llm_v2_config?: FuseSettings;
|
||||
classifier_llm_config?: ClassifierLLMConfig;
|
||||
jev_classifier_config?: JevClassifierConfig;
|
||||
classifier_context_window_size?: number;
|
||||
classifier_context_budget_chars?: number;
|
||||
classifier_context_per_turn_chars?: number;
|
||||
|
|
@ -641,7 +628,11 @@ const ComplexityRouterConfig: React.FC<ComplexityRouterConfigProps> = ({
|
|||
<Card>
|
||||
<CardContent>
|
||||
{!customTierSet && (
|
||||
<NonReasoningTierToggle value={value} onChange={onChange} available={value.classifier_type === "llm"} />
|
||||
<NonReasoningTierToggle
|
||||
value={value}
|
||||
onChange={onChange}
|
||||
available={value.classifier_type === "llm" || value.classifier_type === "jev"}
|
||||
/>
|
||||
)}
|
||||
|
||||
{tierRows.map((row, index) => {
|
||||
|
|
|
|||
|
|
@ -0,0 +1,158 @@
|
|||
import React, { useState } from "react";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { fireEvent, renderWithProviders, screen } from "../../../tests/test-utils";
|
||||
import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized";
|
||||
import ClassificationMethodConfig from "./ClassificationMethodConfig";
|
||||
import AutoRouterClassifierTabs from "./AutoRouterClassifierTabs";
|
||||
import JevEditor from "./JevClassifierConfig";
|
||||
import { type ComplexityRouterConfigValue } from "./ComplexityRouterConfig";
|
||||
import {
|
||||
buildUpdatedComplexityRouterConfig,
|
||||
hydrateComplexityRouterConfig,
|
||||
} from "../edit_auto_router/edit_auto_router_modal";
|
||||
import { applyTierSetAction } from "./tier_set_actions";
|
||||
import { testAutoRouterRouting } from "../networking";
|
||||
import { buildSavedJevConnectionTestRequest } from "./build_auto_router_routing_test_request";
|
||||
|
||||
vi.mock("@/app/(dashboard)/hooks/useAuthorized", () => ({
|
||||
default: vi.fn(() => ({
|
||||
isLoading: false,
|
||||
isAuthorized: true,
|
||||
token: "token",
|
||||
accessToken: "token",
|
||||
userId: "user",
|
||||
userEmail: "user@example.com",
|
||||
userRole: "Admin",
|
||||
userRoleLabel: "Admin",
|
||||
isViewOnly: false,
|
||||
premiumUser: false,
|
||||
disabledPersonalKeyCreation: false,
|
||||
showSSOBanner: false,
|
||||
})),
|
||||
}));
|
||||
|
||||
vi.mock("@/components/networking", async (importOriginal) => ({
|
||||
...(await importOriginal<typeof import("@/components/networking")>()),
|
||||
getComplexityScorerDefaults: vi.fn(async () => ({
|
||||
tier_boundaries: {},
|
||||
token_thresholds: {},
|
||||
dimension_weights: {},
|
||||
})),
|
||||
testAutoRouterRouting: vi.fn(async () => ({ status: "error", error: "fixture" })),
|
||||
}));
|
||||
|
||||
const initial: ComplexityRouterConfigValue = {
|
||||
classifier_type: "llm",
|
||||
classifier_llm_config: { model: "judge", timeout_ms: 1000 },
|
||||
tiers: { SIMPLE: ["fast"], MEDIUM: ["mid"], COMPLEX: ["strong"], REASONING: ["reasoner"] },
|
||||
};
|
||||
|
||||
function Form() {
|
||||
const [value, setValue] = useState(initial);
|
||||
return (
|
||||
<AutoRouterClassifierTabs value={value} onChange={setValue}>
|
||||
<ClassificationMethodConfig
|
||||
value={value}
|
||||
onChange={setValue}
|
||||
modelOptions={[{ value: "judge", label: "judge" }]}
|
||||
effortOptionsByModel={{ judge: ["low"] }}
|
||||
customTechnicalKeywords={[]}
|
||||
onCustomTechnicalKeywordsChange={() => {}}
|
||||
/>
|
||||
<button
|
||||
onClick={() =>
|
||||
setValue(
|
||||
applyTierSetAction(value, [], {
|
||||
kind: "patch",
|
||||
id: "SIMPLE",
|
||||
patch: { name: "QUICK", definition: "Quick tasks" },
|
||||
}).value,
|
||||
)
|
||||
}
|
||||
>
|
||||
Customize tiers
|
||||
</button>
|
||||
<button
|
||||
onClick={() =>
|
||||
setValue(hydrateComplexityRouterConfig(buildUpdatedComplexityRouterConfig({}, value), undefined))
|
||||
}
|
||||
>
|
||||
Save and reload
|
||||
</button>
|
||||
<button
|
||||
onClick={() => {
|
||||
const request = buildSavedJevConnectionTestRequest(buildUpdatedComplexityRouterConfig({}, value));
|
||||
if (request) void testAutoRouterRouting("token", request);
|
||||
}}
|
||||
>
|
||||
Probe current config
|
||||
</button>
|
||||
</AutoRouterClassifierTabs>
|
||||
);
|
||||
}
|
||||
|
||||
describe("JEV classifier editor", () => {
|
||||
afterEach(() => vi.mocked(useAuthorized).mockReset());
|
||||
it("uses built-in JEV without a license and preserves custom tiers and context through reload", () => {
|
||||
renderWithProviders(<Form />);
|
||||
expect(screen.getByLabelText("Classifier Model")).toBeInTheDocument();
|
||||
expect(screen.getByText("Reasoning Effort")).toBeInTheDocument();
|
||||
expect(screen.getByText("Classifier Prompt")).toBeInTheDocument();
|
||||
expect(screen.getByRole("switch", { name: "Use images for classification" })).toBeInTheDocument();
|
||||
fireEvent.click(screen.getByRole("radio", { name: /JEV Classifier/ }));
|
||||
expect(screen.getByRole("tab", { name: "Complexity" })).toHaveAttribute("aria-selected", "true");
|
||||
expect(screen.getByLabelText("JEV Model")).toHaveValue("jev-latest");
|
||||
expect(screen.getByLabelText("JEV Instructions")).toBeDisabled();
|
||||
expect(screen.queryByLabelText("Classifier Model")).not.toBeInTheDocument();
|
||||
expect(screen.queryByText("Reasoning Effort")).not.toBeInTheDocument();
|
||||
expect(screen.queryByText("Classifier Prompt")).not.toBeInTheDocument();
|
||||
expect(screen.queryByRole("switch", { name: "Use images for classification" })).not.toBeInTheDocument();
|
||||
fireEvent.change(screen.getByLabelText("JEV Model"), { target: { value: "jev-test" } });
|
||||
fireEvent.change(screen.getByLabelText("JEV Timeout (ms)"), { target: { value: "4200" } });
|
||||
fireEvent.change(screen.getByLabelText("Context Window Size"), { target: { value: "6" } });
|
||||
fireEvent.change(screen.getByLabelText("Circuit breaker cooldown (seconds)"), { target: { value: "50" } });
|
||||
fireEvent.click(screen.getByRole("switch", { name: "Classifier circuit breaker" }));
|
||||
fireEvent.click(screen.getByRole("button", { name: "Customize tiers" }));
|
||||
fireEvent.click(screen.getByRole("button", { name: "Save and reload" }));
|
||||
expect(screen.getByRole("radio", { name: /JEV Classifier/ })).toBeChecked();
|
||||
expect(screen.getByLabelText("JEV Model")).toHaveValue("jev-test");
|
||||
expect(screen.getByLabelText("JEV Timeout (ms)")).toHaveValue(4200);
|
||||
expect(screen.getByLabelText("Context Window Size")).toHaveValue("6");
|
||||
expect(screen.getByRole("switch", { name: "Classifier circuit breaker" })).not.toBeChecked();
|
||||
fireEvent.click(screen.getByRole("button", { name: "Probe current config" }));
|
||||
expect(testAutoRouterRouting).toHaveBeenCalledWith(
|
||||
"token",
|
||||
expect.objectContaining({
|
||||
complexity_router_config: expect.objectContaining({
|
||||
classifier_type: "jev",
|
||||
jev_classifier_config: {
|
||||
model: "jev-test",
|
||||
timeout_ms: 4200,
|
||||
circuit_breaker_enabled: false,
|
||||
circuit_breaker_cooldown_seconds: 50,
|
||||
},
|
||||
tiers: expect.objectContaining({ QUICK: ["fast"] }),
|
||||
}),
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("allows licensed instructions and can restore built-in instructions", () => {
|
||||
const authorized = useAuthorized();
|
||||
vi.mocked(useAuthorized).mockReturnValue({ ...authorized, premiumUser: true });
|
||||
const LicensedForm = () => {
|
||||
const [value, setValue] = useState<ComplexityRouterConfigValue>({
|
||||
...initial,
|
||||
classifier_type: "jev",
|
||||
jev_classifier_config: { model: "jev-latest", timeout_ms: 3000, instructions: "Existing instructions" },
|
||||
});
|
||||
return <JevEditor value={value} onChange={setValue} />;
|
||||
};
|
||||
renderWithProviders(<LicensedForm />);
|
||||
expect(screen.getByLabelText("JEV Instructions")).toBeEnabled();
|
||||
fireEvent.change(screen.getByLabelText("JEV Instructions"), { target: { value: "New instructions" } });
|
||||
expect(screen.getByLabelText("JEV Instructions")).toHaveValue("New instructions");
|
||||
fireEvent.click(screen.getByRole("button", { name: "Restore built-in JEV instructions" }));
|
||||
expect(screen.getByLabelText("JEV Instructions")).toHaveValue("");
|
||||
});
|
||||
});
|
||||
|
|
@ -0,0 +1,88 @@
|
|||
import React, { useId } from "react";
|
||||
import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized";
|
||||
import { Button } from "@/components/ui/button";
|
||||
import { Input } from "@/components/ui/input";
|
||||
import { Label } from "@/components/ui/label";
|
||||
import { Textarea } from "@/components/ui/textarea";
|
||||
import { SimpleTooltip } from "@/components/ui/tooltip";
|
||||
import ClassifierCircuitBreakerConfig from "./ClassifierCircuitBreakerConfig";
|
||||
import type { ComplexityRouterConfigValue } from "./ComplexityRouterConfig";
|
||||
import { defaultJevClassifierConfig } from "./jev_classifier_config";
|
||||
|
||||
export default function JevClassifierConfig({
|
||||
value,
|
||||
onChange,
|
||||
}: {
|
||||
value: ComplexityRouterConfigValue;
|
||||
onChange: (value: ComplexityRouterConfigValue) => void;
|
||||
}) {
|
||||
const id = useId();
|
||||
const { premiumUser } = useAuthorized();
|
||||
const config = value.jev_classifier_config ?? defaultJevClassifierConfig();
|
||||
const update = (patch: Partial<typeof config>) =>
|
||||
onChange({ ...value, jev_classifier_config: { ...config, ...patch } });
|
||||
|
||||
return (
|
||||
<div className="mt-4 space-y-3">
|
||||
<p className="text-sm text-muted-foreground">
|
||||
Uses TypeSafe System One Choice evaluation with your configured tiers
|
||||
</p>
|
||||
<div>
|
||||
<Label htmlFor={`${id}-model`}>JEV Model</Label>
|
||||
<Input id={`${id}-model`} value={config.model} onChange={(event) => update({ model: event.target.value })} />
|
||||
</div>
|
||||
<div>
|
||||
<Label htmlFor={`${id}-timeout`}>JEV Timeout (ms)</Label>
|
||||
<Input
|
||||
id={`${id}-timeout`}
|
||||
type="number"
|
||||
min={1}
|
||||
step={1}
|
||||
value={config.timeout_ms}
|
||||
onChange={(event) => update({ timeout_ms: Number(event.target.value) })}
|
||||
/>
|
||||
</div>
|
||||
<ClassifierCircuitBreakerConfig
|
||||
value={config}
|
||||
onChange={(next) =>
|
||||
update({
|
||||
circuit_breaker_enabled: next.circuit_breaker_enabled,
|
||||
circuit_breaker_cooldown_seconds: next.circuit_breaker_cooldown_seconds,
|
||||
})
|
||||
}
|
||||
/>
|
||||
<div>
|
||||
<Label htmlFor={`${id}-instructions`}>JEV Instructions</Label>
|
||||
<SimpleTooltip
|
||||
content={!premiumUser ? "Custom JEV instructions require a LiteLLM Enterprise license" : undefined}
|
||||
>
|
||||
<div>
|
||||
<Textarea
|
||||
id={`${id}-instructions`}
|
||||
value={config.instructions ?? ""}
|
||||
disabled={!premiumUser}
|
||||
placeholder="Leave blank to use the built-in instructions"
|
||||
onChange={(event) => update({ instructions: event.target.value || undefined })}
|
||||
/>
|
||||
</div>
|
||||
</SimpleTooltip>
|
||||
{config.instructions && (
|
||||
<Button variant="outline" type="button" onClick={() => update({ instructions: undefined })}>
|
||||
Restore built-in JEV instructions
|
||||
</Button>
|
||||
)}
|
||||
<p className="text-xs text-muted-foreground">
|
||||
Built-in JEV is available without a license and uses the shipped tier criteria
|
||||
{!premiumUser && (
|
||||
<>
|
||||
. Custom instructions require LiteLLM Enterprise. Get a trial key{" "}
|
||||
<a href="https://www.litellm.ai/#pricing" target="_blank" rel="noopener noreferrer" className="underline">
|
||||
here
|
||||
</a>
|
||||
</>
|
||||
)}
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
|
@ -0,0 +1,148 @@
|
|||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { fireEvent, renderWithProviders, screen, waitFor } from "../../../tests/test-utils";
|
||||
import AutoRouterConnectionTest from "./auto_router_connection_test";
|
||||
import AutoRouterRoutingTest from "./AutoRouterRoutingTest";
|
||||
import { buildAutoRouterTestTargets } from "./build_auto_router_test_targets";
|
||||
import {
|
||||
buildSavedJevConnectionTestRequest,
|
||||
JEV_CONNECTION_TEST_PROMPT,
|
||||
} from "./build_auto_router_routing_test_request";
|
||||
import { buildComplexityRouterConfig } from "./build_complexity_router_config";
|
||||
|
||||
vi.mock(
|
||||
"@/app/(dashboard)/hooks/autoRouter/useComplexityScorerDefaults",
|
||||
async () => await import("../../../tests/mocks/complexityScorerDefaults"),
|
||||
);
|
||||
|
||||
const config = buildComplexityRouterConfig({
|
||||
classifierType: "jev",
|
||||
jevClassifierConfig: { model: "jev-latest", timeout_ms: 3000 },
|
||||
tiers: { SIMPLE: ["fast"], MEDIUM: ["mid"], COMPLEX: ["strong"], REASONING: ["reasoner"] },
|
||||
defaultModel: undefined,
|
||||
planModeMinTier: undefined,
|
||||
tierLabels: undefined,
|
||||
classifierLlmConfig: undefined,
|
||||
classifierContextWindowSize: undefined,
|
||||
classifierContextBudgetChars: undefined,
|
||||
classifierContextIncludeAssistantTurns: undefined,
|
||||
classifierFallback: undefined,
|
||||
classificationPrompt: undefined,
|
||||
classificationExamples: undefined,
|
||||
heuristicFirstMaxTier: undefined,
|
||||
classificationMode: undefined,
|
||||
sessionAffinity: false,
|
||||
deploymentAffinity: true,
|
||||
customTechnicalKeywords: [],
|
||||
keywordTierRules: [],
|
||||
semanticMatchingEnabled: false,
|
||||
embeddingModel: undefined,
|
||||
matchThreshold: 0.5,
|
||||
escalationKeywords: [],
|
||||
adaptive: false,
|
||||
adaptiveWeights: { quality: 0.3, cost: 0.7 },
|
||||
tierDistancePenalty: 0.5,
|
||||
adaptiveEligible: "all",
|
||||
returnRawModelName: false,
|
||||
});
|
||||
const request = buildSavedJevConnectionTestRequest(JSON.stringify(config), "fast", "my-router");
|
||||
const targets = buildAutoRouterTestTargets({
|
||||
tiers: Object.entries(config.tiers),
|
||||
semanticMatchingEnabled: false,
|
||||
embeddingModel: undefined,
|
||||
});
|
||||
const response = (cause: string) => ({
|
||||
routed_model: "fast",
|
||||
routed_model_configured: true,
|
||||
routing_decision: {
|
||||
cause,
|
||||
tier: "SIMPLE",
|
||||
classifier_model: "jev-latest",
|
||||
classifier_confidence: 0.8,
|
||||
classifier_probabilities: { SIMPLE: 0.8, REASONING: 0.2 },
|
||||
classifier_cost: 0.00001234,
|
||||
},
|
||||
});
|
||||
|
||||
afterEach(() => vi.unstubAllGlobals());
|
||||
|
||||
describe("JEV network probes", () => {
|
||||
it.each(["jev_classifier", "classifier_fallback", "default_model_fallback", "keyword_match"])(
|
||||
"probes the routing endpoint independently of tier models and checks the cause %s",
|
||||
async (cause) => {
|
||||
const fetchMock = vi.fn<typeof fetch>(
|
||||
async (input) =>
|
||||
new Response(JSON.stringify(String(input).endsWith("/auto_router/test_routing") ? response(cause) : {})),
|
||||
);
|
||||
vi.stubGlobal("fetch", fetchMock);
|
||||
const onTestComplete = vi.fn();
|
||||
renderWithProviders(
|
||||
<AutoRouterConnectionTest
|
||||
accessToken="test-token"
|
||||
targets={targets}
|
||||
jevRequest={request}
|
||||
onTestComplete={onTestComplete}
|
||||
/>,
|
||||
);
|
||||
await waitFor(() => expect(onTestComplete).toHaveBeenCalledOnce());
|
||||
expect(fetchMock).toHaveBeenCalledWith(
|
||||
expect.stringContaining("/auto_router/test_routing"),
|
||||
expect.objectContaining({
|
||||
method: "POST",
|
||||
body: expect.any(String),
|
||||
}),
|
||||
);
|
||||
const routingCall = fetchMock.mock.calls.find(([url]) => String(url).endsWith("/auto_router/test_routing"));
|
||||
expect(JSON.parse(String(routingCall?.[1]?.body))).toEqual({
|
||||
prompt: JEV_CONNECTION_TEST_PROMPT,
|
||||
complexity_router_config: config,
|
||||
default_model: "fast",
|
||||
router_name: "my-router",
|
||||
});
|
||||
expect(fetchMock).toHaveBeenCalledTimes(5);
|
||||
expect(screen.getAllByTestId("test-status-success")).toHaveLength(4);
|
||||
expect(screen.getByRole("status", { name: "JEV connection" })).toHaveTextContent(
|
||||
cause === "jev_classifier"
|
||||
? "JEV classification succeeded"
|
||||
: `JEV was not reached successfully (routing cause: ${cause})`,
|
||||
);
|
||||
},
|
||||
);
|
||||
|
||||
it("shows routing diagnostics from the real networking response", async () => {
|
||||
vi.stubGlobal(
|
||||
"fetch",
|
||||
vi.fn<typeof fetch>(async () => new Response(JSON.stringify(response("jev_classifier")))),
|
||||
);
|
||||
renderWithProviders(
|
||||
<AutoRouterRoutingTest
|
||||
accessToken="token"
|
||||
config={config}
|
||||
defaultModel="fast"
|
||||
routerName="router"
|
||||
teamId={undefined}
|
||||
/>,
|
||||
);
|
||||
fireEvent.change(screen.getByTestId("auto-router-routing-test-prompt"), { target: { value: "Hello" } });
|
||||
fireEvent.click(screen.getByTestId("auto-router-routing-test-send"));
|
||||
expect(await screen.findByText("JEV classifier")).toBeInTheDocument();
|
||||
expect(screen.getByText("jev-latest")).toBeInTheDocument();
|
||||
expect(screen.getByText("80.0%")).toBeInTheDocument();
|
||||
expect(screen.getByText("SIMPLE: 80.0%")).toBeInTheDocument();
|
||||
expect(screen.getByText("REASONING: 20.0%")).toBeInTheDocument();
|
||||
expect(screen.getByText("$0.00001234")).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("reports a classifier endpoint error while still checking downstream models", async () => {
|
||||
vi.stubGlobal(
|
||||
"fetch",
|
||||
vi.fn<typeof fetch>(async (input) =>
|
||||
String(input).endsWith("/auto_router/test_routing")
|
||||
? new Response(JSON.stringify({ detail: "JEV classifier unavailable" }), { status: 503 })
|
||||
: new Response("{}"),
|
||||
),
|
||||
);
|
||||
renderWithProviders(<AutoRouterConnectionTest accessToken="token" targets={targets} jevRequest={request} />);
|
||||
expect(await screen.findByText("JEV classifier unavailable")).toBeInTheDocument();
|
||||
expect(screen.getAllByTestId("test-status-success")).toHaveLength(4);
|
||||
});
|
||||
});
|
||||
|
|
@ -39,7 +39,7 @@ const NonReasoningTierToggle: React.FC<{
|
|||
<span className="block text-xs text-muted-foreground">
|
||||
Adds NON_REASONING below Simple, for operational agent traffic that relays or reformats information rather than
|
||||
reasoning about it. Escalation still moves up out of it when a request needs more.
|
||||
{!available && " Requires the LLM classification method."}
|
||||
{!available && " Requires the LLM or JEV classification method"}
|
||||
</span>
|
||||
<Separator className="my-4" />
|
||||
</>
|
||||
|
|
|
|||
|
|
@ -4,6 +4,9 @@ import { type ComplexityRouterConfigValue, heuristicScoringRole, usesLlmClassifi
|
|||
import { restrictedBy } from "./TierRestrictions";
|
||||
|
||||
const tierConfigIntroText = (value: ComplexityRouterConfigValue): string => {
|
||||
if (value.classifier_type === "jev") {
|
||||
return "JEV classifies each request with TypeSafe System One Choice evaluation and routes it to a tier. Configure which models handle each tier";
|
||||
}
|
||||
if (value.classifier_type === "heuristic_v2") {
|
||||
return "The complexity router classifies each request with a calibrated local four-tier model (no API calls). Configure which model(s) handle each tier.";
|
||||
}
|
||||
|
|
|
|||
|
|
@ -57,7 +57,11 @@ import {
|
|||
import { activeTierName, activeTierRows, getCustomTierRowsError, resolveComplexityDefaultModel } from "./tier_rows";
|
||||
import { tierRowLabel } from "./complexity_router_tiers";
|
||||
import { buildAutoRouterTestTargets, AutoRouterTestTarget } from "./build_auto_router_test_targets";
|
||||
import AutoRouterConnectionTest from "./auto_router_connection_test";
|
||||
import { AutoRouterConnectionTestDialog } from "./auto_router_connection_test";
|
||||
import {
|
||||
buildAutoRouterRoutingTestRequest,
|
||||
JEV_CONNECTION_TEST_PROMPT,
|
||||
} from "./build_auto_router_routing_test_request";
|
||||
import AutoRouterRoutingTest from "./AutoRouterRoutingTest";
|
||||
import { toast } from "@/lib/toast";
|
||||
import {
|
||||
|
|
@ -405,6 +409,7 @@ const AddAutoRouterTab: React.FC<AddAutoRouterTabProps> = ({
|
|||
classificationMode: complexityRouterConfig.classification_mode,
|
||||
tierLabels: complexityRouterConfig.tier_labels,
|
||||
classifierType: complexityRouterConfig.classifier_type,
|
||||
jevClassifierConfig: complexityRouterConfig.jev_classifier_config,
|
||||
capabilityClassifierConfig: complexityRouterConfig.capability_classifier_config,
|
||||
llmV2Config: complexityRouterConfig.llm_v2_config,
|
||||
classifierLlmConfig: complexityRouterConfig.classifier_llm_config,
|
||||
|
|
@ -839,41 +844,31 @@ const AddAutoRouterTab: React.FC<AddAutoRouterTabProps> = ({
|
|||
</DialogContent>
|
||||
</Dialog>
|
||||
|
||||
<Dialog
|
||||
<AutoRouterConnectionTestDialog
|
||||
open={isTestModalVisible}
|
||||
onOpenChange={(open) => {
|
||||
if (!open) {
|
||||
setIsTestModalVisible(false);
|
||||
setIsTestingConnection(false);
|
||||
}
|
||||
onClose={() => {
|
||||
setIsTestModalVisible(false);
|
||||
setIsTestingConnection(false);
|
||||
}}
|
||||
>
|
||||
<DialogContent className="max-h-[calc(100dvh-2rem)] overflow-y-auto sm:max-w-[700px]">
|
||||
<DialogHeader>
|
||||
<DialogTitle>Connection Test Results</DialogTitle>
|
||||
</DialogHeader>
|
||||
{isTestModalVisible && (
|
||||
<AutoRouterConnectionTest
|
||||
key={connectionTestId}
|
||||
accessToken={accessToken}
|
||||
targets={testTargets}
|
||||
onTestComplete={() => setIsTestingConnection(false)}
|
||||
/>
|
||||
)}
|
||||
<DialogFooter>
|
||||
{" "}
|
||||
<Button
|
||||
variant="outline"
|
||||
onClick={() => {
|
||||
setIsTestModalVisible(false);
|
||||
setIsTestingConnection(false);
|
||||
}}
|
||||
>
|
||||
Close
|
||||
</Button>
|
||||
</DialogFooter>
|
||||
</DialogContent>
|
||||
</Dialog>
|
||||
testId={connectionTestId}
|
||||
accessToken={accessToken}
|
||||
targets={testTargets}
|
||||
jevRequest={
|
||||
effectiveClassifierType(complexityRouterConfig) === "jev"
|
||||
? buildAutoRouterRoutingTestRequest({
|
||||
prompt: JEV_CONNECTION_TEST_PROMPT,
|
||||
config: buildComplexityRouterConfig(complexityRouterConfigParams),
|
||||
defaultModel: resolveComplexityDefaultModel(
|
||||
complexityRouterConfig,
|
||||
complexityRouterConfig.default_model,
|
||||
),
|
||||
routerName: watchedName,
|
||||
teamId: requiresTeamScope ? watchedTeamId ?? undefined : undefined,
|
||||
})
|
||||
: undefined
|
||||
}
|
||||
onTestComplete={() => setIsTestingConnection(false)}
|
||||
/>
|
||||
</TooltipProvider>
|
||||
);
|
||||
};
|
||||
|
|
|
|||
|
|
@ -1,12 +1,20 @@
|
|||
import React from "react";
|
||||
import { CircleCheck, CircleX, LoaderCircle } from "lucide-react";
|
||||
|
||||
import { testModelGroupConnection, ModelGroupConnectionResult } from "../networking";
|
||||
import {
|
||||
testModelGroupConnection,
|
||||
ModelGroupConnectionResult,
|
||||
testAutoRouterRouting,
|
||||
AutoRouterRoutingTestRequest,
|
||||
} from "../networking";
|
||||
import { AutoRouterTestTarget } from "./build_auto_router_test_targets";
|
||||
import { Dialog, DialogContent, DialogFooter, DialogHeader, DialogTitle } from "@/components/ui/dialog";
|
||||
import { Button } from "@/components/ui/button";
|
||||
|
||||
interface AutoRouterConnectionTestProps {
|
||||
accessToken: string;
|
||||
targets: AutoRouterTestTarget[];
|
||||
jevRequest?: AutoRouterRoutingTestRequest;
|
||||
onTestComplete?: () => void;
|
||||
}
|
||||
|
||||
|
|
@ -20,15 +28,36 @@ const cleanErrorMessage = (error: string): string => {
|
|||
const AutoRouterConnectionTest: React.FC<AutoRouterConnectionTestProps> = ({
|
||||
accessToken,
|
||||
targets,
|
||||
jevRequest,
|
||||
onTestComplete,
|
||||
}) => {
|
||||
const [results, setResults] = React.useState<TargetResult[]>(() => targets.map(() => ({ status: "pending" })));
|
||||
const [jevResult, setJevResult] = React.useState<TargetResult>({ status: "pending" });
|
||||
|
||||
React.useEffect(() => {
|
||||
let cancelled = false;
|
||||
const probeJev = async () => {
|
||||
if (!jevRequest) return;
|
||||
const response = await testAutoRouterRouting(accessToken, jevRequest);
|
||||
if (cancelled) return;
|
||||
if (response.status === "error") {
|
||||
setJevResult(response);
|
||||
return;
|
||||
}
|
||||
const decision = response.result.routing_decision;
|
||||
setJevResult(
|
||||
decision.cause === "jev_classifier"
|
||||
? { status: "success" }
|
||||
: {
|
||||
status: "error",
|
||||
error: `JEV was not reached successfully (routing cause: ${decision.cause ?? "unknown"})`,
|
||||
},
|
||||
);
|
||||
};
|
||||
const run = async () => {
|
||||
await Promise.all(
|
||||
targets.map(async (target, index) => {
|
||||
await Promise.all([
|
||||
probeJev(),
|
||||
...targets.map(async (target, index) => {
|
||||
const result = target.requestParams
|
||||
? await testModelGroupConnection(accessToken, target.modelGroup, target.mode, target.requestParams)
|
||||
: await testModelGroupConnection(accessToken, target.modelGroup, target.mode);
|
||||
|
|
@ -37,7 +66,7 @@ const AutoRouterConnectionTest: React.FC<AutoRouterConnectionTestProps> = ({
|
|||
result.status === "error" ? { status: "error", error: cleanErrorMessage(result.error) } : result;
|
||||
setResults((prev) => prev.map((r, i) => (i === index ? cleaned : r)));
|
||||
}),
|
||||
);
|
||||
]);
|
||||
if (!cancelled && onTestComplete) onTestComplete();
|
||||
};
|
||||
run();
|
||||
|
|
@ -47,7 +76,7 @@ const AutoRouterConnectionTest: React.FC<AutoRouterConnectionTestProps> = ({
|
|||
// eslint-disable-next-line react-hooks/exhaustive-deps -- probes run once per mount; the parent remounts via `key` to start a fresh test, and re-running on prop identity changes would refire paid requests
|
||||
}, []);
|
||||
|
||||
if (targets.length === 0) {
|
||||
if (targets.length === 0 && !jevRequest) {
|
||||
return (
|
||||
<p className="text-sm text-muted-foreground">
|
||||
No complexity tiers are configured yet, so there is nothing to test.
|
||||
|
|
@ -61,6 +90,16 @@ const AutoRouterConnectionTest: React.FC<AutoRouterConnectionTestProps> = ({
|
|||
Test Connection sends a minimal request to every configured tier, classifier, default, and embedding model. The
|
||||
classifier probe includes its reasoning effort override.
|
||||
</p>
|
||||
{jevRequest && (
|
||||
<div role="status" aria-label="JEV connection" className="rounded-lg border p-3 text-sm">
|
||||
<strong>JEV Classifier</strong>
|
||||
<p>
|
||||
{jevResult.status === "pending" && "Testing JEV classification"}
|
||||
{jevResult.status === "success" && "JEV classification succeeded"}
|
||||
{jevResult.status === "error" && jevResult.error}
|
||||
</p>
|
||||
</div>
|
||||
)}
|
||||
{targets.map((target, index) => {
|
||||
const result = results[index] ?? { status: "pending" };
|
||||
return (
|
||||
|
|
@ -100,3 +139,26 @@ const AutoRouterConnectionTest: React.FC<AutoRouterConnectionTestProps> = ({
|
|||
};
|
||||
|
||||
export default AutoRouterConnectionTest;
|
||||
|
||||
export function AutoRouterConnectionTestDialog({
|
||||
open,
|
||||
onClose,
|
||||
testId,
|
||||
...props
|
||||
}: AutoRouterConnectionTestProps & { open: boolean; onClose: () => void; testId: number }) {
|
||||
return (
|
||||
<Dialog open={open} onOpenChange={(next) => !next && onClose()}>
|
||||
<DialogContent className="max-h-[calc(100dvh-2rem)] overflow-y-auto sm:max-w-[700px]">
|
||||
<DialogHeader>
|
||||
<DialogTitle>Connection Test Results</DialogTitle>
|
||||
</DialogHeader>
|
||||
{open && <AutoRouterConnectionTest key={testId} {...props} />}
|
||||
<DialogFooter>
|
||||
<Button variant="outline" onClick={onClose}>
|
||||
Close
|
||||
</Button>
|
||||
</DialogFooter>
|
||||
</DialogContent>
|
||||
</Dialog>
|
||||
);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,4 +1,9 @@
|
|||
import { buildAutoRouterRoutingTestRequest } from "./build_auto_router_routing_test_request";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import {
|
||||
buildAutoRouterRoutingTestRequest,
|
||||
buildSavedJevConnectionTestRequest,
|
||||
JEV_CONNECTION_TEST_PROMPT,
|
||||
} from "./build_auto_router_routing_test_request";
|
||||
import { ComplexityRouterConfigPayload } from "./build_complexity_router_config";
|
||||
|
||||
const CONFIG = {
|
||||
|
|
@ -15,6 +20,36 @@ const params = {
|
|||
};
|
||||
|
||||
describe("buildAutoRouterRoutingTestRequest", () => {
|
||||
it.each(["object", "json"])("probes saved JEV %s configuration with custom tiers and team context", (format) => {
|
||||
const config = {
|
||||
classifier_type: "jev",
|
||||
jev_classifier_config: { model: "jev-test", timeout_ms: 900 },
|
||||
tiers: { QUICK: ["fast"], DEEP: ["strong"] },
|
||||
tier_definitions: { QUICK: "Simple questions", DEEP: "Complex questions" },
|
||||
fallback_tier: "DEEP",
|
||||
classifier_context_window_size: 4,
|
||||
};
|
||||
expect(
|
||||
buildSavedJevConnectionTestRequest(
|
||||
format === "json" ? JSON.stringify(config) : config,
|
||||
"strong",
|
||||
"saved-router",
|
||||
"team-1",
|
||||
),
|
||||
).toEqual({
|
||||
prompt: JEV_CONNECTION_TEST_PROMPT,
|
||||
complexity_router_config: config,
|
||||
default_model: "strong",
|
||||
router_name: "saved-router",
|
||||
team_id: "team-1",
|
||||
});
|
||||
});
|
||||
it.each([undefined, null, "not json", "[]", {}, { classifier_type: "llm", tiers: {} }, { classifier_type: "jev" }])(
|
||||
"does not build a JEV probe for invalid or other classifier configurations: %j",
|
||||
(config) => {
|
||||
expect(buildSavedJevConnectionTestRequest(config)).toBeUndefined();
|
||||
},
|
||||
);
|
||||
it("sends the prompt with the config being edited", () => {
|
||||
const request = buildAutoRouterRoutingTestRequest(params);
|
||||
|
||||
|
|
|
|||
|
|
@ -1,5 +1,38 @@
|
|||
import { AutoRouterRoutingTestRequest } from "../networking";
|
||||
import { ComplexityRouterConfigPayload } from "./build_complexity_router_config";
|
||||
import { z } from "zod";
|
||||
|
||||
export const JEV_CONNECTION_TEST_PROMPT = "What is 2 plus 2?";
|
||||
|
||||
export const buildSavedJevConnectionTestRequest = (
|
||||
rawConfig: unknown,
|
||||
defaultModel?: string,
|
||||
routerName?: string,
|
||||
teamId?: string,
|
||||
): AutoRouterRoutingTestRequest | undefined => {
|
||||
const parsed: unknown =
|
||||
typeof rawConfig === "string"
|
||||
? (() => {
|
||||
try {
|
||||
return JSON.parse(rawConfig) as unknown;
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
})()
|
||||
: rawConfig;
|
||||
const result = z
|
||||
.object({ classifier_type: z.literal("jev"), tiers: z.record(z.unknown()) })
|
||||
.passthrough()
|
||||
.safeParse(parsed);
|
||||
if (!result.success) return undefined;
|
||||
return {
|
||||
prompt: JEV_CONNECTION_TEST_PROMPT,
|
||||
complexity_router_config: result.data,
|
||||
...(defaultModel && { default_model: defaultModel }),
|
||||
...(routerName && { router_name: routerName }),
|
||||
...(teamId && { team_id: teamId }),
|
||||
};
|
||||
};
|
||||
|
||||
export interface BuildAutoRouterRoutingTestRequestParams {
|
||||
prompt: string;
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
import { describe, expect, it } from "vitest";
|
||||
import {
|
||||
buildComplexityRouterConfig,
|
||||
getPlanModeTierError,
|
||||
|
|
@ -24,6 +25,11 @@ const tiers = {
|
|||
|
||||
const baseParams: BuildComplexityRouterConfigParams = {
|
||||
tiers,
|
||||
defaultModel: undefined,
|
||||
planModeMinTier: undefined,
|
||||
classificationExamples: undefined,
|
||||
heuristicFirstMaxTier: undefined,
|
||||
classificationMode: undefined,
|
||||
tierLabels: undefined,
|
||||
classifierType: "heuristic",
|
||||
classifierLlmConfig: undefined,
|
||||
|
|
@ -48,6 +54,94 @@ const baseParams: BuildComplexityRouterConfigParams = {
|
|||
};
|
||||
|
||||
describe("buildComplexityRouterConfig", () => {
|
||||
it("accepts built-in JEV defaults without an LLM classifier model", () => {
|
||||
expect(getClassifierModelError({ classifier_type: "jev" })).toBeNull();
|
||||
});
|
||||
|
||||
it.each([
|
||||
{ model: "" },
|
||||
{ model: " " },
|
||||
{ timeout_ms: 0 },
|
||||
{ timeout_ms: 1.5 },
|
||||
{ timeout_ms: Number.NaN },
|
||||
{ circuit_breaker_cooldown_seconds: -1 },
|
||||
{ circuit_breaker_cooldown_seconds: Number.POSITIVE_INFINITY },
|
||||
])("rejects invalid JEV settings before saving or testing: %j", (patch) => {
|
||||
expect(
|
||||
getClassifierModelError({
|
||||
classifier_type: "jev",
|
||||
jev_classifier_config: { model: "jev-latest", timeout_ms: 3000, ...patch },
|
||||
}),
|
||||
).toBe("Enter a JEV model, a positive whole-number timeout and a positive cooldown");
|
||||
});
|
||||
|
||||
it.each([false, true])("serializes JEV with shared context and no LLM config, custom tiers: %s", (custom) => {
|
||||
const config = buildComplexityRouterConfig({
|
||||
...baseParams,
|
||||
classifierType: "jev",
|
||||
jevClassifierConfig: {
|
||||
model: "jev-test",
|
||||
timeout_ms: 4500,
|
||||
instructions: " Choose the configured tier ",
|
||||
circuit_breaker_enabled: false,
|
||||
circuit_breaker_cooldown_seconds: 12.5,
|
||||
},
|
||||
classifierLlmConfig: { model: "stale", timeout_ms: 30 },
|
||||
classificationPrompt: "stale prompt",
|
||||
classificationExamples: "stale examples",
|
||||
classifierContextWindowSize: 4,
|
||||
classifierContextBudgetChars: 2000,
|
||||
classifierContextIncludeAssistantTurns: true,
|
||||
classifierFallback: "default_model",
|
||||
...(custom && {
|
||||
customTierSet: {
|
||||
tiers: [
|
||||
{ id: "quick", name: "QUICK", definition: "Short answers", models: ["fast"] },
|
||||
{ id: "review", name: "REVIEW", definition: "Deep review", models: ["strong"] },
|
||||
],
|
||||
fallback_tier_id: "quick",
|
||||
},
|
||||
}),
|
||||
});
|
||||
expect(config.classifier_type).toBe("jev");
|
||||
expect(config.jev_classifier_config).toEqual({
|
||||
model: "jev-test",
|
||||
timeout_ms: 4500,
|
||||
instructions: "Choose the configured tier",
|
||||
circuit_breaker_enabled: false,
|
||||
circuit_breaker_cooldown_seconds: 12.5,
|
||||
});
|
||||
expect(config.classifier_context_window_size).toBe(4);
|
||||
expect(config.classifier_context_budget_chars).toBe(2000);
|
||||
expect(config.classifier_context_include_assistant_turns).toBe(true);
|
||||
expect(config).not.toHaveProperty("classifier_llm_config");
|
||||
expect(config).not.toHaveProperty("classification_prompt");
|
||||
expect(config).not.toHaveProperty("classification_examples");
|
||||
if (custom) {
|
||||
expect(config.tiers).toEqual({ QUICK: ["fast"], REVIEW: ["strong"] });
|
||||
expect(config.fallback_tier).toBe("QUICK");
|
||||
} else {
|
||||
expect(config.classifier_fallback).toBe("default_model");
|
||||
expect(config.tiers).toEqual(tiers);
|
||||
}
|
||||
});
|
||||
|
||||
it("omits blank JEV instructions and ignores stale JEV settings when saving LLM", () => {
|
||||
const jev = buildComplexityRouterConfig({
|
||||
...baseParams,
|
||||
classifierType: "jev",
|
||||
jevClassifierConfig: { model: "jev-latest", timeout_ms: 3000, instructions: " " },
|
||||
});
|
||||
expect(jev.jev_classifier_config).toEqual({ model: "jev-latest", timeout_ms: 3000 });
|
||||
const llm = buildComplexityRouterConfig({
|
||||
...baseParams,
|
||||
classifierType: "llm",
|
||||
classifierLlmConfig: { model: "judge", timeout_ms: 1000 },
|
||||
jevClassifierConfig: jev.jev_classifier_config,
|
||||
});
|
||||
expect(llm).not.toHaveProperty("jev_classifier_config");
|
||||
});
|
||||
|
||||
it.each(["capability", "llm_v2", "heuristic"] as const)(
|
||||
"disables the removed overrides only for forecast creates: %s",
|
||||
(classifierType) => {
|
||||
|
|
|
|||
|
|
@ -6,6 +6,11 @@ import {
|
|||
} from "./forecast_classifier_config";
|
||||
import type { ModelGroup } from "../llm_calls/fetch_models";
|
||||
import { KeywordTierRule } from "./KeywordTierRules";
|
||||
import {
|
||||
type JevClassifierConfig,
|
||||
jevClassifierConfigSchema,
|
||||
normalizeJevClassifierConfig,
|
||||
} from "./jev_classifier_config";
|
||||
import {
|
||||
type CustomTierSet,
|
||||
type TierRow,
|
||||
|
|
@ -44,6 +49,7 @@ import {
|
|||
effectiveTierLabel,
|
||||
heuristicScoringRoleFor,
|
||||
usesLlmClassifier,
|
||||
usesClassifierContext,
|
||||
} from "./ComplexityRouterConfig";
|
||||
|
||||
export type ClassifierVisionConfig = { enabled?: boolean; max_images?: number };
|
||||
|
|
@ -133,7 +139,7 @@ const scorerKnobPayload = ({
|
|||
};
|
||||
|
||||
export interface StoredComplexityRouterConfig {
|
||||
tiers?: Partial<Record<keyof ComplexityTiers, unknown>>;
|
||||
tiers?: Record<string, unknown>;
|
||||
enable_non_reasoning_tier?: boolean;
|
||||
tier_model_configs?: unknown;
|
||||
default_model?: string | null;
|
||||
|
|
@ -147,6 +153,7 @@ export interface StoredComplexityRouterConfig {
|
|||
capability_classifier_config?: unknown;
|
||||
llm_v2_config?: unknown;
|
||||
classifier_llm_config?: ClassifierLLMConfig;
|
||||
jev_classifier_config?: unknown;
|
||||
classifier_context_window_size?: unknown;
|
||||
classifier_context_budget_chars?: unknown;
|
||||
classifier_context_include_assistant_turns?: unknown;
|
||||
|
|
@ -185,6 +192,7 @@ export interface BuildComplexityRouterConfigParams {
|
|||
capabilityClassifierConfig?: CapabilitySettings;
|
||||
llmV2Config?: FuseSettings;
|
||||
classifierLlmConfig: ClassifierLLMConfigWire | undefined;
|
||||
jevClassifierConfig?: JevClassifierConfig;
|
||||
classifierContextWindowSize: number | undefined;
|
||||
classifierContextBudgetChars: number | undefined;
|
||||
classifierContextIncludeAssistantTurns: boolean | undefined;
|
||||
|
|
@ -251,6 +259,7 @@ export interface ComplexityRouterConfigPayload {
|
|||
capability_classifier_config?: CapabilitySettings;
|
||||
llm_v2_config?: FuseSettings;
|
||||
classifier_llm_config?: ClassifierLLMConfig;
|
||||
jev_classifier_config?: JevClassifierConfig;
|
||||
classifier_context_window_size?: number;
|
||||
classifier_context_budget_chars?: number;
|
||||
classifier_context_per_turn_chars?: number;
|
||||
|
|
@ -356,11 +365,16 @@ export const getKeywordTierRulesError = (
|
|||
return `Keyword rule(s) ${orphaned.join(", ")} route to a tier this router no longer has`;
|
||||
};
|
||||
|
||||
// An edited tier set forces the LLM classifier, so the model requirement follows the EFFECTIVE type.
|
||||
// Both forms' submit gates and their submit handlers read this one answer so they cannot drift.
|
||||
export const getClassifierModelError = (
|
||||
config: Pick<ComplexityRouterConfigValue, "custom_tier_set" | "classifier_type" | "classifier_llm_config">,
|
||||
config: Pick<
|
||||
ComplexityRouterConfigValue,
|
||||
"custom_tier_set" | "classifier_type" | "classifier_llm_config" | "jev_classifier_config"
|
||||
>,
|
||||
): string | null => {
|
||||
if (effectiveClassifierType(config) === "jev") {
|
||||
const parsed = jevClassifierConfigSchema.safeParse(config.jev_classifier_config ?? {});
|
||||
return parsed.success ? null : "Enter a JEV model, a positive whole-number timeout and a positive cooldown";
|
||||
}
|
||||
if (!usesLlmClassifier(effectiveClassifierType(config)) || config.classifier_llm_config?.model) return null;
|
||||
return config.custom_tier_set
|
||||
? "Please select a classifier model: an edited tier set routes with the LLM classifier"
|
||||
|
|
@ -395,6 +409,7 @@ export const getSemanticConfigError = ({
|
|||
};
|
||||
|
||||
interface CustomTierWireFieldInputs {
|
||||
classifierType?: ClassifierType;
|
||||
classifierLlmConfig: ClassifierLLMConfigWire | undefined;
|
||||
planModeMinTierId: string | undefined;
|
||||
classificationPrompt: string | undefined;
|
||||
|
|
@ -403,7 +418,13 @@ interface CustomTierWireFieldInputs {
|
|||
|
||||
export const customTierWireFields = (
|
||||
customTierSet: CustomTierSet,
|
||||
{ classifierLlmConfig, planModeMinTierId, classificationPrompt, classificationExamples }: CustomTierWireFieldInputs,
|
||||
{
|
||||
classifierType,
|
||||
classifierLlmConfig,
|
||||
planModeMinTierId,
|
||||
classificationPrompt,
|
||||
classificationExamples,
|
||||
}: CustomTierWireFieldInputs,
|
||||
): Partial<ComplexityRouterConfigPayload> => {
|
||||
const rows = customTierSet.tiers;
|
||||
const fallback = tierRowById(rows, customTierSet.fallback_tier_id);
|
||||
|
|
@ -412,27 +433,30 @@ export const customTierWireFields = (
|
|||
tiers: Object.fromEntries(rows.map((row) => [activeTierName(row), row.models])),
|
||||
tier_definitions: tierDefinitionsFromRows(rows),
|
||||
...(fallback && { fallback_tier: activeTierName(fallback) }),
|
||||
classifier_type: "llm",
|
||||
classifier_type: classifierType === "jev" ? "jev" : "llm",
|
||||
// Rebuilt from the fields an edited tier set allows. The backend rejects system_prompt and
|
||||
// classification_rubric beside tier_definitions, and both live inside this object rather than at
|
||||
// the top level the omit list covers. The opening instructions ride classification_prompt below.
|
||||
...(classifierLlmConfig && {
|
||||
classifier_llm_config: {
|
||||
model: classifierLlmConfig.model,
|
||||
timeout_ms: classifierLlmConfig.timeout_ms,
|
||||
...(classifierLlmConfig.circuit_breaker_enabled !== undefined && {
|
||||
circuit_breaker_enabled: classifierLlmConfig.circuit_breaker_enabled,
|
||||
}),
|
||||
...(classifierLlmConfig.circuit_breaker_cooldown_seconds !== undefined && {
|
||||
circuit_breaker_cooldown_seconds: classifierLlmConfig.circuit_breaker_cooldown_seconds,
|
||||
}),
|
||||
...(classifierLlmConfig.reasoning_effort && { reasoning_effort: classifierLlmConfig.reasoning_effort }),
|
||||
...(classifierLlmConfig.vision && { vision: classifierLlmConfig.vision }),
|
||||
},
|
||||
}),
|
||||
...(classifierType !== "jev" &&
|
||||
classifierLlmConfig && {
|
||||
classifier_llm_config: {
|
||||
model: classifierLlmConfig.model,
|
||||
timeout_ms: classifierLlmConfig.timeout_ms,
|
||||
...(classifierLlmConfig.circuit_breaker_enabled !== undefined && {
|
||||
circuit_breaker_enabled: classifierLlmConfig.circuit_breaker_enabled,
|
||||
}),
|
||||
...(classifierLlmConfig.circuit_breaker_cooldown_seconds !== undefined && {
|
||||
circuit_breaker_cooldown_seconds: classifierLlmConfig.circuit_breaker_cooldown_seconds,
|
||||
}),
|
||||
...(classifierLlmConfig.reasoning_effort && { reasoning_effort: classifierLlmConfig.reasoning_effort }),
|
||||
...(classifierLlmConfig.vision && { vision: classifierLlmConfig.vision }),
|
||||
},
|
||||
}),
|
||||
session_affinity: false,
|
||||
...(classificationPrompt?.trim() && { classification_prompt: classificationPrompt.trim() }),
|
||||
...(classificationExamples?.trim() && { classification_examples: classificationExamples.trim() }),
|
||||
...(classifierType !== "jev" &&
|
||||
classificationPrompt?.trim() && { classification_prompt: classificationPrompt.trim() }),
|
||||
...(classifierType !== "jev" &&
|
||||
classificationExamples?.trim() && { classification_examples: classificationExamples.trim() }),
|
||||
...(floor && { plan_mode_min_tier: activeTierName(floor) }),
|
||||
};
|
||||
};
|
||||
|
|
@ -521,7 +545,7 @@ const classifierWireFields = (
|
|||
| "classifierContextIncludeAssistantTurns"
|
||||
>,
|
||||
): Partial<ComplexityRouterConfigPayload> => {
|
||||
const supportsFallback = usesLlmClassifier(effectiveType) && !isForecastClassifier(effectiveType);
|
||||
const supportsFallback = usesClassifierContext(effectiveType) && !isForecastClassifier(effectiveType);
|
||||
return {
|
||||
...(usesLlmClassifier(effectiveType) &&
|
||||
classifierLlmConfig && {
|
||||
|
|
@ -534,15 +558,15 @@ const classifierWireFields = (
|
|||
heuristicFirstMaxTier?.trim() && { heuristic_first_max_tier: heuristicFirstMaxTier }),
|
||||
...(effectiveType === "hybrid" &&
|
||||
hybridBoundaryMargin !== undefined && { hybrid_boundary_margin: hybridBoundaryMargin }),
|
||||
...(usesLlmClassifier(effectiveType) &&
|
||||
...(usesClassifierContext(effectiveType) &&
|
||||
classifierContextWindowSize !== undefined && {
|
||||
classifier_context_window_size: classifierContextWindowSize,
|
||||
}),
|
||||
...(usesLlmClassifier(effectiveType) &&
|
||||
...(usesClassifierContext(effectiveType) &&
|
||||
classifierContextBudgetChars !== undefined && {
|
||||
classifier_context_budget_chars: classifierContextBudgetChars,
|
||||
}),
|
||||
...(usesLlmClassifier(effectiveType) &&
|
||||
...(usesClassifierContext(effectiveType) &&
|
||||
classifierContextIncludeAssistantTurns !== undefined && {
|
||||
classifier_context_include_assistant_turns: classifierContextIncludeAssistantTurns,
|
||||
}),
|
||||
|
|
@ -560,6 +584,7 @@ export const buildComplexityRouterConfig = ({
|
|||
capabilityClassifierConfig,
|
||||
llmV2Config,
|
||||
classifierLlmConfig,
|
||||
jevClassifierConfig,
|
||||
classifierContextWindowSize,
|
||||
classifierContextBudgetChars,
|
||||
classifierContextIncludeAssistantTurns,
|
||||
|
|
@ -625,9 +650,7 @@ export const buildComplexityRouterConfig = ({
|
|||
classifierContextBudgetChars,
|
||||
classifierContextIncludeAssistantTurns,
|
||||
};
|
||||
// An edited tier set forces the LLM classifier, so llm-only inputs must survive a classifier_type
|
||||
// the form never rewrote. The UI gates the same controls on this, not on the raw value.
|
||||
const effectiveType: ClassifierType = customTierSet ? "llm" : classifierType;
|
||||
const effectiveType = effectiveClassifierType({ custom_tier_set: customTierSet, classifier_type: classifierType });
|
||||
const forecast = isForecastClassifier(effectiveType);
|
||||
|
||||
const supportsOpeningPrompt = !customTierSet && !forecast && usesLlmClassifier(effectiveType);
|
||||
|
|
@ -640,6 +663,7 @@ export const buildComplexityRouterConfig = ({
|
|||
...(planModeMinTier?.trim() && { plan_mode_min_tier: planModeMinTier }),
|
||||
...(cleanedTierLabels && { tier_labels: cleanedTierLabels }),
|
||||
classifier_type: classifierType,
|
||||
...(effectiveType === "jev" && { jev_classifier_config: normalizeJevClassifierConfig(jevClassifierConfig) }),
|
||||
...classifierWireFields(effectiveType, classifierInputs),
|
||||
...(effectiveType === "capability" &&
|
||||
capabilityClassifierConfig && { capability_classifier_config: capabilityClassifierConfig }),
|
||||
|
|
@ -700,6 +724,7 @@ export const buildComplexityRouterConfig = ({
|
|||
Object.entries(payload).filter(([key]) => !CUSTOM_TIER_STRIPPED_KEYS.includes(key)),
|
||||
) as ComplexityRouterConfigPayload;
|
||||
const customTierInputs: CustomTierWireFieldInputs = {
|
||||
classifierType: effectiveType,
|
||||
classifierLlmConfig,
|
||||
planModeMinTierId: planModeMinTier,
|
||||
classificationPrompt,
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
import { describe, expect, it } from "vitest";
|
||||
import type { ComplexityRouterConfigValue } from "./ComplexityRouterConfig";
|
||||
import { effectiveClassifierType, type ComplexityRouterConfigValue } from "./ComplexityRouterConfig";
|
||||
import { transitionClassifierType } from "./classifier_type_transition";
|
||||
import { applyTierSetAction } from "./tier_set_actions";
|
||||
|
||||
const standard: ComplexityRouterConfigValue = {
|
||||
classifier_type: "llm",
|
||||
|
|
@ -13,6 +14,44 @@ const standard: ComplexityRouterConfigValue = {
|
|||
};
|
||||
|
||||
describe("transitionClassifierType", () => {
|
||||
it("switches between LLM and JEV without losing shared routing settings or leaking opposite config", () => {
|
||||
const initial = {
|
||||
...standard,
|
||||
classification_prompt: "LLM only",
|
||||
classification_examples: "LLM examples",
|
||||
enable_non_reasoning_tier: true,
|
||||
tiers: { ...standard.tiers, NON_REASONING: ["fast"] },
|
||||
plan_mode_min_tier: "NON_REASONING",
|
||||
adaptive: true,
|
||||
};
|
||||
const jev = transitionClassifierType(initial, "jev");
|
||||
expect(jev).toMatchObject({
|
||||
classifier_type: "jev",
|
||||
jev_classifier_config: { model: "jev-latest", timeout_ms: 3000 },
|
||||
classifier_context_window_size: 8,
|
||||
classifier_context_budget_chars: 16000,
|
||||
classifier_context_include_assistant_turns: true,
|
||||
classifier_fallback: "default_model",
|
||||
adaptive: true,
|
||||
enable_non_reasoning_tier: true,
|
||||
plan_mode_min_tier: "NON_REASONING",
|
||||
tiers: initial.tiers,
|
||||
});
|
||||
expect(jev.classifier_llm_config).toBeUndefined();
|
||||
expect(jev.classification_prompt).toBeUndefined();
|
||||
expect(jev.classification_examples).toBeUndefined();
|
||||
const custom = applyTierSetAction(jev, [], { kind: "patch", id: "SIMPLE", patch: { name: "QUICK" } }).value;
|
||||
expect(effectiveClassifierType(custom)).toBe("jev");
|
||||
const restored = applyTierSetAction(custom, [], { kind: "restore" }).value;
|
||||
expect(effectiveClassifierType(restored)).toBe("jev");
|
||||
expect(restored.jev_classifier_config).toEqual(jev.jev_classifier_config);
|
||||
const llm = transitionClassifierType(custom, "llm");
|
||||
expect(llm.jev_classifier_config).toBeUndefined();
|
||||
expect(llm.classifier_llm_config).toMatchObject({ model: "" });
|
||||
expect(llm.custom_tier_set).toEqual(custom.custom_tier_set);
|
||||
expect(llm.classifier_context_window_size).toBe(8);
|
||||
});
|
||||
|
||||
it.each(["heuristic_first", "hybrid"] as const)("keeps existing LLM settings when switching to %s", (target) => {
|
||||
const result = transitionClassifierType(standard, target);
|
||||
const expectedSettings = {
|
||||
|
|
|
|||
|
|
@ -8,7 +8,9 @@ import {
|
|||
DEFAULT_HYBRID_BOUNDARY_MARGIN,
|
||||
NEW_CLASSIFIER_CLASSIFICATION_RUBRIC,
|
||||
usesLlmClassifier,
|
||||
usesClassifierContext,
|
||||
} from "./ComplexityRouterConfig";
|
||||
import { defaultJevClassifierConfig } from "./jev_classifier_config";
|
||||
import { isForecastClassifier, prepareForecastClassifier } from "./forecast_classifier_config";
|
||||
import { nonReasoningTierFields } from "./nonReasoningTierFields";
|
||||
|
||||
|
|
@ -22,22 +24,29 @@ export const transitionClassifierType = (
|
|||
const judgeConfig = value.classifier_llm_config ?? { model: "", timeout_ms: DEFAULT_CLASSIFIER_TIMEOUT_MS };
|
||||
const nextValue: ComplexityRouterConfigValue = {
|
||||
...value,
|
||||
jev_classifier_config:
|
||||
classifierType === "jev" ? value.jev_classifier_config ?? defaultJevClassifierConfig() : undefined,
|
||||
classification_prompt: classifierType === "jev" ? undefined : value.classification_prompt,
|
||||
classification_examples: classifierType === "jev" ? undefined : value.classification_examples,
|
||||
classifier_llm_config: usesLlmClassifier(classifierType)
|
||||
? {
|
||||
...judgeConfig,
|
||||
...(startsLlmRubric && { classification_rubric: NEW_CLASSIFIER_CLASSIFICATION_RUBRIC }),
|
||||
}
|
||||
: undefined,
|
||||
classifier_context_window_size: usesLlmClassifier(classifierType)
|
||||
classifier_context_window_size: usesClassifierContext(classifierType)
|
||||
? value.classifier_context_window_size ?? DEFAULT_CLASSIFIER_CONTEXT_WINDOW_SIZE
|
||||
: undefined,
|
||||
classifier_context_budget_chars: usesLlmClassifier(classifierType)
|
||||
classifier_context_budget_chars: usesClassifierContext(classifierType)
|
||||
? value.classifier_context_budget_chars ?? DEFAULT_CLASSIFIER_CONTEXT_BUDGET_CHARS
|
||||
: undefined,
|
||||
classifier_context_include_assistant_turns: usesLlmClassifier(classifierType)
|
||||
classifier_context_per_turn_chars: usesClassifierContext(classifierType)
|
||||
? value.classifier_context_per_turn_chars
|
||||
: undefined,
|
||||
classifier_context_include_assistant_turns: usesClassifierContext(classifierType)
|
||||
? value.classifier_context_include_assistant_turns
|
||||
: undefined,
|
||||
classifier_fallback: usesLlmClassifier(classifierType) ? value.classifier_fallback : undefined,
|
||||
classifier_fallback: usesClassifierContext(classifierType) ? value.classifier_fallback : undefined,
|
||||
heuristic_first_max_tier:
|
||||
classifierType === "heuristic_first"
|
||||
? value.heuristic_first_max_tier ?? DEFAULT_HEURISTIC_FIRST_MAX_TIER
|
||||
|
|
|
|||
|
|
@ -0,0 +1,15 @@
|
|||
export type ClassifierType =
|
||||
| "heuristic"
|
||||
| "heuristic_v2"
|
||||
| "llm"
|
||||
| "jev"
|
||||
| "heuristic_first"
|
||||
| "hybrid"
|
||||
| "capability"
|
||||
| "llm_v2";
|
||||
|
||||
export const usesLlmClassifier = (classifierType: ClassifierType): boolean =>
|
||||
(["llm", "heuristic_first", "hybrid", "capability", "llm_v2"] as const).some((type) => type === classifierType);
|
||||
|
||||
export const usesClassifierContext = (classifierType: ClassifierType): boolean =>
|
||||
classifierType === "jev" || usesLlmClassifier(classifierType);
|
||||
|
|
@ -0,0 +1,28 @@
|
|||
import { z } from "zod";
|
||||
|
||||
export const jevClassifierConfigSchema = z.object({
|
||||
model: z.string().trim().min(1).default("jev-latest"),
|
||||
timeout_ms: z.number().int().positive().default(3000),
|
||||
instructions: z
|
||||
.string()
|
||||
.nullish()
|
||||
.transform((value) => value ?? undefined),
|
||||
circuit_breaker_enabled: z.boolean().optional(),
|
||||
circuit_breaker_cooldown_seconds: z.number().finite().positive().optional(),
|
||||
});
|
||||
|
||||
export type JevClassifierConfig = z.infer<typeof jevClassifierConfigSchema>;
|
||||
|
||||
export const defaultJevClassifierConfig = (): JevClassifierConfig => jevClassifierConfigSchema.parse({});
|
||||
|
||||
export const normalizeJevClassifierConfig = (
|
||||
config: JevClassifierConfig = defaultJevClassifierConfig(),
|
||||
): JevClassifierConfig => ({
|
||||
model: config.model.trim(),
|
||||
timeout_ms: config.timeout_ms,
|
||||
...(config.instructions?.trim() && { instructions: config.instructions.trim() }),
|
||||
...(config.circuit_breaker_enabled !== undefined && { circuit_breaker_enabled: config.circuit_breaker_enabled }),
|
||||
...(config.circuit_breaker_cooldown_seconds !== undefined && {
|
||||
circuit_breaker_cooldown_seconds: config.circuit_breaker_cooldown_seconds,
|
||||
}),
|
||||
});
|
||||
|
|
@ -12,7 +12,7 @@ export const nonReasoningTierFields = (
|
|||
classifierType: ClassifierType,
|
||||
value: ComplexityRouterConfigValue,
|
||||
): Pick<ComplexityRouterConfigValue, "enable_non_reasoning_tier" | "tiers" | "plan_mode_min_tier"> => {
|
||||
if (classifierType === "llm") {
|
||||
if (classifierType === "llm" || classifierType === "jev") {
|
||||
return {
|
||||
enable_non_reasoning_tier: value.enable_non_reasoning_tier,
|
||||
tiers: value.tiers,
|
||||
|
|
|
|||
|
|
@ -145,7 +145,7 @@ export const CUSTOM_TIER_RESTRICTIONS = {
|
|||
heuristicClassifier: {
|
||||
omit: ["heuristic_first_max_tier", "hybrid_boundary_margin"],
|
||||
reason:
|
||||
"The heuristic scorer only produces the built-in tiers, so an edited set needs the LLM classifier. " +
|
||||
"The heuristic scorer only produces the built-in tiers, so an edited set needs the LLM or JEV classifier. " +
|
||||
"Heuristic first and hybrid are out for the same reason: their local scorer decides the traffic it is sure of",
|
||||
},
|
||||
heuristicScoring: {
|
||||
|
|
|
|||
|
|
@ -1,4 +1,6 @@
|
|||
import { describe, expect, it } from "vitest";
|
||||
import { transitionClassifierType } from "../add_model/classifier_type_transition";
|
||||
import { effectiveClassifierType } from "../add_model/ComplexityRouterConfig";
|
||||
|
||||
import {
|
||||
MANAGED_COMPLEXITY_ROUTER_KEYS,
|
||||
|
|
@ -46,6 +48,62 @@ const hydratedState: KeywordMatchingState = {
|
|||
};
|
||||
|
||||
describe("buildUpdatedComplexityRouterConfig keyword matching", () => {
|
||||
it("hydrates nullable JEV instructions without resetting the server configuration", () => {
|
||||
const stored = {
|
||||
classifier_type: "jev" as const,
|
||||
jev_classifier_config: {
|
||||
model: "jev-configured",
|
||||
timeout_ms: 6100,
|
||||
instructions: null,
|
||||
circuit_breaker_enabled: false,
|
||||
},
|
||||
tiers: FORM_VALUE.tiers,
|
||||
};
|
||||
const saved = buildUpdatedComplexityRouterConfig(stored, hydrateComplexityRouterConfig(stored, undefined));
|
||||
expect(saved.jev_classifier_config).toEqual({
|
||||
model: "jev-configured",
|
||||
timeout_ms: 6100,
|
||||
circuit_breaker_enabled: false,
|
||||
});
|
||||
});
|
||||
it.each([false, true])("round trips JEV settings and preserves unmanaged fields, custom: %s", (custom) => {
|
||||
const stored = {
|
||||
...(custom ? storedCustomConfig() : STORED),
|
||||
classifier_llm_config: { model: "stale-judge", timeout_ms: 3000 },
|
||||
classifier_type: "jev" as const,
|
||||
jev_classifier_config: {
|
||||
model: "jev-test",
|
||||
timeout_ms: 4100,
|
||||
instructions: "Judge the request",
|
||||
circuit_breaker_enabled: false,
|
||||
circuit_breaker_cooldown_seconds: 10.5,
|
||||
},
|
||||
classifier_context_window_size: 7,
|
||||
classifier_context_budget_chars: 9000,
|
||||
classifier_context_include_assistant_turns: true,
|
||||
some_future_backend_key: { nested: true },
|
||||
};
|
||||
const hydrated = hydrateComplexityRouterConfig(stored, undefined);
|
||||
expect(effectiveClassifierType(hydrated)).toBe("jev");
|
||||
expect(hydrated.classifier_llm_config).toBeUndefined();
|
||||
expect(hydrated.jev_classifier_config).toEqual(stored.jev_classifier_config);
|
||||
const saved = buildUpdatedComplexityRouterConfig(stored, hydrated);
|
||||
expect(saved).toMatchObject({
|
||||
classifier_type: "jev",
|
||||
jev_classifier_config: stored.jev_classifier_config,
|
||||
classifier_context_window_size: 7,
|
||||
classifier_context_budget_chars: 9000,
|
||||
classifier_context_include_assistant_turns: true,
|
||||
some_future_backend_key: { nested: true },
|
||||
});
|
||||
expect(saved).not.toHaveProperty("classifier_llm_config");
|
||||
const reloaded = hydrateComplexityRouterConfig(saved, undefined);
|
||||
expect(reloaded.jev_classifier_config).toEqual(hydrated.jev_classifier_config);
|
||||
expect(effectiveClassifierType(reloaded)).toBe("jev");
|
||||
const llm = buildUpdatedComplexityRouterConfig(saved, transitionClassifierType(reloaded, "llm"));
|
||||
expect(llm).not.toHaveProperty("jev_classifier_config");
|
||||
});
|
||||
|
||||
it.each(["capability", "llm_v2", "heuristic"] as const)(
|
||||
"handles enabled stored overrides when editing %s with or without keyword form state",
|
||||
(classifier_type) => {
|
||||
|
|
@ -700,7 +758,12 @@ describe("managed keys survive an untouched open-and-save", () => {
|
|||
// tier_definitions and fallback_tier cannot sit beside heuristic_first, which this fixture uses,
|
||||
// and hybrid_boundary_margin belongs to the sibling hybrid type, so no single stored config can
|
||||
// hold every managed key. Each gets its own round trip below.
|
||||
const KEYS_ANOTHER_CLASSIFIER_TYPE_OWNS = new Set(["tier_definitions", "fallback_tier", "hybrid_boundary_margin"]);
|
||||
const KEYS_ANOTHER_CLASSIFIER_TYPE_OWNS = new Set([
|
||||
"tier_definitions",
|
||||
"fallback_tier",
|
||||
"hybrid_boundary_margin",
|
||||
"jev_classifier_config",
|
||||
]);
|
||||
|
||||
// The stall keys are rejected beside the session pinning and user-turn classification this
|
||||
// fixture sets, so they get their own round trip below rather than widening this one.
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
import AutoRouterClassifierTabs from "../add_model/AutoRouterClassifierTabs";
|
||||
import { defaultJevClassifierConfig, jevClassifierConfigSchema } from "../add_model/jev_classifier_config";
|
||||
import type { StoredComplexityRouterConfig } from "../add_model/build_complexity_router_config";
|
||||
export type { StoredComplexityRouterConfig } from "../add_model/build_complexity_router_config";
|
||||
import {
|
||||
|
|
@ -129,7 +130,12 @@ export const hydrateComplexityRouterConfig = (
|
|||
classifier_type: parsedConfig.classifier_type || "heuristic",
|
||||
capability_classifier_config: capabilitySettingsSchema.safeParse(parsedConfig.capability_classifier_config).data,
|
||||
llm_v2_config: fuseSettingsSchema.safeParse(parsedConfig.llm_v2_config).data,
|
||||
classifier_llm_config: parsedConfig.classifier_llm_config,
|
||||
classifier_llm_config: parsedConfig.classifier_type === "jev" ? undefined : parsedConfig.classifier_llm_config,
|
||||
jev_classifier_config:
|
||||
parsedConfig.classifier_type === "jev"
|
||||
? jevClassifierConfigSchema.safeParse(parsedConfig.jev_classifier_config ?? {}).data ??
|
||||
defaultJevClassifierConfig()
|
||||
: undefined,
|
||||
classifier_context_window_size:
|
||||
typeof parsedConfig.classifier_context_window_size === "number"
|
||||
? parsedConfig.classifier_context_window_size
|
||||
|
|
@ -219,6 +225,7 @@ export const MANAGED_COMPLEXITY_ROUTER_KEYS = new Set([
|
|||
"capability_classifier_config",
|
||||
"llm_v2_config",
|
||||
"classifier_llm_config",
|
||||
"jev_classifier_config",
|
||||
"classifier_context_window_size",
|
||||
"classifier_context_budget_chars",
|
||||
"classifier_context_include_assistant_turns",
|
||||
|
|
@ -329,6 +336,7 @@ export const buildUpdatedComplexityRouterConfig = (
|
|||
classificationMode: value.classification_mode,
|
||||
tierLabels: value.tier_labels,
|
||||
classifierType: value.classifier_type,
|
||||
jevClassifierConfig: value.jev_classifier_config,
|
||||
capabilityClassifierConfig: value.capability_classifier_config,
|
||||
llmV2Config: value.llm_v2_config,
|
||||
classifierLlmConfig: value.classifier_llm_config,
|
||||
|
|
|
|||
|
|
@ -15,6 +15,7 @@ import { copyToClipboard as utilCopyToClipboard } from "../utils/dataUtils";
|
|||
import { stripMaskedSecrets } from "../utils/maskedSecretUtils";
|
||||
import { truncateString } from "../utils/textUtils";
|
||||
import AutoRouterConnectionTest from "./add_model/auto_router_connection_test";
|
||||
import { buildSavedJevConnectionTestRequest } from "./add_model/build_auto_router_routing_test_request";
|
||||
import { AutoRouterTestTarget, buildComplexityRouterTestTargets } from "./add_model/build_auto_router_test_targets";
|
||||
import {
|
||||
hasAutoRouterEditor,
|
||||
|
|
@ -846,6 +847,12 @@ export default function ModelInfoView({
|
|||
key={autoRouterTestId}
|
||||
accessToken={accessToken}
|
||||
targets={autoRouterTestTargets}
|
||||
jevRequest={buildSavedJevConnectionTestRequest(
|
||||
(localModelData ?? modelData)?.litellm_params?.complexity_router_config,
|
||||
(localModelData ?? modelData)?.litellm_params?.complexity_router_default_model,
|
||||
(localModelData ?? modelData)?.model_name,
|
||||
(localModelData ?? modelData)?.model_info?.team_id,
|
||||
)}
|
||||
/>
|
||||
)}
|
||||
<DialogFooter>
|
||||
|
|
|
|||
|
|
@ -2326,7 +2326,7 @@ export const testModelGroupConnection = async (
|
|||
|
||||
export interface AutoRouterRoutingTestRequest {
|
||||
prompt: string;
|
||||
complexity_router_config: ComplexityRouterConfigPayload;
|
||||
complexity_router_config: ComplexityRouterConfigPayload | Record<string, unknown>;
|
||||
default_model?: string;
|
||||
router_name?: string;
|
||||
team_id?: string;
|
||||
|
|
|
|||
|
|
@ -103,7 +103,7 @@ describe("RoutingDecisionCard", () => {
|
|||
}}
|
||||
/>,
|
||||
);
|
||||
expect(screen.getByText("Default model, LLM classifier failed")).toBeInTheDocument();
|
||||
expect(screen.getByText("Default model, classifier failed")).toBeInTheDocument();
|
||||
expect(screen.queryByText("Tier")).not.toBeInTheDocument();
|
||||
});
|
||||
|
||||
|
|
@ -120,7 +120,7 @@ describe("RoutingDecisionCard", () => {
|
|||
}}
|
||||
/>,
|
||||
);
|
||||
expect(screen.getByText("Fallback tier, LLM classifier failed")).toBeInTheDocument();
|
||||
expect(screen.getByText("Fallback tier, classifier failed")).toBeInTheDocument();
|
||||
expect(screen.getByText("SECURITY_REVIEW")).toBeInTheDocument();
|
||||
});
|
||||
|
||||
|
|
|
|||
|
|
@ -24,6 +24,9 @@ export interface RoutingDecision {
|
|||
matched_keyword?: string;
|
||||
escalation_keyword?: string;
|
||||
classifier_model?: string;
|
||||
classifier_confidence?: number;
|
||||
classifier_probabilities?: Record<string, number>;
|
||||
classifier_cost?: number;
|
||||
escalated?: boolean;
|
||||
tier_boundaries?: RoutingDecisionTierBoundaries;
|
||||
reasoning_override_min_score?: number;
|
||||
|
|
@ -97,8 +100,8 @@ const CONSTANT_CAUSE_LABELS: Record<string, string> = {
|
|||
quality_tier: "Quality tier mapping",
|
||||
bandit: "Adaptive bandit",
|
||||
default_fallback: "Default model, no route matched",
|
||||
classifier_fallback: "Fallback tier, LLM classifier failed",
|
||||
default_model_fallback: "Default model, LLM classifier failed",
|
||||
classifier_fallback: "Fallback tier, classifier failed",
|
||||
default_model_fallback: "Default model, classifier failed",
|
||||
};
|
||||
|
||||
function describeCause(decision: RoutingDecision): string {
|
||||
|
|
@ -118,6 +121,8 @@ function describeCause(decision: RoutingDecision): string {
|
|||
return describeReasoningOverride(tierLabel, overrideFloor);
|
||||
case "llm_classifier":
|
||||
return classifierModel ? `LLM classifier (${classifierModel})` : "LLM classifier";
|
||||
case "jev_classifier":
|
||||
return "JEV classifier";
|
||||
case "literal_keyword_match":
|
||||
case "keyword":
|
||||
return matchedKeyword ? `Keyword match: "${matchedKeyword}"` : "Keyword match";
|
||||
|
|
@ -208,6 +213,20 @@ export function RoutingDecisionCard({
|
|||
{requestType && <Row label="Request type">{requestType}</Row>}
|
||||
|
||||
<Row label="Decided by">{describeCause(decision)}</Row>
|
||||
{decision.classifier_model && <Row label="Classifier model">{decision.classifier_model}</Row>}
|
||||
{decision.classifier_confidence != null && (
|
||||
<Row label="Confidence">{(decision.classifier_confidence * 100).toFixed(1)}%</Row>
|
||||
)}
|
||||
{decision.classifier_probabilities && (
|
||||
<Row label="Probabilities">
|
||||
{Object.entries(decision.classifier_probabilities).map(([name, probability]) => (
|
||||
<div key={name}>
|
||||
{name}: {(probability * 100).toFixed(1)}%
|
||||
</div>
|
||||
))}
|
||||
</Row>
|
||||
)}
|
||||
{decision.classifier_cost != null && <Row label="Classifier cost">${decision.classifier_cost.toFixed(8)}</Row>}
|
||||
|
||||
{score !== undefined && (
|
||||
<Row label="Score">
|
||||
|
|
|
|||
|
|
@ -680,6 +680,32 @@ describe("autorouter_presets", () => {
|
|||
});
|
||||
|
||||
describe("buildPresetPrefill", () => {
|
||||
it("preserves JEV settings and drops inactive classifier settings when prefilling", () => {
|
||||
const config = {
|
||||
tiers: { SIMPLE: ["fast"], MEDIUM: [], COMPLEX: [], REASONING: [] },
|
||||
classifier_type: "jev" as const,
|
||||
classification_mode: "every_request" as const,
|
||||
session_affinity: false,
|
||||
deployment_affinity: true,
|
||||
modality_routing: false,
|
||||
modality_pin_override: false,
|
||||
jev_classifier_config: { model: "jev-test", timeout_ms: 4000, circuit_breaker_enabled: false },
|
||||
classifier_llm_config: { model: "stale-judge", timeout_ms: 6000 },
|
||||
classifier_context_window_size: 6,
|
||||
};
|
||||
const prefill = buildPresetPrefill(config, groupsOnly(["fast"]));
|
||||
expect(prefill.complexityRouterConfig).toMatchObject({
|
||||
classifier_type: "jev",
|
||||
jev_classifier_config: config.jev_classifier_config,
|
||||
classifier_context_window_size: 6,
|
||||
classifier_llm_config: undefined,
|
||||
});
|
||||
const llmConfig = { ...config, classifier_type: "llm" as const };
|
||||
const llmPrefill = buildPresetPrefill(llmConfig, groupsOnly(["fast"]));
|
||||
expect(llmPrefill.complexityRouterConfig.jev_classifier_config).toBeUndefined();
|
||||
expect(llmPrefill.complexityRouterConfig.classifier_llm_config).toEqual(config.classifier_llm_config);
|
||||
});
|
||||
|
||||
it("prefills a real bundled preset's tiers into the config", () => {
|
||||
const preset = getPresetByKey("anthropic_family")!;
|
||||
const prefill = buildPresetPrefill(
|
||||
|
|
|
|||
|
|
@ -284,10 +284,11 @@ export const buildPresetPrefill = (
|
|||
tier_model_params: resolveParamKeys(hydrateTierModelParams(config.tiers, config.tier_model_configs)),
|
||||
tier_labels: hydrateTierLabels(config.tier_labels),
|
||||
classifier_type: config.classifier_type,
|
||||
classifier_llm_config: config.classifier_llm_config && {
|
||||
...config.classifier_llm_config,
|
||||
model: resolve(config.classifier_llm_config.model),
|
||||
},
|
||||
jev_classifier_config: config.classifier_type === "jev" ? config.jev_classifier_config : undefined,
|
||||
classifier_llm_config:
|
||||
config.classifier_type !== "jev" && config.classifier_llm_config
|
||||
? { ...config.classifier_llm_config, model: resolve(config.classifier_llm_config.model) }
|
||||
: undefined,
|
||||
classifier_context_window_size: config.classifier_context_window_size,
|
||||
classifier_context_budget_chars: config.classifier_context_budget_chars,
|
||||
classifier_context_per_turn_chars: config.classifier_context_per_turn_chars,
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue