mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-09 03:18:44 +00:00
fix(ui): adapt the jev dashboard pieces to stable/1.102.x
(cherry picked from commit 4bfac88281)
This commit is contained in:
mateo-berri
mateo
parent
73e50a09cb
commit
c3c466a5c1
7 changed files with 387 additions and 237 deletions
|
|
@ -1,3 +1,5 @@
|
|||
import { transitionClassifierType } from "./classifier_type_transition";
|
||||
import JevClassifierConfig from "./JevClassifierConfig";
|
||||
import { Info } from "lucide-react";
|
||||
import { SimpleTooltip } from "@/components/ui/tooltip";
|
||||
import { MultiSelect } from "@/components/shared/MultiSelect";
|
||||
|
|
@ -10,40 +12,51 @@ import { RadioGroup, RadioGroupItem } from "@/components/ui/radio-group";
|
|||
import { Switch } from "@/components/ui/switch";
|
||||
import React from "react";
|
||||
import ClassifierPromptEditor from "./ClassifierPromptEditor";
|
||||
import CustomTierPromptEditor from "./CustomTierPromptEditor";
|
||||
import OpeningPromptEditor, { type OpeningPromptSelection } from "./OpeningPromptEditor";
|
||||
import { RestrictedSection, restrictedBy } from "./TierRestrictions";
|
||||
import HeuristicScoringConfig from "./HeuristicScoringConfig";
|
||||
import ClassifierReasoningEffortSelect from "./ClassifierReasoningEffortSelect";
|
||||
import ClassifierCircuitBreakerConfig from "./ClassifierCircuitBreakerConfig";
|
||||
import ClassifierVisionConfig from "./ClassifierVisionConfig";
|
||||
import type { ReasoningEffort } from "./complexity_router_tiers";
|
||||
import { useComplexityScorerDefaults } from "@/app/(dashboard)/hooks/autoRouter/useComplexityScorerDefaults";
|
||||
import {
|
||||
ClassificationFrequency,
|
||||
ClassifierFallback,
|
||||
ClassifierLLMConfig,
|
||||
ClassifierType,
|
||||
ComplexityRouterConfigValue,
|
||||
classificationFrequency,
|
||||
withClassificationFrequency,
|
||||
DEFAULT_CLASSIFIER_CONTEXT_BUDGET_CHARS,
|
||||
MIN_QUOTED_CONTEXT_TURN_CHARS,
|
||||
DEFAULT_CLASSIFIER_CONTEXT_WINDOW_SIZE,
|
||||
DEFAULT_CLASSIFIER_FALLBACK,
|
||||
DEFAULT_CLASSIFIER_TIMEOUT_MS,
|
||||
DEFAULT_CLASSIFICATION_RUBRIC,
|
||||
NEW_CLASSIFIER_CLASSIFICATION_RUBRIC,
|
||||
CLASSIFICATION_RUBRIC_DESCRIPTIONS,
|
||||
CLASSIFICATION_RUBRIC_KEYS,
|
||||
ClassificationRubric,
|
||||
effectiveTierLabel,
|
||||
heuristicScoringRole,
|
||||
usesLlmClassifier,
|
||||
DEFAULT_HEURISTIC_FIRST_MAX_TIER,
|
||||
usesClassifierContext,
|
||||
DEFAULT_HYBRID_BOUNDARY_MARGIN,
|
||||
HEURISTIC_FIRST_MAX_TIER_KEYS,
|
||||
effectiveClassifierType,
|
||||
} from "./ComplexityRouterConfig";
|
||||
|
||||
const DEFAULT_SCORING_EXPLANATION =
|
||||
"The router scores each request across 7 dimensions: token count, code presence, reasoning markers, technical " +
|
||||
"terms, simple indicators, multi-step patterns, and question complexity. The weighted score determines the tier:";
|
||||
"The router scores each request across 7 built-in dimensions: token count, code presence, reasoning markers, technical " +
|
||||
"terms, simple indicators, multi-step patterns, and question complexity, plus any custom dimensions you add. " +
|
||||
"The weighted score determines the tier:";
|
||||
|
||||
const HEURISTIC_V2_EXPLANATION =
|
||||
"The router estimates success probability for all four tiers with the bundled calibrated model, then selects " +
|
||||
"the first tier that meets its trained threshold. It runs locally with no classifier API call.";
|
||||
|
||||
const CLASSIFIER_TIMEOUT_ID = "classifier-timeout-ms";
|
||||
const CLASSIFIER_CONTEXT_WINDOW_SIZE_ID = "classifier-context-window-size";
|
||||
const CLASSIFIER_CONTEXT_BUDGET_CHARS_ID = "classifier-context-budget-chars";
|
||||
const HYBRID_BOUNDARY_MARGIN_ID = "hybrid-boundary-margin";
|
||||
|
||||
const CUSTOM_PROMPT_WITH_HEURISTIC_FALLBACK =
|
||||
"This router classifies with your own prompt, so the tier comes from whatever rubric it states. The four tier " +
|
||||
|
|
@ -60,6 +73,7 @@ const CUSTOM_PROMPT_WITH_DEFAULT_MODEL_FALLBACK =
|
|||
* at all, so the panel must not keep implying a score is involved on either router.
|
||||
*/
|
||||
const scoringExplanation = (value: ComplexityRouterConfigValue): string => {
|
||||
if (value.classifier_type === "heuristic_v2") return HEURISTIC_V2_EXPLANATION;
|
||||
const usesCustomPrompt =
|
||||
usesLlmClassifier(value.classifier_type) && Boolean(value.classifier_llm_config?.system_prompt?.trim());
|
||||
if (!usesCustomPrompt) return DEFAULT_SCORING_EXPLANATION;
|
||||
|
|
@ -112,7 +126,7 @@ const HowClassificationWorks: React.FC<{ value: ComplexityRouterConfigValue }> =
|
|||
<strong className="block mb-2 font-semibold">How Classification Works</strong>
|
||||
<span className="text-[13px] text-muted-foreground">{scoringExplanation(value)}</span>
|
||||
{scorerRuns && ranges && (
|
||||
<ul style={{ marginTop: 8, marginBottom: 0, paddingLeft: 20, fontSize: 13, color: "rgba(0, 0, 0, 0.45)" }}>
|
||||
<ul className="mt-2 pl-5 text-[13px] text-muted-foreground">
|
||||
<li>
|
||||
<strong>{effectiveTierLabel("SIMPLE", value.tier_labels)}</strong>: Score < {ranges.simpleMedium}
|
||||
</li>
|
||||
|
|
@ -145,6 +159,7 @@ interface ClassificationMethodConfigProps {
|
|||
value: ComplexityRouterConfigValue;
|
||||
onChange: (value: ComplexityRouterConfigValue) => void;
|
||||
modelOptions: { value: string; label: string }[];
|
||||
effortOptionsByModel: Record<string, string[] | null | undefined>;
|
||||
customTechnicalKeywords?: string[];
|
||||
onCustomTechnicalKeywordsChange?: (keywords: string[]) => void;
|
||||
showValidationErrors?: boolean;
|
||||
|
|
@ -177,6 +192,17 @@ const ClassifierTypeRadios: React.FC<{
|
|||
</span>
|
||||
</Label>
|
||||
</SimpleTooltip>
|
||||
<SimpleTooltip content={scorerLockedReason}>
|
||||
<Label className="items-start font-normal leading-normal has-data-disabled:cursor-not-allowed has-data-disabled:opacity-50">
|
||||
<RadioGroupItem value="heuristic_v2" className="mt-0.5" disabled={scorerLocked} />
|
||||
<span>
|
||||
<strong className="font-semibold">Heuristic v2</strong>{" "}
|
||||
<span className="text-muted-foreground">
|
||||
uses bundled calibrated four-tier probabilities with no API call
|
||||
</span>
|
||||
</span>
|
||||
</Label>
|
||||
</SimpleTooltip>
|
||||
<Label className="items-start font-normal leading-normal">
|
||||
<RadioGroupItem value="llm" className="mt-0.5" />
|
||||
<span>
|
||||
|
|
@ -184,6 +210,13 @@ const ClassifierTypeRadios: React.FC<{
|
|||
<span className="text-muted-foreground">calls a model to decide the tier (e.g. a small/fast model)</span>
|
||||
</span>
|
||||
</Label>
|
||||
<Label className="items-start font-normal leading-normal">
|
||||
<RadioGroupItem value="jev" className="mt-0.5" />
|
||||
<span>
|
||||
<strong className="font-semibold">JEV Classifier</strong>{" "}
|
||||
<span className="text-muted-foreground">uses TypeSafe System One Choice to decide the tier</span>
|
||||
</span>
|
||||
</Label>
|
||||
<SimpleTooltip content={scorerLockedReason}>
|
||||
<Label className="items-start font-normal leading-normal has-data-disabled:cursor-not-allowed has-data-disabled:opacity-50">
|
||||
<RadioGroupItem value="heuristic_first" className="mt-0.5" disabled={scorerLocked} />
|
||||
|
|
@ -195,6 +228,18 @@ const ClassifierTypeRadios: React.FC<{
|
|||
</span>
|
||||
</Label>
|
||||
</SimpleTooltip>
|
||||
<SimpleTooltip content={scorerLockedReason}>
|
||||
<Label className="items-start font-normal leading-normal has-data-disabled:cursor-not-allowed has-data-disabled:opacity-50">
|
||||
<RadioGroupItem value="hybrid" className="mt-0.5" disabled={scorerLocked} />
|
||||
<span>
|
||||
<strong className="font-semibold">Hybrid</strong>{" "}
|
||||
<span className="text-muted-foreground">
|
||||
keeps the local score at any tier, and only pays for the classifier when that score lands near a tier
|
||||
boundary
|
||||
</span>
|
||||
</span>
|
||||
</Label>
|
||||
</SimpleTooltip>
|
||||
</div>
|
||||
</RadioGroup>
|
||||
);
|
||||
|
|
@ -204,6 +249,7 @@ const ClassificationMethodConfig: React.FC<ClassificationMethodConfigProps> = ({
|
|||
value,
|
||||
onChange,
|
||||
modelOptions,
|
||||
effortOptionsByModel,
|
||||
customTechnicalKeywords,
|
||||
onCustomTechnicalKeywordsChange,
|
||||
showValidationErrors = false,
|
||||
|
|
@ -212,61 +258,84 @@ const ClassificationMethodConfig: React.FC<ClassificationMethodConfigProps> = ({
|
|||
const [draft, setDraft] = React.useState<{ id: string; raw: string } | null>(null);
|
||||
const hasDefaultModel = Boolean(defaultModel);
|
||||
const classifierType = effectiveClassifierType(value);
|
||||
const sessionFrequencyRestriction = restrictedBy(value, "sessionAffinity");
|
||||
const classifierModelMissing =
|
||||
showValidationErrors && usesLlmClassifier(classifierType) && !value.classifier_llm_config?.model;
|
||||
const usesCustomPrompt = Boolean(value.classifier_llm_config?.system_prompt?.trim());
|
||||
const contextBudget = value.classifier_context_budget_chars ?? DEFAULT_CLASSIFIER_CONTEXT_BUDGET_CHARS;
|
||||
const contextBudgetQuotesNothing = contextBudget > 0 && contextBudget < MIN_QUOTED_CONTEXT_TURN_CHARS;
|
||||
const classificationRubric = value.classifier_llm_config?.classification_rubric ?? DEFAULT_CLASSIFICATION_RUBRIC;
|
||||
const classifierModel = value.classifier_llm_config?.model ?? "";
|
||||
const classifierReasoningEffort = value.classifier_llm_config?.reasoning_effort;
|
||||
const explicitlySupportedClassifierEfforts = effortOptionsByModel[classifierModel];
|
||||
|
||||
const handleClassifierTypeChange = (classifierType: ClassifierType) => {
|
||||
const nextValue: ComplexityRouterConfigValue = {
|
||||
...value,
|
||||
classifier_type: classifierType,
|
||||
classifier_llm_config: usesLlmClassifier(classifierType)
|
||||
? value.classifier_llm_config ?? {
|
||||
model: "",
|
||||
timeout_ms: DEFAULT_CLASSIFIER_TIMEOUT_MS,
|
||||
classification_rubric: NEW_CLASSIFIER_CLASSIFICATION_RUBRIC,
|
||||
}
|
||||
: undefined,
|
||||
classifier_context_window_size: usesLlmClassifier(classifierType)
|
||||
? value.classifier_context_window_size ?? DEFAULT_CLASSIFIER_CONTEXT_WINDOW_SIZE
|
||||
: undefined,
|
||||
classifier_context_budget_chars: usesLlmClassifier(classifierType)
|
||||
? value.classifier_context_budget_chars ?? DEFAULT_CLASSIFIER_CONTEXT_BUDGET_CHARS
|
||||
: undefined,
|
||||
classifier_context_include_assistant_turns: usesLlmClassifier(classifierType)
|
||||
? value.classifier_context_include_assistant_turns
|
||||
: undefined,
|
||||
classifier_fallback: usesLlmClassifier(classifierType) ? value.classifier_fallback : undefined,
|
||||
heuristic_first_max_tier:
|
||||
classifierType === "heuristic_first"
|
||||
? value.heuristic_first_max_tier ?? DEFAULT_HEURISTIC_FIRST_MAX_TIER
|
||||
: undefined,
|
||||
};
|
||||
onChange(nextValue);
|
||||
onChange(transitionClassifierType(value, classifierType));
|
||||
};
|
||||
|
||||
const handleHeuristicFirstMaxTierChange = (tier: string) => {
|
||||
onChange({ ...value, heuristic_first_max_tier: tier });
|
||||
};
|
||||
|
||||
const handleClassificationPromptChange = (classificationPrompt: string | undefined) => {
|
||||
onChange({ ...value, classification_prompt: classificationPrompt });
|
||||
const handleHybridBoundaryMarginChange = (raw: string) => {
|
||||
setDraft({ id: HYBRID_BOUNDARY_MARGIN_ID, raw });
|
||||
const parsed = Number(raw);
|
||||
if (raw.trim() === "" || !Number.isFinite(parsed)) return;
|
||||
onChange({ ...value, hybrid_boundary_margin: Math.min(1, Math.max(0, parsed)) });
|
||||
};
|
||||
|
||||
const handleClassifierModelChange = (model: string) => {
|
||||
// One write for everything the prompt dialog owns. The rubric arrives here rather than through the
|
||||
// rubric handler because two onChange calls in one tick would both spread this render's `value`,
|
||||
// so whichever landed second would drop the other's edit.
|
||||
const handleClassificationPromptChange = ({
|
||||
classificationPrompt,
|
||||
classificationExamples,
|
||||
classificationRubric: selectedRubric,
|
||||
}: OpeningPromptSelection) => {
|
||||
const rubricConfig: ClassifierLLMConfig = {
|
||||
...value.classifier_llm_config,
|
||||
model: value.classifier_llm_config?.model ?? "",
|
||||
timeout_ms: value.classifier_llm_config?.timeout_ms ?? DEFAULT_CLASSIFIER_TIMEOUT_MS,
|
||||
classification_rubric: selectedRubric,
|
||||
};
|
||||
const nextValue: ComplexityRouterConfigValue = {
|
||||
...value,
|
||||
...(selectedRubric && { classifier_llm_config: rubricConfig }),
|
||||
classification_prompt: classificationPrompt,
|
||||
classification_examples: classificationExamples,
|
||||
};
|
||||
onChange(nextValue);
|
||||
};
|
||||
|
||||
const handleClassifierModelChange = (model: string | null) => {
|
||||
if (model === null) return;
|
||||
if (model === value.classifier_llm_config?.model) return;
|
||||
const { reasoning_effort: _reasoningEffort, ...classifierLlmConfig } = value.classifier_llm_config ?? {
|
||||
model: "",
|
||||
timeout_ms: DEFAULT_CLASSIFIER_TIMEOUT_MS,
|
||||
};
|
||||
onChange({
|
||||
...value,
|
||||
classifier_llm_config: {
|
||||
...value.classifier_llm_config,
|
||||
...classifierLlmConfig,
|
||||
model,
|
||||
timeout_ms: value.classifier_llm_config?.timeout_ms ?? DEFAULT_CLASSIFIER_TIMEOUT_MS,
|
||||
timeout_ms: classifierLlmConfig.timeout_ms,
|
||||
},
|
||||
});
|
||||
};
|
||||
|
||||
const handleClassifierReasoningEffortChange = (reasoningEffort: ReasoningEffort | undefined) => {
|
||||
if (!value.classifier_llm_config) return;
|
||||
const { reasoning_effort: _reasoningEffort, ...classifierLlmConfig } = value.classifier_llm_config;
|
||||
onChange({
|
||||
...value,
|
||||
classifier_llm_config:
|
||||
reasoningEffort === undefined
|
||||
? classifierLlmConfig
|
||||
: { ...classifierLlmConfig, reasoning_effort: reasoningEffort },
|
||||
});
|
||||
};
|
||||
|
||||
const handleClassifierTimeoutChange = (timeoutMs: number) => {
|
||||
onChange({
|
||||
...value,
|
||||
|
|
@ -306,6 +375,10 @@ const ClassificationMethodConfig: React.FC<ClassificationMethodConfigProps> = ({
|
|||
onChange({ ...value, classifier_fallback: fallback });
|
||||
};
|
||||
|
||||
const handleClassificationFrequencyChange = (frequency: ClassificationFrequency) => {
|
||||
onChange(withClassificationFrequency(value, frequency));
|
||||
};
|
||||
|
||||
const handleClassifierContextWindowSizeChange = (windowSize: number) => {
|
||||
onChange({
|
||||
...value,
|
||||
|
|
@ -368,6 +441,74 @@ const ClassificationMethodConfig: React.FC<ClassificationMethodConfigProps> = ({
|
|||
</div>
|
||||
)}
|
||||
|
||||
{classifierType === "hybrid" && (
|
||||
<div className="mt-4 space-y-2">
|
||||
<strong className="block font-semibold">Boundary margin</strong>
|
||||
<Input
|
||||
id={HYBRID_BOUNDARY_MARGIN_ID}
|
||||
type="text"
|
||||
inputMode="decimal"
|
||||
value={
|
||||
draft?.id === HYBRID_BOUNDARY_MARGIN_ID
|
||||
? draft.raw
|
||||
: String(value.hybrid_boundary_margin ?? DEFAULT_HYBRID_BOUNDARY_MARGIN)
|
||||
}
|
||||
onChange={(event) => handleHybridBoundaryMarginChange(event.target.value)}
|
||||
onBlur={() => setDraft(null)}
|
||||
className="w-full"
|
||||
/>
|
||||
<p className="text-sm text-muted-foreground">
|
||||
A score further than this from every tier boundary routes on the scorer's own tier, however expensive
|
||||
that tier is. A score closer than this, and anything the scorer found no signal for at all, goes to the
|
||||
classifier to break the tie
|
||||
</p>
|
||||
</div>
|
||||
)}
|
||||
|
||||
<div className="mt-4 space-y-2">
|
||||
<strong className="block font-semibold">How often to classify</strong>
|
||||
<RadioGroup
|
||||
value={classificationFrequency(value)}
|
||||
onValueChange={(frequency: unknown) =>
|
||||
handleClassificationFrequencyChange(frequency as ClassificationFrequency)
|
||||
}
|
||||
>
|
||||
<div className="inline-flex flex-col gap-2">
|
||||
<Label className="items-start font-normal leading-normal">
|
||||
<RadioGroupItem value="every_request" className="mt-0.5" />
|
||||
<span>
|
||||
<span>Every request</span>{" "}
|
||||
<span className="text-muted-foreground">: score every turn, tool-result continuations included</span>
|
||||
</span>
|
||||
</Label>
|
||||
<Label className="items-start font-normal leading-normal">
|
||||
<RadioGroupItem value="user_turn" className="mt-0.5" />
|
||||
<span>
|
||||
<span>Every new user message</span>{" "}
|
||||
<span className="text-muted-foreground">
|
||||
: score each new human ask, then hold that tier for the tool calls that follow it
|
||||
</span>
|
||||
</span>
|
||||
</Label>
|
||||
<Label className="items-start font-normal leading-normal">
|
||||
<RadioGroupItem value="session" className="mt-0.5" disabled={Boolean(sessionFrequencyRestriction)} />
|
||||
<span>
|
||||
<span>Once per session</span>{" "}
|
||||
<span className="text-muted-foreground">
|
||||
{sessionFrequencyRestriction?.reason ??
|
||||
": score the first turn only, then hold that tier and its deployment for the whole session"}
|
||||
</span>
|
||||
</span>
|
||||
</Label>
|
||||
</div>
|
||||
</RadioGroup>
|
||||
<p className="text-sm text-muted-foreground">
|
||||
Holding the tier keeps an agent on one model for a whole tool loop and cuts scoring cost. A turn the router
|
||||
cannot match to a held decision, such as one with no session id or an expired one, is scored again
|
||||
</p>
|
||||
</div>
|
||||
|
||||
{classifierType === "jev" && <JevClassifierConfig value={value} onChange={onChange} />}
|
||||
{usesLlmClassifier(classifierType) && (
|
||||
<div className="mt-4 space-y-3">
|
||||
<div>
|
||||
|
|
@ -380,9 +521,16 @@ const ClassificationMethodConfig: React.FC<ClassificationMethodConfigProps> = ({
|
|||
emptyText="No models found"
|
||||
allowClear={false}
|
||||
className={classifierModelMissing ? "border-destructive" : undefined}
|
||||
aria-label="Classifier Model"
|
||||
/>
|
||||
{classifierModelMissing && <span className="text-xs text-destructive">A classifier model is required</span>}
|
||||
</div>
|
||||
<ClassifierReasoningEffortSelect
|
||||
model={classifierModel}
|
||||
value={classifierReasoningEffort}
|
||||
explicitlySupported={explicitlySupportedClassifierEfforts}
|
||||
onChange={handleClassifierReasoningEffortChange}
|
||||
/>
|
||||
<div>
|
||||
<Label htmlFor={CLASSIFIER_TIMEOUT_ID} className="block mb-1 font-semibold">
|
||||
Timeout (ms)
|
||||
|
|
@ -415,60 +563,18 @@ const ClassificationMethodConfig: React.FC<ClassificationMethodConfigProps> = ({
|
|||
value={value.classifier_llm_config ?? { model: "", timeout_ms: DEFAULT_CLASSIFIER_TIMEOUT_MS }}
|
||||
onChange={(classifier_llm_config) => onChange({ ...value, classifier_llm_config })}
|
||||
/>
|
||||
<ClassifierVisionConfig
|
||||
value={value.classifier_llm_config ?? { model: "", timeout_ms: DEFAULT_CLASSIFIER_TIMEOUT_MS }}
|
||||
onChange={(classifier_llm_config) => onChange({ ...value, classifier_llm_config })}
|
||||
/>
|
||||
<div>
|
||||
<div className="flex items-center gap-2 mb-1">
|
||||
<strong className="font-semibold">Classification Rubric</strong>
|
||||
<SimpleTooltip content="Every rubric uses the same four tiers. They differ in the worked examples that show the classifier where the boundary between tiers sits, and the Business rubric also rewrites the tier definitions for business traffic.">
|
||||
<strong className="font-semibold">Classifier Prompt</strong>
|
||||
<SimpleTooltip content="Every rubric uses the same four tiers. They differ in the worked examples that show the classifier where the boundary between tiers sits, and the Business rubric also rewrites the tier definitions for business traffic. Pick the rubric, and write your own opening instructions and calibration examples, inside the prompt editor.">
|
||||
<Info className="size-4 text-muted-foreground" />
|
||||
</SimpleTooltip>
|
||||
</div>
|
||||
<SimpleTooltip
|
||||
content={
|
||||
restrictedBy(value, "classificationRubric")?.reason ??
|
||||
(usesCustomPrompt ? "Your custom prompt replaces the built-in rubric entirely" : undefined)
|
||||
}
|
||||
className="w-full"
|
||||
>
|
||||
<Select
|
||||
items={CLASSIFICATION_RUBRIC_KEYS.map((preset) => ({
|
||||
value: preset,
|
||||
label: CLASSIFICATION_RUBRIC_DESCRIPTIONS[preset].label,
|
||||
}))}
|
||||
value={classificationRubric}
|
||||
onValueChange={(preset: ClassificationRubric | null) =>
|
||||
preset && handleClassificationRubricChange(preset)
|
||||
}
|
||||
disabled={usesCustomPrompt || Boolean(value.custom_tier_set)}
|
||||
>
|
||||
<SelectTrigger aria-label="Classification Rubric" className="w-full">
|
||||
<SelectValue />
|
||||
</SelectTrigger>
|
||||
<SelectContent>
|
||||
{CLASSIFICATION_RUBRIC_KEYS.map((preset) => (
|
||||
<SelectItem key={preset} value={preset}>
|
||||
{CLASSIFICATION_RUBRIC_DESCRIPTIONS[preset].label}
|
||||
</SelectItem>
|
||||
))}
|
||||
</SelectContent>
|
||||
</Select>
|
||||
</SimpleTooltip>
|
||||
<span className="block text-xs text-muted-foreground">
|
||||
{restrictedBy(value, "classificationRubric")?.reason ??
|
||||
(usesCustomPrompt
|
||||
? "Not in use: the custom prompt below is the classifier's entire rubric."
|
||||
: CLASSIFICATION_RUBRIC_DESCRIPTIONS[classificationRubric].description)}
|
||||
</span>
|
||||
</div>
|
||||
<div>
|
||||
<strong className="block mb-1 font-semibold">Classifier Prompt</strong>
|
||||
{value.custom_tier_set ? (
|
||||
<CustomTierPromptEditor
|
||||
classificationPrompt={value.classification_prompt}
|
||||
onChange={handleClassificationPromptChange}
|
||||
tierRows={value.custom_tier_set.tiers}
|
||||
contextWindowSize={value.classifier_context_window_size ?? DEFAULT_CLASSIFIER_CONTEXT_WINDOW_SIZE}
|
||||
/>
|
||||
) : (
|
||||
{!value.custom_tier_set && usesCustomPrompt ? (
|
||||
<ClassifierPromptEditor
|
||||
systemPrompt={value.classifier_llm_config?.system_prompt}
|
||||
onChange={handleClassifierSystemPromptChange}
|
||||
|
|
@ -476,8 +582,29 @@ const ClassificationMethodConfig: React.FC<ClassificationMethodConfigProps> = ({
|
|||
tierLabels={value.tier_labels}
|
||||
classificationRubric={classificationRubric}
|
||||
/>
|
||||
) : (
|
||||
<OpeningPromptEditor
|
||||
classificationPrompt={value.classification_prompt}
|
||||
classificationExamples={value.classification_examples}
|
||||
onChange={handleClassificationPromptChange}
|
||||
tierSource={
|
||||
value.custom_tier_set
|
||||
? { kind: "custom", tierRows: value.custom_tier_set.tiers }
|
||||
: {
|
||||
kind: "builtIn",
|
||||
tierLabels: value.tier_labels,
|
||||
classificationRubric,
|
||||
rubricRestriction: restrictedBy(value, "classificationRubric")?.reason,
|
||||
}
|
||||
}
|
||||
contextWindowSize={value.classifier_context_window_size ?? DEFAULT_CLASSIFIER_CONTEXT_WINDOW_SIZE}
|
||||
/>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
{usesClassifierContext(classifierType) && (
|
||||
<div className="mt-4 space-y-3">
|
||||
<RestrictedSection heading="If the classifier fails" by={restrictedBy(value, "classifierFallback")}>
|
||||
<RadioGroup
|
||||
value={value.classifier_fallback ?? DEFAULT_CLASSIFIER_FALLBACK}
|
||||
|
|
@ -539,9 +666,9 @@ const ClassificationMethodConfig: React.FC<ClassificationMethodConfigProps> = ({
|
|||
className="w-full"
|
||||
/>
|
||||
<span className="text-xs text-muted-foreground">
|
||||
Number of prior user turns (tool output and harness reminders excluded) sent to the classifier as context,
|
||||
so a referring follow-up like "now do the same for the streaming path" is classified against
|
||||
what it refers to. Set to 0 to send only the current message.
|
||||
Number of prior user turns sent to the classifier provider, excluding tool output and harness reminders.
|
||||
LLM and JEV default to 3 turns; JEV sends them to the configured TypeSafe endpoint. Set to 0 to omit
|
||||
conversation history. The current message and selected system text are still sent.
|
||||
</span>
|
||||
</div>
|
||||
<div>
|
||||
|
|
@ -641,18 +768,3 @@ const ClassificationMethodConfig: React.FC<ClassificationMethodConfigProps> = ({
|
|||
};
|
||||
|
||||
export default ClassificationMethodConfig;
|
||||
|
||||
import JevClassifierConfig from "./JevClassifierConfig";
|
||||
usesClassifierContext,
|
||||
<Label className="items-start font-normal leading-normal">
|
||||
<RadioGroupItem value="jev" className="mt-0.5" />
|
||||
<span>
|
||||
<strong className="font-semibold">JEV Classifier</strong>{" "}
|
||||
<span className="text-muted-foreground">uses TypeSafe System One Choice to decide the tier</span>
|
||||
</span>
|
||||
</Label>
|
||||
{classifierType === "jev" && <JevClassifierConfig value={value} onChange={onChange} />}
|
||||
</div>
|
||||
)}
|
||||
{usesClassifierContext(classifierType) && (
|
||||
<div className="mt-4 space-y-3">
|
||||
|
|
@ -3,7 +3,6 @@ import { afterEach, describe, expect, it, vi } from "vitest";
|
|||
import { fireEvent, renderWithProviders, screen } from "../../../tests/test-utils";
|
||||
import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized";
|
||||
import ClassificationMethodConfig from "./ClassificationMethodConfig";
|
||||
import AutoRouterClassifierTabs from "./AutoRouterClassifierTabs";
|
||||
import JevEditor from "./JevClassifierConfig";
|
||||
import { type ComplexityRouterConfigValue } from "./ComplexityRouterConfig";
|
||||
import {
|
||||
|
|
@ -50,7 +49,7 @@ const initial: ComplexityRouterConfigValue = {
|
|||
function Form() {
|
||||
const [value, setValue] = useState(initial);
|
||||
return (
|
||||
<AutoRouterClassifierTabs value={value} onChange={setValue}>
|
||||
<>
|
||||
<ClassificationMethodConfig
|
||||
value={value}
|
||||
onChange={setValue}
|
||||
|
|
@ -90,7 +89,7 @@ function Form() {
|
|||
>
|
||||
Probe current config
|
||||
</button>
|
||||
</AutoRouterClassifierTabs>
|
||||
</>
|
||||
);
|
||||
}
|
||||
|
||||
|
|
@ -103,7 +102,6 @@ describe("JEV classifier editor", () => {
|
|||
expect(screen.getByText("Classifier Prompt")).toBeInTheDocument();
|
||||
expect(screen.getByRole("switch", { name: "Use images for classification" })).toBeInTheDocument();
|
||||
fireEvent.click(screen.getByRole("radio", { name: /JEV Classifier/ }));
|
||||
expect(screen.getByRole("tab", { name: "Complexity" })).toHaveAttribute("aria-selected", "true");
|
||||
expect(screen.getByLabelText("JEV Model")).toHaveValue("jev-latest");
|
||||
expect(screen.getByLabelText("JEV Instructions")).toBeDisabled();
|
||||
expect(screen.queryByLabelText("Classifier Model")).not.toBeInTheDocument();
|
||||
|
|
|
|||
|
|
@ -471,6 +471,17 @@ const AddAutoRouterTab: React.FC<AddAutoRouterTabProps> = ({
|
|||
setIsTestModalVisible(true);
|
||||
};
|
||||
|
||||
const jevConnectionTestParams =
|
||||
effectiveClassifierType(complexityRouterConfig) === "jev"
|
||||
? {
|
||||
prompt: JEV_CONNECTION_TEST_PROMPT,
|
||||
config: buildComplexityRouterConfig(complexityRouterConfigParams),
|
||||
defaultModel: resolveComplexityDefaultModel(complexityRouterConfig, complexityRouterConfig.default_model),
|
||||
routerName: watchedName,
|
||||
teamId: requiresTeamScope ? watchedTeamId ?? undefined : undefined,
|
||||
}
|
||||
: undefined;
|
||||
|
||||
return (
|
||||
<TooltipProvider>
|
||||
<Card>
|
||||
|
|
@ -713,20 +724,7 @@ const AddAutoRouterTab: React.FC<AddAutoRouterTabProps> = ({
|
|||
testId={connectionTestId}
|
||||
accessToken={accessToken}
|
||||
targets={testTargets}
|
||||
jevRequest={
|
||||
effectiveClassifierType(complexityRouterConfig) === "jev"
|
||||
? buildAutoRouterRoutingTestRequest({
|
||||
prompt: JEV_CONNECTION_TEST_PROMPT,
|
||||
config: buildComplexityRouterConfig(complexityRouterConfigParams),
|
||||
defaultModel: resolveComplexityDefaultModel(
|
||||
complexityRouterConfig,
|
||||
complexityRouterConfig.default_model,
|
||||
),
|
||||
routerName: watchedName,
|
||||
teamId: requiresTeamScope ? watchedTeamId ?? undefined : undefined,
|
||||
})
|
||||
: undefined
|
||||
}
|
||||
jevRequest={jevConnectionTestParams && buildAutoRouterRoutingTestRequest(jevConnectionTestParams)}
|
||||
onTestComplete={() => setIsTestingConnection(false)}
|
||||
/>
|
||||
</TooltipProvider>
|
||||
|
|
|
|||
|
|
@ -66,39 +66,6 @@ describe("transitionClassifierType", () => {
|
|||
expect(result).toMatchObject(expectedSettings);
|
||||
});
|
||||
|
||||
it.each(["capability", "llm_v2"] as const)("requires explicit policy input for a new %s classifier", (target) => {
|
||||
const result = transitionClassifierType(standard, target);
|
||||
expect(result.classifier_llm_config).toEqual({ model: "judge", timeout_ms: 20000 });
|
||||
expect(result.classifier_fallback).toBeUndefined();
|
||||
if (target === "capability") {
|
||||
expect(result.capability_classifier_config?.base_threshold).toBeNaN();
|
||||
} else {
|
||||
expect(result.llm_v2_config).toMatchObject({ efficient_profile: "", capable_profile: "", harness: "" });
|
||||
expect(result.llm_v2_config?.max_quality_gap).toBeNaN();
|
||||
}
|
||||
expect(standard.tiers.MEDIUM).toEqual(["middle"]);
|
||||
expect(standard.classifier_llm_config?.classification_rubric).toBe("business");
|
||||
});
|
||||
|
||||
it.each([
|
||||
["capability", "llm"],
|
||||
["capability", "heuristic_first"],
|
||||
["capability", "hybrid"],
|
||||
["llm_v2", "llm"],
|
||||
["llm_v2", "heuristic_first"],
|
||||
["llm_v2", "hybrid"],
|
||||
] as const)("restores the complexity rubric from %s to %s while preserving the judge", (source, target) => {
|
||||
const forecast = transitionClassifierType(standard, source);
|
||||
const result = transitionClassifierType(forecast, target);
|
||||
expect(result.classifier_llm_config).toEqual({
|
||||
model: "judge",
|
||||
timeout_ms: 20000,
|
||||
classification_rubric: "agentic",
|
||||
});
|
||||
expect(result.capability_classifier_config).toBeUndefined();
|
||||
expect(result.llm_v2_config).toBeUndefined();
|
||||
});
|
||||
|
||||
it("clears the inactive non-reasoning pool and plan floor when switching to local classification", () => {
|
||||
const initial: ComplexityRouterConfigValue = {
|
||||
...standard,
|
||||
|
|
|
|||
|
|
@ -11,19 +11,17 @@ import {
|
|||
usesClassifierContext,
|
||||
} from "./ComplexityRouterConfig";
|
||||
import { defaultJevClassifierConfig } from "./jev_classifier_config";
|
||||
import { isForecastClassifier, prepareForecastClassifier } from "./forecast_classifier_config";
|
||||
import { nonReasoningTierFields } from "./nonReasoningTierFields";
|
||||
|
||||
export const transitionClassifierType = (
|
||||
value: ComplexityRouterConfigValue,
|
||||
classifierType: ClassifierType,
|
||||
): ComplexityRouterConfigValue => {
|
||||
const startsLlmRubric =
|
||||
!value.classifier_llm_config ||
|
||||
(isForecastClassifier(value.classifier_type) && !isForecastClassifier(classifierType));
|
||||
const startsLlmRubric = !value.classifier_llm_config;
|
||||
const judgeConfig = value.classifier_llm_config ?? { model: "", timeout_ms: DEFAULT_CLASSIFIER_TIMEOUT_MS };
|
||||
const nextValue: ComplexityRouterConfigValue = {
|
||||
...value,
|
||||
classifier_type: classifierType,
|
||||
jev_classifier_config:
|
||||
classifierType === "jev" ? value.jev_classifier_config ?? defaultJevClassifierConfig() : undefined,
|
||||
classification_prompt: classifierType === "jev" ? undefined : value.classification_prompt,
|
||||
|
|
@ -55,5 +53,5 @@ export const transitionClassifierType = (
|
|||
classifierType === "hybrid" ? value.hybrid_boundary_margin ?? DEFAULT_HYBRID_BOUNDARY_MARGIN : undefined,
|
||||
...nonReasoningTierFields(classifierType, value),
|
||||
};
|
||||
return prepareForecastClassifier(nextValue, classifierType);
|
||||
return nextValue;
|
||||
};
|
||||
|
|
|
|||
|
|
@ -1,15 +1,7 @@
|
|||
export type ClassifierType =
|
||||
| "heuristic"
|
||||
| "heuristic_v2"
|
||||
| "llm"
|
||||
| "jev"
|
||||
| "heuristic_first"
|
||||
| "hybrid"
|
||||
| "capability"
|
||||
| "llm_v2";
|
||||
export type ClassifierType = "heuristic" | "heuristic_v2" | "llm" | "jev" | "heuristic_first" | "hybrid";
|
||||
|
||||
export const usesLlmClassifier = (classifierType: ClassifierType): boolean =>
|
||||
(["llm", "heuristic_first", "hybrid", "capability", "llm_v2"] as const).some((type) => type === classifierType);
|
||||
(["llm", "heuristic_first", "hybrid"] as const).some((type) => type === classifierType);
|
||||
|
||||
export const usesClassifierContext = (classifierType: ClassifierType): boolean =>
|
||||
classifierType === "jev" || usesLlmClassifier(classifierType);
|
||||
|
|
|
|||
|
|
@ -1,5 +1,12 @@
|
|||
import { usesClassifierContext } from "../add_model/classifier_types";
|
||||
import { defaultJevClassifierConfig, jevClassifierConfigSchema } from "../add_model/jev_classifier_config";
|
||||
import React, { useEffect, useMemo, useState } from "react";
|
||||
import { z } from "zod/v4";
|
||||
import {
|
||||
complexityRouterSchema,
|
||||
semanticRouterSchema,
|
||||
EMPTY_FORM_VALUES,
|
||||
type EditAutoRouterFormValues,
|
||||
} from "./editAutoRouterFormSchema";
|
||||
import { toast } from "@/lib/toast";
|
||||
import { CircleHelp } from "lucide-react";
|
||||
import { FieldGroup } from "@/components/ui/field";
|
||||
|
|
@ -13,8 +20,8 @@ import AccessGroupTagsCombobox from "../add_model/AccessGroupTagsCombobox";
|
|||
import ModelChoiceCombobox, { type ModelChoice } from "../add_model/ModelChoiceCombobox";
|
||||
import { modelAvailableCall, modelPatchUpdateCall, validateAutoRouterConfig } from "../networking";
|
||||
import { fetchAvailableModels, ModelGroup } from "@/components/llm_calls/fetch_models";
|
||||
import RouterConfigBuilder from "../add_model/RouterConfigBuilder";
|
||||
import { hydrateTierModelParams, normalizeTierModels } from "../add_model/complexity_router_tiers";
|
||||
import RouterConfigBuilder, { type RouterConfig, serializeRouterConfig } from "../add_model/RouterConfigBuilder";
|
||||
import { hydrateTierModelParams } from "../add_model/complexity_router_tiers";
|
||||
import {
|
||||
type ActiveTierSet,
|
||||
CUSTOM_TIER_OMITTED_KEYS,
|
||||
|
|
@ -28,11 +35,13 @@ import {
|
|||
type BuildComplexityRouterConfigParams,
|
||||
buildComplexityRouterConfig,
|
||||
getClassifierModelError,
|
||||
getClassifierReasoningEffortError,
|
||||
getKeywordTierRulesError,
|
||||
getMissingTiersError,
|
||||
getSemanticConfigError,
|
||||
getPlanModeTierError,
|
||||
getTierLabelsError,
|
||||
hydrateBuiltInTiers,
|
||||
hydrateCustomTierSet,
|
||||
hydratePlanModeMinTier,
|
||||
hydrateTierLabels,
|
||||
|
|
@ -40,7 +49,14 @@ import {
|
|||
} from "../add_model/build_complexity_router_config";
|
||||
import { KeywordTierRule } from "../add_model/KeywordTierRules";
|
||||
import { DEFAULT_MATCH_THRESHOLD } from "../add_model/SemanticKeywordMatching";
|
||||
import {
|
||||
type AutoRouterCompressionState,
|
||||
buildAutoRouterCompressionPatch,
|
||||
DEFAULT_AUTO_ROUTER_COMPRESSION,
|
||||
hydrateAutoRouterCompression,
|
||||
} from "../add_model/buildAutoRouterCompression";
|
||||
import { hydrateKeywordTierRules } from "../add_model/complexity_router_keywords";
|
||||
import { customDimensionsError, hydrateCustomDimensions } from "../add_model/custom_dimensions";
|
||||
import {
|
||||
hydrateDimensionWeights,
|
||||
hydrateReasoningOverrideMinScore,
|
||||
|
|
@ -53,7 +69,8 @@ import ComplexityRouterConfig, {
|
|||
ClassifierLLMConfig,
|
||||
ClassifierType,
|
||||
ComplexityRouterConfigValue,
|
||||
ComplexityTiers,
|
||||
effectiveClassifierType,
|
||||
heuristicScoringRole,
|
||||
DEFAULT_ADAPTIVE_WEIGHTS,
|
||||
DEFAULT_SESSION_AFFINITY,
|
||||
DEFAULT_DEPLOYMENT_AFFINITY,
|
||||
|
|
@ -83,30 +100,45 @@ interface EditAutoRouterModalProps {
|
|||
/** The complexity_router_config as it comes back from the proxy, before any hydration. Fields the
|
||||
* hydrators validate themselves stay `unknown`; the ones assigned straight through carry their type. */
|
||||
export interface StoredComplexityRouterConfig {
|
||||
tiers?: Partial<Record<keyof ComplexityTiers, unknown>>;
|
||||
tiers?: Record<string, unknown>;
|
||||
enable_non_reasoning_tier?: boolean;
|
||||
tier_model_configs?: unknown;
|
||||
default_model?: string | null;
|
||||
plan_mode_min_tier?: unknown;
|
||||
classification_prompt?: unknown;
|
||||
classification_examples?: unknown;
|
||||
heuristic_first_max_tier?: unknown;
|
||||
hybrid_boundary_margin?: unknown;
|
||||
tier_labels?: unknown;
|
||||
classifier_type?: ClassifierType;
|
||||
classifier_llm_config?: ClassifierLLMConfig;
|
||||
jev_classifier_config?: unknown;
|
||||
classifier_context_window_size?: unknown;
|
||||
classifier_context_budget_chars?: unknown;
|
||||
classifier_context_per_turn_chars?: unknown;
|
||||
classifier_context_include_assistant_turns?: unknown;
|
||||
classifier_fallback?: unknown;
|
||||
classification_mode?: unknown;
|
||||
tier_boundaries?: unknown;
|
||||
token_thresholds?: unknown;
|
||||
dimension_weights?: unknown;
|
||||
custom_dimensions?: unknown;
|
||||
reasoning_override_min_score?: unknown;
|
||||
session_affinity?: unknown;
|
||||
session_affinity_ttl_seconds?: unknown;
|
||||
modality_routing?: unknown;
|
||||
modality_pin_override?: unknown;
|
||||
deployment_affinity?: unknown;
|
||||
adaptive?: boolean;
|
||||
adaptive_weights?: AdaptiveRouterWeights;
|
||||
tier_distance_penalty?: number;
|
||||
adaptive_eligible?: AdaptiveEligible;
|
||||
return_raw_model_name?: boolean;
|
||||
enable_context_window_escalation?: unknown;
|
||||
context_window_escalation_buffer?: unknown;
|
||||
stall_escalation_enabled?: unknown;
|
||||
stall_escalation_window?: unknown;
|
||||
stall_escalation_repeat_threshold?: unknown;
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -117,18 +149,14 @@ export const hydrateComplexityRouterConfig = (
|
|||
parsedConfig: StoredComplexityRouterConfig,
|
||||
complexityRouterDefaultModel: string | null | undefined,
|
||||
): ComplexityRouterConfigValue => {
|
||||
const hydratedTiers: ComplexityTiers = {
|
||||
SIMPLE: normalizeTierModels(parsedConfig.tiers?.SIMPLE),
|
||||
MEDIUM: normalizeTierModels(parsedConfig.tiers?.MEDIUM),
|
||||
COMPLEX: normalizeTierModels(parsedConfig.tiers?.COMPLEX),
|
||||
REASONING: normalizeTierModels(parsedConfig.tiers?.REASONING),
|
||||
};
|
||||
|
||||
const builtIn = hydrateBuiltInTiers(parsedConfig.tiers, parsedConfig.enable_non_reasoning_tier);
|
||||
const { tiers: hydratedTiers, enable_non_reasoning_tier } = builtIn;
|
||||
const custom_tier_set = hydrateCustomTierSet(parsedConfig);
|
||||
const activeTiers = { tiers: hydratedTiers, custom_tier_set };
|
||||
const activeTiers = { ...builtIn, custom_tier_set };
|
||||
|
||||
return {
|
||||
tiers: hydratedTiers,
|
||||
enable_non_reasoning_tier,
|
||||
custom_tier_set,
|
||||
tier_model_params: tierParamsByRowId(
|
||||
hydrateTierModelParams(parsedConfig.tiers, parsedConfig.tier_model_configs),
|
||||
|
|
@ -138,7 +166,12 @@ export const hydrateComplexityRouterConfig = (
|
|||
plan_mode_min_tier: hydratePlanModeMinTier(parsedConfig.plan_mode_min_tier, custom_tier_set),
|
||||
tier_labels: hydrateTierLabels(parsedConfig.tier_labels),
|
||||
classifier_type: parsedConfig.classifier_type || "heuristic",
|
||||
classifier_llm_config: parsedConfig.classifier_llm_config,
|
||||
classifier_llm_config: parsedConfig.classifier_type === "jev" ? undefined : parsedConfig.classifier_llm_config,
|
||||
jev_classifier_config:
|
||||
parsedConfig.classifier_type === "jev"
|
||||
? jevClassifierConfigSchema.safeParse(parsedConfig.jev_classifier_config ?? {}).data ??
|
||||
defaultJevClassifierConfig()
|
||||
: undefined,
|
||||
classifier_context_window_size:
|
||||
typeof parsedConfig.classifier_context_window_size === "number"
|
||||
? parsedConfig.classifier_context_window_size
|
||||
|
|
@ -147,6 +180,10 @@ export const hydrateComplexityRouterConfig = (
|
|||
typeof parsedConfig.classifier_context_budget_chars === "number"
|
||||
? parsedConfig.classifier_context_budget_chars
|
||||
: undefined,
|
||||
classifier_context_per_turn_chars:
|
||||
typeof parsedConfig.classifier_context_per_turn_chars === "number"
|
||||
? parsedConfig.classifier_context_per_turn_chars
|
||||
: undefined,
|
||||
classifier_context_include_assistant_turns:
|
||||
typeof parsedConfig.classifier_context_include_assistant_turns === "boolean"
|
||||
? parsedConfig.classifier_context_include_assistant_turns
|
||||
|
|
@ -159,16 +196,35 @@ export const hydrateComplexityRouterConfig = (
|
|||
typeof parsedConfig.classification_prompt === "string" && parsedConfig.classification_prompt.trim() !== ""
|
||||
? parsedConfig.classification_prompt
|
||||
: undefined,
|
||||
classification_examples:
|
||||
typeof parsedConfig.classification_examples === "string" && parsedConfig.classification_examples.trim() !== ""
|
||||
? parsedConfig.classification_examples
|
||||
: undefined,
|
||||
heuristic_first_max_tier:
|
||||
typeof parsedConfig.heuristic_first_max_tier === "string" && parsedConfig.heuristic_first_max_tier.trim() !== ""
|
||||
? parsedConfig.heuristic_first_max_tier
|
||||
: undefined,
|
||||
hybrid_boundary_margin:
|
||||
typeof parsedConfig.hybrid_boundary_margin === "number" ? parsedConfig.hybrid_boundary_margin : undefined,
|
||||
classification_mode:
|
||||
parsedConfig.classification_mode === "user_turn" || parsedConfig.classification_mode === "every_request"
|
||||
? parsedConfig.classification_mode
|
||||
: undefined,
|
||||
tier_boundaries: hydrateTierBoundaries(parsedConfig.tier_boundaries),
|
||||
token_thresholds: hydrateTokenThresholds(parsedConfig.token_thresholds),
|
||||
dimension_weights: hydrateDimensionWeights(parsedConfig.dimension_weights),
|
||||
custom_dimensions: hydrateCustomDimensions(parsedConfig.custom_dimensions),
|
||||
reasoning_override_min_score: hydrateReasoningOverrideMinScore(parsedConfig.reasoning_override_min_score),
|
||||
session_affinity:
|
||||
typeof parsedConfig.session_affinity === "boolean" ? parsedConfig.session_affinity : DEFAULT_SESSION_AFFINITY,
|
||||
session_affinity_ttl_seconds:
|
||||
typeof parsedConfig.session_affinity_ttl_seconds === "number" &&
|
||||
Number.isFinite(parsedConfig.session_affinity_ttl_seconds)
|
||||
? parsedConfig.session_affinity_ttl_seconds
|
||||
: undefined,
|
||||
modality_routing: typeof parsedConfig.modality_routing === "boolean" ? parsedConfig.modality_routing : false,
|
||||
modality_pin_override:
|
||||
typeof parsedConfig.modality_pin_override === "boolean" ? parsedConfig.modality_pin_override : false,
|
||||
deployment_affinity:
|
||||
typeof parsedConfig.deployment_affinity === "boolean"
|
||||
? parsedConfig.deployment_affinity
|
||||
|
|
@ -178,11 +234,27 @@ export const hydrateComplexityRouterConfig = (
|
|||
tier_distance_penalty: parsedConfig.tier_distance_penalty,
|
||||
adaptive_eligible: parsedConfig.adaptive_eligible || "all",
|
||||
return_raw_model_name: parsedConfig.return_raw_model_name || false,
|
||||
enable_context_window_escalation:
|
||||
typeof parsedConfig.enable_context_window_escalation === "boolean"
|
||||
? parsedConfig.enable_context_window_escalation
|
||||
: undefined,
|
||||
context_window_escalation_buffer:
|
||||
typeof parsedConfig.context_window_escalation_buffer === "number"
|
||||
? parsedConfig.context_window_escalation_buffer
|
||||
: undefined,
|
||||
stall_escalation_enabled: parsedConfig.stall_escalation_enabled === true || undefined,
|
||||
stall_escalation_window:
|
||||
typeof parsedConfig.stall_escalation_window === "number" ? parsedConfig.stall_escalation_window : undefined,
|
||||
stall_escalation_repeat_threshold:
|
||||
typeof parsedConfig.stall_escalation_repeat_threshold === "number"
|
||||
? parsedConfig.stall_escalation_repeat_threshold
|
||||
: undefined,
|
||||
};
|
||||
};
|
||||
|
||||
export const MANAGED_COMPLEXITY_ROUTER_KEYS = new Set([
|
||||
"tiers",
|
||||
"enable_non_reasoning_tier",
|
||||
"tier_definitions",
|
||||
"fallback_tier",
|
||||
"tier_model_configs",
|
||||
|
|
@ -191,13 +263,20 @@ export const MANAGED_COMPLEXITY_ROUTER_KEYS = new Set([
|
|||
"tier_labels",
|
||||
"classifier_type",
|
||||
"classifier_llm_config",
|
||||
"jev_classifier_config",
|
||||
"classifier_context_window_size",
|
||||
"classifier_context_budget_chars",
|
||||
"classifier_context_include_assistant_turns",
|
||||
"classifier_fallback",
|
||||
"classification_prompt",
|
||||
"classification_examples",
|
||||
"heuristic_first_max_tier",
|
||||
"hybrid_boundary_margin",
|
||||
"classification_mode",
|
||||
"session_affinity",
|
||||
"session_affinity_ttl_seconds",
|
||||
"modality_routing",
|
||||
"modality_pin_override",
|
||||
"deployment_affinity",
|
||||
"adaptive",
|
||||
"adaptive_weights",
|
||||
|
|
@ -207,7 +286,13 @@ export const MANAGED_COMPLEXITY_ROUTER_KEYS = new Set([
|
|||
"tier_boundaries",
|
||||
"token_thresholds",
|
||||
"dimension_weights",
|
||||
"custom_dimensions",
|
||||
"reasoning_override_min_score",
|
||||
"enable_context_window_escalation",
|
||||
"context_window_escalation_buffer",
|
||||
"stall_escalation_enabled",
|
||||
"stall_escalation_window",
|
||||
"stall_escalation_repeat_threshold",
|
||||
]);
|
||||
|
||||
// Managed only when the caller passes the corresponding state. A caller that does not render
|
||||
|
|
@ -254,8 +339,8 @@ export interface KeywordMatchingState {
|
|||
}
|
||||
|
||||
// A custom save drops the stored keys an edited tier set forbids. classification_prompt needs no
|
||||
// entry here: it is a managed key, so a built-in save already drops it through isManaged and the
|
||||
// built-in branch of the builder never re-emits it.
|
||||
// entry here: it is a managed key, so every save rewrites it from form state and the builder
|
||||
// re-emits it on both branches only when the form still holds one.
|
||||
const customTierDroppedKeys = (value: ComplexityRouterConfigValue): readonly string[] =>
|
||||
value.custom_tier_set ? CUSTOM_TIER_OMITTED_KEYS : [];
|
||||
|
||||
|
|
@ -266,6 +351,8 @@ export const buildUpdatedComplexityRouterConfig = (
|
|||
keywordMatching?: KeywordMatchingState,
|
||||
): Record<string, unknown> => {
|
||||
const isManaged = (key: string): boolean => {
|
||||
if (key === "classifier_context_per_turn_chars")
|
||||
return !usesClassifierContext(effectiveClassifierType(value)) || Object.prototype.hasOwnProperty.call(value, key);
|
||||
if (MANAGED_COMPLEXITY_ROUTER_KEYS.has(key)) return true;
|
||||
if (keywordMatching !== undefined && KEYWORD_MATCHING_KEYS.has(key)) return true;
|
||||
return customTechnicalKeywords !== undefined && key === "custom_technical_keywords";
|
||||
|
|
@ -277,19 +364,28 @@ export const buildUpdatedComplexityRouterConfig = (
|
|||
|
||||
const builderParams: BuildComplexityRouterConfigParams = {
|
||||
tiers: value.tiers,
|
||||
enableNonReasoningTier: value.enable_non_reasoning_tier,
|
||||
customTierSet: value.custom_tier_set,
|
||||
defaultModel: value.default_model,
|
||||
planModeMinTier: value.plan_mode_min_tier,
|
||||
classificationPrompt: value.classification_prompt,
|
||||
classificationExamples: value.classification_examples,
|
||||
heuristicFirstMaxTier: value.heuristic_first_max_tier,
|
||||
hybridBoundaryMargin: value.hybrid_boundary_margin,
|
||||
classificationMode: value.classification_mode,
|
||||
tierLabels: value.tier_labels,
|
||||
classifierType: value.classifier_type,
|
||||
jevClassifierConfig: value.jev_classifier_config,
|
||||
classifierLlmConfig: value.classifier_llm_config,
|
||||
classifierContextWindowSize: value.classifier_context_window_size,
|
||||
classifierContextBudgetChars: value.classifier_context_budget_chars,
|
||||
classifierContextPerTurnChars: value.classifier_context_per_turn_chars,
|
||||
classifierContextIncludeAssistantTurns: value.classifier_context_include_assistant_turns,
|
||||
classifierFallback: value.classifier_fallback,
|
||||
sessionAffinity: value.session_affinity ?? DEFAULT_SESSION_AFFINITY,
|
||||
sessionAffinityTtlSeconds: value.session_affinity_ttl_seconds,
|
||||
modalityRouting: value.modality_routing ?? false,
|
||||
modalityPinOverride: value.modality_pin_override ?? false,
|
||||
deploymentAffinity: value.deployment_affinity ?? DEFAULT_DEPLOYMENT_AFFINITY,
|
||||
customTechnicalKeywords: customTechnicalKeywords ?? [],
|
||||
keywordTierRules: keywordMatching?.keywordTierRules ?? [],
|
||||
|
|
@ -305,8 +401,14 @@ export const buildUpdatedComplexityRouterConfig = (
|
|||
tierBoundaries: value.tier_boundaries,
|
||||
tokenThresholds: value.token_thresholds,
|
||||
dimensionWeights: value.dimension_weights,
|
||||
customDimensions: value.custom_dimensions,
|
||||
reasoningOverrideMinScore: value.reasoning_override_min_score,
|
||||
tierModelParams: value.tier_model_params,
|
||||
enableContextWindowEscalation: value.enable_context_window_escalation,
|
||||
contextWindowEscalationBuffer: value.context_window_escalation_buffer,
|
||||
stallEscalationEnabled: value.stall_escalation_enabled,
|
||||
stallEscalationWindow: value.stall_escalation_window,
|
||||
stallEscalationRepeatThreshold: value.stall_escalation_repeat_threshold,
|
||||
};
|
||||
const built = buildComplexityRouterConfig(builderParams);
|
||||
|
||||
|
|
@ -321,35 +423,6 @@ export const buildUpdatedComplexityRouterConfig = (
|
|||
};
|
||||
};
|
||||
|
||||
const sharedShape = {
|
||||
auto_router_name: z.string().min(1, "Auto router name is required"),
|
||||
model_access_group: z.array(z.string()),
|
||||
};
|
||||
|
||||
const complexityRouterShape = {
|
||||
...sharedShape,
|
||||
auto_router_default_model: z.string(),
|
||||
auto_router_embedding_model: z.string(),
|
||||
};
|
||||
|
||||
const semanticRouterShape = {
|
||||
...sharedShape,
|
||||
auto_router_default_model: z.string().min(1, "Default model is required"),
|
||||
auto_router_embedding_model: z.string().min(1, "Embedding model is required"),
|
||||
};
|
||||
|
||||
const complexityRouterSchema = z.object(complexityRouterShape);
|
||||
const semanticRouterSchema = z.object(semanticRouterShape);
|
||||
|
||||
type EditAutoRouterFormValues = z.infer<typeof semanticRouterSchema>;
|
||||
|
||||
const EMPTY_FORM_VALUES: EditAutoRouterFormValues = {
|
||||
auto_router_name: "",
|
||||
auto_router_default_model: "",
|
||||
auto_router_embedding_model: "",
|
||||
model_access_group: [],
|
||||
};
|
||||
|
||||
const labelWithHint = (label: string, hint: string): React.ReactNode => (
|
||||
<>
|
||||
{label}
|
||||
|
|
@ -373,13 +446,16 @@ const EditAutoRouterModal: React.FC<EditAutoRouterModalProps> = ({
|
|||
const [modelInfo, setModelInfo] = useState<ModelGroup[]>([]);
|
||||
const [showValidationErrors, setShowValidationErrors] = useState<boolean>(false);
|
||||
const [editingTiers, setEditingTiers] = useState(false);
|
||||
const [routerConfig, setRouterConfig] = useState<any>(null);
|
||||
const [routerConfig, setRouterConfig] = useState<RouterConfig | null>(null);
|
||||
const [customTechnicalKeywords, setCustomTechnicalKeywords] = useState<string[]>([]);
|
||||
const [keywordTierRules, setKeywordTierRules] = useState<KeywordTierRule[]>([]);
|
||||
const [escalationKeywords, setEscalationKeywords] = useState<string[]>([]);
|
||||
const [semanticMatchingEnabled, setSemanticMatchingEnabled] = useState<boolean>(false);
|
||||
const [embeddingModel, setEmbeddingModel] = useState<string | undefined>(undefined);
|
||||
const [matchThreshold, setMatchThreshold] = useState<number>(DEFAULT_MATCH_THRESHOLD);
|
||||
const [autoRouterCompression, setAutoRouterCompression] = useState<AutoRouterCompressionState>(
|
||||
DEFAULT_AUTO_ROUTER_COMPRESSION,
|
||||
);
|
||||
const [complexityRouterConfig, setComplexityRouterConfig] = useState<ComplexityRouterConfigValue>({
|
||||
tiers: { SIMPLE: [], MEDIUM: [], COMPLEX: [], REASONING: [] },
|
||||
classifier_type: "heuristic",
|
||||
|
|
@ -405,7 +481,10 @@ const EditAutoRouterModal: React.FC<EditAutoRouterModalProps> = ({
|
|||
: null) ?? getTierLabelsError(complexityRouterConfig.tier_labels)) ??
|
||||
getPlanModeTierError(complexityRouterConfig.plan_mode_min_tier, activeTierRows(complexityRouterConfig)) ??
|
||||
getKeywordTierRulesError(keywordTierRules, activeTierRows(complexityRouterConfig)) ??
|
||||
getClassifierModelError(complexityRouterConfig);
|
||||
getClassifierModelError(complexityRouterConfig) ??
|
||||
(heuristicScoringRole(complexityRouterConfig) === "decides"
|
||||
? customDimensionsError(complexityRouterConfig.custom_dimensions)
|
||||
: null);
|
||||
|
||||
useEffect(() => {
|
||||
if (isVisible && modelData) {
|
||||
|
|
@ -472,6 +551,12 @@ const EditAutoRouterModal: React.FC<EditAutoRouterModalProps> = ({
|
|||
setMatchThreshold(
|
||||
typeof parsedConfig.match_threshold === "number" ? parsedConfig.match_threshold : DEFAULT_MATCH_THRESHOLD,
|
||||
);
|
||||
setAutoRouterCompression(
|
||||
hydrateAutoRouterCompression({
|
||||
auto_router_routing_compression: modelData.litellm_params?.auto_router_routing_compression,
|
||||
auto_router_model_compression: modelData.litellm_params?.auto_router_model_compression,
|
||||
}),
|
||||
);
|
||||
|
||||
form.reset({
|
||||
...EMPTY_FORM_VALUES,
|
||||
|
|
@ -496,8 +581,8 @@ const EditAutoRouterModal: React.FC<EditAutoRouterModalProps> = ({
|
|||
// Set form values
|
||||
form.reset({
|
||||
auto_router_name: modelData.model_name,
|
||||
auto_router_default_model: modelData.litellm_params?.auto_router_default_model || "",
|
||||
auto_router_embedding_model: modelData.litellm_params?.auto_router_embedding_model || "",
|
||||
auto_router_default_model: modelData.litellm_params?.auto_router_default_model || null,
|
||||
auto_router_embedding_model: modelData.litellm_params?.auto_router_embedding_model || null,
|
||||
model_access_group: modelData.model_info?.access_groups || [],
|
||||
});
|
||||
} catch (error) {
|
||||
|
|
@ -519,12 +604,22 @@ const EditAutoRouterModal: React.FC<EditAutoRouterModalProps> = ({
|
|||
toast.fromError(tierSetError);
|
||||
return;
|
||||
}
|
||||
const classifierError = getClassifierModelError(complexityRouterConfig);
|
||||
const classifierError =
|
||||
getClassifierModelError(complexityRouterConfig) ??
|
||||
(heuristicScoringRole(complexityRouterConfig) === "decides"
|
||||
? customDimensionsError(complexityRouterConfig.custom_dimensions)
|
||||
: null);
|
||||
if (classifierError) {
|
||||
setShowValidationErrors(true);
|
||||
toast.fromError(classifierError);
|
||||
return;
|
||||
}
|
||||
const classifierEffortError = getClassifierReasoningEffortError(complexityRouterConfig, modelInfo);
|
||||
if (classifierEffortError) {
|
||||
setShowValidationErrors(true);
|
||||
toast.fromError(classifierEffortError);
|
||||
return;
|
||||
}
|
||||
// Same guards the create form applies (add_auto_router_tab.tsx). The backend rejects a
|
||||
// keyword rule with no keyword, and semantic_keyword_matching without an embedding model
|
||||
// or keyword rules (complexity_router/config.py), so without these a save fails as a raw
|
||||
|
|
@ -578,6 +673,7 @@ const EditAutoRouterModal: React.FC<EditAutoRouterModalProps> = ({
|
|||
...modelData.litellm_params,
|
||||
complexity_router_config: updatedConfig,
|
||||
complexity_router_default_model: defaultModel,
|
||||
...buildAutoRouterCompressionPatch(autoRouterCompression, modelData.litellm_params ?? {}),
|
||||
};
|
||||
const updatedModelInfo = {
|
||||
...modelData.model_info,
|
||||
|
|
@ -604,7 +700,7 @@ const EditAutoRouterModal: React.FC<EditAutoRouterModalProps> = ({
|
|||
// Prepare the updated litellm_params
|
||||
const updatedLitellmParams = {
|
||||
...modelData.litellm_params,
|
||||
auto_router_config: JSON.stringify(routerConfig),
|
||||
auto_router_config: serializeRouterConfig(routerConfig),
|
||||
auto_router_default_model: values.auto_router_default_model,
|
||||
auto_router_embedding_model: values.auto_router_embedding_model || undefined,
|
||||
};
|
||||
|
|
@ -643,7 +739,7 @@ const EditAutoRouterModal: React.FC<EditAutoRouterModalProps> = ({
|
|||
})();
|
||||
} catch (error) {
|
||||
console.error("Error updating auto router:", error);
|
||||
toast.fromError("Failed to update auto router configuration");
|
||||
toast.fromError(error);
|
||||
} finally {
|
||||
setLoading(false);
|
||||
}
|
||||
|
|
@ -699,6 +795,8 @@ const EditAutoRouterModal: React.FC<EditAutoRouterModalProps> = ({
|
|||
onMatchThresholdChange={setMatchThreshold}
|
||||
escalationKeywords={escalationKeywords}
|
||||
onEscalationKeywordsChange={setEscalationKeywords}
|
||||
autoRouterCompression={autoRouterCompression}
|
||||
onAutoRouterCompressionChange={setAutoRouterCompression}
|
||||
/>
|
||||
</div>
|
||||
) : (
|
||||
|
|
@ -794,16 +892,3 @@ const EditAutoRouterModal: React.FC<EditAutoRouterModalProps> = ({
|
|||
};
|
||||
|
||||
export default EditAutoRouterModal;
|
||||
|
||||
import { usesClassifierContext } from "../add_model/classifier_types";
|
||||
import { defaultJevClassifierConfig, jevClassifierConfigSchema } from "../add_model/jev_classifier_config";
|
||||
: undefined,
|
||||
classifier_context_per_turn_chars:
|
||||
typeof parsedConfig.classifier_context_per_turn_chars === "number"
|
||||
? parsedConfig.classifier_context_per_turn_chars
|
||||
"jev_classifier_config",
|
||||
if (key === "classifier_context_per_turn_chars") {
|
||||
return !usesClassifierContext(effectiveClassifierType(value)) || Object.prototype.hasOwnProperty.call(value, key);
|
||||
}
|
||||
jevClassifierConfig: value.jev_classifier_config,
|
||||
classifierContextPerTurnChars: value.classifier_context_per_turn_chars,
|
||||
Loading…
Add table
Reference in a new issue