From bbd45bf37d715e048e15f09e782255893cc9422d Mon Sep 17 00:00:00 2001 From: Tin Chi Lo Date: Wed, 26 Aug 2026 17:45:11 -0700 Subject: [PATCH] feat(ui): edit the auto-router tier set with custom classifier-defined tiers The auto-router form was fixed at the four built-in complexity tiers. The backend has taken tier_definitions since #37226, so the only way to add a tier was hand-written raw config plus a proxy restart. Edit tiers turns the existing tier list into an editor: rename a built-in, remove one, restore it, or add a tier with a name, the definition the LLM classifier routes on, and its models. A Fallback Tier picks where classifier failures land. Every input an edited tier set forbids lives in one table beside the payload keys it owns, so a control cannot grey out while its value still ships: display names, escalation, adaptive routing, session pinning, the heuristic classifier, the replacement classifier prompt, the rubric preset, and the classifier-failure fallback. The scorer knobs go too, as inert rather than rejected. Per-model params live on the tier row that owns them, so a row's models and its reasoning efforts cannot be keyed differently and travel together through a rename, a removal, and a return to the built-in tiers. Both forms dry-run POST /auto_router/validate_complexity_router_config against the exact payload the save will send, and fail open on a transport error since the write gate stays authoritative. A form that never opens the editor submits a byte-identical payload. Resolves LIT-5214 --- .../CostOptimizationView.activity.test.tsx | 2 + .../_components/TierTurnsChart.test.tsx | 14 + .../_components/TierTurnsChart.tsx | 7 +- .../add_model/ClassificationMethodConfig.tsx | 153 +++--- .../add_model/ComplexityRouterConfig.test.tsx | 248 ++++++++++ .../add_model/ComplexityRouterConfig.tsx | 454 ++++++++++++++---- .../components/add_model/KeywordTierRules.tsx | 9 +- .../add_model/add_auto_router_tab.test.tsx | 18 + .../add_model/add_auto_router_tab.tsx | 64 ++- .../build_complexity_router_config.test.ts | 181 ++++++- .../build_complexity_router_config.ts | 151 +++++- .../add_model/complexity_router_tiers.test.ts | 20 + .../add_model/complexity_router_tiers.ts | 22 +- .../components/add_model/tier_rows.test.ts | 157 +++++- .../src/components/add_model/tier_rows.ts | 154 +++++- ...d_updated_complexity_router_config.test.ts | 48 +- .../edit_auto_router_modal.test.tsx | 72 +++ .../edit_auto_router_modal.tsx | 95 ++-- .../src/components/networking.tsx | 24 + 19 files changed, 1636 insertions(+), 257 deletions(-) diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/CostOptimizationView.activity.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/CostOptimizationView.activity.test.tsx index 2d46ca48adb..1cc7bec13d1 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/CostOptimizationView.activity.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/CostOptimizationView.activity.test.tsx @@ -32,6 +32,8 @@ vi.mock("@/components/shared/charts", () => ({ DonutChart: () =>
, BarChart: () =>
, CustomLegend: () =>
, + chartColorValue: (color: string) => color, + DEFAULT_COLOR_CYCLE: ["blue", "cyan", "sky", "indigo", "violet", "purple", "fuchsia", "slate"], SEQUENTIAL_COLOR_RAMP: ["indigo"], })); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/TierTurnsChart.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/TierTurnsChart.test.tsx index 057eb54ee4e..da4af8baf29 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/TierTurnsChart.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/TierTurnsChart.test.tsx @@ -6,6 +6,7 @@ import type { AutoRouterDeployment } from "@/app/(dashboard)/hooks/models/useMod vi.mock("@/components/shared/charts", () => ({ DonutChart: ({ label }: { label: string }) =>
{label}
, + DEFAULT_COLOR_CYCLE: ["blue", "cyan", "sky", "indigo", "violet", "purple", "fuchsia", "slate"], SEQUENTIAL_COLOR_RAMP: ["indigo", "blue"], chartColorValue: (color: string) => color, })); @@ -111,6 +112,19 @@ describe("TierTurnsChart", () => { expect(screen.getByText("gpt-4o-mini")).toBeInTheDocument(); }); + it("lists a custom tier's models, which the built-in name guard used to hide", () => { + render( + , + ); + + expect(screen.getByText(/SECURITY_REVIEW/)).toBeInTheDocument(); + expect(screen.getByText("o1-preview")).toBeInTheDocument(); + expect(screen.getByText("gpt-4o-mini")).toBeInTheDocument(); + }); + it("omits the model line for a tier with no configured models", () => { render(); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/TierTurnsChart.tsx b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/TierTurnsChart.tsx index e55ebc07656..44cca6331b8 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/TierTurnsChart.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/TierTurnsChart.tsx @@ -11,7 +11,7 @@ import { type ComplexityTiers, } from "@/components/add_model/ComplexityRouterConfig"; import { normalizeTierModels } from "@/components/add_model/complexity_router_tiers"; -import { chartColorValue, DonutChart, type ChartColor } from "@/components/shared/charts"; +import { chartColorValue, DEFAULT_COLOR_CYCLE, DonutChart } from "@/components/shared/charts"; import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card"; import { viewGroup, type BenchmarkView } from "./autoRouterBenchmarks"; @@ -71,7 +71,6 @@ const tierModelsFor = ( routerType: string, autoRouters: readonly AutoRouterDeployment[], ): string[] => { - if (!isComplexityTier(tier)) return []; const deployment = deploymentFor(routerName, routerType, autoRouters); if (!deployment) return []; const config = asRecord(deployment.litellm_params?.complexity_router_config); @@ -84,8 +83,6 @@ interface TierTurnsChartProps { autoRouters: readonly AutoRouterDeployment[]; } -const TIER_DONUT_COLORS: readonly ChartColor[] = ["#c7d2fe", "#1e293b", "#d4b483", "#87a878"]; - const TierTurnsChart: React.FC = ({ view, autoRouters }) => { const group = viewGroup(view); const entries = Object.entries(group?.tier_turns ?? {}).filter(([, turns]) => turns > 0); @@ -98,7 +95,7 @@ const TierTurnsChart: React.FC = ({ view, autoRouters }) => turns, models: tierModelsFor(tier, group.router_name, group.router_type, autoRouters), })); - const colors = slices.map((_, idx) => TIER_DONUT_COLORS[idx % TIER_DONUT_COLORS.length]); + const colors = slices.map((_, idx) => DEFAULT_COLOR_CYCLE[idx % DEFAULT_COLOR_CYCLE.length]); return ( diff --git a/ui/litellm-dashboard/src/components/add_model/ClassificationMethodConfig.tsx b/ui/litellm-dashboard/src/components/add_model/ClassificationMethodConfig.tsx index cea967f5966..8a982556a51 100644 --- a/ui/litellm-dashboard/src/components/add_model/ClassificationMethodConfig.tsx +++ b/ui/litellm-dashboard/src/components/add_model/ClassificationMethodConfig.tsx @@ -31,6 +31,10 @@ import { usesLlmClassifier, DEFAULT_HEURISTIC_FIRST_MAX_TIER, HEURISTIC_FIRST_MAX_TIER_KEYS, + effectiveClassifierType, + heuristicScoringRole, + restrictedBy, + Restricted, } from "./ComplexityRouterConfig"; const DEFAULT_SCORING_EXPLANATION = @@ -51,7 +55,12 @@ const CUSTOM_PROMPT_WITH_DEFAULT_MODEL_FALLBACK = * decides the tier, and pairing one with the default-model fallback means the heuristic never runs * at all, so the panel must not keep implying a score is involved on either router. */ +const CUSTOM_TIER_SET_EXPLANATION = + "The LLM classifier reads your tier definitions and picks the tier a request belongs to. The built-in " + + "seven-dimension scorer does not run, so its score ranges do not apply here."; + const scoringExplanation = (value: ComplexityRouterConfigValue): string => { + if (value.custom_tier_set) return CUSTOM_TIER_SET_EXPLANATION; const usesCustomPrompt = usesLlmClassifier(value.classifier_type) && Boolean(value.classifier_llm_config?.system_prompt?.trim()); if (!usesCustomPrompt) return DEFAULT_SCORING_EXPLANATION; @@ -89,6 +98,7 @@ const boundaryRanges = ( const HowClassificationWorks: React.FC<{ value: ComplexityRouterConfigValue }> = ({ value }) => { // The shipped boundaries come from the proxy, so this card cannot state ranges the router stopped using. const { data: scorerDefaults, isError } = useComplexityScorerDefaults(); + const scorerRuns = heuristicScoringRole(value) !== "never"; const ranges = boundaryRanges( scorerDefaults?.tier_boundaries, value.tier_boundaries, @@ -100,7 +110,7 @@ const HowClassificationWorks: React.FC<{ value: ComplexityRouterConfigValue }> = How Classification Works {scoringExplanation(value)} - {ranges && ( + {scorerRuns && ranges && (
  • {effectiveTierLabel("SIMPLE", value.tier_labels)}: Score < {ranges.simpleMedium} @@ -151,8 +161,9 @@ const ClassificationMethodConfig: React.FC = ({ defaultModel, }) => { const hasDefaultModel = Boolean(defaultModel); + const classifierType = effectiveClassifierType(value); const classifierModelMissing = - showValidationErrors && usesLlmClassifier(value.classifier_type) && !value.classifier_llm_config?.model; + showValidationErrors && usesLlmClassifier(classifierType) && !value.classifier_llm_config?.model; const usesCustomPrompt = Boolean(value.classifier_llm_config?.system_prompt?.trim()); const contextBudget = value.classifier_context_budget_chars ?? DEFAULT_CLASSIFIER_CONTEXT_BUDGET_CHARS; const contextBudgetQuotesNothing = contextBudget > 0 && contextBudget < MIN_QUOTED_CONTEXT_TURN_CHARS; @@ -265,20 +276,22 @@ const ClassificationMethodConfig: React.FC = ({ return ( <> handleClassifierTypeChange(classifierType as ClassifierType)} className="w-full" >
    - + + - + +
    - {value.classifier_type === "heuristic_first" && ( + {classifierType === "heuristic_first" && (
    Decide locally up to = ({ onValueChange={(preset: ClassificationRubric | null) => preset && handleClassificationRubricChange(preset) } - disabled={usesCustomPrompt} + disabled={usesCustomPrompt || Boolean(value.custom_tier_set)} > @@ -388,57 +406,62 @@ const ClassificationMethodConfig: React.FC = ({ - {usesCustomPrompt - ? "Not in use: the custom prompt below is the classifier's entire rubric." - : CLASSIFICATION_RUBRIC_DESCRIPTIONS[classificationRubric].description} + {restrictedBy(value, "classificationRubric")?.reason ?? + (usesCustomPrompt + ? "Not in use: the custom prompt below is the classifier's entire rubric." + : CLASSIFICATION_RUBRIC_DESCRIPTIONS[classificationRubric].description)}
    Classifier Prompt - + + +
    If the classifier fails - handleClassifierFallbackChange(fallback as ClassifierFallback)} - > -
    - - + +
    +     + + Applies when the classifier call errors, times out, or returns an unparseable response. + +
    Context Window Size diff --git a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.test.tsx b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.test.tsx index 2925c28cc5e..f9572371b17 100644 --- a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.test.tsx +++ b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.test.tsx @@ -1056,3 +1056,251 @@ describe("ComplexityRouterConfig custom technical keywords", () => { expect(screen.queryByText("Custom Technical Keywords")).not.toBeInTheDocument(); }); }); + +describe("ComplexityRouterConfig tier editing", () => { + const renderEditor = ( + value?: ComplexityRouterConfigValue, + props: Partial> = {}, + ) => { + const onChange = vi.fn(); + const view = renderWithProviders( + , + ); + return { ...view, committed: () => onChange.mock.calls[0][0] as ComplexityRouterConfigValue, onChange }; + }; + + const customValue: ComplexityRouterConfigValue = { + ...defaultValue, + classifier_type: "llm", + classifier_llm_config: { model: "gpt-4", timeout_ms: 3000 }, + custom_tier_set: { + tiers: [ + { id: "CASUAL", name: "CASUAL", definition: "small talk", models: ["gpt-3.5-turbo"] }, + { id: "sec", name: "SECURITY_REVIEW", definition: "audits", models: ["gpt-4"] }, + ], + fallback_tier_id: "CASUAL", + }, + }; + + it("offers Edit tiers only when the parent owns the editor flag", () => { + renderWithProviders(); + expect(screen.queryByRole("button", { name: "Edit tiers" })).not.toBeInTheDocument(); + }); + + it("renders the four built-in tiers before any edit, unchanged", () => { + renderWithProviders(); + expect(screen.getByRole("button", { name: "Edit tiers" })).toBeInTheDocument(); + expect(screen.getByText("Tier 1 of 4", { exact: false })).toHaveTextContent("SIMPLE"); + }); + + it("adds a row and moves the form into an edited tier set, which the built-in record never leaves", () => { + const { committed } = renderEditor(); + fireEvent.click(screen.getByRole("button", { name: "Add tier" })); + const next = committed(); + expect(next.custom_tier_set?.tiers).toHaveLength(5); + expect(next.tiers).toEqual(defaultValue.tiers); + }); + + it("renames a built-in tier straight from the editor, which is what makes the set custom", () => { + const { committed } = renderEditor(); + fireEvent.change(screen.getByLabelText("Name for tier 3"), { target: { value: "SECURITY_REVIEW" } }); + const next = committed(); + expect(next.custom_tier_set?.tiers.map((row) => row.name)).toEqual([ + "SIMPLE", + "MEDIUM", + "SECURITY_REVIEW", + "REASONING", + ]); + expect(next.tiers).toEqual(defaultValue.tiers); + }); + + it("opening the editor and changing nothing leaves the router on the built-in tiers", () => { + const { onChange } = renderEditor(); + expect(screen.getByRole("button", { name: "Done" })).toBeEnabled(); + expect(onChange).not.toHaveBeenCalled(); + }); + + it("swaps the display-name field for the tier-name field while the editor is open", () => { + const { rerender } = renderWithProviders(); + expect(screen.getByLabelText("Display name for the Simple tier")).toBeInTheDocument(); + rerender(); + expect(screen.queryByLabelText("Display name for the Simple tier")).not.toBeInTheDocument(); + expect(screen.getByLabelText("Name for tier 1")).toBeInTheDocument(); + }); + + it("leaves no custom row's effort settings behind when the operator returns to the built-in tiers", () => { + const onChange = vi.fn(); + const withEfforts: ComplexityRouterConfigValue = { + ...customValue, + tier_model_params: { sec: { "gpt-4": { reasoning_effort: "high" } }, SIMPLE: { "gpt-3.5-turbo": {} } }, + }; + const { committed } = renderEditor(withEfforts); + fireEvent.click(screen.getByRole("button", { name: "Use built-in tiers" })); + const next = committed(); + expect(next.custom_tier_set).toBeUndefined(); + expect(Object.keys(next.tier_model_params ?? {})).toEqual(["SIMPLE"]); + }); + + it("keeps a renamed built-in row's effort settings when the operator reverts to the built-in tiers", () => { + const renamed: ComplexityRouterConfigValue = { + ...customValue, + tier_model_params: { COMPLEX: { "gpt-4": { reasoning_effort: "high" } } }, + custom_tier_set: { + tiers: [ + { id: "SIMPLE", name: "SIMPLE", definition: "", models: ["gpt-3.5-turbo"] }, + { id: "COMPLEX", name: "DEEP_WORK", definition: "renamed built-in", models: ["gpt-4"] }, + ], + fallback_tier_id: "SIMPLE", + }, + }; + const { committed } = renderEditor(renamed); + fireEvent.click(screen.getByRole("button", { name: "Use built-in tiers" })); + const next = committed(); + expect(next.tiers.COMPLEX).toEqual(["gpt-4"]); + expect(next.tier_model_params).toEqual({ COMPLEX: { "gpt-4": { reasoning_effort: "high" } } }); + }); + + it("refuses to restore the built-in tiers when doing so would pass the tier limit", () => { + const nearLimit: ComplexityRouterConfigValue = { + ...customValue, + custom_tier_set: { + tiers: Array.from({ length: 6 }, (_, index) => ({ + id: `row-${index}`, + name: `TIER_${index}`, + definition: "d", + models: ["gpt-4"], + })), + fallback_tier_id: "row-0", + }, + }; + renderEditor(nearLimit); + expect(screen.getByRole("button", { name: "Restore defaults" })).toBeDisabled(); + }); + + it("stops describing the heuristic scorer once an edited tier set replaces it", () => { + renderWithProviders(); + fireEvent.click(screen.getByText("Advanced: Classification Method")); + expect(screen.getByText("The LLM classifier reads your tier definitions", { exact: false })).toBeInTheDocument(); + expect(screen.queryByText("scores each request across 7 dimensions", { exact: false })).not.toBeInTheDocument(); + }); + + it("says why a custom row is blocked instead of only reddening its border", () => { + const missingDefinition: ComplexityRouterConfigValue = { + ...customValue, + custom_tier_set: { + tiers: [customValue.custom_tier_set!.tiers[0], { id: "b", name: "AUDIT", definition: "", models: ["gpt-4"] }], + fallback_tier_id: "CASUAL", + }, + }; + renderEditor(missingDefinition, { showValidationErrors: true }); + expect(screen.getByText("A definition is required", { exact: false })).toBeInTheDocument(); + }); + + it("keeps Done disabled while a row is incomplete and says what is missing", async () => { + const incomplete: ComplexityRouterConfigValue = { + ...customValue, + custom_tier_set: { + tiers: [customValue.custom_tier_set!.tiers[0], { id: "new", name: "", definition: "", models: [] }], + fallback_tier_id: "CASUAL", + }, + }; + renderEditor(incomplete); + expect(screen.getByRole("button", { name: "Done" })).toBeDisabled(); + }); + + it("enables Done once every row carries a name, a definition and a model", () => { + renderEditor(customValue); + expect(screen.getByRole("button", { name: "Done" })).toBeEnabled(); + }); + + it("refuses to remove a row that would take the set below the backend's minimum", () => { + renderEditor(customValue); + expect(screen.getByRole("button", { name: "Remove the CASUAL tier" })).toBeDisabled(); + }); + + it("keeps a definition on one line, because the backend rejects a newline in it", () => { + const { committed } = renderEditor(customValue); + fireEvent.change(screen.getByLabelText("Definition for tier 2"), { target: { value: "audits\nand reviews" } }); + const next = committed(); + expect(next.custom_tier_set?.tiers[1].definition).toBe("audits and reviews"); + }); + + it("moves a keyword rule with the tier it points at when that tier is renamed", () => { + const onKeywordTierRulesChange = vi.fn(); + renderWithProviders( + , + ); + fireEvent.change(screen.getByLabelText("Name for tier 2"), { target: { value: "AUDIT" } }); + expect(onKeywordTierRulesChange).toHaveBeenCalledWith([{ id: "r1", keywords: ["audit"], tier: "AUDIT" }]); + }); + + it("re-points the fallback tier when the row it named is removed, never leaving it dangling", () => { + const threeRows: ComplexityRouterConfigValue = { + ...customValue, + custom_tier_set: { + tiers: [ + ...customValue.custom_tier_set!.tiers, + { id: "third", name: "MEDIUM", definition: "", models: ["gpt-4"] }, + ], + fallback_tier_id: "sec", + }, + }; + const { committed } = renderEditor(threeRows); + fireEvent.click(screen.getByRole("button", { name: "Remove the SECURITY_REVIEW tier" })); + const next = committed(); + expect(next.custom_tier_set?.tiers.some((row) => row.id === next.custom_tier_set?.fallback_tier_id)).toBe(true); + }); + + it("turns off a plan-mode floor whose row was removed, rather than leaving it pointing at nothing", () => { + const withFloor: ComplexityRouterConfigValue = { + ...customValue, + plan_mode_min_tier: "sec", + custom_tier_set: { + tiers: [ + ...customValue.custom_tier_set!.tiers, + { id: "third", name: "BULK", definition: "d", models: ["gpt-4"] }, + ], + fallback_tier_id: "CASUAL", + }, + }; + const { committed } = renderEditor(withFloor); + fireEvent.click(screen.getByRole("button", { name: "Remove the SECURITY_REVIEW tier" })); + expect(committed().plan_mode_min_tier).toBeUndefined(); + }); + + it("replaces the display-name inputs with the reason an edited tier set forbids them", () => { + renderWithProviders(); + expect(screen.queryByLabelText("Display name for the Simple tier")).not.toBeInTheDocument(); + expect(screen.getByText("Display names rename the built-in tiers", { exact: false })).toBeInTheDocument(); + expect(screen.getByLabelText("Fallback tier")).toBeInTheDocument(); + }); + + it("disables session pinning and says why, rather than letting a stripped value look saved", () => { + renderWithProviders(); + fireEvent.click(screen.getByText("Advanced: Affinity")); + expect(screen.getByLabelText("Pin a session to its first model")).toHaveAttribute("data-disabled"); + expect( + screen.getByText("Session pinning escalates along the built-in tier ladder", { exact: false }), + ).toBeInTheDocument(); + }); + + it("leaves built-in routers with their display-name inputs and no restriction copy", () => { + renderWithProviders(); + expect(screen.getByLabelText("Display name for the Simple tier")).toBeInTheDocument(); + expect(screen.queryByText("Display names rename the built-in tiers", { exact: false })).not.toBeInTheDocument(); + }); +}); diff --git a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx index de6c9cd72fb..5091d722ab7 100644 --- a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx +++ b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx @@ -2,12 +2,37 @@ import { SimpleTooltip } from "@/components/ui/tooltip"; import { MultiSelect } from "@/components/shared/MultiSelect"; import { SearchSelect } from "@/components/shared/SearchSelect"; import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select"; -import { ChevronRight, Info, X } from "lucide-react"; +import { ChevronRight, Info, Plus, Trash2, X } from "lucide-react"; import { Switch } from "@/components/ui/switch"; import { Card, CardContent } from "@/components/ui/card"; import { Collapsible, CollapsibleContent, CollapsibleTrigger } from "@/components/ui/collapsible"; import { InputGroup, InputGroupAddon, InputGroupButton, InputGroupInput } from "@/components/ui/input-group"; import { Separator } from "@/components/ui/separator"; +import { Button } from "@/components/ui/button"; +import { Input } from "@/components/ui/input"; +import { Textarea } from "@/components/ui/textarea"; +import { + type CustomTierSet, + type TierRestriction, + type ActiveTierRow, + type TierRow, + rowParamsByTier, + CUSTOM_TIER_RESTRICTIONS, + MAX_TIER_COUNT, + MAX_TIER_DEFINITION_CHARS, + MAX_TIER_NAME_CHARS, + MIN_TIER_COUNT, + TIER_ORDER, + activeTierName, + activeTierRows, + getCustomTierRowsError, + isBuiltInTierName, + resolveComplexityDefaultModel, + restoredBuiltInRows, + sameTierIdentity, + tierRowById, + tierRowByName, +} from "./tier_rows"; import React from "react"; import { ModelGroup } from "@/components/llm_calls/fetch_models"; import AdaptiveRoutingConfig from "./AdaptiveRoutingConfig"; @@ -18,15 +43,16 @@ import { TierModelParamsByTier, pruneTierModelParams, setTierModelReasoningEffort, + tierRowLabel, } from "./complexity_router_tiers"; import TierModelEffortRows from "./TierModelEffortRows"; import EscalationKeywords from "./EscalationKeywords"; import KeywordTierRules, { KeywordTierRule } from "./KeywordTierRules"; import SemanticKeywordMatching from "./SemanticKeywordMatching"; import { type DimensionWeights, type TierBoundaries, type TokenThresholds } from "./heuristic_scoring_knobs"; -import { type TierRow, activeTierRows, resolveComplexityDefaultModel } from "./tier_rows"; export type { DimensionWeights, TierBoundaries, TokenThresholds }; +export type { CustomTierSet, TierRow } from "./tier_rows"; export const DEFAULT_CLASSIFIER_TIMEOUT_MS = 3000; export const DEFAULT_TIER_DISTANCE_PENALTY = 0.5; @@ -133,7 +159,48 @@ export const heuristicScoringRoleFor = ( }; export const heuristicScoringRole = (value: ComplexityRouterConfigValue): HeuristicScoringRole => - heuristicScoringRoleFor(value.classifier_type, value.classifier_fallback); + value.custom_tier_set ? "never" : heuristicScoringRoleFor(value.classifier_type, value.classifier_fallback); + +// Derived, never written into the value, so undoing a tier edit reverts the form with nothing left behind. +export const effectiveClassifierType = ( + value: Pick, +): ClassifierType => (value.custom_tier_set ? "llm" : value.classifier_type); + +export const restrictedBy = ( + value: Pick, + key: keyof typeof CUSTOM_TIER_RESTRICTIONS, +): TierRestriction | undefined => (value.custom_tier_set ? CUSTOM_TIER_RESTRICTIONS[key] : undefined); + +export const Restricted: React.FC<{ by: TierRestriction | undefined; children: React.ReactNode }> = ({ + by, + children, +}) => (by ? {by.reason} : <>{children}); + +const rowOrigin = (row: TierRow, editing: boolean): string => { + if (!editing) return row.id; + return isBuiltInTierName(row.name) ? "built-in" : "custom"; +}; + +const TierRowSelect: React.FC<{ + label: string; + options: { value: string; label: string }[]; + value: string | null; + onValueChange: (rowId: string) => void; + placeholder?: string; +}> = ({ label, options, value, onValueChange, placeholder }) => ( + +); export type AdaptiveEligible = "all" | "classified_tier"; @@ -141,6 +208,7 @@ export type ComplexityTierLabels = Partial export interface ComplexityRouterConfigValue { tiers: ComplexityTiers; + custom_tier_set?: CustomTierSet; tier_labels?: ComplexityTierLabels; /** An explicit pin. Unset means the default tracks the tiers - see resolveComplexityDefaultModel. */ default_model?: string; @@ -155,7 +223,7 @@ export interface ComplexityRouterConfigValue { heuristic_first_max_tier?: string; session_affinity?: boolean; deployment_affinity?: boolean; - /** Tier floor for coding-agent plan-mode requests. Unset means detection is off, matching the backend. */ + /** Plan-mode floor as a tier ROW ID, unset meaning off. The wire carries the row's name. */ plan_mode_min_tier?: string; adaptive?: boolean; adaptive_weights?: AdaptiveRouterWeights; @@ -186,6 +254,9 @@ interface ComplexityRouterConfigProps { modelInfo: ModelGroup[]; value: ComplexityRouterConfigValue; onChange: (value: ComplexityRouterConfigValue) => void; + /** Parent-owned: this component unmounts when its section collapses. */ + editingTiers?: boolean; + onEditingTiersChange?: (editing: boolean) => void; customTechnicalKeywords?: string[]; onCustomTechnicalKeywordsChange?: (keywords: string[]) => void; // Optional: the edit-auto-router modal doesn't yet support editing keyword tier @@ -246,6 +317,8 @@ const ComplexityRouterConfig: React.FC = ({ modelInfo, value, onChange, + editingTiers, + onEditingTiersChange, customTechnicalKeywords, onCustomTechnicalKeywordsChange, keywordTierRules = [], @@ -260,13 +333,113 @@ const ComplexityRouterConfig: React.FC = ({ onEscalationKeywordsChange, showValidationErrors = false, }) => { + const customTierSet = value.custom_tier_set; const tierRows = activeTierRows(value); - const planModeTierOptions = tierRows - .filter((row) => row.models.length > 0) - .map((row) => ({ value: row.id, label: effectiveTierLabel(row.id as keyof ComplexityTiers, value.tier_labels) })); + const currentFallbackId = customTierSet?.fallback_tier_id ?? "MEDIUM"; + const tierRowsError = customTierSet ? getCustomTierRowsError(customTierSet) : null; + + const planModeRows = tierRows.filter((row) => row.models.length > 0); + const planModeTierOptions = planModeRows.map((row) => ({ + value: row.id, + label: tierRowLabel(row, value.tier_labels), + })); const derivedDefaultModel = resolveComplexityDefaultModel(value); + const emptyTiersHint = customTierSet + ? "Add a model to your fallback tier" + : "Add a model to the Simple or Medium tier"; + const defaultModelPlaceholder = derivedDefaultModel ? `Derived from tiers: ${derivedDefaultModel}` : emptyTiersHint; const defaultModel = resolveComplexityDefaultModel(value, value.default_model); + // The sole tier-set writer: it owns where rows live and reconciles both row-id pointers, so a + // fallback re-points and a floor whose row is gone turns off in the same write. + const commitTierRows = (rows: TierRow[], fallbackTierId: string, base: ComplexityRouterConfigValue = value) => { + const floorGone = base.plan_mode_min_tier !== undefined && !rows.some((row) => row.id === base.plan_mode_min_tier); + const next = floorGone ? { ...base, plan_mode_min_tier: undefined } : base; + if (!next.custom_tier_set) { + onChange({ ...next, tiers: { ...next.tiers, ...Object.fromEntries(rows.map((row) => [row.id, row.models])) } }); + return; + } + const fallback_tier_id = rows.some((row) => row.id === fallbackTierId) + ? fallbackTierId + : (tierRowByName(rows, "MEDIUM") ?? rows[0])?.id ?? ""; + onChange({ ...next, custom_tier_set: { tiers: rows, fallback_tier_id } }); + }; + + const asCustomBase = (base: ComplexityRouterConfigValue): ComplexityRouterConfigValue => + base.custom_tier_set + ? base + : { ...base, custom_tier_set: { tiers: activeTierRows(base), fallback_tier_id: "MEDIUM" } }; + + const setRowModels = (row: TierRow, models: string[]) => + commitTierRows( + tierRows.map((candidate) => (candidate.id === row.id ? { ...candidate, models } : candidate)), + currentFallbackId, + { ...value, tier_model_params: pruneTierModelParams(value.tier_model_params, row.id, models) }, + ); + + const updateTierRow = (id: string, patch: Partial>) => { + const renamedFrom = patch.name === undefined ? undefined : tierRowById(tierRows, id)?.name; + const sharedName = + renamedFrom !== undefined && tierRows.some((row) => row.id !== id && sameTierIdentity(row.name, renamedFrom)); + if (renamedFrom !== undefined && !sharedName && onKeywordTierRulesChange) { + onKeywordTierRulesChange( + keywordTierRules.map((rule) => + rule.tier === renamedFrom.trim() ? { ...rule, tier: (patch.name ?? "").trim() } : rule, + ), + ); + } + commitTierRows( + tierRows.map((row) => (row.id === id ? { ...row, ...patch } : row)), + currentFallbackId, + asCustomBase(value), + ); + }; + + const addCustomTier = () => + commitTierRows( + [...tierRows, { id: crypto.randomUUID(), name: "", definition: "", models: [] }], + currentFallbackId, + asCustomBase(value), + ); + + const removeTierRow = (id: string) => { + const removed = tierRowById(tierRows, id); + const snapshot = + removed && (TIER_ORDER as string[]).includes(id) + ? { ...value, tiers: { ...value.tiers, [id]: removed.models } } + : value; + commitTierRows( + tierRows.filter((row) => row.id !== id), + currentFallbackId, + asCustomBase(snapshot), + ); + }; + + const restorableRows = restoredBuiltInRows(tierRows, value.tiers); + + const restoreDefaultTiers = () => commitTierRows(restorableRows, currentFallbackId); + + // Models and params both come from these rows, so the two cannot be keyed differently. + const exitToBuiltInTiers = () => { + const { custom_tier_set: _dropped, ...rest } = value; + const builtInRows: ActiveTierRow[] = TIER_ORDER.map( + (tier) => + tierRowById(tierRows, tier) ?? { + id: tier, + name: tier, + definition: "", + models: value.tiers[tier], + params: value.tier_model_params?.[tier] ?? {}, + }, + ); + const restored: ComplexityRouterConfigValue = { + ...rest, + tier_model_params: rowParamsByTier(builtInRows), + tiers: { ...value.tiers, ...Object.fromEntries(builtInRows.map((row) => [row.id, row.models])) }, + }; + commitTierRows(activeTierRows(restored), "", restored); + }; + // An absent list means the proxy does not send the field yet, so every level is offered as before. // An empty list is the group's own answer that its deployments share no level, and is left empty. const effortOptionsByModel: Record = Object.fromEntries( @@ -284,19 +457,7 @@ const ComplexityRouterConfig: React.FC = ({ label: model.model_group, })); - const handleTierChange = (tier: keyof ComplexityTiers, models: string[]) => { - onChange({ - ...value, - tiers: { ...value.tiers, [tier]: models }, - tier_model_params: pruneTierModelParams(value.tier_model_params, tier, models), - }); - }; - - const handleTierModelEffortChange = ( - tier: keyof ComplexityTiers, - model: string, - effort: ReasoningEffort | undefined, - ) => { + const handleTierModelEffortChange = (tier: string, model: string, effort: ReasoningEffort | undefined) => { onChange({ ...value, tier_model_params: setTierModelReasoningEffort(value.tier_model_params, tier, model, effort), @@ -326,61 +487,120 @@ const ComplexityRouterConfig: React.FC = ({
    - The complexity router automatically classifies requests by complexity using rule-based scoring (no API calls, - <1ms latency). Configure which model(s) handle each tier. + {heuristicScoringRole(value) === "never" + ? "The complexity router classifies each request with your classifier model and routes it to that tier. Configure which model(s) handle each tier." + : "The complexity router automatically classifies requests by complexity using rule-based scoring (no API calls, <1ms latency). Configure which model(s) handle each tier."} - Rename a tier to use your own vocabulary in the dashboard and your spend logs. Renaming doesn't change how - requests are classified, and callers never see these names. - {usesLlmClassifier(value.classifier_type) && + {restrictedBy(value, "displayNames")?.reason ?? + "Rename a tier to use your own vocabulary in the dashboard and your spend logs. Renaming doesn't change how requests are classified, and callers never see these names."} + {!customTierSet && + usesLlmClassifier(value.classifier_type) && " Your classifier model reads these names, so clearer ones can sharpen its choices."} - {tierRows.map((row: TierRow, index) => { - const tier = row.id as keyof ComplexityTiers; - const tierInfo = TIER_DESCRIPTIONS[tier]; - const label = effectiveTierLabel(tier, value.tier_labels); + {tierRows.map((row, index) => { + const builtIn = TIER_ORDER.find((tier) => tier === row.id); + const tierInfo = builtIn ? TIER_DESCRIPTIONS[builtIn] : undefined; + const label = tierRowLabel(row, value.tier_labels); const tierMissing = showValidationErrors && row.models.length === 0; + const definitionMissing = + showValidationErrors && Boolean(customTierSet) && !row.definition.trim() && !isBuiltInTierName(row.name); return (
    {index > 0 && }
    {label} Tier - + - Tier {index + 1} of {tierRows.length} · {row.id} + Tier {index + 1} of {tierRows.length} · {rowOrigin(row, Boolean(customTierSet))} -
    - Examples: {tierInfo.examples} - - handleTierLabelChange(tier, event.target.value)} - placeholder={`Display name (default: ${tierInfo.label})`} - aria-label={`Display name for the ${tierInfo.label} tier`} - /> - {value.tier_labels?.[tier] && ( - - handleTierLabelChange(tier, "")} - > - - - + {editingTiers && ( + )} - +
    + {tierInfo && !customTierSet && ( + Examples: {tierInfo.examples} + )} + {editingTiers && ( + <> + updateTierRow(row.id, { name: event.target.value })} + placeholder="Tier name, e.g. SECURITY_REVIEW" + aria-label={`Name for tier ${index + 1}`} + maxLength={MAX_TIER_NAME_CHARS} + className="mb-2" + /> +