diff --git a/.gitignore b/.gitignore index 201e02f2189..7e171f6cf3a 100644 --- a/.gitignore +++ b/.gitignore @@ -145,3 +145,4 @@ crash.*.log ui/litellm-dashboard/out/ litellm.log +litellm-round.log diff --git a/ui/litellm-dashboard/src/components/add_model/ClassificationMethodConfig.tsx b/ui/litellm-dashboard/src/components/add_model/ClassificationMethodConfig.tsx index ef6e521de42..f52bc75c0e1 100644 --- a/ui/litellm-dashboard/src/components/add_model/ClassificationMethodConfig.tsx +++ b/ui/litellm-dashboard/src/components/add_model/ClassificationMethodConfig.tsx @@ -26,6 +26,7 @@ import { CLASSIFICATION_RUBRIC_KEYS, ClassificationRubric, effectiveTierLabel, + effectiveClassifierType, } from "./ComplexityRouterConfig"; const DEFAULT_SCORING_EXPLANATION = @@ -146,8 +147,10 @@ const ClassificationMethodConfig: React.FC = ({ defaultModel, }) => { const hasDefaultModel = Boolean(defaultModel); + const hasCustomTierSet = Boolean(value.custom_tier_set); + const classifierType = effectiveClassifierType(value); const classifierModelMissing = - showValidationErrors && value.classifier_type === "llm" && !value.classifier_llm_config?.model; + showValidationErrors && classifierType === "llm" && !value.classifier_llm_config?.model; const usesCustomPrompt = Boolean(value.classifier_llm_config?.system_prompt?.trim()); const classificationRubric = value.classifier_llm_config?.classification_rubric ?? DEFAULT_CLASSIFICATION_RUBRIC; @@ -252,20 +255,28 @@ const ClassificationMethodConfig: React.FC = ({ return ( <> handleClassifierTypeChange(classifierType as ClassifierType)} className="w-full" >
- + +
- {value.classifier_type === "llm" && ( + {classifierType === "llm" && (
Classifier Model @@ -348,13 +359,20 @@ const ClassificationMethodConfig: React.FC = ({
Classifier Prompt - + {hasCustomTierSet ? ( + + Unavailable with an edited tier set: a replacement prompt would drop the tier definitions the classifier + routes on, along with the injection guard. Your tier definitions are the rubric. + + ) : ( + + )}
If the classifier fails diff --git a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.test.tsx b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.test.tsx index 0a848ed7deb..1cbc24a40ff 100644 --- a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.test.tsx +++ b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.test.tsx @@ -28,6 +28,8 @@ const baseProps = { modelInfo: mockModelInfo, value: defaultValue, onChange: vi.fn(), + editingTiers: false, + onEditingTiersChange: vi.fn(), keywordTierRules: [], onKeywordTierRulesChange: vi.fn(), semanticMatchingEnabled: false, @@ -943,3 +945,110 @@ describe("ComplexityRouterConfig reasoning effort gating", () => { ).toHaveTextContent("low"); }); }); + +describe("edit tiers", () => { + const customValue: ComplexityRouterConfigValue = { + ...defaultValue, + classifier_type: "llm", + classifier_llm_config: { model: "gpt-3.5-turbo", timeout_ms: 3000 }, + custom_tier_set: { + tiers: [ + { id: "SIMPLE", name: "SIMPLE", definition: "", models: ["gpt-3.5-turbo"] }, + { id: "COMPLEX", name: "COMPLEX", definition: "", models: ["gpt-4"] }, + { id: "sec", name: "AUDIT", definition: "security audits", models: ["claude-3-opus"] }, + ], + fallback_tier_id: "COMPLEX", + }, + }; + + // Remove snapshots in-editor models so Restore returns those; only "Use built-in tiers" exits. + // One pass: row limits, rules following a rename (they point by NAME), and forced-off inputs. + it("edits a custom tier set: rows, keyword rules, and the inputs it forces off", async () => { + const onChange = vi.fn(); + const onKeywordTierRulesChange = vi.fn(); + renderWithProviders( + , + ); + const name = screen.getByRole("textbox", { name: "Name for tier 3" }); + const definition = screen.getByRole("textbox", { name: "Definition for tier 3" }); + expect(name).toHaveValue("AUDIT"); + expect(name).toHaveAttribute("maxLength", "64"); + expect(definition).toHaveAttribute("maxLength", "500"); + expect(screen.queryByLabelText("Display name for the Simple tier")).not.toBeInTheDocument(); + expect(screen.queryByRole("combobox", { name: /Reasoning effort/ })).not.toBeInTheDocument(); + + fireEvent.change(definition, { target: { value: "audits\nand\r\nreviews" } }); + expect((onChange.mock.calls.at(-1)?.[0] as ComplexityRouterConfigValue).custom_tier_set?.tiers[2].definition).toBe( + "audits and reviews", + ); + + fireEvent.click(screen.getByText("Advanced: Keyword/Semantic Matching")); + expect(screen.getByRole("combobox", { name: "Route keyword rule 1 to tier" })).toHaveTextContent("AUDIT"); + fireEvent.change(name, { target: { value: "SECREV" } }); + expect(onKeywordTierRulesChange).toHaveBeenLastCalledWith([{ id: "rule-1", keywords: ["scan"], tier: "SECREV" }]); + + await userEvent.click(screen.getByText("Advanced: Affinity")); + expect(screen.getByLabelText("Pin a session to its first model")).toHaveAttribute("data-disabled"); + expect(screen.getByText(/escalation walks the built-in tier ladder/)).toBeInTheDocument(); + + fireEvent.click(screen.getByText("Advanced: Classification Method")); + expect(screen.getByRole("radio", { name: /rule-based scoring/ })).toHaveAttribute("aria-disabled", "true"); + expect(screen.getByText(/Unavailable with an edited tier set: a replacement prompt/)).toBeInTheDocument(); + + fireEvent.click(screen.getByText("Advanced: Plan-Mode Override")); + const floor = screen.getByRole("switch", { name: "Route plan-mode requests to a minimum tier" }); + expect(floor).toHaveAttribute("aria-disabled", "true"); + expect(floor).toBeChecked(); + expect(screen.queryByRole("combobox", { name: "Plan-mode minimum tier" })).not.toBeInTheDocument(); + }); + + it("leaves keyword rules alone when the old tier name is shared by another row", () => { + const onKeywordTierRulesChange = vi.fn(); + const tiers = customValue.custom_tier_set!.tiers.map((row) => ({ ...row, name: "AUDIT" })); + const shared = { ...customValue, custom_tier_set: { tiers, fallback_tier_id: tiers[0].id } }; + const rules = [{ id: "rule-1", keywords: ["scan"], tier: "AUDIT" }]; + renderWithProviders( + , + ); + fireEvent.change(screen.getByRole("textbox", { name: "Name for tier 3" }), { target: { value: "SECREV" } }); + expect(onKeywordTierRulesChange).not.toHaveBeenCalled(); + }); + + it("removes, restores, and exits the tier set without losing in-editor models", async () => { + const onChange = vi.fn(); + const step = (value: ComplexityRouterConfigValue, button: string) => { + onChange.mockClear(); + renderWithProviders(); + return userEvent + .click(screen.getAllByRole("button", { name: button })[0]) + .then(() => onChange.mock.calls.at(-1)?.[0] as ComplexityRouterConfigValue); + }; + const removed = await step({ ...defaultValue, plan_mode_min_tier: "MEDIUM" }, "Remove the MEDIUM tier"); + expect(removed.plan_mode_min_tier).toBeUndefined(); + expect(removed.custom_tier_set?.tiers.map((r) => r.id)).toEqual(["SIMPLE", "COMPLEX", "REASONING"]); + expect(removed.custom_tier_set?.fallback_tier_id).toBe("SIMPLE"); + + const restored = await step(removed, "Restore defaults"); + expect(restored.custom_tier_set?.tiers.map((r) => r.id)).toEqual(["SIMPLE", "MEDIUM", "COMPLEX", "REASONING"]); + expect(restored.custom_tier_set?.tiers[1].models).toEqual(defaultValue.tiers.MEDIUM); + + expect(await step(restored, "Use built-in tiers")).toEqual(defaultValue); + }); +}); diff --git a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx index fc731e2c77f..20e2fd4dac9 100644 --- a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx +++ b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx @@ -2,12 +2,25 @@ import { SimpleTooltip } from "@/components/ui/tooltip"; import { MultiSelect } from "@/components/shared/MultiSelect"; import { SearchSelect } from "@/components/shared/SearchSelect"; import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select"; -import { ChevronRight, Info, X } from "lucide-react"; +import { ChevronRight, Info, Plus, Trash2, X } from "lucide-react"; import { Switch } from "@/components/ui/switch"; +import { Button } from "@/components/ui/button"; +import { Input } from "@/components/ui/input"; +import { Textarea } from "@/components/ui/textarea"; import { Card, CardContent } from "@/components/ui/card"; import { Collapsible, CollapsibleContent, CollapsibleTrigger } from "@/components/ui/collapsible"; import { InputGroup, InputGroupAddon, InputGroupButton, InputGroupInput } from "@/components/ui/input-group"; import { Separator } from "@/components/ui/separator"; +import { + type CustomTierSet, + type TierDraft, + MAX_TIER_COUNT, + MIN_TIER_COUNT, + findTierByName, + getCustomTierRowsError, + isBuiltInTierName, + tierNamesMatch, +} from "./custom_tier_set"; import React from "react"; import { ModelGroup } from "@/components/llm_calls/fetch_models"; import AdaptiveRoutingConfig from "./AdaptiveRoutingConfig"; @@ -15,6 +28,7 @@ import ClassificationMethodConfig from "./ClassificationMethodConfig"; import { ReasoningEffort, TierModelParamsByTier, + customTierDefaultModel, pruneTierModelParams, resolveComplexityDefaultModel, setTierModelReasoningEffort, @@ -27,11 +41,15 @@ import SemanticKeywordMatching from "./SemanticKeywordMatching"; import { type DimensionWeights, type TierBoundaries, type TokenThresholds } from "./heuristic_scoring_knobs"; export type { DimensionWeights, TierBoundaries, TokenThresholds }; +export type { CustomTierSet, TierDraft } from "./custom_tier_set"; +export { MAX_TIER_COUNT, MIN_TIER_COUNT, getCustomTierRowsError, isBuiltInTierName } from "./custom_tier_set"; export const DEFAULT_CLASSIFIER_TIMEOUT_MS = 3000; export const DEFAULT_TIER_DISTANCE_PENALTY = 0.5; export const DEFAULT_CLASSIFIER_CONTEXT_WINDOW_SIZE = 3; export const DEFAULT_CLASSIFIER_CONTEXT_PER_TURN_CHARS = 200; +export const MAX_TIER_NAME_CHARS = 64; +export const MAX_TIER_DESCRIPTION_CHARS = 500; export const DEFAULT_SESSION_AFFINITY = false; export const DEFAULT_DEPLOYMENT_AFFINITY = true; @@ -123,15 +141,26 @@ export const heuristicScoringRoleFor = ( }; export const heuristicScoringRole = (value: ComplexityRouterConfigValue): HeuristicScoringRole => - heuristicScoringRoleFor(value.classifier_type, value.classifier_fallback); + value.custom_tier_set ? "never" : heuristicScoringRoleFor(value.classifier_type, value.classifier_fallback); export type AdaptiveEligible = "all" | "classified_tier"; export type ComplexityTierLabels = Partial>; +const defaultModelPlaceholder = (derived: string | undefined, hasCustomTierSet: boolean): string => { + if (derived) return `Derived from tiers: ${derived}`; + return hasCustomTierSet ? "Add a model to your fallback tier" : "Add a model to the Simple or Medium tier"; +}; + +/** Derived, never written into the value, so an undone tier edit reverts the form with nothing left behind. */ +export const effectiveClassifierType = ( + value: Pick, +): ClassifierType => (value.custom_tier_set ? "llm" : value.classifier_type); + export interface ComplexityRouterConfigValue { tiers: ComplexityTiers; tier_labels?: ComplexityTierLabels; + custom_tier_set?: CustomTierSet; /** An explicit pin. Unset means the default tracks the tiers - see resolveComplexityDefaultModel. */ default_model?: string; classifier_type: ClassifierType; @@ -142,7 +171,7 @@ export interface ComplexityRouterConfigValue { classifier_fallback?: ClassifierFallback; session_affinity?: boolean; deployment_affinity?: boolean; - /** Tier floor for coding-agent plan-mode requests. Unset means detection is off, matching the backend. */ + /** Plan-mode floor as a tier ROW ID (unset = off); the wire carries the row's name. */ plan_mode_min_tier?: string; adaptive?: boolean; adaptive_weights?: AdaptiveRouterWeights; @@ -173,6 +202,9 @@ interface ComplexityRouterConfigProps { modelInfo: ModelGroup[]; value: ComplexityRouterConfigValue; onChange: (value: ComplexityRouterConfigValue) => void; + /** Parent-owned: this component unmounts on section collapse. */ + editingTiers?: boolean; + onEditingTiersChange?: (editing: boolean) => void; customTechnicalKeywords?: string[]; onCustomTechnicalKeywordsChange?: (keywords: string[]) => void; // Optional: the edit-auto-router modal doesn't yet support editing keyword tier @@ -229,6 +261,8 @@ const ComplexityRouterConfig: React.FC = ({ modelInfo, value, onChange, + editingTiers, + onEditingTiersChange, customTechnicalKeywords, onCustomTechnicalKeywordsChange, keywordTierRules = [], @@ -243,12 +277,89 @@ const ComplexityRouterConfig: React.FC = ({ onEscalationKeywordsChange, showValidationErrors = false, }) => { + const customTierSet = value.custom_tier_set; + + const builtInRow = (tier: keyof ComplexityTiers): TierDraft => ({ + id: tier, + name: tier, + definition: "", + models: value.tiers[tier], + }); + const tierRows: TierDraft[] = customTierSet?.tiers ?? TIER_KEYS.map(builtInRow); + + // Sole tier-set writer: reconciles both row-id pointers (fallback re-points, dead floor clears). + const commitTierRows = (rows: TierDraft[], fallbackTierId: string, base: ComplexityRouterConfigValue = value) => { + const fallback_tier_id = rows.some((row) => row.id === fallbackTierId) + ? fallbackTierId + : (findTierByName(rows, "MEDIUM") ?? rows[0])?.id ?? ""; + const floorGone = base.plan_mode_min_tier !== undefined && !rows.some((r) => r.id === base.plan_mode_min_tier); + const floor = floorGone ? { plan_mode_min_tier: undefined } : {}; + onChange({ ...base, ...floor, custom_tier_set: { tiers: rows, fallback_tier_id } }); + }; + + const currentFallbackId = customTierSet?.fallback_tier_id ?? "MEDIUM"; + const tierRowsError = customTierSet ? getCustomTierRowsError(customTierSet) : null; + + const removeTierRow = (id: string) => { + const removed = tierRows.find((row) => row.id === id); + const snapshotBase = + removed && (TIER_KEYS as string[]).includes(removed.id) + ? { ...value, tiers: { ...value.tiers, [removed.id]: removed.models } } + : value; + commitTierRows( + tierRows.filter((row) => row.id !== id), + currentFallbackId, + snapshotBase, + ); + }; + + const restoreDefaultTiers = () => { + const restoredInCanonicalOrder = [ + ...TIER_KEYS.map((builtIn) => tierRows.find((row) => row.id === builtIn) ?? builtInRow(builtIn)), + ...tierRows.filter((row) => !(TIER_KEYS as string[]).includes(row.id)), + ]; + commitTierRows(restoredInCanonicalOrder, currentFallbackId); + }; + + const exitToBuiltInTiers = () => { + const { custom_tier_set: _cleared, ...rest } = value; + const modelsFor = (tier: keyof ComplexityTiers) => + tierRows.find((row) => row.id === tier)?.models ?? value.tiers[tier]; + onChange({ ...rest, tiers: { ...value.tiers, ...Object.fromEntries(TIER_KEYS.map((t) => [t, modelsFor(t)])) } }); + }; + + const addCustomTier = () => { + commitTierRows([...tierRows, { id: crypto.randomUUID(), name: "", definition: "", models: [] }], currentFallbackId); + }; + + const updateTierRow = (id: string, patch: Partial>) => { + const renamedTo = patch.name; + const renamedFrom = tierRows.find((candidate) => candidate.id === id)?.name; + // Skip when another row already answers to the old name: which rules were this row's is unknowable. + const shared = tierRows.some( + (r) => r.id !== id && renamedFrom !== undefined && tierNamesMatch(r.name, renamedFrom), + ); + if (renamedTo !== undefined && renamedFrom !== undefined && !shared && onKeywordTierRulesChange) { + onKeywordTierRulesChange( + keywordTierRules.map((rule) => (tierNamesMatch(rule.tier, renamedFrom) ? { ...rule, tier: renamedTo } : rule)), + ); + } + commitTierRows( + tierRows.map((candidate) => (candidate.id === id ? { ...candidate, ...patch } : candidate)), + currentFallbackId, + ); + }; + const planModeTiers = planModeEligibleTiers(value.tiers); const planModeTierOptions = tierOptions(value.tier_labels).filter((option) => (planModeTiers as string[]).includes(option.value), ); - const derivedDefaultModel = resolveComplexityDefaultModel(value.tiers); - const defaultModel = resolveComplexityDefaultModel(value.tiers, value.default_model); + const derivedDefaultModel = customTierSet + ? customTierDefaultModel(customTierSet) + : resolveComplexityDefaultModel(value.tiers); + const defaultModel = customTierSet + ? customTierDefaultModel(customTierSet, value.default_model) + : resolveComplexityDefaultModel(value.tiers, value.default_model); // Embedding models can't serve a chat-completion role, so they're excluded here. const reasoningModels = new Set( @@ -270,11 +381,7 @@ const ComplexityRouterConfig: React.FC = ({ }); }; - const handleTierModelEffortChange = ( - tier: keyof ComplexityTiers, - model: string, - effort: ReasoningEffort | undefined, - ) => { + const handleTierModelEffortChange = (tier: string, model: string, effort: ReasoningEffort | undefined) => { onChange({ ...value, tier_model_params: setTierModelReasoningEffort(value.tier_model_params, tier, model, effort), @@ -309,77 +416,227 @@ const ComplexityRouterConfig: React.FC = ({ - Rename a tier to use your own vocabulary in the dashboard and your spend logs. Renaming doesn't change how - requests are classified, and callers never see these names. - {value.classifier_type === "llm" && + {customTierSet + ? "Display names are unavailable with an edited tier set: your tier names appear as-is." + : "Rename a tier to use your own vocabulary in the dashboard and your spend logs. Renaming doesn't change how requests are classified, and callers never see these names."} + {!customTierSet && + value.classifier_type === "llm" && " Your classifier model reads these names, so clearer ones can sharpen its choices."} - {TIER_KEYS.map((tier, index) => { - const tierInfo = TIER_DESCRIPTIONS[tier]; - const label = effectiveTierLabel(tier, value.tier_labels); - const tierMissing = showValidationErrors && value.tiers[tier].length === 0; + {tierRows.map((row, index) => { + const builtInKey = TIER_KEYS.find((tier) => tier === row.id); + const tierInfo = builtInKey ? TIER_DESCRIPTIONS[builtInKey] : undefined; + const label = customTierSet + ? row.name.trim() || "New" + : effectiveTierLabel(row.id as keyof ComplexityTiers, value.tier_labels); + const nameMissing = showValidationErrors && Boolean(customTierSet) && !row.name.trim(); + const definitionMissing = + showValidationErrors && Boolean(customTierSet) && !row.definition.trim() && !isBuiltInTierName(row.name); + const modelsMissing = showValidationErrors && row.models.length === 0; return ( -
+
{index > 0 && }
{label} Tier - + - Tier {index + 1} of {TIER_KEYS.length} · {tier} + Tier {index + 1} of {tierRows.length} · {!customTierSet && row.id} + {customTierSet && (isBuiltInTierName(row.name) ? "built-in" : "custom")} -
- Examples: {tierInfo.examples} - - handleTierLabelChange(tier, event.target.value)} - placeholder={`Display name (default: ${tierInfo.label})`} - aria-label={`Display name for the ${tierInfo.label} tier`} - /> - {value.tier_labels?.[tier] && ( - - handleTierLabelChange(tier, "")} - > - - - + {editingTiers && ( + )} - +
+ {tierInfo && ( + Examples: {tierInfo.examples} + )} + {customTierSet && editingTiers && ( + <> + updateTierRow(row.id, { name: event.target.value })} + placeholder="Tier name, e.g. SECURITY_REVIEW" + aria-label={`Name for tier ${index + 1}`} + maxLength={MAX_TIER_NAME_CHARS} + className={nameMissing ? "mb-2 border-destructive" : "mb-2"} + /> +