mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-09 22:31:41 +00:00
fix(auto_router): clear the non-reasoning tier when the classifier changes
Three review findings, all in the dashboard. Switching off the LLM classifier left the toggle checked but disabled, so the flag could not be cleared and every save was refused by the backend. The classifier-change handler now drops the flag and the tier's pool the same way it already drops the other classifier-specific keys. builtInTierInfo resolved rows against the four-tier order, so the new row rendered with no description, no examples and no rename field. It now resolves against every built-in tier, and the duplicate BUILT_IN_TIER_ORDER constant is gone in favour of the one in tier_rows. The preset schema widening is reverted. It was speculative, no published catalog carries the tier, and prefill would have discarded it while the route-wide null exclusion changed the endpoint's passthrough contract for every other field. Also splits NonReasoningTierToggle and TierConfigIntro into their own files to get ComplexityRouterConfig.tsx back under the max-lines limit, and applies ruff format to the two backend files CI flagged.
This commit is contained in:
parent
629b464cd6
commit
91f1d98fb0
12 changed files with 171 additions and 91 deletions
|
|
@ -534,9 +534,6 @@ async def get_autorouter_presets(
|
|||
"/public/autorouter_presets",
|
||||
tags=["public", "auto router"], # mutable-ok: FastAPI route tags take a list
|
||||
response_model=dict[str, AutoRouterPresetRecord],
|
||||
# An optional tier a preset does not set must not reach the dashboard as a null pool, which the
|
||||
# template picker would render as an empty tier row the operator never asked for.
|
||||
response_model_exclude_none=True,
|
||||
)
|
||||
async def get_public_autorouter_presets() -> Mapping[str, AutoRouterPresetRecord]:
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -2334,9 +2334,7 @@ class ComplexityRouter(CustomLogger):
|
|||
distance = 0
|
||||
else:
|
||||
model_tiers = self._model_tiers.get(model, (classified_tier,))
|
||||
distance = min(
|
||||
abs(severity_order.index(model_tier) - classified_idx) for model_tier in model_tiers
|
||||
)
|
||||
distance = min(abs(severity_order.index(model_tier) - classified_idx) for model_tier in model_tiers)
|
||||
score = quality_weight * quality_sample + cost_weight * cost_score - penalty_weight * distance
|
||||
candidate_scores.append(
|
||||
{
|
||||
|
|
|
|||
|
|
@ -78,6 +78,7 @@ def tier_severity_order(non_reasoning_enabled: bool) -> tuple[ComplexityTier, ..
|
|||
"""The built-in ladder for one router, tier 0 included only when it opted in."""
|
||||
return NON_REASONING_TIER_SEVERITY_ORDER if non_reasoning_enabled else TIER_SEVERITY_ORDER
|
||||
|
||||
|
||||
DEFAULT_TIER_DISTANCE_PENALTY: Final[float] = 0.5
|
||||
|
||||
DEFAULT_CLASSIFIER_CONTEXT_WINDOW_SIZE: Final[int] = 3
|
||||
|
|
|
|||
|
|
@ -74,14 +74,10 @@ class SupportedEndpointsResponse(BaseModel):
|
|||
|
||||
|
||||
class AutoRouterPresetTiers(BaseModel):
|
||||
"""The built-in tiers the dashboard's preset prefill can apply.
|
||||
"""Exactly the four built-in tiers the dashboard's preset prefill can apply.
|
||||
|
||||
extra="forbid" on purpose: a tier name this dashboard cannot apply would grey out or crash the
|
||||
picker, so such a catalog is rejected wholesale and the bundled one serves instead. NON_REASONING
|
||||
is optional rather than required so this proxy accepts both a catalog that ships the opt-in fifth
|
||||
tier and the four-tier catalogs that predate it. It stays None when unset rather than defaulting
|
||||
to an empty pool, so a four-tier preset serves the tier set it was published with instead of
|
||||
growing a key the dashboard would render as an empty fifth tier row.
|
||||
picker, so such a catalog is rejected wholesale and the bundled one serves instead.
|
||||
"""
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
|
@ -90,7 +86,6 @@ class AutoRouterPresetTiers(BaseModel):
|
|||
MEDIUM: Sequence[str]
|
||||
COMPLEX: Sequence[str]
|
||||
REASONING: Sequence[str]
|
||||
NON_REASONING: Sequence[str] | None = None
|
||||
|
||||
|
||||
class AutoRouterPresetConfig(BaseModel):
|
||||
|
|
|
|||
|
|
@ -236,6 +236,22 @@ const ClassifierTypeRadios: React.FC<{
|
|||
);
|
||||
};
|
||||
|
||||
/**
|
||||
* The NON_REASONING keys a classifier switch should carry forward, or clear. Leaving the flag set
|
||||
* under a classifier that cannot emit the tier produces a config the backend refuses on save, and
|
||||
* the switch is disabled there, so the operator would have no way to undo it.
|
||||
*/
|
||||
export const nonReasoningTierFields = (
|
||||
classifierType: ClassifierType,
|
||||
value: ComplexityRouterConfigValue,
|
||||
): Pick<ComplexityRouterConfigValue, "enable_non_reasoning_tier" | "tiers"> => {
|
||||
if (classifierType === "llm") {
|
||||
return { enable_non_reasoning_tier: value.enable_non_reasoning_tier, tiers: value.tiers };
|
||||
}
|
||||
const { NON_REASONING: _cleared, ...keptTiers } = value.tiers;
|
||||
return { enable_non_reasoning_tier: undefined, tiers: keptTiers };
|
||||
};
|
||||
|
||||
const ClassificationMethodConfig: React.FC<ClassificationMethodConfigProps> = ({
|
||||
value,
|
||||
onChange,
|
||||
|
|
@ -287,6 +303,10 @@ const ClassificationMethodConfig: React.FC<ClassificationMethodConfigProps> = ({
|
|||
: undefined,
|
||||
hybrid_boundary_margin:
|
||||
classifierType === "hybrid" ? value.hybrid_boundary_margin ?? DEFAULT_HYBRID_BOUNDARY_MARGIN : undefined,
|
||||
// Only the LLM classifier can produce NON_REASONING, and the backend rejects the flag
|
||||
// beside any other type. Clearing it here (with the tier's own pool) is what keeps a
|
||||
// switch away from LLM from stranding a config that can never be saved.
|
||||
...nonReasoningTierFields(classifierType, value),
|
||||
};
|
||||
onChange(nextValue);
|
||||
};
|
||||
|
|
|
|||
|
|
@ -5,6 +5,8 @@ import { ChevronRight, Info, Plus, Trash2, X } from "lucide-react";
|
|||
import { Switch } from "@/components/ui/switch";
|
||||
|
||||
import { AffinityControls } from "./AffinityControls";
|
||||
import NonReasoningTierToggle from "./NonReasoningTierToggle";
|
||||
import TierConfigIntro from "./TierConfigIntro";
|
||||
import TierRowSelect from "./TierRowSelect";
|
||||
import { ModalityRoutingControls } from "./ModalityRoutingControls";
|
||||
import { Card, CardContent } from "@/components/ui/card";
|
||||
|
|
@ -20,6 +22,7 @@ import {
|
|||
MAX_TIER_COUNT,
|
||||
MAX_TIER_DEFINITION_CHARS,
|
||||
MAX_TIER_NAME_CHARS,
|
||||
ALL_BUILT_IN_TIERS,
|
||||
MIN_TIER_COUNT,
|
||||
TIER_ORDER,
|
||||
activeTierName,
|
||||
|
|
@ -200,34 +203,10 @@ const defaultModelPlaceholderFor = (derivedDefaultModel: string | undefined, isC
|
|||
};
|
||||
|
||||
const builtInTierInfo = (rowId: string): { label: string; description: string; examples: string } | undefined => {
|
||||
const builtIn = TIER_ORDER.find((tier) => tier === rowId);
|
||||
const builtIn = ALL_BUILT_IN_TIERS.find((tier) => tier === rowId);
|
||||
return builtIn ? TIER_DESCRIPTIONS[builtIn] : undefined;
|
||||
};
|
||||
|
||||
const tierConfigIntroText = (value: ComplexityRouterConfigValue): string => {
|
||||
if (value.classifier_type === "heuristic_v2") {
|
||||
return "The complexity router classifies each request with a calibrated local four-tier model (no API calls). Configure which model(s) handle each tier.";
|
||||
}
|
||||
if (heuristicScoringRole(value) === "never") {
|
||||
return "The complexity router classifies each request with your classifier model and routes it to that tier. Configure which model(s) handle each tier.";
|
||||
}
|
||||
return "The complexity router automatically classifies requests by complexity using rule-based scoring (no API calls, <1ms latency). Configure which model(s) handle each tier.";
|
||||
};
|
||||
|
||||
const TierConfigIntro: React.FC<{ value: ComplexityRouterConfigValue }> = ({ value }) => (
|
||||
<>
|
||||
<span className="block mb-6 text-muted-foreground">{tierConfigIntroText(value)}</span>
|
||||
|
||||
<span className="block mb-4 text-xs text-muted-foreground">
|
||||
{restrictedBy(value, "displayNames")?.reason ??
|
||||
"Rename a tier to use your own vocabulary in the dashboard and your spend logs. Renaming doesn't change how requests are classified, and callers never see these names."}
|
||||
{!value.custom_tier_set &&
|
||||
usesLlmClassifier(value.classifier_type) &&
|
||||
" Your classifier model reads these names, so clearer ones can sharpen its choices."}
|
||||
</span>
|
||||
</>
|
||||
);
|
||||
|
||||
const TierSetToolbar: React.FC<{
|
||||
editing: boolean;
|
||||
isCustomSet: boolean;
|
||||
|
|
@ -534,13 +513,6 @@ export const TIER_DESCRIPTIONS: Record<
|
|||
/** Every built-in tier name, including the opt-in one, for label and membership checks. */
|
||||
export const TIER_KEYS = Object.keys(TIER_DESCRIPTIONS) as Array<keyof ComplexityTiers>;
|
||||
|
||||
/**
|
||||
* The four-tier ladder in ascending severity, which is what a router sends unless it opted into
|
||||
* NON_REASONING. Mirrors TIER_SEVERITY_ORDER in the backend config; use tierOrderFor to get the
|
||||
* ladder one router actually renders.
|
||||
*/
|
||||
export const BUILT_IN_TIER_ORDER: Array<keyof ComplexityTiers> = ["SIMPLE", "MEDIUM", "COMPLEX", "REASONING"];
|
||||
|
||||
export const effectiveTierLabel = (tier: keyof ComplexityTiers, tierLabels: ComplexityTierLabels | undefined): string =>
|
||||
tierLabels?.[tier]?.trim() || TIER_DESCRIPTIONS[tier].label;
|
||||
|
||||
|
|
@ -555,42 +527,7 @@ export const DEFAULT_HYBRID_BOUNDARY_MARGIN = 0.03;
|
|||
* NON_REASONING, which the backend refuses alongside heuristic_first because the local scorer
|
||||
* cannot produce it.
|
||||
*/
|
||||
export const HEURISTIC_FIRST_MAX_TIER_KEYS = BUILT_IN_TIER_ORDER.slice(0, -1);
|
||||
|
||||
/**
|
||||
* The opt-in fifth tier. Only offered on the LLM classification method, matching the backend: the
|
||||
* heuristic scorers cannot produce the tier, so enabling it there would buy a rubric bullet and a
|
||||
* model pool that no request ever reaches.
|
||||
*/
|
||||
const NonReasoningTierToggle: React.FC<{
|
||||
value: ComplexityRouterConfigValue;
|
||||
onChange: (value: ComplexityRouterConfigValue) => void;
|
||||
available: boolean;
|
||||
}> = ({ value, onChange, available }) => (
|
||||
<>
|
||||
<div className="flex items-center gap-2 mt-4 mb-2">
|
||||
<Switch
|
||||
checked={value.enable_non_reasoning_tier === true}
|
||||
disabled={!available}
|
||||
onCheckedChange={(enabled) => {
|
||||
const { NON_REASONING: _dropped, ...keptTiers } = value.tiers;
|
||||
onChange({
|
||||
...value,
|
||||
enable_non_reasoning_tier: enabled ? true : undefined,
|
||||
tiers: enabled ? { ...keptTiers, NON_REASONING: value.tiers.NON_REASONING ?? [] } : keptTiers,
|
||||
});
|
||||
}}
|
||||
aria-label="Add a non-reasoning tier"
|
||||
/>
|
||||
<strong className="font-semibold">Add a non-reasoning tier</strong>
|
||||
</div>
|
||||
<span className="block text-xs mb-3 text-muted-foreground">
|
||||
Adds NON_REASONING below Simple, for operational agent traffic that relays or reformats information rather than
|
||||
reasoning about it. Escalation still moves up out of it when a request needs more.
|
||||
{!available && " Requires the LLM classification method."}
|
||||
</span>
|
||||
</>
|
||||
);
|
||||
export const HEURISTIC_FIRST_MAX_TIER_KEYS = TIER_ORDER.slice(0, -1);
|
||||
|
||||
const PlanModeOverrideControls: React.FC<{
|
||||
value: ComplexityRouterConfigValue;
|
||||
|
|
|
|||
|
|
@ -0,0 +1,49 @@
|
|||
import React from "react";
|
||||
|
||||
import { Switch } from "@/components/ui/switch";
|
||||
|
||||
import type { ComplexityRouterConfigValue } from "./ComplexityRouterConfig";
|
||||
|
||||
/**
|
||||
* The opt-in fifth tier. Only offered on the LLM classification method, matching the backend: the
|
||||
* heuristic scorers cannot produce the tier, so enabling it there would buy a rubric bullet and a
|
||||
* model pool that no request ever reaches.
|
||||
*/
|
||||
const NonReasoningTierToggle: React.FC<{
|
||||
value: ComplexityRouterConfigValue;
|
||||
onChange: (value: ComplexityRouterConfigValue) => void;
|
||||
available: boolean;
|
||||
}> = ({ value, onChange, available }) => {
|
||||
// Turning it off drops the tier's key rather than leaving an empty pool, which the backend
|
||||
// rejects; turning it back on restores whatever pool the form still held.
|
||||
const handleToggle = (enabled: boolean): void => {
|
||||
const { NON_REASONING: existingPool, ...keptTiers } = value.tiers;
|
||||
const next: ComplexityRouterConfigValue = {
|
||||
...value,
|
||||
enable_non_reasoning_tier: enabled ? true : undefined,
|
||||
tiers: enabled ? { ...keptTiers, NON_REASONING: existingPool ?? [] } : keptTiers,
|
||||
};
|
||||
onChange(next);
|
||||
};
|
||||
|
||||
return (
|
||||
<>
|
||||
<div className="flex items-center gap-2 mt-4 mb-2">
|
||||
<Switch
|
||||
checked={value.enable_non_reasoning_tier === true}
|
||||
disabled={!available}
|
||||
onCheckedChange={handleToggle}
|
||||
aria-label="Add a non-reasoning tier"
|
||||
/>
|
||||
<strong className="font-semibold">Add a non-reasoning tier</strong>
|
||||
</div>
|
||||
<span className="block text-xs mb-3 text-muted-foreground">
|
||||
Adds NON_REASONING below Simple, for operational agent traffic that relays or reformats information rather than
|
||||
reasoning about it. Escalation still moves up out of it when a request needs more.
|
||||
{!available && " Requires the LLM classification method."}
|
||||
</span>
|
||||
</>
|
||||
);
|
||||
};
|
||||
|
||||
export default NonReasoningTierToggle;
|
||||
|
|
@ -0,0 +1,30 @@
|
|||
import React from "react";
|
||||
|
||||
import { type ComplexityRouterConfigValue, heuristicScoringRole, usesLlmClassifier } from "./ComplexityRouterConfig";
|
||||
import { restrictedBy } from "./TierRestrictions";
|
||||
|
||||
const tierConfigIntroText = (value: ComplexityRouterConfigValue): string => {
|
||||
if (value.classifier_type === "heuristic_v2") {
|
||||
return "The complexity router classifies each request with a calibrated local four-tier model (no API calls). Configure which model(s) handle each tier.";
|
||||
}
|
||||
if (heuristicScoringRole(value) === "never") {
|
||||
return "The complexity router classifies each request with your classifier model and routes it to that tier. Configure which model(s) handle each tier.";
|
||||
}
|
||||
return "The complexity router automatically classifies requests by complexity using rule-based scoring (no API calls, <1ms latency). Configure which model(s) handle each tier.";
|
||||
};
|
||||
|
||||
const TierConfigIntro: React.FC<{ value: ComplexityRouterConfigValue }> = ({ value }) => (
|
||||
<>
|
||||
<span className="block mb-6 text-muted-foreground">{tierConfigIntroText(value)}</span>
|
||||
|
||||
<span className="block mb-4 text-xs text-muted-foreground">
|
||||
{restrictedBy(value, "displayNames")?.reason ??
|
||||
"Rename a tier to use your own vocabulary in the dashboard and your spend logs. Renaming doesn't change how requests are classified, and callers never see these names."}
|
||||
{!value.custom_tier_set &&
|
||||
usesLlmClassifier(value.classifier_type) &&
|
||||
" Your classifier model reads these names, so clearer ones can sharpen its choices."}
|
||||
</span>
|
||||
</>
|
||||
);
|
||||
|
||||
export default TierConfigIntro;
|
||||
|
|
@ -0,0 +1,52 @@
|
|||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import type { ComplexityRouterConfigValue } from "./ComplexityRouterConfig";
|
||||
import { nonReasoningTierFields } from "./ClassificationMethodConfig";
|
||||
|
||||
const enabledValue: ComplexityRouterConfigValue = {
|
||||
classifier_type: "llm",
|
||||
enable_non_reasoning_tier: true,
|
||||
tiers: {
|
||||
NON_REASONING: ["relay-cheap"],
|
||||
SIMPLE: ["gpt-4o-mini"],
|
||||
MEDIUM: ["gpt-4o"],
|
||||
COMPLEX: ["sonnet"],
|
||||
REASONING: ["opus"],
|
||||
},
|
||||
};
|
||||
|
||||
describe("nonReasoningTierFields", () => {
|
||||
it("keeps the tier and its pool while the classifier stays LLM", () => {
|
||||
expect(nonReasoningTierFields("llm", enabledValue)).toEqual({
|
||||
enable_non_reasoning_tier: true,
|
||||
tiers: enabledValue.tiers,
|
||||
});
|
||||
});
|
||||
|
||||
it.each(["heuristic", "heuristic_v2", "heuristic_first", "hybrid"] as const)(
|
||||
"clears the flag and the tier when the classifier becomes %s",
|
||||
(classifierType) => {
|
||||
// Leaving the flag set under a classifier that cannot emit the tier is a config the backend
|
||||
// refuses, and the switch is disabled there, so the operator could never undo it.
|
||||
const cleared = nonReasoningTierFields(classifierType, enabledValue);
|
||||
expect(cleared.enable_non_reasoning_tier).toBeUndefined();
|
||||
expect(cleared.tiers).not.toHaveProperty("NON_REASONING");
|
||||
},
|
||||
);
|
||||
|
||||
it("leaves the other tiers untouched when it clears", () => {
|
||||
const { NON_REASONING: _dropped, ...expectedTiers } = enabledValue.tiers;
|
||||
expect(nonReasoningTierFields("heuristic", enabledValue).tiers).toEqual(expectedTiers);
|
||||
});
|
||||
|
||||
it("is a no-op for a router that never enabled the tier", () => {
|
||||
const fourTier: ComplexityRouterConfigValue = {
|
||||
classifier_type: "heuristic",
|
||||
tiers: { SIMPLE: ["gpt-4o-mini"], MEDIUM: ["gpt-4o"], COMPLEX: ["sonnet"], REASONING: ["opus"] },
|
||||
};
|
||||
expect(nonReasoningTierFields("heuristic", fourTier)).toEqual({
|
||||
enable_non_reasoning_tier: undefined,
|
||||
tiers: fourTier.tiers,
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
@ -190,8 +190,15 @@ describe("the opt-in non-reasoning tier", () => {
|
|||
});
|
||||
|
||||
it("renders an enabled tier with no models as an empty row rather than crashing", () => {
|
||||
const emptyTierZeroRow: ActiveTierRow = {
|
||||
id: "NON_REASONING",
|
||||
name: "NON_REASONING",
|
||||
definition: "",
|
||||
models: [],
|
||||
params: {},
|
||||
};
|
||||
const rows = activeTierRows({ tiers, enable_non_reasoning_tier: true });
|
||||
expect(rows[0]).toEqual({ id: "NON_REASONING", name: "NON_REASONING", definition: "", models: [], params: {} });
|
||||
expect(rows[0]).toEqual(emptyTierZeroRow);
|
||||
});
|
||||
|
||||
it("counts as a built-in name either way, so a custom set cannot claim the name", () => {
|
||||
|
|
|
|||
10
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
10
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
|
|
@ -23573,22 +23573,16 @@ export interface components {
|
|||
};
|
||||
/**
|
||||
* AutoRouterPresetTiers
|
||||
* @description The built-in tiers the dashboard's preset prefill can apply.
|
||||
* @description Exactly the four built-in tiers the dashboard's preset prefill can apply.
|
||||
*
|
||||
* extra="forbid" on purpose: a tier name this dashboard cannot apply would grey out or crash the
|
||||
* picker, so such a catalog is rejected wholesale and the bundled one serves instead. NON_REASONING
|
||||
* is optional rather than required so this proxy accepts both a catalog that ships the opt-in fifth
|
||||
* tier and the four-tier catalogs that predate it. It stays None when unset rather than defaulting
|
||||
* to an empty pool, so a four-tier preset serves the tier set it was published with instead of
|
||||
* growing a key the dashboard would render as an empty fifth tier row.
|
||||
* picker, so such a catalog is rejected wholesale and the bundled one serves instead.
|
||||
*/
|
||||
AutoRouterPresetTiers: {
|
||||
/** Complex */
|
||||
COMPLEX: string[];
|
||||
/** Medium */
|
||||
MEDIUM: string[];
|
||||
/** Non Reasoning */
|
||||
NON_REASONING?: string[] | null;
|
||||
/** Reasoning */
|
||||
REASONING: string[];
|
||||
/** Simple */
|
||||
|
|
|
|||
File diff suppressed because one or more lines are too long
Loading…
Add table
Reference in a new issue