From a233ba910d536d39cf8ba376aa97ea2822380a3d Mon Sep 17 00:00:00 2001 From: Tin Chi Lo Date: Mon, 21 Sep 2026 08:37:23 -0700 Subject: [PATCH] feat(auto-router): configure heuristic v2 success threshold --- .../complexity_router/README.md | 16 +++- .../complexity_router/complexity_router.py | 5 +- .../complexity_router/config.py | 12 +++ .../complexity_router/tier_predictor.py | 7 +- .../router_strategy/test_complexity_router.py | 89 +++++++++++++++++-- .../add_model/AutoRouterRoutingTest.test.tsx | 19 +++- .../add_model/AutoRouterRoutingTest.tsx | 8 +- .../add_model/ClassificationMethodConfig.tsx | 85 +++++++++++++++++- .../add_model/ComplexityRouterConfig.test.tsx | 45 ++++++++++ .../add_model/ComplexityRouterConfig.tsx | 5 +- .../add_model/add_auto_router_tab.test.tsx | 75 ++++++++++++++++ .../add_model/add_auto_router_tab.tsx | 3 + .../build_complexity_router_config.test.ts | 34 +++++++ .../build_complexity_router_config.ts | 13 +++ ...d_updated_complexity_router_config.test.ts | 29 ++++++ ...dit_auto_router_modal.integration.test.tsx | 68 ++++++++++++++ .../edit_auto_router_modal.tsx | 9 ++ .../src/lib/autorouter_presets.test.ts | 11 +++ .../src/lib/autorouter_presets.ts | 1 + ui/litellm-dashboard/src/lib/http/schema.d.ts | 5 ++ 20 files changed, 518 insertions(+), 21 deletions(-) diff --git a/litellm/router_strategy/complexity_router/README.md b/litellm/router_strategy/complexity_router/README.md index 6505746bca1..f023d5001d9 100644 --- a/litellm/router_strategy/complexity_router/README.md +++ b/litellm/router_strategy/complexity_router/README.md @@ -191,6 +191,7 @@ model_list: model: auto_router/complexity_router complexity_router_config: classifier_type: heuristic_v2 + heuristic_v2_success_threshold: 0.9 tiers: SIMPLE: luna MEDIUM: terra @@ -201,9 +202,18 @@ model_list: No classifier model call or per-model training data is required. The classifier uses global tier quality, request-type quality, and similar-request cohorts from the bundled UltraFeedback artifact. It estimates success at every tier, enforces -monotonic probabilities, and returns the first tier meeting the trained 0.75 -threshold. The existing complexity-router tier pool then selects and dispatches -a model from that tier +monotonic probabilities, and returns the first tier meeting the success threshold, +or REASONING if no tier meets it. The existing complexity-router tier pool then +selects and dispatches a model from that tier + +Set `heuristic_v2_success_threshold` to a value from 0 to 1 to override the +artifact's threshold. For example, `0.9` requires a predicted success probability +of at least 90%. Higher thresholds favor more capable tiers. Omit the setting or +set it to `null` to use the artifact's `routing_threshold`, which is `0.75` for +the bundled artifact. The override leaves the predicted probabilities unchanged + +In the dashboard, select Heuristic v2 under Advanced: Classification Method and +set Success threshold. Clear the field to restore the artifact's default Spend logs record `routing_decision.cause: heuristic_v2`, the detected request type, and all four predicted probabilities. Existing `classifier_type: heuristic` diff --git a/litellm/router_strategy/complexity_router/complexity_router.py b/litellm/router_strategy/complexity_router/complexity_router.py index a3d6ccbd437..83fcfdfc329 100644 --- a/litellm/router_strategy/complexity_router/complexity_router.py +++ b/litellm/router_strategy/complexity_router/complexity_router.py @@ -1429,7 +1429,10 @@ class ComplexityRouter(CustomLogger): _ClassifierCircuitBreaker(circuit_breaker_cooldown) if circuit_breaker_cooldown is not None else None ) self._tier_success_predictor: TierSuccessPredictor | None = ( - TierSuccessPredictor(resolve_tier_artifact(self.config.heuristic_v2_artifact)) + TierSuccessPredictor( + resolve_tier_artifact(self.config.heuristic_v2_artifact), + routing_threshold=self.config.heuristic_v2_success_threshold, + ) if self.config.classifier_type == "heuristic_v2" else None ) diff --git a/litellm/router_strategy/complexity_router/config.py b/litellm/router_strategy/complexity_router/config.py index aa39dff8c53..0b2caa93665 100644 --- a/litellm/router_strategy/complexity_router/config.py +++ b/litellm/router_strategy/complexity_router/config.py @@ -1036,6 +1036,18 @@ class ComplexityRouterConfig(BaseModel): "UltraFeedback artifact is selected by default; an inline trained artifact may replace it" ), ) + heuristic_v2_success_threshold: float | None = Field( + default=None, + strict=True, + ge=0.0, + le=1.0, + description=( + "Minimum predicted success probability for classifier_type 'heuristic_v2' to select a tier. " + "The first tier meeting this threshold is selected, or REASONING if none meets it. " + "When omitted or null, uses the artifact's routing_threshold (0.75 for the bundled artifact). " + "Other classifier types ignore this setting" + ), + ) classifier_llm_config: ClassifierLLMConfig | None = Field( default=None, description=( diff --git a/litellm/router_strategy/complexity_router/tier_predictor.py b/litellm/router_strategy/complexity_router/tier_predictor.py index 764f6e6ad56..7775c36e795 100644 --- a/litellm/router_strategy/complexity_router/tier_predictor.py +++ b/litellm/router_strategy/complexity_router/tier_predictor.py @@ -108,8 +108,9 @@ class TierPrediction: class TierSuccessPredictor: - def __init__(self, artifact: TrainedTierArtifact) -> None: + def __init__(self, artifact: TrainedTierArtifact, *, routing_threshold: float | None = None) -> None: self._artifact = artifact + self._routing_threshold: Final = artifact.routing_threshold if routing_threshold is None else routing_threshold self._global: Mapping[int, TierGlobalStatistic] = MappingProxyType( {stat.tier: stat for stat in artifact.global_statistics} ) @@ -122,7 +123,7 @@ class TierSuccessPredictor: @property def routing_threshold(self) -> float: - return self._artifact.routing_threshold + return self._routing_threshold def predict(self, prompt: str, request_type: RequestType) -> TierPrediction: cohort: Final = similarity_cohort(prompt, request_type) @@ -132,7 +133,7 @@ class TierSuccessPredictor: {int(tier): probability for tier, probability in zip(_TIERS, monotonic)} ) required_tier: Final = next( - (tier for tier in _TIERS if probabilities[tier] >= self._artifact.routing_threshold), + (tier for tier in _TIERS if probabilities[tier] >= self.routing_threshold), 4, ) return TierPrediction(probabilities=probabilities, required_tier=required_tier) diff --git a/tests/test_litellm/router_strategy/test_complexity_router.py b/tests/test_litellm/router_strategy/test_complexity_router.py index ecd25ff654f..90ab39f601c 100644 --- a/tests/test_litellm/router_strategy/test_complexity_router.py +++ b/tests/test_litellm/router_strategy/test_complexity_router.py @@ -3722,16 +3722,37 @@ class TestLLMClassifier: @pytest.mark.asyncio @pytest.mark.parametrize("redact", (False, True)) + @pytest.mark.parametrize( + "override,threshold,tier,model", + ( + ({}, 0.8, "COMPLEX", "complex-model"), + ({"heuristic_v2_success_threshold": None}, 0.8, "COMPLEX", "complex-model"), + ({"heuristic_v2_success_threshold": 0.0}, 0.0, "SIMPLE", "simple-model"), + ({"heuristic_v2_success_threshold": 21 / 102}, 21 / 102, "MEDIUM", "medium-model"), + ({"heuristic_v2_success_threshold": 0.95}, 0.95, "REASONING", "reasoning-model"), + ({"heuristic_v2_success_threshold": 1.0}, 1.0, "REASONING", "reasoning-model"), + ), + ids=("omitted", "null", "zero", "inclusive", "higher", "no-tier-passes"), + ) async def test_heuristic_v2_routes_directly_to_predicted_builtin_tier( - self, mock_router_instance: MagicMock, redact: bool, monkeypatch: pytest.MonkeyPatch + self, + mock_router_instance: MagicMock, + redact: bool, + monkeypatch: pytest.MonkeyPatch, + override: Mapping[str, float | None], + threshold: float, + tier: str, + model: str, ) -> None: monkeypatch.setattr(litellm, "turn_off_message_logging", redact) - router = ComplexityRouter( + artifact: Final = _heuristic_v2_artifact() + router: Final = ComplexityRouter( model_name="tier-router", litellm_router_instance=mock_router_instance, complexity_router_config={ "classifier_type": "heuristic_v2", - "heuristic_v2_artifact": _heuristic_v2_artifact(), + "heuristic_v2_artifact": artifact, + **override, "tiers": { "SIMPLE": "simple-model", "MEDIUM": "medium-model", @@ -3741,15 +3762,15 @@ class TestLLMClassifier: }, ) - response = await router.async_pre_routing_hook( + response: Final = await router.async_pre_routing_hook( model="tier-router", request_kwargs={}, messages=[{"role": "user", "content": "Handle this new request"}], ) assert response is not None - assert response.model == "complex-model" - assert response.routing_decision["tier"] == "COMPLEX" + assert response.model == model + assert response.routing_decision["tier"] == tier assert response.routing_decision["cause"] == "heuristic_v2" assert response.routing_decision["signals"] == [ "request-type:general", @@ -3769,10 +3790,62 @@ class TestLLMClassifier: "COMPLEX": 91 / 102, "REASONING": 100 / 102, }, - "threshold": 0.8, - "predicted_tier": "COMPLEX", + "threshold": threshold, + "predicted_tier": tier, "request_type": "general", } + assert artifact.routing_threshold == 0.8 + + @pytest.mark.parametrize("threshold", (-0.01, 1.01, math.nan, math.inf, -math.inf, True, "0.95")) + def test_heuristic_v2_success_threshold_rejects_invalid_values(self, threshold: float | bool | str) -> None: + with pytest.raises(ValidationError, match="heuristic_v2_success_threshold"): + ComplexityRouterConfig.model_validate( + {"classifier_type": "heuristic_v2", "heuristic_v2_success_threshold": threshold} + ) + + @pytest.mark.asyncio + async def test_heuristic_v2_threshold_reload_and_rejected_update_keep_router_isolated(self) -> None: + artifact: Final = _heuristic_v2_artifact() + + def deployment(threshold: float, name: str = "editable") -> Deployment: + return Deployment( + model_name=name, + litellm_params=LiteLLM_Params( + model="auto_router/complexity_router", + complexity_router_config={ + "classifier_type": "heuristic_v2", + "heuristic_v2_artifact": artifact.model_dump(), + "heuristic_v2_success_threshold": threshold, + "session_affinity": False, + "tiers": {"SIMPLE": "simple-model", "REASONING": "reasoning-model"}, + }, + ), + model_info={"id": name}, + ) + + router: Final = Router( + model_list=[ + deployment(0.95).model_dump(exclude_none=True), + deployment(0.95, "unchanged").model_dump(exclude_none=True), + ], + ignore_invalid_deployments=True, + ) + + async def routed_threshold(name: str) -> tuple[str, float]: + response: Final = await router.async_pre_routing_hook( + model=name, + request_kwargs={}, + messages=[{"role": "user", "content": "Handle this new request"}], + ) + assert response is not None and response.routing_decision is not None + return response.model, response.routing_decision["heuristic_v2_forecast"]["threshold"] + + assert await routed_threshold("editable") == ("reasoning-model", 0.95) + assert router.upsert_deployment(deployment(0.0)) is not None + assert await routed_threshold("editable") == ("simple-model", 0.0) + assert await routed_threshold("unchanged") == ("reasoning-model", 0.95) + assert router.upsert_deployment(deployment(1.01)) is None + assert await routed_threshold("editable") == ("simple-model", 0.0) def test_heuristic_v2_needs_no_classifier_model(self): config = ComplexityRouterConfig(classifier_type="heuristic_v2") diff --git a/ui/litellm-dashboard/src/components/add_model/AutoRouterRoutingTest.test.tsx b/ui/litellm-dashboard/src/components/add_model/AutoRouterRoutingTest.test.tsx index 74e7193c8b0..86d161d1c8e 100644 --- a/ui/litellm-dashboard/src/components/add_model/AutoRouterRoutingTest.test.tsx +++ b/ui/litellm-dashboard/src/components/add_model/AutoRouterRoutingTest.test.tsx @@ -15,7 +15,8 @@ vi.mock("../networking", () => ({ const CONFIG = { tiers: { SIMPLE: ["cheap"], MEDIUM: ["mid"], COMPLEX: ["strong"], REASONING: ["o3"] }, - classifier_type: "heuristic", + classifier_type: "heuristic_v2", + heuristic_v2_success_threshold: 0, } as unknown as ComplexityRouterConfigPayload; const Harness = () => ( @@ -62,6 +63,22 @@ describe("AutoRouterRoutingTest", () => { expect(screen.getByTestId("auto-router-routing-test-send")).toBeDisabled(); }); + it("blocks previewing an invalid success threshold instead of sending NaN as null", () => { + renderWithProviders( + , + ); + fireEvent.change(screen.getByTestId("auto-router-routing-test-prompt"), { target: { value: "hello" } }); + expect(screen.getByTestId("auto-router-routing-test-send")).toBeDisabled(); + expect(screen.getByText("Success threshold must be a number between 0 and 1")).toBeVisible(); + expect(testAutoRouterRouting).not.toHaveBeenCalled(); + }); + it("routes the typed prompt through the config being edited and shows where it landed", async () => { const user = userEvent.setup(); vi.mocked(testAutoRouterRouting).mockResolvedValue(successResponse); diff --git a/ui/litellm-dashboard/src/components/add_model/AutoRouterRoutingTest.tsx b/ui/litellm-dashboard/src/components/add_model/AutoRouterRoutingTest.tsx index 00b2e75dfd1..2b6c06e9a96 100644 --- a/ui/litellm-dashboard/src/components/add_model/AutoRouterRoutingTest.tsx +++ b/ui/litellm-dashboard/src/components/add_model/AutoRouterRoutingTest.tsx @@ -5,7 +5,7 @@ import { Button } from "@/components/ui/button"; import { Textarea } from "@/components/ui/textarea"; import RoutingDecisionCard from "@/components/view_logs/LogDetailsDrawer/RoutingDecisionCard"; import { AutoRouterRoutingTestResult, testAutoRouterRouting } from "../networking"; -import { ComplexityRouterConfigPayload } from "./build_complexity_router_config"; +import { ComplexityRouterConfigPayload, getHeuristicV2SuccessThresholdError } from "./build_complexity_router_config"; import { buildAutoRouterRoutingTestRequest } from "./build_auto_router_routing_test_request"; interface AutoRouterRoutingTestProps { @@ -31,8 +31,10 @@ const AutoRouterRoutingTest: React.FC = ({ }) => { const [prompt, setPrompt] = React.useState(""); const [state, setState] = React.useState({ status: "idle" }); + const configError = getHeuristicV2SuccessThresholdError(config.heuristic_v2_success_threshold); const send = async () => { + if (configError) return; setState({ status: "running" }); const params = { prompt, config, defaultModel, routerName, teamId }; const request = buildAutoRouterRoutingTestRequest(params); @@ -62,13 +64,15 @@ const AutoRouterRoutingTest: React.FC = ({
+ {configError &&

{configError}

} + {state.status === "failed" && (
> = ({ + value, + onChange, +}) => { + const threshold = value.heuristic_v2_success_threshold; + if (effectiveClassifierType(value) === "heuristic_v2" || threshold === undefined) return null; + const error = getHeuristicV2SuccessThresholdError(threshold); + return ( +
+

+ Heuristic v2 success threshold (inactive):{" "} + + {Number.isFinite(threshold) ? threshold : "Invalid value"} + +

+

Only used when Heuristic v2 is selected

+ {error && ( +

+ {error} +

+ )} + +
+ ); +}; + const ClassifierTypeRadios: React.FC<{ value: ComplexityRouterConfigValue; classifierType: ClassifierType; @@ -259,6 +296,12 @@ const ClassificationMethodConfig: React.FC = ({ const classifierModel = value.classifier_llm_config?.model ?? ""; const classifierReasoningEffort = value.classifier_llm_config?.reasoning_effort; const explicitlySupportedClassifierEfforts = effortOptionsByModel[classifierModel]; + const successThresholdError = getHeuristicV2SuccessThresholdError(value.heuristic_v2_success_threshold); + const successThresholdDraft = + draft?.id === HEURISTIC_V2_SUCCESS_THRESHOLD_ID && + Object.is(value.heuristic_v2_success_threshold, draft.raw.trim() === "" ? undefined : Number(draft.raw)) + ? draft.raw + : null; const handleClassifierTypeChange = (classifierType: ClassifierType) => { onChange(transitionClassifierType(value, classifierType)); @@ -275,6 +318,14 @@ const ClassificationMethodConfig: React.FC = ({ onChange({ ...value, hybrid_boundary_margin: Math.min(1, Math.max(0, parsed)) }); }; + const handleSuccessThresholdChange = (raw: string) => { + setDraft({ id: HEURISTIC_V2_SUCCESS_THRESHOLD_ID, raw }); + onChange({ + ...value, + heuristic_v2_success_threshold: raw.trim() === "" ? undefined : Number(raw), + }); + }; + // One write for everything the prompt dialog owns. The rubric arrives here rather than through the // rubric handler because two onChange calls in one tick would both spread this render's `value`, // so whichever landed second would drop the other's edit. @@ -407,6 +458,36 @@ const ClassificationMethodConfig: React.FC = ({ <> + {classifierType === "heuristic_v2" && ( +
+ + handleSuccessThresholdChange(event.target.value)} + onBlur={() => { + if (!successThresholdError) setDraft(null); + }} + aria-invalid={Boolean(successThresholdError)} + aria-describedby={`${HEURISTIC_V2_SUCCESS_THRESHOLD_ID}-help${successThresholdError ? ` ${HEURISTIC_V2_SUCCESS_THRESHOLD_ID}-error` : ""}`} + /> +

+ Minimum predicted success probability, from 0 to 1. Higher values favor more capable tiers. Leave blank to + use the artifact default +

+ {successThresholdError && ( + + )} +
+ )} + {classifierType === "heuristic_first" && (
Decide locally up to diff --git a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.test.tsx b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.test.tsx index e91ff1d59c1..70658b787f0 100644 --- a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.test.tsx +++ b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.test.tsx @@ -153,6 +153,51 @@ describe("ComplexityRouterConfig", () => { expect(screen.queryByText(/Score < 0.15/)).not.toBeInTheDocument(); }); + it.each<[string, Partial]>([ + ["heuristic", { classifier_type: "heuristic" }], + ["LLM", { classifier_type: "llm" }], + ["heuristic first", { classifier_type: "heuristic_first" }], + ["hybrid", { classifier_type: "hybrid" }], + ["Capability", { classifier_type: "capability" }], + ["Fuse v2", { classifier_type: "llm_v2" }], + [ + "custom tiers", + { + classifier_type: "heuristic_v2", + custom_tier_set: { + tiers: [{ id: "review", name: "REVIEW", definition: "Review code", models: ["gpt-4"] }], + fallback_tier_id: "review", + }, + }, + ], + ])("shows and clears an invalid inactive threshold under %s", (_label, overrides) => { + const value = { ...defaultValue, ...overrides, heuristic_v2_success_threshold: Number.NaN }; + const onChange = vi.fn(); + renderWithProviders(); + const retained = screen.getByRole("region", { name: "Inactive Heuristic v2 threshold" }); + expect(within(retained).getByRole("status", { name: "Retained Heuristic v2 threshold" })).toHaveTextContent( + "Invalid value", + ); + expect(within(retained).getByRole("alert")).toHaveTextContent("Success threshold must be a number between 0 and 1"); + fireEvent.click(within(retained).getByRole("button", { name: "Clear Heuristic v2 threshold" })); + expect(onChange).toHaveBeenCalledWith({ ...value, heuristic_v2_success_threshold: undefined }); + }); + + it("shows an inactive zero threshold until explicitly cleared and hides the summary for active or absent values", () => { + const onChange = vi.fn(); + const value = { ...defaultValue, heuristic_v2_success_threshold: 0 }; + const { rerender } = renderWithProviders( + , + ); + expect(screen.getByRole("status", { name: "Retained Heuristic v2 threshold" })).toHaveTextContent("0"); + expect(screen.queryByRole("alert")).not.toBeInTheDocument(); + expect(onChange).not.toHaveBeenCalled(); + rerender(); + expect(screen.queryByRole("region", { name: "Inactive Heuristic v2 threshold" })).not.toBeInTheDocument(); + rerender(); + expect(screen.queryByRole("region", { name: "Inactive Heuristic v2 threshold" })).not.toBeInTheDocument(); + }); + it("should show classifier fields and use the configured values when classifier_type is llm", () => { const llmValue: ComplexityRouterConfigValue = { ...defaultValue, diff --git a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx index f6b50ce20bc..8216df139aa 100644 --- a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx +++ b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx @@ -37,7 +37,7 @@ import { import React from "react"; import { ModelGroup } from "@/components/llm_calls/fetch_models"; import AdaptiveRoutingConfig from "./AdaptiveRoutingConfig"; -import ClassificationMethodConfig from "./ClassificationMethodConfig"; +import ClassificationMethodConfig, { InactiveHeuristicV2Threshold } from "./ClassificationMethodConfig"; import ContextWindowEscalationConfig from "./ContextWindowEscalationConfig"; import ResponseFormatControls from "./ResponseFormatControls"; import StallEscalationConfig from "./StallEscalationConfig"; @@ -374,6 +374,7 @@ export interface ComplexityRouterConfigValue { /** An explicit pin. Unset means the default tracks the tiers - see resolveComplexityDefaultModel. */ default_model?: string; classifier_type: ClassifierType; + heuristic_v2_success_threshold?: number; capability_classifier_config?: CapabilitySettings; llm_v2_config?: FuseSettings; classifier_llm_config?: ClassifierLLMConfig; @@ -618,6 +619,8 @@ const ComplexityRouterConfig: React.FC = ({ )}
+ + {forecast ? ( <> { }); }); + it("blocks invalid success thresholds and creates a heuristic v2 router with explicit zero", async () => { + const user = userEvent.setup(); + vi.mocked(getMissingTiersError).mockReturnValue(null); + renderWithProviders(); + fireEvent.change(screen.getByLabelText("Auto Router Name"), { target: { value: "threshold-router" } }); + expandDetailedConfiguration(); + await user.click(screen.getByText("Advanced: Classification Method")); + await user.click(screen.getByRole("radio", { name: /^Heuristic v2/ })); + + const threshold = screen.getByRole("textbox", { name: "Success threshold" }); + expect(threshold).toHaveValue(""); + fireEvent.change(threshold, { target: { value: "invalid" } }); + fireEvent.blur(threshold); + expect(threshold).toHaveValue("invalid"); + expect(threshold).toHaveAttribute("aria-invalid", "true"); + expect(screen.getByRole("button", { name: "Add Auto Router" })).toBeDisabled(); + expect(screen.getByTestId("auto-router-test-routing-btn")).toBeDisabled(); + + await user.click(screen.getByRole("radio", { name: /^Heuristic \(default\)/ })); + expect(screen.getByRole("button", { name: "Add Auto Router" })).toBeDisabled(); + await user.click(screen.getByRole("radio", { name: /^Heuristic v2/ })); + fireEvent.change(screen.getByRole("textbox", { name: "Success threshold" }), { target: { value: "1.01" } }); + expect(screen.getByRole("button", { name: "Add Auto Router" })).toBeDisabled(); + fireEvent.change(screen.getByRole("textbox", { name: "Success threshold" }), { target: { value: "0" } }); + await user.click(screen.getByRole("button", { name: "Add Auto Router" })); + + await waitFor(() => expect(handleAddAutoRouterSubmit).toHaveBeenCalledOnce()); + expect(vi.mocked(handleAddAutoRouterSubmit).mock.calls.at(-1)?.[0].complexity_router_config).toMatchObject({ + classifier_type: "heuristic_v2", + heuristic_v2_success_threshold: 0, + }); + }); + + it("clears an invalid threshold draft when automatic setup replaces the configuration", async () => { + const user = userEvent.setup(); + mockFetchAvailableModels.mockResolvedValue(ALL_FAMILY_MODELS); + renderWithProviders(); + const automaticSetup = await screen.findByRole("button", { name: "Configure automatically" }); + await waitFor(() => expect(automaticSetup).toBeEnabled()); + await user.click(automaticSetup); + fireEvent.change(screen.getByLabelText("Auto Router Name"), { target: { value: "reset-threshold-router" } }); + await user.click(screen.getByText("Advanced: Classification Method")); + fireEvent.change(screen.getByRole("textbox", { name: "Success threshold" }), { target: { value: "1.1" } }); + expect(screen.getByRole("button", { name: "Add Auto Router" })).toBeDisabled(); + + await user.click(automaticSetup); + expect(screen.getByRole("textbox", { name: "Success threshold" })).toHaveValue(""); + expect(screen.getByRole("textbox", { name: "Success threshold" })).toHaveAttribute("aria-invalid", "false"); + await user.click(screen.getByRole("button", { name: "Add Auto Router" })); + await waitFor(() => expect(handleAddAutoRouterSubmit).toHaveBeenCalledOnce()); + expect(vi.mocked(handleAddAutoRouterSubmit).mock.calls.at(-1)?.[0].complexity_router_config).not.toHaveProperty( + "heuristic_v2_success_threshold", + ); + }); + + it("clears an invalid inactive threshold before creating the router", async () => { + const user = userEvent.setup(); + vi.mocked(getMissingTiersError).mockReturnValue(null); + renderWithProviders(); + fireEvent.change(screen.getByLabelText("Auto Router Name"), { target: { value: "clear-threshold-router" } }); + expandDetailedConfiguration(); + await user.click(screen.getByText("Advanced: Classification Method")); + await user.click(screen.getByRole("radio", { name: /^Heuristic v2/ })); + fireEvent.change(screen.getByRole("textbox", { name: "Success threshold" }), { target: { value: "invalid" } }); + await user.click(screen.getByRole("radio", { name: /^Heuristic \(default\)/ })); + expect(screen.getByRole("button", { name: "Add Auto Router" })).toBeDisabled(); + await user.click(screen.getByRole("button", { name: "Clear Heuristic v2 threshold" })); + expect(screen.queryByRole("region", { name: "Inactive Heuristic v2 threshold" })).not.toBeInTheDocument(); + await user.click(screen.getByRole("button", { name: "Add Auto Router" })); + await waitFor(() => expect(handleAddAutoRouterSubmit).toHaveBeenCalledOnce()); + expect(vi.mocked(handleAddAutoRouterSubmit).mock.calls.at(-1)?.[0].complexity_router_config).not.toHaveProperty( + "heuristic_v2_success_threshold", + ); + }); + it("carries a context-window escalation opt-out through to the create payload", async () => { const user = userEvent.setup(); vi.mocked(getMissingTiersError).mockReturnValue(null); diff --git a/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx b/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx index 126d9ba2311..57a6201bc7b 100644 --- a/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx +++ b/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx @@ -47,6 +47,7 @@ import { buildComplexityRouterConfig, getKeywordTierRulesError, getClassifierModelError, + getHeuristicV2SuccessThresholdError, getClassifierReasoningEffortError, getMissingTiersError, getPlanModeTierError, @@ -146,6 +147,7 @@ export const getSubmitBlockedReason = ( getPlanModeTierError(config.plan_mode_min_tier, activeTierRows(config)) ?? getKeywordTierRulesError(keywordTierRules, activeTierRows(config)) ?? getClassifierModelError(config) ?? + getHeuristicV2SuccessThresholdError(config.heuristic_v2_success_threshold) ?? (heuristicScoringRole(config) === "decides" ? customDimensionsError(config.custom_dimensions) : null) ?? getClassifierReasoningEffortError(config, modelInfo) ?? getReferencedModelsError(referencedModelsParams, availability) @@ -405,6 +407,7 @@ const AddAutoRouterTab: React.FC = ({ classificationMode: complexityRouterConfig.classification_mode, tierLabels: complexityRouterConfig.tier_labels, classifierType: complexityRouterConfig.classifier_type, + heuristicV2SuccessThreshold: complexityRouterConfig.heuristic_v2_success_threshold, capabilityClassifierConfig: complexityRouterConfig.capability_classifier_config, llmV2Config: complexityRouterConfig.llm_v2_config, classifierLlmConfig: complexityRouterConfig.classifier_llm_config, diff --git a/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.test.ts b/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.test.ts index 2990878d086..63fed7c7175 100644 --- a/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.test.ts +++ b/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.test.ts @@ -4,6 +4,7 @@ import { normalizeClassifierLlmConfig, getKeywordTierRulesError, getClassifierModelError, + getHeuristicV2SuccessThresholdError, getClassifierReasoningEffortError, getMissingTiersError, hydrateCustomTierSet, @@ -211,8 +212,28 @@ describe("buildComplexityRouterConfig", () => { expect(config.classifier_llm_config).toBeUndefined(); expect(config.classifier_context_window_size).toBeUndefined(); expect(config.classifier_fallback).toBeUndefined(); + expect(config).not.toHaveProperty("heuristic_v2_success_threshold"); }); + it.each([0, 0.95, 1])("serializes a heuristic v2 success threshold of %s", (heuristicV2SuccessThreshold) => { + const config = buildComplexityRouterConfig({ + ...baseParams, + classifierType: "heuristic_v2", + heuristicV2SuccessThreshold, + }); + expect(config.heuristic_v2_success_threshold).toBe(heuristicV2SuccessThreshold); + }); + + it.each(["heuristic", "llm", "heuristic_first", "hybrid", "capability", "llm_v2"] as const)( + "retains the inactive success threshold under %s", + (classifierType) => { + expect( + buildComplexityRouterConfig({ ...baseParams, classifierType, heuristicV2SuccessThreshold: 0.91 }) + .heuristic_v2_success_threshold, + ).toBe(0.91); + }, + ); + it("includes classifier_context_window_size and classifier_context_budget_chars only when classifier_type is llm", () => { const params: BuildComplexityRouterConfigParams = { ...baseParams, @@ -884,6 +905,19 @@ describe("buildComplexityRouterConfig tier model params", () => { }); }); +describe("getHeuristicV2SuccessThresholdError", () => { + it.each([undefined, 0, 0.95, 1])("accepts the optional probability %s", (threshold) => { + expect(getHeuristicV2SuccessThresholdError(threshold)).toBeNull(); + }); + + it.each([-0.01, 1.01, Number.NaN, Number.POSITIVE_INFINITY, Number.NEGATIVE_INFINITY])( + "rejects invalid success threshold %s", + (threshold) => { + expect(getHeuristicV2SuccessThresholdError(threshold)).toBe("Success threshold must be a number between 0 and 1"); + }, + ); +}); + describe("getClassifierModelError", () => { it("stays quiet for a heuristic router, which needs no classifier model", () => { expect(getClassifierModelError({ classifier_type: "heuristic" })).toBeNull(); diff --git a/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.ts b/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.ts index 8a377c17ad7..05dc327968c 100644 --- a/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.ts +++ b/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.ts @@ -144,6 +144,7 @@ export interface StoredComplexityRouterConfig { hybrid_boundary_margin?: unknown; tier_labels?: unknown; classifier_type?: ClassifierType; + heuristic_v2_success_threshold?: unknown; capability_classifier_config?: unknown; llm_v2_config?: unknown; classifier_llm_config?: ClassifierLLMConfig; @@ -182,6 +183,7 @@ export interface BuildComplexityRouterConfigParams { planModeMinTier: string | undefined; tierLabels: ComplexityTierLabels | undefined; classifierType: ClassifierType; + heuristicV2SuccessThreshold?: number; capabilityClassifierConfig?: CapabilitySettings; llmV2Config?: FuseSettings; classifierLlmConfig: ClassifierLLMConfigWire | undefined; @@ -248,6 +250,7 @@ export interface ComplexityRouterConfigPayload { plan_mode_min_tier?: string; tier_labels?: ComplexityTierLabels; classifier_type: ClassifierType; + heuristic_v2_success_threshold?: number; capability_classifier_config?: CapabilitySettings; llm_v2_config?: FuseSettings; classifier_llm_config?: ClassifierLLMConfig; @@ -356,6 +359,12 @@ export const getKeywordTierRulesError = ( return `Keyword rule(s) ${orphaned.join(", ")} route to a tier this router no longer has`; }; +export const getHeuristicV2SuccessThresholdError = (threshold: number | undefined): string | null => { + if (threshold === undefined) return null; + const validProbability = Number.isFinite(threshold) && threshold >= 0 && threshold <= 1; + return validProbability ? null : "Success threshold must be a number between 0 and 1"; +}; + // An edited tier set forces the LLM classifier, so the model requirement follows the EFFECTIVE type. // Both forms' submit gates and their submit handlers read this one answer so they cannot drift. export const getClassifierModelError = ( @@ -557,6 +566,7 @@ export const buildComplexityRouterConfig = ({ planModeMinTier, tierLabels, classifierType, + heuristicV2SuccessThreshold, capabilityClassifierConfig, llmV2Config, classifierLlmConfig, @@ -640,6 +650,9 @@ export const buildComplexityRouterConfig = ({ ...(planModeMinTier?.trim() && { plan_mode_min_tier: planModeMinTier }), ...(cleanedTierLabels && { tier_labels: cleanedTierLabels }), classifier_type: classifierType, + ...(heuristicV2SuccessThreshold !== undefined && { + heuristic_v2_success_threshold: heuristicV2SuccessThreshold, + }), ...classifierWireFields(effectiveType, classifierInputs), ...(effectiveType === "capability" && capabilityClassifierConfig && { capability_classifier_config: capabilityClassifierConfig }), diff --git a/ui/litellm-dashboard/src/components/edit_auto_router/build_updated_complexity_router_config.test.ts b/ui/litellm-dashboard/src/components/edit_auto_router/build_updated_complexity_router_config.test.ts index 4ae6efbb12d..e4b4cbafdf6 100644 --- a/ui/litellm-dashboard/src/components/edit_auto_router/build_updated_complexity_router_config.test.ts +++ b/ui/litellm-dashboard/src/components/edit_auto_router/build_updated_complexity_router_config.test.ts @@ -46,6 +46,34 @@ const hydratedState: KeywordMatchingState = { }; describe("buildUpdatedComplexityRouterConfig keyword matching", () => { + it.each([0, 0.92, 1])("hydrates and saves a success threshold of %s without changing the artifact", (threshold) => { + const stored = { + ...STORED, + classifier_type: "heuristic_v2" as const, + heuristic_v2_success_threshold: threshold, + heuristic_v2_artifact: { routing_threshold: 0.82, custom_metadata: "retained" }, + }; + const hydrated = hydrateComplexityRouterConfig(stored, undefined); + expect(hydrated.heuristic_v2_success_threshold).toBe(threshold); + const saved = buildUpdatedComplexityRouterConfig(stored, hydrated); + expect(saved.heuristic_v2_success_threshold).toBe(threshold); + expect(saved.heuristic_v2_artifact).toEqual(stored.heuristic_v2_artifact); + + const cleared = buildUpdatedComplexityRouterConfig(stored, { + ...hydrated, + heuristic_v2_success_threshold: undefined, + }); + expect(cleared).not.toHaveProperty("heuristic_v2_success_threshold"); + expect(cleared.heuristic_v2_artifact).toEqual(stored.heuristic_v2_artifact); + }); + + it.each([undefined, null])("keeps an inherited success threshold %s omitted after saving", (threshold) => { + const stored = { ...STORED, heuristic_v2_success_threshold: threshold }; + const hydrated = hydrateComplexityRouterConfig(stored, undefined); + expect(hydrated.heuristic_v2_success_threshold).toBeUndefined(); + expect(buildUpdatedComplexityRouterConfig(stored, hydrated)).not.toHaveProperty("heuristic_v2_success_threshold"); + }); + it.each(["capability", "llm_v2", "heuristic"] as const)( "handles enabled stored overrides when editing %s with or without keyword form state", (classifier_type) => { @@ -669,6 +697,7 @@ describe("managed keys survive an untouched open-and-save", () => { plan_mode_min_tier: "COMPLEX", tier_labels: { SIMPLE: "Cheap" }, classifier_type: "heuristic_first", + heuristic_v2_success_threshold: 0.89, heuristic_first_max_tier: "SIMPLE", classifier_llm_config: { model: "gpt-4o-mini", timeout_ms: 3000, reasoning_effort: "low" }, classifier_context_window_size: 5, diff --git a/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.integration.test.tsx b/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.integration.test.tsx index 0bb3340ac09..34db61483cf 100644 --- a/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.integration.test.tsx +++ b/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.integration.test.tsx @@ -132,6 +132,74 @@ describe("EditAutoRouterModal keyword matching", () => { expect(await screen.findByText(/Keyword\/Semantic Matching/i)).toBeInTheDocument(); }); + it.each(["0", ""])("hydrates the saved threshold and saves an edit to '%s'", async (raw) => { + const user = userEvent.setup(); + renderModal({ + modelData: { + ...MODEL_DATA, + litellm_params: { + ...MODEL_DATA.litellm_params, + complexity_router_config: { + ...STORED_CONFIG, + classifier_type: "heuristic_v2", + heuristic_v2_success_threshold: 0.91, + }, + }, + }, + }); + await user.click(await screen.findByText("Advanced: Classification Method")); + const threshold = screen.getByRole("textbox", { name: "Success threshold" }); + expect(threshold).toHaveValue("0.91"); + fireEvent.change(threshold, { target: { value: raw } }); + await user.click(screen.getByRole("button", { name: "Save Changes" })); + await waitFor(() => expect(modelPatchUpdateCall).toHaveBeenCalledOnce()); + if (raw === "") expect(savedConfig()).not.toHaveProperty("heuristic_v2_success_threshold"); + else expect(savedConfig().heuristic_v2_success_threshold).toBe(0); + }); + + it("blocks an invalid threshold edit and retains a corrected value when switching classifiers", async () => { + const user = userEvent.setup(); + renderModal({ + modelData: { + ...MODEL_DATA, + litellm_params: { + ...MODEL_DATA.litellm_params, + complexity_router_config: { + ...STORED_CONFIG, + classifier_type: "heuristic_v2", + heuristic_v2_success_threshold: 0.91, + }, + }, + }, + }); + await user.click(await screen.findByText("Advanced: Classification Method")); + fireEvent.change(screen.getByRole("textbox", { name: "Success threshold" }), { target: { value: "-0.1" } }); + expect(screen.getByRole("button", { name: "Save Changes" })).toBeDisabled(); + expect(modelPatchUpdateCall).not.toHaveBeenCalled(); + + fireEvent.change(screen.getByRole("textbox", { name: "Success threshold" }), { target: { value: "0.88" } }); + await user.click(screen.getByRole("radio", { name: /^Heuristic \(default\)/ })); + expect(screen.queryByRole("textbox", { name: "Success threshold" })).not.toBeInTheDocument(); + await user.click(screen.getByRole("button", { name: "Save Changes" })); + await waitFor(() => expect(modelPatchUpdateCall).toHaveBeenCalledOnce()); + expect(savedConfig()).toMatchObject({ classifier_type: "heuristic", heuristic_v2_success_threshold: 0.88 }); + }); + + it("clears an invalid inactive threshold before saving the router", async () => { + const user = userEvent.setup(); + renderModal(); + await user.click(await screen.findByText("Advanced: Classification Method")); + await user.click(screen.getByRole("radio", { name: /^Heuristic v2/ })); + fireEvent.change(screen.getByRole("textbox", { name: "Success threshold" }), { target: { value: "1.1" } }); + await user.click(screen.getByRole("radio", { name: /^Heuristic \(default\)/ })); + expect(screen.getByRole("button", { name: "Save Changes" })).toBeDisabled(); + await user.click(screen.getByRole("button", { name: "Clear Heuristic v2 threshold" })); + expect(screen.queryByRole("region", { name: "Inactive Heuristic v2 threshold" })).not.toBeInTheDocument(); + await user.click(screen.getByRole("button", { name: "Save Changes" })); + await waitFor(() => expect(modelPatchUpdateCall).toHaveBeenCalledOnce()); + expect(savedConfig()).not.toHaveProperty("heuristic_v2_success_threshold"); + }); + // These keys are rewritten from form state on save, so if the modal renders the controls // without hydrating them, an untouched save silently wipes the stored configuration. This // drives the real component; a test of the payload builder alone cannot see that bug. diff --git a/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.tsx b/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.tsx index e25c7f07dd7..5991049f4fc 100644 --- a/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.tsx +++ b/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.tsx @@ -42,6 +42,7 @@ import { type BuildComplexityRouterConfigParams, buildComplexityRouterConfig, getClassifierModelError, + getHeuristicV2SuccessThresholdError, getClassifierReasoningEffortError, getKeywordTierRulesError, getMissingTiersError, @@ -127,6 +128,10 @@ export const hydrateComplexityRouterConfig = ( plan_mode_min_tier: hydratePlanModeMinTier(parsedConfig.plan_mode_min_tier, custom_tier_set), tier_labels: hydrateTierLabels(parsedConfig.tier_labels), classifier_type: parsedConfig.classifier_type || "heuristic", + heuristic_v2_success_threshold: + typeof parsedConfig.heuristic_v2_success_threshold === "number" + ? parsedConfig.heuristic_v2_success_threshold + : undefined, capability_classifier_config: capabilitySettingsSchema.safeParse(parsedConfig.capability_classifier_config).data, llm_v2_config: fuseSettingsSchema.safeParse(parsedConfig.llm_v2_config).data, classifier_llm_config: parsedConfig.classifier_llm_config, @@ -227,6 +232,7 @@ export const MANAGED_COMPLEXITY_ROUTER_KEYS = new Set([ "classification_examples", "heuristic_first_max_tier", "hybrid_boundary_margin", + "heuristic_v2_success_threshold", "classification_mode", "session_affinity", "session_affinity_ttl_seconds", @@ -329,6 +335,7 @@ export const buildUpdatedComplexityRouterConfig = ( classificationMode: value.classification_mode, tierLabels: value.tier_labels, classifierType: value.classifier_type, + heuristicV2SuccessThreshold: value.heuristic_v2_success_threshold, capabilityClassifierConfig: value.capability_classifier_config, llmV2Config: value.llm_v2_config, classifierLlmConfig: value.classifier_llm_config, @@ -427,6 +434,7 @@ const EditAutoRouterModal: React.FC = ({ getPlanModeTierError(complexityRouterConfig.plan_mode_min_tier, activeTierRows(complexityRouterConfig)) ?? getKeywordTierRulesError(keywordTierRules, activeTierRows(complexityRouterConfig)) ?? getClassifierModelError(complexityRouterConfig) ?? + getHeuristicV2SuccessThresholdError(complexityRouterConfig.heuristic_v2_success_threshold) ?? getForecastConfigError(complexityRouterConfig) ?? (heuristicScoringRole(complexityRouterConfig) === "decides" ? customDimensionsError(complexityRouterConfig.custom_dimensions) @@ -559,6 +567,7 @@ const EditAutoRouterModal: React.FC = ({ } const classifierError = getClassifierModelError(complexityRouterConfig) ?? + getHeuristicV2SuccessThresholdError(complexityRouterConfig.heuristic_v2_success_threshold) ?? getForecastConfigError(complexityRouterConfig) ?? (heuristicScoringRole(complexityRouterConfig) === "decides" ? customDimensionsError(complexityRouterConfig.custom_dimensions) diff --git a/ui/litellm-dashboard/src/lib/autorouter_presets.test.ts b/ui/litellm-dashboard/src/lib/autorouter_presets.test.ts index fed11454c23..7c49b15e279 100644 --- a/ui/litellm-dashboard/src/lib/autorouter_presets.test.ts +++ b/ui/litellm-dashboard/src/lib/autorouter_presets.test.ts @@ -680,6 +680,17 @@ describe("autorouter_presets", () => { }); describe("buildPresetPrefill", () => { + it.each([undefined, 0, 0.95])("carries a preset's success threshold %s into the form", (threshold) => { + const preset = getPresetByKey("anthropic_family")!; + const config = { + ...preset.complexity_router_config, + classifier_type: "heuristic_v2" as const, + heuristic_v2_success_threshold: threshold, + }; + const prefill = buildPresetPrefill(config, groupsOnly(getRequiredModelsInPreset(preset))); + expect(prefill.complexityRouterConfig.heuristic_v2_success_threshold).toBe(threshold); + }); + it("prefills a real bundled preset's tiers into the config", () => { const preset = getPresetByKey("anthropic_family")!; const prefill = buildPresetPrefill( diff --git a/ui/litellm-dashboard/src/lib/autorouter_presets.ts b/ui/litellm-dashboard/src/lib/autorouter_presets.ts index 02096cada41..f085b4760a9 100644 --- a/ui/litellm-dashboard/src/lib/autorouter_presets.ts +++ b/ui/litellm-dashboard/src/lib/autorouter_presets.ts @@ -284,6 +284,7 @@ export const buildPresetPrefill = ( tier_model_params: resolveParamKeys(hydrateTierModelParams(config.tiers, config.tier_model_configs)), tier_labels: hydrateTierLabels(config.tier_labels), classifier_type: config.classifier_type, + heuristic_v2_success_threshold: config.heuristic_v2_success_threshold, classifier_llm_config: config.classifier_llm_config && { ...config.classifier_llm_config, model: resolve(config.classifier_llm_config.model), diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index d62a758e3a8..aa71adfad42 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -36684,6 +36684,11 @@ export interface components { * @default ultrafeedback */ heuristic_v2_artifact: components["schemas"]["TrainedTierArtifact"] | "ultrafeedback"; + /** + * Heuristic V2 Success Threshold + * @description Minimum predicted success probability for classifier_type 'heuristic_v2' to select a tier. The first tier meeting this threshold is selected, or REASONING if none meets it. When omitted or null, uses the artifact's routing_threshold (0.75 for the bundled artifact). Other classifier types ignore this setting + */ + heuristic_v2_success_threshold?: number | null; /** * Housekeeping Patterns * @description Additional case-sensitive literal sentinels that mark a request as client housekeeping, on top of the built-in conversation-title ones. For clients whose wording the built-ins don't cover, or after a client release changes its strings.