feat(ui): pick the auto router's custom classifier from a config-declared registry

Adds a top-level classifier_plugins config key mapping names to dotted paths,
resolved at startup into litellm.classifier_plugin_registry with the same
load-time checks classifier_plugin already gets. A before-validator on
ComplexityRouterConfig.classifier_plugin resolves string values through the
registry, so the save-time write gate, router deployment init, and direct
construction all accept a registered name and reject an unknown one with a
clean error; DB-stored auto-routers therefore carry plain names and no proxy
read path changes. GET /auto_router/classifier_plugins lists the names.

The dashboard's classification method picker gains a Custom classifier option
with a plugin dropdown fed by that endpoint, a plugin timeout field, and the
fallback picker now shown for both non-heuristic modes. Create and edit
serialize and validate identically, the models table labels custom routers
correctly, and the routing decision card names the classifier_plugin cause
and stops implying the LLM classifier failed on plugin fallbacks
This commit is contained in:
Tin Chi Lo 2026-08-18 15:25:15 -07:00
parent d03ef8be03
commit 72671282df
26 changed files with 939 additions and 60 deletions

View file

@ -358,6 +358,11 @@ blocked_user_list: Optional[Union[str, List]] = None
banned_keywords_list: Optional[Union[str, List]] = None
llm_guard_mode: Literal["all", "key-specific", "request-specific"] = "all"
guardrail_name_config_map: Dict[str, GuardrailItem] = {}
# Complexity-router classifier plugins declared under the proxy config's top-level
# `classifier_plugins` key, keyed by the name the Admin UI selects and saved models
# reference. Populated at proxy startup; ComplexityRouterConfig resolves string
# classifier_plugin values through it.
classifier_plugin_registry: Dict[str, "ClassifierPlugin"] = {} # mutable-ok: populated at proxy startup
include_cost_in_streaming_usage: bool = False
reasoning_auto_summary: bool = False
### PROMPTS ####
@ -488,6 +493,8 @@ priority_reservation: Optional[Dict[str, Union[float, "PriorityReservationDict"]
# priority_reservation_settings is lazy-loaded via __getattr__
# Only declare for type checking - at runtime __getattr__ handles it
if TYPE_CHECKING:
from litellm.types.router import ClassifierPlugin
priority_reservation_settings: Optional["PriorityReservationSettings"] = None

View file

@ -39,6 +39,7 @@ from litellm.types.management_endpoints.auto_router_endpoints import (
AutoRouterCacheStats,
AutoRouterRoutingTestRequest,
AutoRouterRoutingTestResponse,
ClassifierPluginsListResponse,
RequestComplexityRouterConfig,
ShadowEvalJobResponse,
ShadowEvalResult,
@ -369,6 +370,22 @@ def _summed_agg_row(rows: Sequence[_SessionAggRow]) -> _SessionAggRow:
)
@router.get(
"/auto_router/classifier_plugins",
tags=("auto router",),
dependencies=(Depends(user_api_key_auth),),
response_model=ClassifierPluginsListResponse,
)
async def list_classifier_plugins(
user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)],
) -> ClassifierPluginsListResponse:
"""Registered classifier plugin names, for the Admin UI's custom classifier picker."""
import litellm
_require_admin_viewer(user_api_key_dict, "list classifier plugins")
return ClassifierPluginsListResponse(classifier_plugins=tuple(sorted(litellm.classifier_plugin_registry)))
@router.get(
"/auto_router/benchmarks",
tags=("auto router",),

View file

@ -4089,6 +4089,9 @@ def resolve_classifier_plugin(
sync `def classify` passes the runtime_checkable isinstance and would only fail on the
first classified request, so reject it here where the error names the config key.
"""
registered: Final = litellm.classifier_plugin_registry.get(plugin_path)
if registered is not None:
return registered
resolved: Final = get_instance_fn(value=plugin_path, config_file_path=config_file_path)
if not isinstance(resolved, ClassifierPlugin) or not inspect.iscoroutinefunction(
getattr(resolved, "classify", None)
@ -5489,6 +5492,20 @@ class ProxyConfig:
# Load vector stores from config
litellm.vector_store_registry.load_vector_stores_from_config(vector_store_registry_config)
## CLASSIFIER PLUGINS (complexity-router custom classifiers, picked by name in the Admin UI)
classifier_plugins_config: Final = config.get("classifier_plugins", None)
if classifier_plugins_config:
if not isinstance(classifier_plugins_config, dict):
raise TypeError("classifier_plugins must map plugin names to dotted paths")
for plugin_name, plugin_path in classifier_plugins_config.items():
if not isinstance(plugin_path, str):
raise TypeError(f"classifier_plugins.{plugin_name} must be a dotted-path string")
litellm.classifier_plugin_registry[str(plugin_name)] = resolve_classifier_plugin(
plugin_path=plugin_path,
config_file_path=config_file_path,
source_label=f"classifier_plugins.{plugin_name}",
)
## WORKER REGISTRY (Global Control Plane)
worker_registry_config: Final = config.get("worker_registry", None)
if worker_registry_config:

View file

@ -546,12 +546,29 @@ class ComplexityRouterConfig(BaseModel):
classifier_plugin: ClassifierPlugin | None = Field(
default=None,
description=(
"Custom classifier deciding the tier; required when classifier_type is 'custom'. In the proxy "
"config, a dotted path to a ClassifierPlugin instance (resolved at startup, like plugins). Its "
"classify(context) receives the request messages and metadata (caller identity included) and "
"Custom classifier deciding the tier; required when classifier_type is 'custom'. A name from "
"the proxy config's top-level classifier_plugins registry (what the Admin UI saves), or in the "
"proxy config a dotted path to a ClassifierPlugin instance (resolved at startup, like plugins). "
"Its classify(context) receives the request messages and metadata (caller identity included) and "
"returns the name of the tier to route to, or None to decline and let classifier_fallback decide."
),
)
@field_validator("classifier_plugin", mode="before")
@classmethod
def _resolve_registered_classifier_plugin(cls, value: object) -> object:
if not isinstance(value, str):
return value
import litellm
registered: Final = litellm.classifier_plugin_registry.get(value)
if registered is None:
raise ValueError(
f"{value!r} is not a registered classifier plugin; declare it under "
"classifier_plugins in the proxy config"
)
return registered
classifier_plugin_timeout_ms: int = Field(
default=3000,
gt=0,

View file

@ -27,6 +27,14 @@ class RequestComplexityRouterConfig(ComplexityRouterConfig):
)
class ClassifierPluginsListResponse(BaseModel):
"""Names from the proxy config's classifier_plugins registry, for the custom classifier picker."""
classifier_plugins: tuple[str, ...] = Field(
description="Registered classifier plugin names an auto-router's classifier_plugin may reference",
)
class AutoRouterRoutingTestRequest(BaseModel):
"""A single prompt to classify against a complexity-router config that need not be saved yet."""

View file

@ -293,6 +293,21 @@ def test_classifier_plugin_is_not_settable_over_http():
_request("what is 2+2", classifier_type="custom", classifier_plugin="my_module.instance")
@pytest.mark.asyncio
async def test_list_classifier_plugins_returns_sorted_registry_names(monkeypatch):
import litellm
from litellm.proxy.management_endpoints.auto_router_endpoints import list_classifier_plugins
class _Classifier:
async def classify(self, context):
return "SIMPLE"
monkeypatch.setitem(litellm.classifier_plugin_registry, "tier-by-team", _Classifier())
monkeypatch.setitem(litellm.classifier_plugin_registry, "spend-aware", _Classifier())
response = await list_classifier_plugins(user_api_key_dict=ADMIN)
assert response.classifier_plugins == ("spend-aware", "tier-by-team")
class TestAutoRouterBenchmarks:
from litellm.proxy.management_endpoints.auto_router_endpoints import _SessionAggRow

View file

@ -246,6 +246,22 @@ def test_resolve_complexity_router_plugins_rejects_synchronous_classify_method(t
)
def test_resolve_classifier_plugin_prefers_the_registry_over_dotted_import(monkeypatch):
import litellm
from litellm.proxy.proxy_server import resolve_classifier_plugin
class _Classifier:
async def classify(self, context):
return "SIMPLE"
instance = _Classifier()
monkeypatch.setitem(litellm.classifier_plugin_registry, "tier-by-team", instance)
resolved = resolve_classifier_plugin(
plugin_path="tier-by-team", config_file_path=None, source_label="classifier_plugins.tier-by-team"
)
assert resolved is instance
def test_resolve_complexity_router_plugins_leaves_live_classifier_instance_alone():
class _Classifier:
async def classify(self, context):

View file

@ -4233,6 +4233,52 @@ class TestClassifierPluginConfig:
)
class _RegistryClassifier:
async def classify(self, context):
return "COMPLEX"
class TestClassifierPluginRegistry:
"""String classifier_plugin values resolve through litellm.classifier_plugin_registry."""
def test_registered_name_resolves_to_the_instance(self, monkeypatch):
instance = _RegistryClassifier()
monkeypatch.setitem(litellm.classifier_plugin_registry, "tier-by-team", instance)
config = ComplexityRouterConfig(
tiers={"SIMPLE": "gpt-4o-mini"}, classifier_type="custom", classifier_plugin="tier-by-team"
)
assert config.classifier_plugin is instance
def test_unknown_name_is_rejected_at_validation(self):
with pytest.raises(ValidationError, match="not a registered classifier plugin"):
ComplexityRouterConfig(
tiers={"SIMPLE": "gpt-4o-mini"}, classifier_type="custom", classifier_plugin="no-such-plugin"
)
def test_live_instance_passes_through_untouched(self):
instance = _RegistryClassifier()
config = ComplexityRouterConfig(
tiers={"SIMPLE": "gpt-4o-mini"}, classifier_type="custom", classifier_plugin=instance
)
assert config.classifier_plugin is instance
@pytest.mark.asyncio
async def test_registry_named_plugin_classifies_end_to_end(self, mock_router_instance, monkeypatch):
monkeypatch.setitem(litellm.classifier_plugin_registry, "tier-by-team", _RegistryClassifier())
router = ComplexityRouter(
model_name="test-complexity-router",
litellm_router_instance=mock_router_instance,
complexity_router_config={
"tiers": {"SIMPLE": "gpt-4o-mini", "COMPLEX": "claude-sonnet-4-20250514"},
"classifier_type": "custom",
"classifier_plugin": "tier-by-team",
},
)
outcome = await router.aclassify("hello")
assert outcome.cause == "classifier_plugin"
assert outcome.tier == ComplexityTier.COMPLEX
class TestClassifierPlugin:
"""classifier_type='custom': an operator hook decides the tier."""

View file

@ -138,6 +138,38 @@ def test_validate_rejects_ambiguous_tier_labels(tier_labels, expected_fragment):
assert expected_fragment in violation
def test_validate_rejects_an_unregistered_classifier_plugin_name():
"""A stored name the registry does not hold must be refused at write time, not at load."""
violation = validate_complexity_router_config_write(
complexity_router_config={
"tiers": VALID_TIERS,
"classifier_type": "custom",
"classifier_plugin": "no-such-plugin",
}
)
assert violation is not None
assert "complexity_router_config is invalid" in violation
assert "not a registered classifier plugin" in violation
def test_validate_accepts_a_registered_classifier_plugin_name(monkeypatch):
import litellm
class _Classifier:
async def classify(self, context):
return "SIMPLE"
monkeypatch.setitem(litellm.classifier_plugin_registry, "tier-by-team", _Classifier())
violation = validate_complexity_router_config_write(
complexity_router_config={
"tiers": VALID_TIERS,
"classifier_type": "custom",
"classifier_plugin": "tier-by-team",
}
)
assert violation is None
@pytest.mark.parametrize(
"complexity_router_config",
[

View file

@ -0,0 +1,13 @@
import { $api } from "@/lib/http/api";
export const useClassifierPlugins = () =>
$api.useQuery(
"get",
"/auto_router/classifier_plugins",
{},
{
// Registration is config-file-only, so the list changes only on a proxy reload.
staleTime: 5 * 60 * 1000,
select: (data) => data.classifier_plugins,
},
);

View file

@ -99,6 +99,23 @@ describe("autoRouterRows", () => {
expect(row.typeLabel).toBe("LLM Classifier");
});
it("labels a router using a custom classifier plugin", () => {
const row = toAutoRouterRow(
{
...complexityDeployment,
litellm_params: {
...complexityDeployment.litellm_params,
complexity_router_config: { tiers: {}, classifier_type: "custom", classifier_plugin: "tier-by-team" },
},
},
0,
ADMIN,
null,
);
expect(row.typeLabel).toBe("Custom classifier");
});
it("treats a deployment carrying complexity_router_config as complexity even off the canonical model string", () => {
expect(isComplexityRouter({ model: "auto_router/legacy", complexity_router_config: { tiers: {} } })).toBe(true);
});

View file

@ -54,8 +54,11 @@ const asStringArray = (value: unknown): string[] =>
const dedupe = (models: string[]): string[] => Array.from(new Set(models));
export const complexityTypeLabel = (config: Record<string, unknown>): string =>
config.classifier_type === "llm" ? "LLM Classifier" : "Heuristic";
export const complexityTypeLabel = (config: Record<string, unknown>): string => {
if (config.classifier_type === "llm") return "LLM Classifier";
if (config.classifier_type === "custom") return "Custom classifier";
return "Heuristic";
};
interface Presentation {
typeLabel: string;

View file

@ -0,0 +1,167 @@
import { fireEvent, renderWithProviders, screen } from "../../../tests/test-utils";
import userEvent from "@testing-library/user-event";
import { afterEach, describe, expect, it, vi } from "vitest";
import { useClassifierPlugins } from "@/app/(dashboard)/hooks/autoRouter/useClassifierPlugins";
import ClassificationMethodConfig from "./ClassificationMethodConfig";
import { ComplexityRouterConfigValue } from "./ComplexityRouterConfig";
import { LOADED_CLASSIFIER_PLUGINS_QUERY } from "../../../tests/mocks/classifierPlugins";
vi.mock(
"@/app/(dashboard)/hooks/autoRouter/useComplexityScorerDefaults",
async () => await import("../../../tests/mocks/complexityScorerDefaults"),
);
vi.mock(
"@/app/(dashboard)/hooks/autoRouter/useClassifierPlugins",
async () => await import("../../../tests/mocks/classifierPlugins"),
);
const BASE: ComplexityRouterConfigValue = {
tiers: { SIMPLE: ["gpt-4o-mini"], MEDIUM: ["gpt-4o"], COMPLEX: ["o3"], REASONING: ["o3"] },
classifier_type: "heuristic",
};
const CUSTOM: ComplexityRouterConfigValue = {
...BASE,
classifier_type: "custom",
classifier_plugin: "tier-by-team",
classifier_plugin_timeout_ms: 3000,
};
const MODEL_OPTIONS = [{ value: "gpt-4o-mini", label: "gpt-4o-mini" }];
const render = (value: ComplexityRouterConfigValue, onChange = vi.fn()) => {
renderWithProviders(
<ClassificationMethodConfig value={value} onChange={onChange} modelOptions={MODEL_OPTIONS} defaultModel="gpt-4o" />,
);
return onChange;
};
const lastValue = (onChange: ReturnType<typeof vi.fn>) =>
onChange.mock.calls.at(-1)?.[0] as ComplexityRouterConfigValue | undefined;
describe("ClassificationMethodConfig classification method radios", () => {
it("offers the custom classifier alongside the heuristic and the LLM classifier", () => {
render(BASE);
expect(screen.getByRole("radio", { name: /Heuristic/ })).toBeInTheDocument();
expect(screen.getByRole("radio", { name: /LLM Classifier/ })).toBeInTheDocument();
expect(screen.getByRole("radio", { name: /Custom classifier/ })).toBeEnabled();
});
// Only the custom branch may carry a plugin: leaving one behind would have the backend reject the
// save outright (classifier_plugin set with a non-custom classifier_type).
it("clears the plugin when the operator switches back to the heuristic", async () => {
const onChange = render(CUSTOM);
await userEvent.click(screen.getByRole("radio", { name: /Heuristic/ }));
expect(lastValue(onChange)).toMatchObject({
classifier_type: "heuristic",
classifier_plugin: undefined,
classifier_plugin_timeout_ms: undefined,
classifier_fallback: undefined,
});
});
it("stamps the default plugin timeout when custom is selected", async () => {
const onChange = render(BASE);
await userEvent.click(screen.getByRole("radio", { name: /Custom classifier/ }));
expect(lastValue(onChange)).toMatchObject({ classifier_type: "custom", classifier_plugin_timeout_ms: 3000 });
});
});
describe("ClassificationMethodConfig custom classifier controls", () => {
it("does not show the plugin controls until custom is the selected method", () => {
render(BASE);
expect(screen.queryByRole("combobox", { name: "Classifier Plugin" })).not.toBeInTheDocument();
});
it("offers the names the proxy registered", async () => {
render({ ...CUSTOM, classifier_plugin: undefined });
fireEvent.mouseDown(screen.getByRole("combobox", { name: "Classifier Plugin" }));
expect(await screen.findByTitle("tier-by-team")).toBeInTheDocument();
expect(screen.getByTitle("spend-aware")).toBeInTheDocument();
});
it("records the plugin the operator picks", async () => {
const onChange = render({ ...CUSTOM, classifier_plugin: undefined });
fireEvent.mouseDown(screen.getByRole("combobox", { name: "Classifier Plugin" }));
await userEvent.click(await screen.findByTitle("spend-aware"));
expect(lastValue(onChange)).toMatchObject({ classifier_plugin: "spend-aware", classifier_plugin_timeout_ms: 3000 });
});
it("emits an edited plugin timeout", () => {
const onChange = render(CUSTOM);
fireEvent.change(screen.getByRole("spinbutton", { name: "Plugin Timeout (ms)" }), { target: { value: "750" } });
expect(lastValue(onChange)).toMatchObject({ classifier_plugin_timeout_ms: 750 });
});
// The backend applies classifier_fallback to a plugin exactly as it does to an LLM classifier, so
// gating this picker on the LLM branch would leave the plugin's failure path unconfigurable.
it("offers the fallback picker, which is not LLM-only", () => {
render(CUSTOM);
expect(screen.getByRole("radio", { name: /Score with the heuristic/ })).toBeInTheDocument();
expect(screen.getByRole("radio", { name: /Route to the default model/ })).toBeInTheDocument();
});
it("says the score no longer decides the tier under a plugin", () => {
render(CUSTOM);
expect(screen.getByText(/classifies with your own classifier plugin/)).toBeInTheDocument();
});
it("reports a missing plugin once the form has been submitted", () => {
renderWithProviders(
<ClassificationMethodConfig
value={{ ...CUSTOM, classifier_plugin: undefined }}
onChange={vi.fn()}
modelOptions={MODEL_OPTIONS}
showValidationErrors
/>,
);
expect(screen.getByText("A classifier plugin is required")).toBeInTheDocument();
});
});
describe("ClassificationMethodConfig when the proxy registered no plugins", () => {
const empty = { data: [], isPending: false, isError: false, refetch: vi.fn() };
const failing = { data: undefined, isPending: false, isError: true, refetch: vi.fn() };
afterEach(() => vi.mocked(useClassifierPlugins).mockReturnValue(LOADED_CLASSIFIER_PLUGINS_QUERY));
it("disables the custom radio and says how to enable it", () => {
vi.mocked(useClassifierPlugins).mockReturnValue(empty as never);
render(BASE);
expect(screen.getByRole("radio", { name: /Custom classifier/ })).toBeDisabled();
expect(
screen.getByText("Declare classifier_plugins in the proxy config to enable custom classifiers"),
).toBeInTheDocument();
});
// A fetch that never landed leaves the registry unknown, not empty. Blanking the select or greying
// out the radio here would clear an already-configured plugin on the operator's next save.
it("keeps a stored plugin name selectable when the fetch failed", () => {
vi.mocked(useClassifierPlugins).mockReturnValue(failing as never);
render(CUSTOM);
expect(screen.getByRole("radio", { name: /Custom classifier/ })).toBeEnabled();
expect(screen.getByRole("combobox", { name: "Classifier Plugin" })).toBeInTheDocument();
expect(screen.getByTitle("tier-by-team")).toBeInTheDocument();
expect(
screen.queryByText("Declare classifier_plugins in the proxy config to enable custom classifiers"),
).not.toBeInTheDocument();
});
});

View file

@ -3,6 +3,7 @@ import { Select as AntdSelect, Card, InputNumber, Radio, Space, Switch, Tooltip,
import React from "react";
import ClassifierPromptEditor from "./ClassifierPromptEditor";
import HeuristicScoringConfig from "./HeuristicScoringConfig";
import { useClassifierPlugins } from "@/app/(dashboard)/hooks/autoRouter/useClassifierPlugins";
import { useComplexityScorerDefaults } from "@/app/(dashboard)/hooks/autoRouter/useComplexityScorerDefaults";
import {
ClassifierFallback,
@ -11,6 +12,7 @@ import {
DEFAULT_CLASSIFIER_CONTEXT_PER_TURN_CHARS,
DEFAULT_CLASSIFIER_CONTEXT_WINDOW_SIZE,
DEFAULT_CLASSIFIER_FALLBACK,
DEFAULT_CLASSIFIER_PLUGIN_TIMEOUT_MS,
DEFAULT_CLASSIFIER_TIMEOUT_MS,
DEFAULT_CLASSIFICATION_RUBRIC,
NEW_CLASSIFIER_CLASSIFICATION_RUBRIC,
@ -35,18 +37,29 @@ const CUSTOM_PROMPT_WITH_DEFAULT_MODEL_FALLBACK =
"names stay fixed. The scoring below no longer runs at all, since a failed classifier routes to the default " +
"model instead:";
const PLUGIN_WITH_HEURISTIC_FALLBACK =
"This router classifies with your own classifier plugin, so the tier comes from whatever the plugin decides. The " +
"four tier names stay fixed. The scoring below is the heuristic, which now runs only when the plugin fails:";
const PLUGIN_WITH_DEFAULT_MODEL_FALLBACK =
"This router classifies with your own classifier plugin, so the tier comes from whatever the plugin decides. The " +
"four tier names stay fixed. The scoring below no longer runs at all, since a failed plugin routes to the default " +
"model instead:";
/**
* What the scoring breakdown below it actually describes. A custom prompt means the score no longer
* decides the tier, and pairing one with the default-model fallback means the heuristic never runs
* at all, so the panel must not keep implying a score is involved on either router.
* What the scoring breakdown below it actually describes. A custom prompt or a classifier plugin means
* the score no longer decides the tier, and pairing either with the default-model fallback means the
* heuristic never runs at all, so the panel must not keep implying a score is involved.
*/
const scoringExplanation = (value: ComplexityRouterConfigValue): string => {
const fallsBackToDefaultModel = value.classifier_fallback === "default_model";
if (value.classifier_type === "custom") {
return fallsBackToDefaultModel ? PLUGIN_WITH_DEFAULT_MODEL_FALLBACK : PLUGIN_WITH_HEURISTIC_FALLBACK;
}
const usesCustomPrompt =
value.classifier_type === "llm" && Boolean(value.classifier_llm_config?.system_prompt?.trim());
if (!usesCustomPrompt) return DEFAULT_SCORING_EXPLANATION;
return value.classifier_fallback === "default_model"
? CUSTOM_PROMPT_WITH_DEFAULT_MODEL_FALLBACK
: CUSTOM_PROMPT_WITH_HEURISTIC_FALLBACK;
return fallsBackToDefaultModel ? CUSTOM_PROMPT_WITH_DEFAULT_MODEL_FALLBACK : CUSTOM_PROMPT_WITH_HEURISTIC_FALLBACK;
};
/**
@ -105,6 +118,43 @@ const HowClassificationWorks: React.FC<{ value: ComplexityRouterConfigValue }> =
);
};
const ClassifierFallbackPicker: React.FC<{
fallback: ClassifierFallback | undefined;
onChange: (fallback: ClassifierFallback) => void;
defaultModel: string | undefined;
}> = ({ fallback, onChange, defaultModel }) => (
<div>
<Text strong style={{ display: "block", marginBottom: 4 }}>
If the classifier fails
</Text>
<Radio.Group value={fallback ?? DEFAULT_CLASSIFIER_FALLBACK} onChange={(e) => onChange(e.target.value)}>
<Space direction="vertical">
<Radio value="heuristic">
<Text>Score with the heuristic</Text>{" "}
<Text type="secondary"> right when the classifier grades complexity too</Text>
</Radio>
<Radio value="default_model" disabled={!defaultModel}>
<Tooltip
title={
defaultModel
? "Change it from the Default Model select."
: "Set a default model on this router to use this option"
}
>
<span>
<Text>Route to the default model{defaultModel ? ` (${defaultModel})` : ""}</Text>{" "}
<Text type="secondary"> right when your classifier grades something other than complexity</Text>
</span>
</Tooltip>
</Radio>
</Space>
</Radio.Group>
<Text type="secondary" style={{ display: "block", fontSize: 12 }}>
Applies when the classifier call errors, times out, or returns an unparseable response.
</Text>
</div>
);
interface ClassificationMethodConfigProps {
value: ComplexityRouterConfigValue;
onChange: (value: ComplexityRouterConfigValue) => void;
@ -125,12 +175,21 @@ const ClassificationMethodConfig: React.FC<ClassificationMethodConfigProps> = ({
showValidationErrors = false,
defaultModel,
}) => {
const hasDefaultModel = Boolean(defaultModel);
const classifierModelMissing =
showValidationErrors && value.classifier_type === "llm" && !value.classifier_llm_config?.model;
const classifierPluginMissing =
showValidationErrors && value.classifier_type === "custom" && !value.classifier_plugin;
const usesCustomPrompt = Boolean(value.classifier_llm_config?.system_prompt?.trim());
const classificationRubric = value.classifier_llm_config?.classification_rubric ?? DEFAULT_CLASSIFICATION_RUBRIC;
const { data: registeredPlugins, isError: pluginsError } = useClassifierPlugins();
// A failed fetch leaves the registry unknown, not empty: offering the stored name anyway is what keeps
// opening this form on an existing custom router from silently clearing its plugin on the next save.
const registryIsEmpty = !pluginsError && registeredPlugins !== undefined && registeredPlugins.length === 0;
const pluginOptions = Array.from(
new Set([...(registeredPlugins ?? []), ...(value.classifier_plugin ? [value.classifier_plugin] : [])]),
).map((name) => ({ value: name, label: name }));
const handleClassifierTypeChange = (classifierType: ClassifierType) => {
const nextValue: ComplexityRouterConfigValue = {
...value,
@ -153,7 +212,12 @@ const ClassificationMethodConfig: React.FC<ClassificationMethodConfigProps> = ({
: undefined,
classifier_context_include_assistant_turns:
classifierType === "llm" ? value.classifier_context_include_assistant_turns : undefined,
classifier_fallback: classifierType === "llm" ? value.classifier_fallback : undefined,
classifier_fallback: classifierType === "heuristic" ? undefined : value.classifier_fallback,
classifier_plugin: classifierType === "custom" ? value.classifier_plugin : undefined,
classifier_plugin_timeout_ms:
classifierType === "custom"
? value.classifier_plugin_timeout_ms ?? DEFAULT_CLASSIFIER_PLUGIN_TIMEOUT_MS
: undefined,
};
onChange(nextValue);
};
@ -180,6 +244,18 @@ const ClassificationMethodConfig: React.FC<ClassificationMethodConfigProps> = ({
});
};
const handleClassifierPluginChange = (classifierPlugin: string) => {
onChange({
...value,
classifier_plugin: classifierPlugin,
classifier_plugin_timeout_ms: value.classifier_plugin_timeout_ms ?? DEFAULT_CLASSIFIER_PLUGIN_TIMEOUT_MS,
});
};
const handleClassifierPluginTimeoutChange = (timeoutMs: number | null) => {
onChange({ ...value, classifier_plugin_timeout_ms: timeoutMs ?? DEFAULT_CLASSIFIER_PLUGIN_TIMEOUT_MS });
};
const handleClassificationRubricChange = (classificationRubric: ClassificationRubric) => {
onChange({
...value,
@ -245,6 +321,15 @@ const ClassificationMethodConfig: React.FC<ClassificationMethodConfigProps> = ({
<Text strong>LLM Classifier</Text>{" "}
<Text type="secondary"> use a model to decide the tier (e.g. a small/fast model)</Text>
</Radio>
<Radio value="custom" disabled={registryIsEmpty && value.classifier_type !== "custom"}>
<Text strong>Custom classifier</Text>{" "}
<Text type="secondary"> let a classifier plugin registered in the proxy config decide the tier</Text>
</Radio>
{registryIsEmpty && (
<Text type="secondary" style={{ fontSize: 12 }}>
Declare classifier_plugins in the proxy config to enable custom classifiers
</Text>
)}
</Space>
</Radio.Group>
@ -321,39 +406,11 @@ const ClassificationMethodConfig: React.FC<ClassificationMethodConfigProps> = ({
classificationRubric={classificationRubric}
/>
</div>
<div>
<Text strong style={{ display: "block", marginBottom: 4 }}>
If the classifier fails
</Text>
<Radio.Group
value={value.classifier_fallback ?? DEFAULT_CLASSIFIER_FALLBACK}
onChange={(e) => handleClassifierFallbackChange(e.target.value)}
>
<Space direction="vertical">
<Radio value="heuristic">
<Text>Score with the heuristic</Text>{" "}
<Text type="secondary"> right when the classifier grades complexity too</Text>
</Radio>
<Radio value="default_model" disabled={!hasDefaultModel}>
<Tooltip
title={
hasDefaultModel
? "Change it from the Default Model select."
: "Set a default model on this router to use this option"
}
>
<span>
<Text>Route to the default model{defaultModel ? ` (${defaultModel})` : ""}</Text>{" "}
<Text type="secondary"> right when your prompt grades something other than complexity</Text>
</span>
</Tooltip>
</Radio>
</Space>
</Radio.Group>
<Text type="secondary" style={{ display: "block", fontSize: 12 }}>
Applies when the classifier call errors, times out, or returns an unparseable response.
</Text>
</div>
<ClassifierFallbackPicker
fallback={value.classifier_fallback}
onChange={handleClassifierFallbackChange}
defaultModel={defaultModel}
/>
<div>
<Text strong style={{ display: "block", marginBottom: 4 }}>
Context Window Size
@ -407,6 +464,56 @@ const ClassificationMethodConfig: React.FC<ClassificationMethodConfigProps> = ({
</div>
)}
{value.classifier_type === "custom" && (
<div className="mt-4 space-y-3">
<div>
<Text strong style={{ display: "block", marginBottom: 4 }}>
Classifier Plugin
</Text>
<AntdSelect
value={value.classifier_plugin || undefined}
onChange={handleClassifierPluginChange}
placeholder="Select a classifier plugin registered in the proxy config"
showSearch
style={{ width: "100%" }}
options={pluginOptions}
status={classifierPluginMissing ? "error" : undefined}
aria-label="Classifier Plugin"
/>
{classifierPluginMissing && (
<Text type="danger" style={{ fontSize: 12 }}>
A classifier plugin is required
</Text>
)}
{pluginsError && (
<Text type="secondary" style={{ display: "block", fontSize: 12 }}>
The registered plugin names could not be loaded from the proxy.
</Text>
)}
</div>
<div>
<Text strong style={{ display: "block", marginBottom: 4 }}>
Plugin Timeout (ms)
</Text>
<InputNumber
value={value.classifier_plugin_timeout_ms ?? DEFAULT_CLASSIFIER_PLUGIN_TIMEOUT_MS}
onChange={handleClassifierPluginTimeoutChange}
min={1}
style={{ width: "100%" }}
aria-label="Plugin Timeout (ms)"
/>
<Text type="secondary" style={{ fontSize: 12 }}>
How long the plugin has to classify before it fails and the fallback below takes over.
</Text>
</div>
<ClassifierFallbackPicker
fallback={value.classifier_fallback}
onChange={handleClassifierFallbackChange}
defaultModel={defaultModel}
/>
</div>
)}
{value.classifier_type === "heuristic" && (
<div className="mt-4">
<div className="flex items-center gap-2 mb-1">

View file

@ -15,6 +15,7 @@ export type { DimensionWeights, TierBoundaries, TokenThresholds };
const { Text } = Typography;
export const DEFAULT_CLASSIFIER_TIMEOUT_MS = 3000;
export const DEFAULT_CLASSIFIER_PLUGIN_TIMEOUT_MS = 3000;
export const DEFAULT_TIER_DISTANCE_PENALTY = 0.5;
export const DEFAULT_CLASSIFIER_CONTEXT_WINDOW_SIZE = 3;
export const DEFAULT_CLASSIFIER_CONTEXT_PER_TURN_CHARS = 200;
@ -73,7 +74,7 @@ export interface ClassifierLLMConfig {
system_prompt?: string;
}
export type ClassifierType = "heuristic" | "llm";
export type ClassifierType = "heuristic" | "llm" | "custom";
export type ClassifierFallback = "heuristic" | "default_model";
@ -90,8 +91,8 @@ export type HeuristicScoringRole = "decides" | "fallback_only" | "never";
/**
* Whether the heuristic scorer runs on this router at all, which is what gates its knobs. An LLM
* classifier still falls back to the scorer unless the fallback is the default model, so the gate cannot be
* a plain classifier_type check.
* classifier or a classifier plugin still falls back to the scorer unless the fallback is the default
* model, so the gate cannot be a plain classifier_type check.
*/
export const heuristicScoringRoleFor = (
classifierType: ClassifierType,
@ -115,6 +116,9 @@ export interface ComplexityRouterConfigValue {
default_model?: string;
classifier_type: ClassifierType;
classifier_llm_config?: ClassifierLLMConfig;
/** A name from the proxy config's classifier_plugins registry. Required when classifier_type is "custom". */
classifier_plugin?: string;
classifier_plugin_timeout_ms?: number;
classifier_context_window_size?: number;
classifier_context_per_turn_chars?: number;
classifier_context_include_assistant_turns?: boolean;

View file

@ -13,6 +13,11 @@ vi.mock(
async () => await import("../../../tests/mocks/complexityScorerDefaults"),
);
vi.mock(
"@/app/(dashboard)/hooks/autoRouter/useClassifierPlugins",
async () => await import("../../../tests/mocks/classifierPlugins"),
);
const ANTHROPIC_PRESET = getPresetByKey("anthropic_family")!;
const ANTHROPIC_TIERS = ANTHROPIC_PRESET.complexity_router_config.tiers;
@ -285,6 +290,50 @@ describe("AddAutoRouterTab", () => {
});
});
it("carries a selected classifier plugin and its timeout through to the create payload", async () => {
const user = userEvent.setup();
vi.mocked(getMissingTiersError).mockReturnValue(null);
renderWithProviders(<Harness />);
await user.type(screen.getByPlaceholderText(/smart_router/i), "plugin-router");
expandDetailedConfiguration();
await user.click(screen.getByText("Advanced: Classification Method"));
await user.click(await screen.findByRole("radio", { name: /Custom classifier/ }));
fireEvent.mouseDown(await screen.findByRole("combobox", { name: "Classifier Plugin" }));
await user.click(await screen.findByTitle("tier-by-team"));
await user.click(screen.getByRole("button", { name: /add auto router/i }));
await waitFor(() => expect(handleAddAutoRouterSubmit).toHaveBeenCalled());
expect(vi.mocked(handleAddAutoRouterSubmit).mock.calls.at(-1)?.[0].complexity_router_config).toMatchObject({
classifier_type: "custom",
classifier_plugin: "tier-by-team",
classifier_plugin_timeout_ms: 3000,
});
});
// The backend rejects classifier_type "custom" with no classifier_plugin, so the form has to say
// what is missing rather than let the save come back as a raw 400.
it("blocks a submit that selects the custom classifier without a plugin", async () => {
const user = userEvent.setup();
vi.mocked(getMissingTiersError).mockReturnValue(null);
renderWithProviders(<Harness />);
await user.type(screen.getByPlaceholderText(/smart_router/i), "plugin-router");
expandDetailedConfiguration();
await user.click(screen.getByText("Advanced: Classification Method"));
await user.click(await screen.findByRole("radio", { name: /Custom classifier/ }));
await user.click(screen.getByRole("button", { name: /add auto router/i }));
await waitFor(() =>
expect(toast.fromError).toHaveBeenCalledWith("Please select a classifier plugin, or switch back to Heuristic"),
);
expect(handleAddAutoRouterSubmit).not.toHaveBeenCalled();
});
it("carries session affinity turned on through to the create payload", async () => {
const user = userEvent.setup();
vi.mocked(getMissingTiersError).mockReturnValue(null);

View file

@ -341,6 +341,8 @@ const AddAutoRouterTab: React.FC<AddAutoRouterTabProps> = ({
tierLabels: complexityRouterConfig.tier_labels,
classifierType: complexityRouterConfig.classifier_type,
classifierLlmConfig: complexityRouterConfig.classifier_llm_config,
classifierPlugin: complexityRouterConfig.classifier_plugin,
classifierPluginTimeoutMs: complexityRouterConfig.classifier_plugin_timeout_ms,
classifierContextWindowSize: complexityRouterConfig.classifier_context_window_size,
classifierContextPerTurnChars: complexityRouterConfig.classifier_context_per_turn_chars,
classifierContextIncludeAssistantTurns: complexityRouterConfig.classifier_context_include_assistant_turns,
@ -364,7 +366,7 @@ const AddAutoRouterTab: React.FC<AddAutoRouterTabProps> = ({
};
const submitRecommendedRouter = async (name: string) => {
const { tiers, tierLabels, classifierType, classifierLlmConfig } = complexityRouterConfigParams;
const { tiers, tierLabels, classifierType, classifierLlmConfig, classifierPlugin } = complexityRouterConfigParams;
const missingTiersError = getMissingTiersError(tiers);
if (missingTiersError) {
@ -386,6 +388,12 @@ const AddAutoRouterTab: React.FC<AddAutoRouterTabProps> = ({
return;
}
if (classifierType === "custom" && !classifierPlugin) {
setShowValidationErrors(true);
toast.fromError("Please select a classifier plugin, or switch back to Heuristic");
return;
}
const keywordRulesError = getKeywordTierRulesError(keywordTierRules);
if (keywordRulesError) {
setShowValidationErrors(true);

View file

@ -22,6 +22,8 @@ const baseParams: BuildComplexityRouterConfigParams = {
tierLabels: undefined,
classifierType: "heuristic",
classifierLlmConfig: undefined,
classifierPlugin: undefined,
classifierPluginTimeoutMs: undefined,
classifierContextWindowSize: undefined,
classifierContextPerTurnChars: undefined,
classifierContextIncludeAssistantTurns: undefined,
@ -93,6 +95,86 @@ describe("buildComplexityRouterConfig", () => {
expect(config.classifier_llm_config).toBeUndefined();
});
it("includes the plugin and its timeout only when classifier_type is custom", () => {
const config = buildComplexityRouterConfig({
...baseParams,
classifierType: "custom",
classifierPlugin: "tier-by-team",
classifierPluginTimeoutMs: 1500,
});
expect(config.classifier_type).toBe("custom");
expect(config.classifier_plugin).toBe("tier-by-team");
expect(config.classifier_plugin_timeout_ms).toBe(1500);
expect(config.classifier_llm_config).toBeUndefined();
});
// The backend rejects classifier_plugin alongside a non-custom classifier_type, so a selection
// left on the form after the operator switched away must never reach the wire.
it("omits the plugin and its timeout when classifier_type is heuristic even if they linger in state", () => {
const config = buildComplexityRouterConfig({
...baseParams,
classifierPlugin: "tier-by-team",
classifierPluginTimeoutMs: 1500,
});
expect(config.classifier_plugin).toBeUndefined();
expect(config.classifier_plugin_timeout_ms).toBeUndefined();
});
it("omits the plugin and its timeout when classifier_type is llm even if they linger in state", () => {
const config = buildComplexityRouterConfig({
...baseParams,
classifierType: "llm",
classifierLlmConfig: { model: "gpt-4o-mini", timeout_ms: 3000 },
classifierPlugin: "tier-by-team",
classifierPluginTimeoutMs: 1500,
});
expect(config.classifier_plugin).toBeUndefined();
expect(config.classifier_plugin_timeout_ms).toBeUndefined();
});
// The plugin's failure path is configurable exactly like the LLM classifier's, so the fallback
// gate is "not the heuristic" rather than "llm".
it("emits classifier_fallback for a custom classifier as well as an llm one", () => {
const custom = buildComplexityRouterConfig({
...baseParams,
classifierType: "custom",
classifierPlugin: "tier-by-team",
classifierFallback: "default_model",
});
const llm = buildComplexityRouterConfig({
...baseParams,
classifierType: "llm",
classifierLlmConfig: { model: "gpt-4o-mini", timeout_ms: 3000 },
classifierFallback: "default_model",
});
expect(custom.classifier_fallback).toBe("default_model");
expect(llm.classifier_fallback).toBe("default_model");
});
it("omits classifier_fallback for the heuristic, which has nothing to fall back from", () => {
const config = buildComplexityRouterConfig({ ...baseParams, classifierFallback: "default_model" });
expect(config.classifier_fallback).toBeUndefined();
});
// heuristicScoringRoleFor keys off the fallback, not the classifier type, so a plugin routing its
// failures to the default model must drop the scorer knobs the same way an LLM classifier does.
it("drops the scorer knobs on a custom classifier that never scores", () => {
const config = buildComplexityRouterConfig({
...baseParams,
classifierType: "custom",
classifierPlugin: "tier-by-team",
classifierFallback: "default_model",
tokenThresholds: { simple: 25, complex: 900 },
});
expect(config.token_thresholds).toBeUndefined();
});
it("includes classifier_context_window_size and classifier_context_per_turn_chars only when classifier_type is llm", () => {
const params: BuildComplexityRouterConfigParams = {
...baseParams,

View file

@ -75,6 +75,8 @@ export interface BuildComplexityRouterConfigParams {
tierLabels: ComplexityTierLabels | undefined;
classifierType: ClassifierType;
classifierLlmConfig: ClassifierLLMConfig | undefined;
classifierPlugin: string | undefined;
classifierPluginTimeoutMs: number | undefined;
classifierContextWindowSize: number | undefined;
classifierContextPerTurnChars: number | undefined;
classifierContextIncludeAssistantTurns: boolean | undefined;
@ -104,6 +106,8 @@ export interface ComplexityRouterConfigPayload {
tier_labels?: ComplexityTierLabels;
classifier_type: ClassifierType;
classifier_llm_config?: ClassifierLLMConfig;
classifier_plugin?: string;
classifier_plugin_timeout_ms?: number;
classifier_context_window_size?: number;
classifier_context_per_turn_chars?: number;
classifier_context_include_assistant_turns?: boolean;
@ -210,6 +214,8 @@ export const buildComplexityRouterConfig = ({
tierLabels,
classifierType,
classifierLlmConfig,
classifierPlugin,
classifierPluginTimeoutMs,
classifierContextWindowSize,
classifierContextPerTurnChars,
classifierContextIncludeAssistantTurns,
@ -245,7 +251,11 @@ export const buildComplexityRouterConfig = ({
classifier_type: classifierType,
...(classifierType === "llm" &&
classifierLlmConfig && { classifier_llm_config: normalizeClassifierLlmConfig(classifierLlmConfig) }),
...(classifierType === "llm" && classifierFallback !== undefined && { classifier_fallback: classifierFallback }),
...(classifierType === "custom" && classifierPlugin && { classifier_plugin: classifierPlugin }),
...(classifierType === "custom" &&
classifierPluginTimeoutMs !== undefined && { classifier_plugin_timeout_ms: classifierPluginTimeoutMs }),
...(classifierType !== "heuristic" &&
classifierFallback !== undefined && { classifier_fallback: classifierFallback }),
...(classifierType === "llm" &&
classifierContextWindowSize !== undefined && {
classifier_context_window_size: classifierContextWindowSize,

View file

@ -106,6 +106,40 @@ describe("buildUpdatedComplexityRouterConfig", () => {
expect(updatedConfig.custom_technical_keywords).toBeUndefined();
});
it("writes the plugin and its timeout for a custom classifier", () => {
const updatedConfig = buildUpdatedComplexityRouterConfig(storedConfig, {
tiers,
classifier_type: "custom" as const,
classifier_plugin: "tier-by-team",
classifier_plugin_timeout_ms: 1500,
classifier_fallback: "default_model" as const,
});
expect(updatedConfig).toMatchObject({
classifier_type: "custom",
classifier_plugin: "tier-by-team",
classifier_plugin_timeout_ms: 1500,
classifier_fallback: "default_model",
});
expect(updatedConfig.classifier_llm_config).toBeUndefined();
});
// Both keys are managed, so switching off custom has to remove them: the backend rejects a stored
// classifier_plugin sitting next to a non-custom classifier_type.
it("removes a stored plugin and its timeout when the classifier is no longer custom", () => {
const storedCustom = JSON.stringify({
...storedConfigValue,
classifier_type: "custom",
classifier_plugin: "tier-by-team",
classifier_plugin_timeout_ms: 1500,
});
const updatedConfig = buildUpdatedComplexityRouterConfig(storedCustom, classifiedTierValue);
expect(updatedConfig.classifier_plugin).toBeUndefined();
expect(updatedConfig.classifier_plugin_timeout_ms).toBeUndefined();
});
it("preserves a tier configured with more than one model as a pool", () => {
const multiModelValue = {
...classifiedTierValue,

View file

@ -10,6 +10,11 @@ vi.mock(
async () => await import("../../../tests/mocks/complexityScorerDefaults"),
);
vi.mock(
"@/app/(dashboard)/hooks/autoRouter/useClassifierPlugins",
async () => await import("../../../tests/mocks/classifierPlugins"),
);
const { modelPatchUpdateCall, modelAvailableCall, getAutoRouterClassifierDefaultPromptCall } = vi.hoisted(() => ({
modelPatchUpdateCall: vi.fn().mockResolvedValue({}),
modelAvailableCall: vi.fn().mockResolvedValue({ data: [] }),
@ -484,6 +489,93 @@ describe("EditAutoRouterModal deployment affinity", () => {
});
});
describe("EditAutoRouterModal custom classifier plugin", () => {
beforeEach(() => {
modelPatchUpdateCall.mockClear();
});
const STORED_PLUGIN_CONFIG = {
tiers: { SIMPLE: ["gpt-4o-mini"], MEDIUM: ["gpt-4o-mini"], COMPLEX: ["gpt-4o-mini"], REASONING: ["gpt-4o-mini"] },
classifier_type: "custom",
classifier_plugin: "tier-by-team",
classifier_plugin_timeout_ms: 1500,
classifier_fallback: "heuristic",
};
const renderPluginModal = () =>
renderWithProviders(
<EditAutoRouterModal
isVisible
onCancel={vi.fn()}
onSuccess={vi.fn()}
modelData={{
...MODEL_DATA,
litellm_params: { ...MODEL_DATA.litellm_params, complexity_router_config: STORED_PLUGIN_CONFIG },
}}
accessToken="token"
userRole="Admin"
/>,
);
// Every one of these keys is rewritten from form state on save, so a missing hydration line would
// silently wipe the operator's plugin the first time they opened this modal for anything else.
it("preserves a stored plugin, its timeout, and its fallback through an untouched open-and-save", async () => {
const user = userEvent.setup();
renderPluginModal();
await user.click(await screen.findByText("Advanced: Classification Method"));
expect(await screen.findByRole("combobox", { name: "Classifier Plugin" })).toBeInTheDocument();
expect(screen.getByRole("spinbutton", { name: "Plugin Timeout (ms)" })).toHaveValue("1500");
await user.click(screen.getByRole("button", { name: /save changes/i }));
await waitFor(() => expect(modelPatchUpdateCall).toHaveBeenCalled());
const config = savedConfig();
expect(config.classifier_type).toBe("custom");
expect(config.classifier_plugin).toBe("tier-by-team");
expect(config.classifier_plugin_timeout_ms).toBe(1500);
expect(config.classifier_fallback).toBe("heuristic");
});
it("persists a switch to another registered plugin", async () => {
const user = userEvent.setup();
renderPluginModal();
await user.click(await screen.findByText("Advanced: Classification Method"));
fireEvent.mouseDown(await screen.findByRole("combobox", { name: "Classifier Plugin" }));
await user.click(await screen.findByTitle("spend-aware"));
await user.click(screen.getByRole("button", { name: /save changes/i }));
await waitFor(() => expect(modelPatchUpdateCall).toHaveBeenCalled());
expect(savedConfig().classifier_plugin).toBe("spend-aware");
});
it("blocks a save that leaves the custom classifier with no plugin", async () => {
const user = userEvent.setup();
renderWithProviders(
<EditAutoRouterModal
isVisible
onCancel={vi.fn()}
onSuccess={vi.fn()}
modelData={{
...MODEL_DATA,
litellm_params: {
...MODEL_DATA.litellm_params,
complexity_router_config: { ...STORED_PLUGIN_CONFIG, classifier_plugin: undefined },
},
}}
accessToken="token"
userRole="Admin"
/>,
);
await user.click(await screen.findByRole("button", { name: /save changes/i }));
await waitFor(() => expect(toast.fromError).toHaveBeenCalled());
expect(modelPatchUpdateCall).not.toHaveBeenCalled();
});
});
describe("EditAutoRouterModal custom classifier prompt and fallback", () => {
beforeEach(() => {
modelPatchUpdateCall.mockClear();

View file

@ -70,6 +70,8 @@ const MANAGED_COMPLEXITY_ROUTER_KEYS = new Set([
"tier_labels",
"classifier_type",
"classifier_llm_config",
"classifier_plugin",
"classifier_plugin_timeout_ms",
"classifier_context_window_size",
"classifier_context_per_turn_chars",
"classifier_context_include_assistant_turns",
@ -157,7 +159,14 @@ export const buildUpdatedComplexityRouterConfig = (
...(value.classifier_type === "llm" && value.classifier_llm_config
? { classifier_llm_config: normalizeClassifierLlmConfig(value.classifier_llm_config) }
: {}),
...(value.classifier_type === "llm" &&
...(value.classifier_type === "custom" && value.classifier_plugin
? { classifier_plugin: value.classifier_plugin }
: {}),
...(value.classifier_type === "custom" &&
value.classifier_plugin_timeout_ms !== undefined && {
classifier_plugin_timeout_ms: value.classifier_plugin_timeout_ms,
}),
...(value.classifier_type !== "heuristic" &&
value.classifier_fallback !== undefined && { classifier_fallback: value.classifier_fallback }),
...(value.classifier_type === "llm" &&
value.classifier_context_window_size !== undefined && {
@ -348,6 +357,12 @@ const EditAutoRouterModal: React.FC<EditAutoRouterModalProps> = ({
tier_labels: hydrateTierLabels(parsedConfig.tier_labels),
classifier_type: parsedConfig.classifier_type || "heuristic",
classifier_llm_config: parsedConfig.classifier_llm_config,
classifier_plugin:
typeof parsedConfig.classifier_plugin === "string" ? parsedConfig.classifier_plugin : undefined,
classifier_plugin_timeout_ms:
typeof parsedConfig.classifier_plugin_timeout_ms === "number"
? parsedConfig.classifier_plugin_timeout_ms
: undefined,
classifier_context_window_size:
typeof parsedConfig.classifier_context_window_size === "number"
? parsedConfig.classifier_context_window_size
@ -435,7 +450,7 @@ const EditAutoRouterModal: React.FC<EditAutoRouterModalProps> = ({
const saveValues = async (values: EditAutoRouterFormValues) => {
if (isComplexityRouterModel) {
const { tiers, classifier_type, classifier_llm_config } = complexityRouterConfig;
const { tiers, classifier_type, classifier_llm_config, classifier_plugin } = complexityRouterConfig;
if (Object.values(tiers).every((models) => models.length === 0)) {
setShowValidationErrors(true);
toast.fromError("Please select at least one model for a complexity tier");
@ -446,6 +461,11 @@ const EditAutoRouterModal: React.FC<EditAutoRouterModalProps> = ({
toast.fromError("Please select a classifier model, or switch back to Heuristic");
return;
}
if (classifier_type === "custom" && !classifier_plugin) {
setShowValidationErrors(true);
toast.fromError("Please select a classifier plugin, or switch back to Heuristic");
return;
}
// Same guards the create form applies (add_auto_router_tab.tsx). The backend rejects a
// keyword rule with no keyword, and semantic_keyword_matching without an embedding model
// or keyword rules (complexity_router/config.py), so without these a save fails as a raw

View file

@ -85,6 +85,31 @@ describe("RoutingDecisionCard", () => {
expect(screen.queryByText("Score")).not.toBeInTheDocument();
});
it("names the custom classifier plugin instead of rendering its raw cause string", () => {
render(
<RoutingDecisionCard
decision={{
router_model_name: "plugin-router",
router_type: "complexity",
routed_model: "claude-sonnet",
cause: "classifier_plugin",
tier: "COMPLEX",
signals: ["classifier-plugin:COMPLEX"],
}}
/>,
);
expect(screen.getByText("Custom classifier plugin")).toBeInTheDocument();
expect(screen.queryByText("classifier_plugin")).not.toBeInTheDocument();
});
// The plugin chose the tier, so pairing a score with a boundary band would claim the heuristic did.
it("does not claim the score chose the tier on a plugin-decided row", () => {
render(<RoutingDecisionCard decision={{ ...heuristic, cause: "classifier_plugin", score: 0.2 }} />);
expect(screen.getByText("0.20")).toBeInTheDocument();
expect(screen.queryByText(/SIMPLE|MEDIUM|COMPLEX|at or above/)).not.toBeInTheDocument();
});
it("explains a route that fell back to the default model after the classifier failed", () => {
// No tier is recorded on this path, so the card must not show a Tier row: nothing
// about the request produced one, the classifier never answered.
@ -99,7 +124,7 @@ describe("RoutingDecisionCard", () => {
}}
/>,
);
expect(screen.getByText("Default model, LLM classifier failed")).toBeInTheDocument();
expect(screen.getByText("Default model, the classifier failed")).toBeInTheDocument();
expect(screen.queryByText("Tier")).not.toBeInTheDocument();
});
@ -116,7 +141,7 @@ describe("RoutingDecisionCard", () => {
}}
/>,
);
expect(screen.getByText("Fallback tier, LLM classifier failed")).toBeInTheDocument();
expect(screen.getByText("Fallback tier, the classifier failed")).toBeInTheDocument();
expect(screen.getByText("SECURITY_REVIEW")).toBeInTheDocument();
});

View file

@ -75,6 +75,8 @@ function describeCause(decision: RoutingDecision): string {
return `Heuristic, ${tierLabel ?? "REASONING"} override (2 or more reasoning markers)`;
case "llm_classifier":
return classifierModel ? `LLM classifier (${classifierModel})` : "LLM classifier";
case "classifier_plugin":
return "Custom classifier plugin";
case "literal_keyword_match":
return matchedKeyword ? `Keyword match: "${matchedKeyword}"` : "Keyword match";
case "semantic_keyword_match":
@ -94,9 +96,9 @@ function describeCause(decision: RoutingDecision): string {
case "default_fallback":
return "Default model, no route matched";
case "classifier_fallback":
return "Fallback tier, LLM classifier failed";
return "Fallback tier, the classifier failed";
case "default_model_fallback":
return "Default model, LLM classifier failed";
return "Default model, the classifier failed";
default:
return cause ?? "Unknown";
}
@ -145,11 +147,14 @@ export function RoutingDecisionCard({
tier_boundaries: tierBoundaries,
} = decision;
// On an override row the score did not decide the tier, so showing it against a
// boundary would claim something untrue. Keyed off the cause rather than a marker
// On an override row, or one a plugin decided, the score did not choose the tier, so showing
// it against a boundary would claim something untrue. Keyed off the cause rather than a marker
// inside `signals`, which redaction can remove.
const scoreExplanation =
score !== undefined && decision.cause !== "reasoning_override" && decision.cause !== "plan_mode"
score !== undefined &&
decision.cause !== "reasoning_override" &&
decision.cause !== "plan_mode" &&
decision.cause !== "classifier_plugin"
? describeScoreAgainstBoundaries(score, tierBoundaries, tierLabel !== undefined)
: null;

View file

@ -807,6 +807,26 @@ export interface paths {
patch?: never;
trace?: never;
};
"/auto_router/classifier_plugins": {
parameters: {
query?: never;
header?: never;
path?: never;
cookie?: never;
};
/**
* List Classifier Plugins
* @description Registered classifier plugin names, for the Admin UI's custom classifier picker.
*/
get: operations["list_classifier_plugins_auto_router_classifier_plugins_get"];
put?: never;
post?: never;
delete?: never;
options?: never;
head?: never;
patch?: never;
trace?: never;
};
"/auto_router/shadow_eval": {
parameters: {
query?: never;
@ -23592,6 +23612,17 @@ export interface components {
*/
timeout_ms: number;
};
/**
* ClassifierPluginsListResponse
* @description Names from the proxy config's classifier_plugins registry, for the custom classifier picker.
*/
ClassifierPluginsListResponse: {
/**
* Classifier Plugins
* @description Registered classifier plugin names an auto-router's classifier_plugin may reference
*/
classifier_plugins: string[];
};
/**
* CloudZeroExportRequest
* @description Request model for CloudZero export operations
@ -37806,6 +37837,26 @@ export interface operations {
};
};
};
list_classifier_plugins_auto_router_classifier_plugins_get: {
parameters: {
query?: never;
header?: never;
path?: never;
cookie?: never;
};
requestBody?: never;
responses: {
/** @description Successful Response */
200: {
headers: {
[name: string]: unknown;
};
content: {
"application/json": components["schemas"]["ClassifierPluginsListResponse"];
};
};
};
};
list_shadow_eval_jobs_auto_router_shadow_eval_get: {
parameters: {
query?: {

View file

@ -0,0 +1,17 @@
import { vi } from "vitest";
/**
* Stubs the proxy's registered classifier plugin names for any test that renders the auto-router tree.
* Exported as a vi.fn so a test can override the query state, which is how the empty-registry and
* failed-fetch paths are covered.
*/
export const REGISTERED_CLASSIFIER_PLUGINS = ["spend-aware", "tier-by-team"];
export const LOADED_CLASSIFIER_PLUGINS_QUERY = {
data: REGISTERED_CLASSIFIER_PLUGINS,
isPending: false,
isError: false,
refetch: vi.fn(),
};
export const useClassifierPlugins = vi.fn(() => LOADED_CLASSIFIER_PLUGINS_QUERY);