mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-12 23:01:41 +00:00
feat(ui): pick the auto router's custom classifier from a config-declared registry
Adds a top-level classifier_plugins config key mapping names to dotted paths, resolved at startup into litellm.classifier_plugin_registry with the same load-time checks classifier_plugin already gets. A before-validator on ComplexityRouterConfig.classifier_plugin resolves string values through the registry, so the save-time write gate, router deployment init, and direct construction all accept a registered name and reject an unknown one with a clean error; DB-stored auto-routers therefore carry plain names and no proxy read path changes. GET /auto_router/classifier_plugins lists the names. The dashboard's classification method picker gains a Custom classifier option with a plugin dropdown fed by that endpoint, a plugin timeout field, and the fallback picker now shown for both non-heuristic modes. Create and edit serialize and validate identically, the models table labels custom routers correctly, and the routing decision card names the classifier_plugin cause and stops implying the LLM classifier failed on plugin fallbacks
This commit is contained in:
parent
d03ef8be03
commit
72671282df
26 changed files with 939 additions and 60 deletions
|
|
@ -358,6 +358,11 @@ blocked_user_list: Optional[Union[str, List]] = None
|
|||
banned_keywords_list: Optional[Union[str, List]] = None
|
||||
llm_guard_mode: Literal["all", "key-specific", "request-specific"] = "all"
|
||||
guardrail_name_config_map: Dict[str, GuardrailItem] = {}
|
||||
# Complexity-router classifier plugins declared under the proxy config's top-level
|
||||
# `classifier_plugins` key, keyed by the name the Admin UI selects and saved models
|
||||
# reference. Populated at proxy startup; ComplexityRouterConfig resolves string
|
||||
# classifier_plugin values through it.
|
||||
classifier_plugin_registry: Dict[str, "ClassifierPlugin"] = {} # mutable-ok: populated at proxy startup
|
||||
include_cost_in_streaming_usage: bool = False
|
||||
reasoning_auto_summary: bool = False
|
||||
### PROMPTS ####
|
||||
|
|
@ -488,6 +493,8 @@ priority_reservation: Optional[Dict[str, Union[float, "PriorityReservationDict"]
|
|||
# priority_reservation_settings is lazy-loaded via __getattr__
|
||||
# Only declare for type checking - at runtime __getattr__ handles it
|
||||
if TYPE_CHECKING:
|
||||
from litellm.types.router import ClassifierPlugin
|
||||
|
||||
priority_reservation_settings: Optional["PriorityReservationSettings"] = None
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -39,6 +39,7 @@ from litellm.types.management_endpoints.auto_router_endpoints import (
|
|||
AutoRouterCacheStats,
|
||||
AutoRouterRoutingTestRequest,
|
||||
AutoRouterRoutingTestResponse,
|
||||
ClassifierPluginsListResponse,
|
||||
RequestComplexityRouterConfig,
|
||||
ShadowEvalJobResponse,
|
||||
ShadowEvalResult,
|
||||
|
|
@ -369,6 +370,22 @@ def _summed_agg_row(rows: Sequence[_SessionAggRow]) -> _SessionAggRow:
|
|||
)
|
||||
|
||||
|
||||
@router.get(
|
||||
"/auto_router/classifier_plugins",
|
||||
tags=("auto router",),
|
||||
dependencies=(Depends(user_api_key_auth),),
|
||||
response_model=ClassifierPluginsListResponse,
|
||||
)
|
||||
async def list_classifier_plugins(
|
||||
user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)],
|
||||
) -> ClassifierPluginsListResponse:
|
||||
"""Registered classifier plugin names, for the Admin UI's custom classifier picker."""
|
||||
import litellm
|
||||
|
||||
_require_admin_viewer(user_api_key_dict, "list classifier plugins")
|
||||
return ClassifierPluginsListResponse(classifier_plugins=tuple(sorted(litellm.classifier_plugin_registry)))
|
||||
|
||||
|
||||
@router.get(
|
||||
"/auto_router/benchmarks",
|
||||
tags=("auto router",),
|
||||
|
|
|
|||
|
|
@ -4089,6 +4089,9 @@ def resolve_classifier_plugin(
|
|||
sync `def classify` passes the runtime_checkable isinstance and would only fail on the
|
||||
first classified request, so reject it here where the error names the config key.
|
||||
"""
|
||||
registered: Final = litellm.classifier_plugin_registry.get(plugin_path)
|
||||
if registered is not None:
|
||||
return registered
|
||||
resolved: Final = get_instance_fn(value=plugin_path, config_file_path=config_file_path)
|
||||
if not isinstance(resolved, ClassifierPlugin) or not inspect.iscoroutinefunction(
|
||||
getattr(resolved, "classify", None)
|
||||
|
|
@ -5489,6 +5492,20 @@ class ProxyConfig:
|
|||
# Load vector stores from config
|
||||
litellm.vector_store_registry.load_vector_stores_from_config(vector_store_registry_config)
|
||||
|
||||
## CLASSIFIER PLUGINS (complexity-router custom classifiers, picked by name in the Admin UI)
|
||||
classifier_plugins_config: Final = config.get("classifier_plugins", None)
|
||||
if classifier_plugins_config:
|
||||
if not isinstance(classifier_plugins_config, dict):
|
||||
raise TypeError("classifier_plugins must map plugin names to dotted paths")
|
||||
for plugin_name, plugin_path in classifier_plugins_config.items():
|
||||
if not isinstance(plugin_path, str):
|
||||
raise TypeError(f"classifier_plugins.{plugin_name} must be a dotted-path string")
|
||||
litellm.classifier_plugin_registry[str(plugin_name)] = resolve_classifier_plugin(
|
||||
plugin_path=plugin_path,
|
||||
config_file_path=config_file_path,
|
||||
source_label=f"classifier_plugins.{plugin_name}",
|
||||
)
|
||||
|
||||
## WORKER REGISTRY (Global Control Plane)
|
||||
worker_registry_config: Final = config.get("worker_registry", None)
|
||||
if worker_registry_config:
|
||||
|
|
|
|||
|
|
@ -546,12 +546,29 @@ class ComplexityRouterConfig(BaseModel):
|
|||
classifier_plugin: ClassifierPlugin | None = Field(
|
||||
default=None,
|
||||
description=(
|
||||
"Custom classifier deciding the tier; required when classifier_type is 'custom'. In the proxy "
|
||||
"config, a dotted path to a ClassifierPlugin instance (resolved at startup, like plugins). Its "
|
||||
"classify(context) receives the request messages and metadata (caller identity included) and "
|
||||
"Custom classifier deciding the tier; required when classifier_type is 'custom'. A name from "
|
||||
"the proxy config's top-level classifier_plugins registry (what the Admin UI saves), or in the "
|
||||
"proxy config a dotted path to a ClassifierPlugin instance (resolved at startup, like plugins). "
|
||||
"Its classify(context) receives the request messages and metadata (caller identity included) and "
|
||||
"returns the name of the tier to route to, or None to decline and let classifier_fallback decide."
|
||||
),
|
||||
)
|
||||
|
||||
@field_validator("classifier_plugin", mode="before")
|
||||
@classmethod
|
||||
def _resolve_registered_classifier_plugin(cls, value: object) -> object:
|
||||
if not isinstance(value, str):
|
||||
return value
|
||||
import litellm
|
||||
|
||||
registered: Final = litellm.classifier_plugin_registry.get(value)
|
||||
if registered is None:
|
||||
raise ValueError(
|
||||
f"{value!r} is not a registered classifier plugin; declare it under "
|
||||
"classifier_plugins in the proxy config"
|
||||
)
|
||||
return registered
|
||||
|
||||
classifier_plugin_timeout_ms: int = Field(
|
||||
default=3000,
|
||||
gt=0,
|
||||
|
|
|
|||
|
|
@ -27,6 +27,14 @@ class RequestComplexityRouterConfig(ComplexityRouterConfig):
|
|||
)
|
||||
|
||||
|
||||
class ClassifierPluginsListResponse(BaseModel):
|
||||
"""Names from the proxy config's classifier_plugins registry, for the custom classifier picker."""
|
||||
|
||||
classifier_plugins: tuple[str, ...] = Field(
|
||||
description="Registered classifier plugin names an auto-router's classifier_plugin may reference",
|
||||
)
|
||||
|
||||
|
||||
class AutoRouterRoutingTestRequest(BaseModel):
|
||||
"""A single prompt to classify against a complexity-router config that need not be saved yet."""
|
||||
|
||||
|
|
|
|||
|
|
@ -293,6 +293,21 @@ def test_classifier_plugin_is_not_settable_over_http():
|
|||
_request("what is 2+2", classifier_type="custom", classifier_plugin="my_module.instance")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_list_classifier_plugins_returns_sorted_registry_names(monkeypatch):
|
||||
import litellm
|
||||
from litellm.proxy.management_endpoints.auto_router_endpoints import list_classifier_plugins
|
||||
|
||||
class _Classifier:
|
||||
async def classify(self, context):
|
||||
return "SIMPLE"
|
||||
|
||||
monkeypatch.setitem(litellm.classifier_plugin_registry, "tier-by-team", _Classifier())
|
||||
monkeypatch.setitem(litellm.classifier_plugin_registry, "spend-aware", _Classifier())
|
||||
response = await list_classifier_plugins(user_api_key_dict=ADMIN)
|
||||
assert response.classifier_plugins == ("spend-aware", "tier-by-team")
|
||||
|
||||
|
||||
class TestAutoRouterBenchmarks:
|
||||
from litellm.proxy.management_endpoints.auto_router_endpoints import _SessionAggRow
|
||||
|
||||
|
|
|
|||
|
|
@ -246,6 +246,22 @@ def test_resolve_complexity_router_plugins_rejects_synchronous_classify_method(t
|
|||
)
|
||||
|
||||
|
||||
def test_resolve_classifier_plugin_prefers_the_registry_over_dotted_import(monkeypatch):
|
||||
import litellm
|
||||
from litellm.proxy.proxy_server import resolve_classifier_plugin
|
||||
|
||||
class _Classifier:
|
||||
async def classify(self, context):
|
||||
return "SIMPLE"
|
||||
|
||||
instance = _Classifier()
|
||||
monkeypatch.setitem(litellm.classifier_plugin_registry, "tier-by-team", instance)
|
||||
resolved = resolve_classifier_plugin(
|
||||
plugin_path="tier-by-team", config_file_path=None, source_label="classifier_plugins.tier-by-team"
|
||||
)
|
||||
assert resolved is instance
|
||||
|
||||
|
||||
def test_resolve_complexity_router_plugins_leaves_live_classifier_instance_alone():
|
||||
class _Classifier:
|
||||
async def classify(self, context):
|
||||
|
|
|
|||
|
|
@ -4233,6 +4233,52 @@ class TestClassifierPluginConfig:
|
|||
)
|
||||
|
||||
|
||||
class _RegistryClassifier:
|
||||
async def classify(self, context):
|
||||
return "COMPLEX"
|
||||
|
||||
|
||||
class TestClassifierPluginRegistry:
|
||||
"""String classifier_plugin values resolve through litellm.classifier_plugin_registry."""
|
||||
|
||||
def test_registered_name_resolves_to_the_instance(self, monkeypatch):
|
||||
instance = _RegistryClassifier()
|
||||
monkeypatch.setitem(litellm.classifier_plugin_registry, "tier-by-team", instance)
|
||||
config = ComplexityRouterConfig(
|
||||
tiers={"SIMPLE": "gpt-4o-mini"}, classifier_type="custom", classifier_plugin="tier-by-team"
|
||||
)
|
||||
assert config.classifier_plugin is instance
|
||||
|
||||
def test_unknown_name_is_rejected_at_validation(self):
|
||||
with pytest.raises(ValidationError, match="not a registered classifier plugin"):
|
||||
ComplexityRouterConfig(
|
||||
tiers={"SIMPLE": "gpt-4o-mini"}, classifier_type="custom", classifier_plugin="no-such-plugin"
|
||||
)
|
||||
|
||||
def test_live_instance_passes_through_untouched(self):
|
||||
instance = _RegistryClassifier()
|
||||
config = ComplexityRouterConfig(
|
||||
tiers={"SIMPLE": "gpt-4o-mini"}, classifier_type="custom", classifier_plugin=instance
|
||||
)
|
||||
assert config.classifier_plugin is instance
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_registry_named_plugin_classifies_end_to_end(self, mock_router_instance, monkeypatch):
|
||||
monkeypatch.setitem(litellm.classifier_plugin_registry, "tier-by-team", _RegistryClassifier())
|
||||
router = ComplexityRouter(
|
||||
model_name="test-complexity-router",
|
||||
litellm_router_instance=mock_router_instance,
|
||||
complexity_router_config={
|
||||
"tiers": {"SIMPLE": "gpt-4o-mini", "COMPLEX": "claude-sonnet-4-20250514"},
|
||||
"classifier_type": "custom",
|
||||
"classifier_plugin": "tier-by-team",
|
||||
},
|
||||
)
|
||||
outcome = await router.aclassify("hello")
|
||||
assert outcome.cause == "classifier_plugin"
|
||||
assert outcome.tier == ComplexityTier.COMPLEX
|
||||
|
||||
|
||||
class TestClassifierPlugin:
|
||||
"""classifier_type='custom': an operator hook decides the tier."""
|
||||
|
||||
|
|
|
|||
|
|
@ -138,6 +138,38 @@ def test_validate_rejects_ambiguous_tier_labels(tier_labels, expected_fragment):
|
|||
assert expected_fragment in violation
|
||||
|
||||
|
||||
def test_validate_rejects_an_unregistered_classifier_plugin_name():
|
||||
"""A stored name the registry does not hold must be refused at write time, not at load."""
|
||||
violation = validate_complexity_router_config_write(
|
||||
complexity_router_config={
|
||||
"tiers": VALID_TIERS,
|
||||
"classifier_type": "custom",
|
||||
"classifier_plugin": "no-such-plugin",
|
||||
}
|
||||
)
|
||||
assert violation is not None
|
||||
assert "complexity_router_config is invalid" in violation
|
||||
assert "not a registered classifier plugin" in violation
|
||||
|
||||
|
||||
def test_validate_accepts_a_registered_classifier_plugin_name(monkeypatch):
|
||||
import litellm
|
||||
|
||||
class _Classifier:
|
||||
async def classify(self, context):
|
||||
return "SIMPLE"
|
||||
|
||||
monkeypatch.setitem(litellm.classifier_plugin_registry, "tier-by-team", _Classifier())
|
||||
violation = validate_complexity_router_config_write(
|
||||
complexity_router_config={
|
||||
"tiers": VALID_TIERS,
|
||||
"classifier_type": "custom",
|
||||
"classifier_plugin": "tier-by-team",
|
||||
}
|
||||
)
|
||||
assert violation is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"complexity_router_config",
|
||||
[
|
||||
|
|
|
|||
|
|
@ -0,0 +1,13 @@
|
|||
import { $api } from "@/lib/http/api";
|
||||
|
||||
export const useClassifierPlugins = () =>
|
||||
$api.useQuery(
|
||||
"get",
|
||||
"/auto_router/classifier_plugins",
|
||||
{},
|
||||
{
|
||||
// Registration is config-file-only, so the list changes only on a proxy reload.
|
||||
staleTime: 5 * 60 * 1000,
|
||||
select: (data) => data.classifier_plugins,
|
||||
},
|
||||
);
|
||||
|
|
@ -99,6 +99,23 @@ describe("autoRouterRows", () => {
|
|||
expect(row.typeLabel).toBe("LLM Classifier");
|
||||
});
|
||||
|
||||
it("labels a router using a custom classifier plugin", () => {
|
||||
const row = toAutoRouterRow(
|
||||
{
|
||||
...complexityDeployment,
|
||||
litellm_params: {
|
||||
...complexityDeployment.litellm_params,
|
||||
complexity_router_config: { tiers: {}, classifier_type: "custom", classifier_plugin: "tier-by-team" },
|
||||
},
|
||||
},
|
||||
0,
|
||||
ADMIN,
|
||||
null,
|
||||
);
|
||||
|
||||
expect(row.typeLabel).toBe("Custom classifier");
|
||||
});
|
||||
|
||||
it("treats a deployment carrying complexity_router_config as complexity even off the canonical model string", () => {
|
||||
expect(isComplexityRouter({ model: "auto_router/legacy", complexity_router_config: { tiers: {} } })).toBe(true);
|
||||
});
|
||||
|
|
|
|||
|
|
@ -54,8 +54,11 @@ const asStringArray = (value: unknown): string[] =>
|
|||
|
||||
const dedupe = (models: string[]): string[] => Array.from(new Set(models));
|
||||
|
||||
export const complexityTypeLabel = (config: Record<string, unknown>): string =>
|
||||
config.classifier_type === "llm" ? "LLM Classifier" : "Heuristic";
|
||||
export const complexityTypeLabel = (config: Record<string, unknown>): string => {
|
||||
if (config.classifier_type === "llm") return "LLM Classifier";
|
||||
if (config.classifier_type === "custom") return "Custom classifier";
|
||||
return "Heuristic";
|
||||
};
|
||||
|
||||
interface Presentation {
|
||||
typeLabel: string;
|
||||
|
|
|
|||
|
|
@ -0,0 +1,167 @@
|
|||
import { fireEvent, renderWithProviders, screen } from "../../../tests/test-utils";
|
||||
import userEvent from "@testing-library/user-event";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { useClassifierPlugins } from "@/app/(dashboard)/hooks/autoRouter/useClassifierPlugins";
|
||||
import ClassificationMethodConfig from "./ClassificationMethodConfig";
|
||||
import { ComplexityRouterConfigValue } from "./ComplexityRouterConfig";
|
||||
import { LOADED_CLASSIFIER_PLUGINS_QUERY } from "../../../tests/mocks/classifierPlugins";
|
||||
|
||||
vi.mock(
|
||||
"@/app/(dashboard)/hooks/autoRouter/useComplexityScorerDefaults",
|
||||
async () => await import("../../../tests/mocks/complexityScorerDefaults"),
|
||||
);
|
||||
|
||||
vi.mock(
|
||||
"@/app/(dashboard)/hooks/autoRouter/useClassifierPlugins",
|
||||
async () => await import("../../../tests/mocks/classifierPlugins"),
|
||||
);
|
||||
|
||||
const BASE: ComplexityRouterConfigValue = {
|
||||
tiers: { SIMPLE: ["gpt-4o-mini"], MEDIUM: ["gpt-4o"], COMPLEX: ["o3"], REASONING: ["o3"] },
|
||||
classifier_type: "heuristic",
|
||||
};
|
||||
|
||||
const CUSTOM: ComplexityRouterConfigValue = {
|
||||
...BASE,
|
||||
classifier_type: "custom",
|
||||
classifier_plugin: "tier-by-team",
|
||||
classifier_plugin_timeout_ms: 3000,
|
||||
};
|
||||
|
||||
const MODEL_OPTIONS = [{ value: "gpt-4o-mini", label: "gpt-4o-mini" }];
|
||||
|
||||
const render = (value: ComplexityRouterConfigValue, onChange = vi.fn()) => {
|
||||
renderWithProviders(
|
||||
<ClassificationMethodConfig value={value} onChange={onChange} modelOptions={MODEL_OPTIONS} defaultModel="gpt-4o" />,
|
||||
);
|
||||
return onChange;
|
||||
};
|
||||
|
||||
const lastValue = (onChange: ReturnType<typeof vi.fn>) =>
|
||||
onChange.mock.calls.at(-1)?.[0] as ComplexityRouterConfigValue | undefined;
|
||||
|
||||
describe("ClassificationMethodConfig classification method radios", () => {
|
||||
it("offers the custom classifier alongside the heuristic and the LLM classifier", () => {
|
||||
render(BASE);
|
||||
|
||||
expect(screen.getByRole("radio", { name: /Heuristic/ })).toBeInTheDocument();
|
||||
expect(screen.getByRole("radio", { name: /LLM Classifier/ })).toBeInTheDocument();
|
||||
expect(screen.getByRole("radio", { name: /Custom classifier/ })).toBeEnabled();
|
||||
});
|
||||
|
||||
// Only the custom branch may carry a plugin: leaving one behind would have the backend reject the
|
||||
// save outright (classifier_plugin set with a non-custom classifier_type).
|
||||
it("clears the plugin when the operator switches back to the heuristic", async () => {
|
||||
const onChange = render(CUSTOM);
|
||||
|
||||
await userEvent.click(screen.getByRole("radio", { name: /Heuristic/ }));
|
||||
|
||||
expect(lastValue(onChange)).toMatchObject({
|
||||
classifier_type: "heuristic",
|
||||
classifier_plugin: undefined,
|
||||
classifier_plugin_timeout_ms: undefined,
|
||||
classifier_fallback: undefined,
|
||||
});
|
||||
});
|
||||
|
||||
it("stamps the default plugin timeout when custom is selected", async () => {
|
||||
const onChange = render(BASE);
|
||||
|
||||
await userEvent.click(screen.getByRole("radio", { name: /Custom classifier/ }));
|
||||
|
||||
expect(lastValue(onChange)).toMatchObject({ classifier_type: "custom", classifier_plugin_timeout_ms: 3000 });
|
||||
});
|
||||
});
|
||||
|
||||
describe("ClassificationMethodConfig custom classifier controls", () => {
|
||||
it("does not show the plugin controls until custom is the selected method", () => {
|
||||
render(BASE);
|
||||
|
||||
expect(screen.queryByRole("combobox", { name: "Classifier Plugin" })).not.toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("offers the names the proxy registered", async () => {
|
||||
render({ ...CUSTOM, classifier_plugin: undefined });
|
||||
|
||||
fireEvent.mouseDown(screen.getByRole("combobox", { name: "Classifier Plugin" }));
|
||||
|
||||
expect(await screen.findByTitle("tier-by-team")).toBeInTheDocument();
|
||||
expect(screen.getByTitle("spend-aware")).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("records the plugin the operator picks", async () => {
|
||||
const onChange = render({ ...CUSTOM, classifier_plugin: undefined });
|
||||
|
||||
fireEvent.mouseDown(screen.getByRole("combobox", { name: "Classifier Plugin" }));
|
||||
await userEvent.click(await screen.findByTitle("spend-aware"));
|
||||
|
||||
expect(lastValue(onChange)).toMatchObject({ classifier_plugin: "spend-aware", classifier_plugin_timeout_ms: 3000 });
|
||||
});
|
||||
|
||||
it("emits an edited plugin timeout", () => {
|
||||
const onChange = render(CUSTOM);
|
||||
|
||||
fireEvent.change(screen.getByRole("spinbutton", { name: "Plugin Timeout (ms)" }), { target: { value: "750" } });
|
||||
|
||||
expect(lastValue(onChange)).toMatchObject({ classifier_plugin_timeout_ms: 750 });
|
||||
});
|
||||
|
||||
// The backend applies classifier_fallback to a plugin exactly as it does to an LLM classifier, so
|
||||
// gating this picker on the LLM branch would leave the plugin's failure path unconfigurable.
|
||||
it("offers the fallback picker, which is not LLM-only", () => {
|
||||
render(CUSTOM);
|
||||
|
||||
expect(screen.getByRole("radio", { name: /Score with the heuristic/ })).toBeInTheDocument();
|
||||
expect(screen.getByRole("radio", { name: /Route to the default model/ })).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("says the score no longer decides the tier under a plugin", () => {
|
||||
render(CUSTOM);
|
||||
|
||||
expect(screen.getByText(/classifies with your own classifier plugin/)).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("reports a missing plugin once the form has been submitted", () => {
|
||||
renderWithProviders(
|
||||
<ClassificationMethodConfig
|
||||
value={{ ...CUSTOM, classifier_plugin: undefined }}
|
||||
onChange={vi.fn()}
|
||||
modelOptions={MODEL_OPTIONS}
|
||||
showValidationErrors
|
||||
/>,
|
||||
);
|
||||
|
||||
expect(screen.getByText("A classifier plugin is required")).toBeInTheDocument();
|
||||
});
|
||||
});
|
||||
|
||||
describe("ClassificationMethodConfig when the proxy registered no plugins", () => {
|
||||
const empty = { data: [], isPending: false, isError: false, refetch: vi.fn() };
|
||||
const failing = { data: undefined, isPending: false, isError: true, refetch: vi.fn() };
|
||||
|
||||
afterEach(() => vi.mocked(useClassifierPlugins).mockReturnValue(LOADED_CLASSIFIER_PLUGINS_QUERY));
|
||||
|
||||
it("disables the custom radio and says how to enable it", () => {
|
||||
vi.mocked(useClassifierPlugins).mockReturnValue(empty as never);
|
||||
render(BASE);
|
||||
|
||||
expect(screen.getByRole("radio", { name: /Custom classifier/ })).toBeDisabled();
|
||||
expect(
|
||||
screen.getByText("Declare classifier_plugins in the proxy config to enable custom classifiers"),
|
||||
).toBeInTheDocument();
|
||||
});
|
||||
|
||||
// A fetch that never landed leaves the registry unknown, not empty. Blanking the select or greying
|
||||
// out the radio here would clear an already-configured plugin on the operator's next save.
|
||||
it("keeps a stored plugin name selectable when the fetch failed", () => {
|
||||
vi.mocked(useClassifierPlugins).mockReturnValue(failing as never);
|
||||
render(CUSTOM);
|
||||
|
||||
expect(screen.getByRole("radio", { name: /Custom classifier/ })).toBeEnabled();
|
||||
expect(screen.getByRole("combobox", { name: "Classifier Plugin" })).toBeInTheDocument();
|
||||
expect(screen.getByTitle("tier-by-team")).toBeInTheDocument();
|
||||
expect(
|
||||
screen.queryByText("Declare classifier_plugins in the proxy config to enable custom classifiers"),
|
||||
).not.toBeInTheDocument();
|
||||
});
|
||||
});
|
||||
|
|
@ -3,6 +3,7 @@ import { Select as AntdSelect, Card, InputNumber, Radio, Space, Switch, Tooltip,
|
|||
import React from "react";
|
||||
import ClassifierPromptEditor from "./ClassifierPromptEditor";
|
||||
import HeuristicScoringConfig from "./HeuristicScoringConfig";
|
||||
import { useClassifierPlugins } from "@/app/(dashboard)/hooks/autoRouter/useClassifierPlugins";
|
||||
import { useComplexityScorerDefaults } from "@/app/(dashboard)/hooks/autoRouter/useComplexityScorerDefaults";
|
||||
import {
|
||||
ClassifierFallback,
|
||||
|
|
@ -11,6 +12,7 @@ import {
|
|||
DEFAULT_CLASSIFIER_CONTEXT_PER_TURN_CHARS,
|
||||
DEFAULT_CLASSIFIER_CONTEXT_WINDOW_SIZE,
|
||||
DEFAULT_CLASSIFIER_FALLBACK,
|
||||
DEFAULT_CLASSIFIER_PLUGIN_TIMEOUT_MS,
|
||||
DEFAULT_CLASSIFIER_TIMEOUT_MS,
|
||||
DEFAULT_CLASSIFICATION_RUBRIC,
|
||||
NEW_CLASSIFIER_CLASSIFICATION_RUBRIC,
|
||||
|
|
@ -35,18 +37,29 @@ const CUSTOM_PROMPT_WITH_DEFAULT_MODEL_FALLBACK =
|
|||
"names stay fixed. The scoring below no longer runs at all, since a failed classifier routes to the default " +
|
||||
"model instead:";
|
||||
|
||||
const PLUGIN_WITH_HEURISTIC_FALLBACK =
|
||||
"This router classifies with your own classifier plugin, so the tier comes from whatever the plugin decides. The " +
|
||||
"four tier names stay fixed. The scoring below is the heuristic, which now runs only when the plugin fails:";
|
||||
|
||||
const PLUGIN_WITH_DEFAULT_MODEL_FALLBACK =
|
||||
"This router classifies with your own classifier plugin, so the tier comes from whatever the plugin decides. The " +
|
||||
"four tier names stay fixed. The scoring below no longer runs at all, since a failed plugin routes to the default " +
|
||||
"model instead:";
|
||||
|
||||
/**
|
||||
* What the scoring breakdown below it actually describes. A custom prompt means the score no longer
|
||||
* decides the tier, and pairing one with the default-model fallback means the heuristic never runs
|
||||
* at all, so the panel must not keep implying a score is involved on either router.
|
||||
* What the scoring breakdown below it actually describes. A custom prompt or a classifier plugin means
|
||||
* the score no longer decides the tier, and pairing either with the default-model fallback means the
|
||||
* heuristic never runs at all, so the panel must not keep implying a score is involved.
|
||||
*/
|
||||
const scoringExplanation = (value: ComplexityRouterConfigValue): string => {
|
||||
const fallsBackToDefaultModel = value.classifier_fallback === "default_model";
|
||||
if (value.classifier_type === "custom") {
|
||||
return fallsBackToDefaultModel ? PLUGIN_WITH_DEFAULT_MODEL_FALLBACK : PLUGIN_WITH_HEURISTIC_FALLBACK;
|
||||
}
|
||||
const usesCustomPrompt =
|
||||
value.classifier_type === "llm" && Boolean(value.classifier_llm_config?.system_prompt?.trim());
|
||||
if (!usesCustomPrompt) return DEFAULT_SCORING_EXPLANATION;
|
||||
return value.classifier_fallback === "default_model"
|
||||
? CUSTOM_PROMPT_WITH_DEFAULT_MODEL_FALLBACK
|
||||
: CUSTOM_PROMPT_WITH_HEURISTIC_FALLBACK;
|
||||
return fallsBackToDefaultModel ? CUSTOM_PROMPT_WITH_DEFAULT_MODEL_FALLBACK : CUSTOM_PROMPT_WITH_HEURISTIC_FALLBACK;
|
||||
};
|
||||
|
||||
/**
|
||||
|
|
@ -105,6 +118,43 @@ const HowClassificationWorks: React.FC<{ value: ComplexityRouterConfigValue }> =
|
|||
);
|
||||
};
|
||||
|
||||
const ClassifierFallbackPicker: React.FC<{
|
||||
fallback: ClassifierFallback | undefined;
|
||||
onChange: (fallback: ClassifierFallback) => void;
|
||||
defaultModel: string | undefined;
|
||||
}> = ({ fallback, onChange, defaultModel }) => (
|
||||
<div>
|
||||
<Text strong style={{ display: "block", marginBottom: 4 }}>
|
||||
If the classifier fails
|
||||
</Text>
|
||||
<Radio.Group value={fallback ?? DEFAULT_CLASSIFIER_FALLBACK} onChange={(e) => onChange(e.target.value)}>
|
||||
<Space direction="vertical">
|
||||
<Radio value="heuristic">
|
||||
<Text>Score with the heuristic</Text>{" "}
|
||||
<Text type="secondary">— right when the classifier grades complexity too</Text>
|
||||
</Radio>
|
||||
<Radio value="default_model" disabled={!defaultModel}>
|
||||
<Tooltip
|
||||
title={
|
||||
defaultModel
|
||||
? "Change it from the Default Model select."
|
||||
: "Set a default model on this router to use this option"
|
||||
}
|
||||
>
|
||||
<span>
|
||||
<Text>Route to the default model{defaultModel ? ` (${defaultModel})` : ""}</Text>{" "}
|
||||
<Text type="secondary">— right when your classifier grades something other than complexity</Text>
|
||||
</span>
|
||||
</Tooltip>
|
||||
</Radio>
|
||||
</Space>
|
||||
</Radio.Group>
|
||||
<Text type="secondary" style={{ display: "block", fontSize: 12 }}>
|
||||
Applies when the classifier call errors, times out, or returns an unparseable response.
|
||||
</Text>
|
||||
</div>
|
||||
);
|
||||
|
||||
interface ClassificationMethodConfigProps {
|
||||
value: ComplexityRouterConfigValue;
|
||||
onChange: (value: ComplexityRouterConfigValue) => void;
|
||||
|
|
@ -125,12 +175,21 @@ const ClassificationMethodConfig: React.FC<ClassificationMethodConfigProps> = ({
|
|||
showValidationErrors = false,
|
||||
defaultModel,
|
||||
}) => {
|
||||
const hasDefaultModel = Boolean(defaultModel);
|
||||
const classifierModelMissing =
|
||||
showValidationErrors && value.classifier_type === "llm" && !value.classifier_llm_config?.model;
|
||||
const classifierPluginMissing =
|
||||
showValidationErrors && value.classifier_type === "custom" && !value.classifier_plugin;
|
||||
const usesCustomPrompt = Boolean(value.classifier_llm_config?.system_prompt?.trim());
|
||||
const classificationRubric = value.classifier_llm_config?.classification_rubric ?? DEFAULT_CLASSIFICATION_RUBRIC;
|
||||
|
||||
const { data: registeredPlugins, isError: pluginsError } = useClassifierPlugins();
|
||||
// A failed fetch leaves the registry unknown, not empty: offering the stored name anyway is what keeps
|
||||
// opening this form on an existing custom router from silently clearing its plugin on the next save.
|
||||
const registryIsEmpty = !pluginsError && registeredPlugins !== undefined && registeredPlugins.length === 0;
|
||||
const pluginOptions = Array.from(
|
||||
new Set([...(registeredPlugins ?? []), ...(value.classifier_plugin ? [value.classifier_plugin] : [])]),
|
||||
).map((name) => ({ value: name, label: name }));
|
||||
|
||||
const handleClassifierTypeChange = (classifierType: ClassifierType) => {
|
||||
const nextValue: ComplexityRouterConfigValue = {
|
||||
...value,
|
||||
|
|
@ -153,7 +212,12 @@ const ClassificationMethodConfig: React.FC<ClassificationMethodConfigProps> = ({
|
|||
: undefined,
|
||||
classifier_context_include_assistant_turns:
|
||||
classifierType === "llm" ? value.classifier_context_include_assistant_turns : undefined,
|
||||
classifier_fallback: classifierType === "llm" ? value.classifier_fallback : undefined,
|
||||
classifier_fallback: classifierType === "heuristic" ? undefined : value.classifier_fallback,
|
||||
classifier_plugin: classifierType === "custom" ? value.classifier_plugin : undefined,
|
||||
classifier_plugin_timeout_ms:
|
||||
classifierType === "custom"
|
||||
? value.classifier_plugin_timeout_ms ?? DEFAULT_CLASSIFIER_PLUGIN_TIMEOUT_MS
|
||||
: undefined,
|
||||
};
|
||||
onChange(nextValue);
|
||||
};
|
||||
|
|
@ -180,6 +244,18 @@ const ClassificationMethodConfig: React.FC<ClassificationMethodConfigProps> = ({
|
|||
});
|
||||
};
|
||||
|
||||
const handleClassifierPluginChange = (classifierPlugin: string) => {
|
||||
onChange({
|
||||
...value,
|
||||
classifier_plugin: classifierPlugin,
|
||||
classifier_plugin_timeout_ms: value.classifier_plugin_timeout_ms ?? DEFAULT_CLASSIFIER_PLUGIN_TIMEOUT_MS,
|
||||
});
|
||||
};
|
||||
|
||||
const handleClassifierPluginTimeoutChange = (timeoutMs: number | null) => {
|
||||
onChange({ ...value, classifier_plugin_timeout_ms: timeoutMs ?? DEFAULT_CLASSIFIER_PLUGIN_TIMEOUT_MS });
|
||||
};
|
||||
|
||||
const handleClassificationRubricChange = (classificationRubric: ClassificationRubric) => {
|
||||
onChange({
|
||||
...value,
|
||||
|
|
@ -245,6 +321,15 @@ const ClassificationMethodConfig: React.FC<ClassificationMethodConfigProps> = ({
|
|||
<Text strong>LLM Classifier</Text>{" "}
|
||||
<Text type="secondary">— use a model to decide the tier (e.g. a small/fast model)</Text>
|
||||
</Radio>
|
||||
<Radio value="custom" disabled={registryIsEmpty && value.classifier_type !== "custom"}>
|
||||
<Text strong>Custom classifier</Text>{" "}
|
||||
<Text type="secondary">— let a classifier plugin registered in the proxy config decide the tier</Text>
|
||||
</Radio>
|
||||
{registryIsEmpty && (
|
||||
<Text type="secondary" style={{ fontSize: 12 }}>
|
||||
Declare classifier_plugins in the proxy config to enable custom classifiers
|
||||
</Text>
|
||||
)}
|
||||
</Space>
|
||||
</Radio.Group>
|
||||
|
||||
|
|
@ -321,39 +406,11 @@ const ClassificationMethodConfig: React.FC<ClassificationMethodConfigProps> = ({
|
|||
classificationRubric={classificationRubric}
|
||||
/>
|
||||
</div>
|
||||
<div>
|
||||
<Text strong style={{ display: "block", marginBottom: 4 }}>
|
||||
If the classifier fails
|
||||
</Text>
|
||||
<Radio.Group
|
||||
value={value.classifier_fallback ?? DEFAULT_CLASSIFIER_FALLBACK}
|
||||
onChange={(e) => handleClassifierFallbackChange(e.target.value)}
|
||||
>
|
||||
<Space direction="vertical">
|
||||
<Radio value="heuristic">
|
||||
<Text>Score with the heuristic</Text>{" "}
|
||||
<Text type="secondary">— right when the classifier grades complexity too</Text>
|
||||
</Radio>
|
||||
<Radio value="default_model" disabled={!hasDefaultModel}>
|
||||
<Tooltip
|
||||
title={
|
||||
hasDefaultModel
|
||||
? "Change it from the Default Model select."
|
||||
: "Set a default model on this router to use this option"
|
||||
}
|
||||
>
|
||||
<span>
|
||||
<Text>Route to the default model{defaultModel ? ` (${defaultModel})` : ""}</Text>{" "}
|
||||
<Text type="secondary">— right when your prompt grades something other than complexity</Text>
|
||||
</span>
|
||||
</Tooltip>
|
||||
</Radio>
|
||||
</Space>
|
||||
</Radio.Group>
|
||||
<Text type="secondary" style={{ display: "block", fontSize: 12 }}>
|
||||
Applies when the classifier call errors, times out, or returns an unparseable response.
|
||||
</Text>
|
||||
</div>
|
||||
<ClassifierFallbackPicker
|
||||
fallback={value.classifier_fallback}
|
||||
onChange={handleClassifierFallbackChange}
|
||||
defaultModel={defaultModel}
|
||||
/>
|
||||
<div>
|
||||
<Text strong style={{ display: "block", marginBottom: 4 }}>
|
||||
Context Window Size
|
||||
|
|
@ -407,6 +464,56 @@ const ClassificationMethodConfig: React.FC<ClassificationMethodConfigProps> = ({
|
|||
</div>
|
||||
)}
|
||||
|
||||
{value.classifier_type === "custom" && (
|
||||
<div className="mt-4 space-y-3">
|
||||
<div>
|
||||
<Text strong style={{ display: "block", marginBottom: 4 }}>
|
||||
Classifier Plugin
|
||||
</Text>
|
||||
<AntdSelect
|
||||
value={value.classifier_plugin || undefined}
|
||||
onChange={handleClassifierPluginChange}
|
||||
placeholder="Select a classifier plugin registered in the proxy config"
|
||||
showSearch
|
||||
style={{ width: "100%" }}
|
||||
options={pluginOptions}
|
||||
status={classifierPluginMissing ? "error" : undefined}
|
||||
aria-label="Classifier Plugin"
|
||||
/>
|
||||
{classifierPluginMissing && (
|
||||
<Text type="danger" style={{ fontSize: 12 }}>
|
||||
A classifier plugin is required
|
||||
</Text>
|
||||
)}
|
||||
{pluginsError && (
|
||||
<Text type="secondary" style={{ display: "block", fontSize: 12 }}>
|
||||
The registered plugin names could not be loaded from the proxy.
|
||||
</Text>
|
||||
)}
|
||||
</div>
|
||||
<div>
|
||||
<Text strong style={{ display: "block", marginBottom: 4 }}>
|
||||
Plugin Timeout (ms)
|
||||
</Text>
|
||||
<InputNumber
|
||||
value={value.classifier_plugin_timeout_ms ?? DEFAULT_CLASSIFIER_PLUGIN_TIMEOUT_MS}
|
||||
onChange={handleClassifierPluginTimeoutChange}
|
||||
min={1}
|
||||
style={{ width: "100%" }}
|
||||
aria-label="Plugin Timeout (ms)"
|
||||
/>
|
||||
<Text type="secondary" style={{ fontSize: 12 }}>
|
||||
How long the plugin has to classify before it fails and the fallback below takes over.
|
||||
</Text>
|
||||
</div>
|
||||
<ClassifierFallbackPicker
|
||||
fallback={value.classifier_fallback}
|
||||
onChange={handleClassifierFallbackChange}
|
||||
defaultModel={defaultModel}
|
||||
/>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{value.classifier_type === "heuristic" && (
|
||||
<div className="mt-4">
|
||||
<div className="flex items-center gap-2 mb-1">
|
||||
|
|
|
|||
|
|
@ -15,6 +15,7 @@ export type { DimensionWeights, TierBoundaries, TokenThresholds };
|
|||
const { Text } = Typography;
|
||||
|
||||
export const DEFAULT_CLASSIFIER_TIMEOUT_MS = 3000;
|
||||
export const DEFAULT_CLASSIFIER_PLUGIN_TIMEOUT_MS = 3000;
|
||||
export const DEFAULT_TIER_DISTANCE_PENALTY = 0.5;
|
||||
export const DEFAULT_CLASSIFIER_CONTEXT_WINDOW_SIZE = 3;
|
||||
export const DEFAULT_CLASSIFIER_CONTEXT_PER_TURN_CHARS = 200;
|
||||
|
|
@ -73,7 +74,7 @@ export interface ClassifierLLMConfig {
|
|||
system_prompt?: string;
|
||||
}
|
||||
|
||||
export type ClassifierType = "heuristic" | "llm";
|
||||
export type ClassifierType = "heuristic" | "llm" | "custom";
|
||||
|
||||
export type ClassifierFallback = "heuristic" | "default_model";
|
||||
|
||||
|
|
@ -90,8 +91,8 @@ export type HeuristicScoringRole = "decides" | "fallback_only" | "never";
|
|||
|
||||
/**
|
||||
* Whether the heuristic scorer runs on this router at all, which is what gates its knobs. An LLM
|
||||
* classifier still falls back to the scorer unless the fallback is the default model, so the gate cannot be
|
||||
* a plain classifier_type check.
|
||||
* classifier or a classifier plugin still falls back to the scorer unless the fallback is the default
|
||||
* model, so the gate cannot be a plain classifier_type check.
|
||||
*/
|
||||
export const heuristicScoringRoleFor = (
|
||||
classifierType: ClassifierType,
|
||||
|
|
@ -115,6 +116,9 @@ export interface ComplexityRouterConfigValue {
|
|||
default_model?: string;
|
||||
classifier_type: ClassifierType;
|
||||
classifier_llm_config?: ClassifierLLMConfig;
|
||||
/** A name from the proxy config's classifier_plugins registry. Required when classifier_type is "custom". */
|
||||
classifier_plugin?: string;
|
||||
classifier_plugin_timeout_ms?: number;
|
||||
classifier_context_window_size?: number;
|
||||
classifier_context_per_turn_chars?: number;
|
||||
classifier_context_include_assistant_turns?: boolean;
|
||||
|
|
|
|||
|
|
@ -13,6 +13,11 @@ vi.mock(
|
|||
async () => await import("../../../tests/mocks/complexityScorerDefaults"),
|
||||
);
|
||||
|
||||
vi.mock(
|
||||
"@/app/(dashboard)/hooks/autoRouter/useClassifierPlugins",
|
||||
async () => await import("../../../tests/mocks/classifierPlugins"),
|
||||
);
|
||||
|
||||
const ANTHROPIC_PRESET = getPresetByKey("anthropic_family")!;
|
||||
const ANTHROPIC_TIERS = ANTHROPIC_PRESET.complexity_router_config.tiers;
|
||||
|
||||
|
|
@ -285,6 +290,50 @@ describe("AddAutoRouterTab", () => {
|
|||
});
|
||||
});
|
||||
|
||||
it("carries a selected classifier plugin and its timeout through to the create payload", async () => {
|
||||
const user = userEvent.setup();
|
||||
vi.mocked(getMissingTiersError).mockReturnValue(null);
|
||||
|
||||
renderWithProviders(<Harness />);
|
||||
|
||||
await user.type(screen.getByPlaceholderText(/smart_router/i), "plugin-router");
|
||||
expandDetailedConfiguration();
|
||||
await user.click(screen.getByText("Advanced: Classification Method"));
|
||||
await user.click(await screen.findByRole("radio", { name: /Custom classifier/ }));
|
||||
fireEvent.mouseDown(await screen.findByRole("combobox", { name: "Classifier Plugin" }));
|
||||
await user.click(await screen.findByTitle("tier-by-team"));
|
||||
|
||||
await user.click(screen.getByRole("button", { name: /add auto router/i }));
|
||||
|
||||
await waitFor(() => expect(handleAddAutoRouterSubmit).toHaveBeenCalled());
|
||||
expect(vi.mocked(handleAddAutoRouterSubmit).mock.calls.at(-1)?.[0].complexity_router_config).toMatchObject({
|
||||
classifier_type: "custom",
|
||||
classifier_plugin: "tier-by-team",
|
||||
classifier_plugin_timeout_ms: 3000,
|
||||
});
|
||||
});
|
||||
|
||||
// The backend rejects classifier_type "custom" with no classifier_plugin, so the form has to say
|
||||
// what is missing rather than let the save come back as a raw 400.
|
||||
it("blocks a submit that selects the custom classifier without a plugin", async () => {
|
||||
const user = userEvent.setup();
|
||||
vi.mocked(getMissingTiersError).mockReturnValue(null);
|
||||
|
||||
renderWithProviders(<Harness />);
|
||||
|
||||
await user.type(screen.getByPlaceholderText(/smart_router/i), "plugin-router");
|
||||
expandDetailedConfiguration();
|
||||
await user.click(screen.getByText("Advanced: Classification Method"));
|
||||
await user.click(await screen.findByRole("radio", { name: /Custom classifier/ }));
|
||||
|
||||
await user.click(screen.getByRole("button", { name: /add auto router/i }));
|
||||
|
||||
await waitFor(() =>
|
||||
expect(toast.fromError).toHaveBeenCalledWith("Please select a classifier plugin, or switch back to Heuristic"),
|
||||
);
|
||||
expect(handleAddAutoRouterSubmit).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("carries session affinity turned on through to the create payload", async () => {
|
||||
const user = userEvent.setup();
|
||||
vi.mocked(getMissingTiersError).mockReturnValue(null);
|
||||
|
|
|
|||
|
|
@ -341,6 +341,8 @@ const AddAutoRouterTab: React.FC<AddAutoRouterTabProps> = ({
|
|||
tierLabels: complexityRouterConfig.tier_labels,
|
||||
classifierType: complexityRouterConfig.classifier_type,
|
||||
classifierLlmConfig: complexityRouterConfig.classifier_llm_config,
|
||||
classifierPlugin: complexityRouterConfig.classifier_plugin,
|
||||
classifierPluginTimeoutMs: complexityRouterConfig.classifier_plugin_timeout_ms,
|
||||
classifierContextWindowSize: complexityRouterConfig.classifier_context_window_size,
|
||||
classifierContextPerTurnChars: complexityRouterConfig.classifier_context_per_turn_chars,
|
||||
classifierContextIncludeAssistantTurns: complexityRouterConfig.classifier_context_include_assistant_turns,
|
||||
|
|
@ -364,7 +366,7 @@ const AddAutoRouterTab: React.FC<AddAutoRouterTabProps> = ({
|
|||
};
|
||||
|
||||
const submitRecommendedRouter = async (name: string) => {
|
||||
const { tiers, tierLabels, classifierType, classifierLlmConfig } = complexityRouterConfigParams;
|
||||
const { tiers, tierLabels, classifierType, classifierLlmConfig, classifierPlugin } = complexityRouterConfigParams;
|
||||
|
||||
const missingTiersError = getMissingTiersError(tiers);
|
||||
if (missingTiersError) {
|
||||
|
|
@ -386,6 +388,12 @@ const AddAutoRouterTab: React.FC<AddAutoRouterTabProps> = ({
|
|||
return;
|
||||
}
|
||||
|
||||
if (classifierType === "custom" && !classifierPlugin) {
|
||||
setShowValidationErrors(true);
|
||||
toast.fromError("Please select a classifier plugin, or switch back to Heuristic");
|
||||
return;
|
||||
}
|
||||
|
||||
const keywordRulesError = getKeywordTierRulesError(keywordTierRules);
|
||||
if (keywordRulesError) {
|
||||
setShowValidationErrors(true);
|
||||
|
|
|
|||
|
|
@ -22,6 +22,8 @@ const baseParams: BuildComplexityRouterConfigParams = {
|
|||
tierLabels: undefined,
|
||||
classifierType: "heuristic",
|
||||
classifierLlmConfig: undefined,
|
||||
classifierPlugin: undefined,
|
||||
classifierPluginTimeoutMs: undefined,
|
||||
classifierContextWindowSize: undefined,
|
||||
classifierContextPerTurnChars: undefined,
|
||||
classifierContextIncludeAssistantTurns: undefined,
|
||||
|
|
@ -93,6 +95,86 @@ describe("buildComplexityRouterConfig", () => {
|
|||
expect(config.classifier_llm_config).toBeUndefined();
|
||||
});
|
||||
|
||||
it("includes the plugin and its timeout only when classifier_type is custom", () => {
|
||||
const config = buildComplexityRouterConfig({
|
||||
...baseParams,
|
||||
classifierType: "custom",
|
||||
classifierPlugin: "tier-by-team",
|
||||
classifierPluginTimeoutMs: 1500,
|
||||
});
|
||||
|
||||
expect(config.classifier_type).toBe("custom");
|
||||
expect(config.classifier_plugin).toBe("tier-by-team");
|
||||
expect(config.classifier_plugin_timeout_ms).toBe(1500);
|
||||
expect(config.classifier_llm_config).toBeUndefined();
|
||||
});
|
||||
|
||||
// The backend rejects classifier_plugin alongside a non-custom classifier_type, so a selection
|
||||
// left on the form after the operator switched away must never reach the wire.
|
||||
it("omits the plugin and its timeout when classifier_type is heuristic even if they linger in state", () => {
|
||||
const config = buildComplexityRouterConfig({
|
||||
...baseParams,
|
||||
classifierPlugin: "tier-by-team",
|
||||
classifierPluginTimeoutMs: 1500,
|
||||
});
|
||||
|
||||
expect(config.classifier_plugin).toBeUndefined();
|
||||
expect(config.classifier_plugin_timeout_ms).toBeUndefined();
|
||||
});
|
||||
|
||||
it("omits the plugin and its timeout when classifier_type is llm even if they linger in state", () => {
|
||||
const config = buildComplexityRouterConfig({
|
||||
...baseParams,
|
||||
classifierType: "llm",
|
||||
classifierLlmConfig: { model: "gpt-4o-mini", timeout_ms: 3000 },
|
||||
classifierPlugin: "tier-by-team",
|
||||
classifierPluginTimeoutMs: 1500,
|
||||
});
|
||||
|
||||
expect(config.classifier_plugin).toBeUndefined();
|
||||
expect(config.classifier_plugin_timeout_ms).toBeUndefined();
|
||||
});
|
||||
|
||||
// The plugin's failure path is configurable exactly like the LLM classifier's, so the fallback
|
||||
// gate is "not the heuristic" rather than "llm".
|
||||
it("emits classifier_fallback for a custom classifier as well as an llm one", () => {
|
||||
const custom = buildComplexityRouterConfig({
|
||||
...baseParams,
|
||||
classifierType: "custom",
|
||||
classifierPlugin: "tier-by-team",
|
||||
classifierFallback: "default_model",
|
||||
});
|
||||
const llm = buildComplexityRouterConfig({
|
||||
...baseParams,
|
||||
classifierType: "llm",
|
||||
classifierLlmConfig: { model: "gpt-4o-mini", timeout_ms: 3000 },
|
||||
classifierFallback: "default_model",
|
||||
});
|
||||
|
||||
expect(custom.classifier_fallback).toBe("default_model");
|
||||
expect(llm.classifier_fallback).toBe("default_model");
|
||||
});
|
||||
|
||||
it("omits classifier_fallback for the heuristic, which has nothing to fall back from", () => {
|
||||
const config = buildComplexityRouterConfig({ ...baseParams, classifierFallback: "default_model" });
|
||||
|
||||
expect(config.classifier_fallback).toBeUndefined();
|
||||
});
|
||||
|
||||
// heuristicScoringRoleFor keys off the fallback, not the classifier type, so a plugin routing its
|
||||
// failures to the default model must drop the scorer knobs the same way an LLM classifier does.
|
||||
it("drops the scorer knobs on a custom classifier that never scores", () => {
|
||||
const config = buildComplexityRouterConfig({
|
||||
...baseParams,
|
||||
classifierType: "custom",
|
||||
classifierPlugin: "tier-by-team",
|
||||
classifierFallback: "default_model",
|
||||
tokenThresholds: { simple: 25, complex: 900 },
|
||||
});
|
||||
|
||||
expect(config.token_thresholds).toBeUndefined();
|
||||
});
|
||||
|
||||
it("includes classifier_context_window_size and classifier_context_per_turn_chars only when classifier_type is llm", () => {
|
||||
const params: BuildComplexityRouterConfigParams = {
|
||||
...baseParams,
|
||||
|
|
|
|||
|
|
@ -75,6 +75,8 @@ export interface BuildComplexityRouterConfigParams {
|
|||
tierLabels: ComplexityTierLabels | undefined;
|
||||
classifierType: ClassifierType;
|
||||
classifierLlmConfig: ClassifierLLMConfig | undefined;
|
||||
classifierPlugin: string | undefined;
|
||||
classifierPluginTimeoutMs: number | undefined;
|
||||
classifierContextWindowSize: number | undefined;
|
||||
classifierContextPerTurnChars: number | undefined;
|
||||
classifierContextIncludeAssistantTurns: boolean | undefined;
|
||||
|
|
@ -104,6 +106,8 @@ export interface ComplexityRouterConfigPayload {
|
|||
tier_labels?: ComplexityTierLabels;
|
||||
classifier_type: ClassifierType;
|
||||
classifier_llm_config?: ClassifierLLMConfig;
|
||||
classifier_plugin?: string;
|
||||
classifier_plugin_timeout_ms?: number;
|
||||
classifier_context_window_size?: number;
|
||||
classifier_context_per_turn_chars?: number;
|
||||
classifier_context_include_assistant_turns?: boolean;
|
||||
|
|
@ -210,6 +214,8 @@ export const buildComplexityRouterConfig = ({
|
|||
tierLabels,
|
||||
classifierType,
|
||||
classifierLlmConfig,
|
||||
classifierPlugin,
|
||||
classifierPluginTimeoutMs,
|
||||
classifierContextWindowSize,
|
||||
classifierContextPerTurnChars,
|
||||
classifierContextIncludeAssistantTurns,
|
||||
|
|
@ -245,7 +251,11 @@ export const buildComplexityRouterConfig = ({
|
|||
classifier_type: classifierType,
|
||||
...(classifierType === "llm" &&
|
||||
classifierLlmConfig && { classifier_llm_config: normalizeClassifierLlmConfig(classifierLlmConfig) }),
|
||||
...(classifierType === "llm" && classifierFallback !== undefined && { classifier_fallback: classifierFallback }),
|
||||
...(classifierType === "custom" && classifierPlugin && { classifier_plugin: classifierPlugin }),
|
||||
...(classifierType === "custom" &&
|
||||
classifierPluginTimeoutMs !== undefined && { classifier_plugin_timeout_ms: classifierPluginTimeoutMs }),
|
||||
...(classifierType !== "heuristic" &&
|
||||
classifierFallback !== undefined && { classifier_fallback: classifierFallback }),
|
||||
...(classifierType === "llm" &&
|
||||
classifierContextWindowSize !== undefined && {
|
||||
classifier_context_window_size: classifierContextWindowSize,
|
||||
|
|
|
|||
|
|
@ -106,6 +106,40 @@ describe("buildUpdatedComplexityRouterConfig", () => {
|
|||
expect(updatedConfig.custom_technical_keywords).toBeUndefined();
|
||||
});
|
||||
|
||||
it("writes the plugin and its timeout for a custom classifier", () => {
|
||||
const updatedConfig = buildUpdatedComplexityRouterConfig(storedConfig, {
|
||||
tiers,
|
||||
classifier_type: "custom" as const,
|
||||
classifier_plugin: "tier-by-team",
|
||||
classifier_plugin_timeout_ms: 1500,
|
||||
classifier_fallback: "default_model" as const,
|
||||
});
|
||||
|
||||
expect(updatedConfig).toMatchObject({
|
||||
classifier_type: "custom",
|
||||
classifier_plugin: "tier-by-team",
|
||||
classifier_plugin_timeout_ms: 1500,
|
||||
classifier_fallback: "default_model",
|
||||
});
|
||||
expect(updatedConfig.classifier_llm_config).toBeUndefined();
|
||||
});
|
||||
|
||||
// Both keys are managed, so switching off custom has to remove them: the backend rejects a stored
|
||||
// classifier_plugin sitting next to a non-custom classifier_type.
|
||||
it("removes a stored plugin and its timeout when the classifier is no longer custom", () => {
|
||||
const storedCustom = JSON.stringify({
|
||||
...storedConfigValue,
|
||||
classifier_type: "custom",
|
||||
classifier_plugin: "tier-by-team",
|
||||
classifier_plugin_timeout_ms: 1500,
|
||||
});
|
||||
|
||||
const updatedConfig = buildUpdatedComplexityRouterConfig(storedCustom, classifiedTierValue);
|
||||
|
||||
expect(updatedConfig.classifier_plugin).toBeUndefined();
|
||||
expect(updatedConfig.classifier_plugin_timeout_ms).toBeUndefined();
|
||||
});
|
||||
|
||||
it("preserves a tier configured with more than one model as a pool", () => {
|
||||
const multiModelValue = {
|
||||
...classifiedTierValue,
|
||||
|
|
|
|||
|
|
@ -10,6 +10,11 @@ vi.mock(
|
|||
async () => await import("../../../tests/mocks/complexityScorerDefaults"),
|
||||
);
|
||||
|
||||
vi.mock(
|
||||
"@/app/(dashboard)/hooks/autoRouter/useClassifierPlugins",
|
||||
async () => await import("../../../tests/mocks/classifierPlugins"),
|
||||
);
|
||||
|
||||
const { modelPatchUpdateCall, modelAvailableCall, getAutoRouterClassifierDefaultPromptCall } = vi.hoisted(() => ({
|
||||
modelPatchUpdateCall: vi.fn().mockResolvedValue({}),
|
||||
modelAvailableCall: vi.fn().mockResolvedValue({ data: [] }),
|
||||
|
|
@ -484,6 +489,93 @@ describe("EditAutoRouterModal deployment affinity", () => {
|
|||
});
|
||||
});
|
||||
|
||||
describe("EditAutoRouterModal custom classifier plugin", () => {
|
||||
beforeEach(() => {
|
||||
modelPatchUpdateCall.mockClear();
|
||||
});
|
||||
|
||||
const STORED_PLUGIN_CONFIG = {
|
||||
tiers: { SIMPLE: ["gpt-4o-mini"], MEDIUM: ["gpt-4o-mini"], COMPLEX: ["gpt-4o-mini"], REASONING: ["gpt-4o-mini"] },
|
||||
classifier_type: "custom",
|
||||
classifier_plugin: "tier-by-team",
|
||||
classifier_plugin_timeout_ms: 1500,
|
||||
classifier_fallback: "heuristic",
|
||||
};
|
||||
|
||||
const renderPluginModal = () =>
|
||||
renderWithProviders(
|
||||
<EditAutoRouterModal
|
||||
isVisible
|
||||
onCancel={vi.fn()}
|
||||
onSuccess={vi.fn()}
|
||||
modelData={{
|
||||
...MODEL_DATA,
|
||||
litellm_params: { ...MODEL_DATA.litellm_params, complexity_router_config: STORED_PLUGIN_CONFIG },
|
||||
}}
|
||||
accessToken="token"
|
||||
userRole="Admin"
|
||||
/>,
|
||||
);
|
||||
|
||||
// Every one of these keys is rewritten from form state on save, so a missing hydration line would
|
||||
// silently wipe the operator's plugin the first time they opened this modal for anything else.
|
||||
it("preserves a stored plugin, its timeout, and its fallback through an untouched open-and-save", async () => {
|
||||
const user = userEvent.setup();
|
||||
renderPluginModal();
|
||||
|
||||
await user.click(await screen.findByText("Advanced: Classification Method"));
|
||||
expect(await screen.findByRole("combobox", { name: "Classifier Plugin" })).toBeInTheDocument();
|
||||
expect(screen.getByRole("spinbutton", { name: "Plugin Timeout (ms)" })).toHaveValue("1500");
|
||||
|
||||
await user.click(screen.getByRole("button", { name: /save changes/i }));
|
||||
|
||||
await waitFor(() => expect(modelPatchUpdateCall).toHaveBeenCalled());
|
||||
const config = savedConfig();
|
||||
expect(config.classifier_type).toBe("custom");
|
||||
expect(config.classifier_plugin).toBe("tier-by-team");
|
||||
expect(config.classifier_plugin_timeout_ms).toBe(1500);
|
||||
expect(config.classifier_fallback).toBe("heuristic");
|
||||
});
|
||||
|
||||
it("persists a switch to another registered plugin", async () => {
|
||||
const user = userEvent.setup();
|
||||
renderPluginModal();
|
||||
|
||||
await user.click(await screen.findByText("Advanced: Classification Method"));
|
||||
fireEvent.mouseDown(await screen.findByRole("combobox", { name: "Classifier Plugin" }));
|
||||
await user.click(await screen.findByTitle("spend-aware"));
|
||||
await user.click(screen.getByRole("button", { name: /save changes/i }));
|
||||
|
||||
await waitFor(() => expect(modelPatchUpdateCall).toHaveBeenCalled());
|
||||
expect(savedConfig().classifier_plugin).toBe("spend-aware");
|
||||
});
|
||||
|
||||
it("blocks a save that leaves the custom classifier with no plugin", async () => {
|
||||
const user = userEvent.setup();
|
||||
renderWithProviders(
|
||||
<EditAutoRouterModal
|
||||
isVisible
|
||||
onCancel={vi.fn()}
|
||||
onSuccess={vi.fn()}
|
||||
modelData={{
|
||||
...MODEL_DATA,
|
||||
litellm_params: {
|
||||
...MODEL_DATA.litellm_params,
|
||||
complexity_router_config: { ...STORED_PLUGIN_CONFIG, classifier_plugin: undefined },
|
||||
},
|
||||
}}
|
||||
accessToken="token"
|
||||
userRole="Admin"
|
||||
/>,
|
||||
);
|
||||
|
||||
await user.click(await screen.findByRole("button", { name: /save changes/i }));
|
||||
|
||||
await waitFor(() => expect(toast.fromError).toHaveBeenCalled());
|
||||
expect(modelPatchUpdateCall).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
describe("EditAutoRouterModal custom classifier prompt and fallback", () => {
|
||||
beforeEach(() => {
|
||||
modelPatchUpdateCall.mockClear();
|
||||
|
|
|
|||
|
|
@ -70,6 +70,8 @@ const MANAGED_COMPLEXITY_ROUTER_KEYS = new Set([
|
|||
"tier_labels",
|
||||
"classifier_type",
|
||||
"classifier_llm_config",
|
||||
"classifier_plugin",
|
||||
"classifier_plugin_timeout_ms",
|
||||
"classifier_context_window_size",
|
||||
"classifier_context_per_turn_chars",
|
||||
"classifier_context_include_assistant_turns",
|
||||
|
|
@ -157,7 +159,14 @@ export const buildUpdatedComplexityRouterConfig = (
|
|||
...(value.classifier_type === "llm" && value.classifier_llm_config
|
||||
? { classifier_llm_config: normalizeClassifierLlmConfig(value.classifier_llm_config) }
|
||||
: {}),
|
||||
...(value.classifier_type === "llm" &&
|
||||
...(value.classifier_type === "custom" && value.classifier_plugin
|
||||
? { classifier_plugin: value.classifier_plugin }
|
||||
: {}),
|
||||
...(value.classifier_type === "custom" &&
|
||||
value.classifier_plugin_timeout_ms !== undefined && {
|
||||
classifier_plugin_timeout_ms: value.classifier_plugin_timeout_ms,
|
||||
}),
|
||||
...(value.classifier_type !== "heuristic" &&
|
||||
value.classifier_fallback !== undefined && { classifier_fallback: value.classifier_fallback }),
|
||||
...(value.classifier_type === "llm" &&
|
||||
value.classifier_context_window_size !== undefined && {
|
||||
|
|
@ -348,6 +357,12 @@ const EditAutoRouterModal: React.FC<EditAutoRouterModalProps> = ({
|
|||
tier_labels: hydrateTierLabels(parsedConfig.tier_labels),
|
||||
classifier_type: parsedConfig.classifier_type || "heuristic",
|
||||
classifier_llm_config: parsedConfig.classifier_llm_config,
|
||||
classifier_plugin:
|
||||
typeof parsedConfig.classifier_plugin === "string" ? parsedConfig.classifier_plugin : undefined,
|
||||
classifier_plugin_timeout_ms:
|
||||
typeof parsedConfig.classifier_plugin_timeout_ms === "number"
|
||||
? parsedConfig.classifier_plugin_timeout_ms
|
||||
: undefined,
|
||||
classifier_context_window_size:
|
||||
typeof parsedConfig.classifier_context_window_size === "number"
|
||||
? parsedConfig.classifier_context_window_size
|
||||
|
|
@ -435,7 +450,7 @@ const EditAutoRouterModal: React.FC<EditAutoRouterModalProps> = ({
|
|||
|
||||
const saveValues = async (values: EditAutoRouterFormValues) => {
|
||||
if (isComplexityRouterModel) {
|
||||
const { tiers, classifier_type, classifier_llm_config } = complexityRouterConfig;
|
||||
const { tiers, classifier_type, classifier_llm_config, classifier_plugin } = complexityRouterConfig;
|
||||
if (Object.values(tiers).every((models) => models.length === 0)) {
|
||||
setShowValidationErrors(true);
|
||||
toast.fromError("Please select at least one model for a complexity tier");
|
||||
|
|
@ -446,6 +461,11 @@ const EditAutoRouterModal: React.FC<EditAutoRouterModalProps> = ({
|
|||
toast.fromError("Please select a classifier model, or switch back to Heuristic");
|
||||
return;
|
||||
}
|
||||
if (classifier_type === "custom" && !classifier_plugin) {
|
||||
setShowValidationErrors(true);
|
||||
toast.fromError("Please select a classifier plugin, or switch back to Heuristic");
|
||||
return;
|
||||
}
|
||||
// Same guards the create form applies (add_auto_router_tab.tsx). The backend rejects a
|
||||
// keyword rule with no keyword, and semantic_keyword_matching without an embedding model
|
||||
// or keyword rules (complexity_router/config.py), so without these a save fails as a raw
|
||||
|
|
|
|||
|
|
@ -85,6 +85,31 @@ describe("RoutingDecisionCard", () => {
|
|||
expect(screen.queryByText("Score")).not.toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("names the custom classifier plugin instead of rendering its raw cause string", () => {
|
||||
render(
|
||||
<RoutingDecisionCard
|
||||
decision={{
|
||||
router_model_name: "plugin-router",
|
||||
router_type: "complexity",
|
||||
routed_model: "claude-sonnet",
|
||||
cause: "classifier_plugin",
|
||||
tier: "COMPLEX",
|
||||
signals: ["classifier-plugin:COMPLEX"],
|
||||
}}
|
||||
/>,
|
||||
);
|
||||
expect(screen.getByText("Custom classifier plugin")).toBeInTheDocument();
|
||||
expect(screen.queryByText("classifier_plugin")).not.toBeInTheDocument();
|
||||
});
|
||||
|
||||
// The plugin chose the tier, so pairing a score with a boundary band would claim the heuristic did.
|
||||
it("does not claim the score chose the tier on a plugin-decided row", () => {
|
||||
render(<RoutingDecisionCard decision={{ ...heuristic, cause: "classifier_plugin", score: 0.2 }} />);
|
||||
|
||||
expect(screen.getByText("0.20")).toBeInTheDocument();
|
||||
expect(screen.queryByText(/SIMPLE|MEDIUM|COMPLEX|at or above/)).not.toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("explains a route that fell back to the default model after the classifier failed", () => {
|
||||
// No tier is recorded on this path, so the card must not show a Tier row: nothing
|
||||
// about the request produced one, the classifier never answered.
|
||||
|
|
@ -99,7 +124,7 @@ describe("RoutingDecisionCard", () => {
|
|||
}}
|
||||
/>,
|
||||
);
|
||||
expect(screen.getByText("Default model, LLM classifier failed")).toBeInTheDocument();
|
||||
expect(screen.getByText("Default model, the classifier failed")).toBeInTheDocument();
|
||||
expect(screen.queryByText("Tier")).not.toBeInTheDocument();
|
||||
});
|
||||
|
||||
|
|
@ -116,7 +141,7 @@ describe("RoutingDecisionCard", () => {
|
|||
}}
|
||||
/>,
|
||||
);
|
||||
expect(screen.getByText("Fallback tier, LLM classifier failed")).toBeInTheDocument();
|
||||
expect(screen.getByText("Fallback tier, the classifier failed")).toBeInTheDocument();
|
||||
expect(screen.getByText("SECURITY_REVIEW")).toBeInTheDocument();
|
||||
});
|
||||
|
||||
|
|
|
|||
|
|
@ -75,6 +75,8 @@ function describeCause(decision: RoutingDecision): string {
|
|||
return `Heuristic, ${tierLabel ?? "REASONING"} override (2 or more reasoning markers)`;
|
||||
case "llm_classifier":
|
||||
return classifierModel ? `LLM classifier (${classifierModel})` : "LLM classifier";
|
||||
case "classifier_plugin":
|
||||
return "Custom classifier plugin";
|
||||
case "literal_keyword_match":
|
||||
return matchedKeyword ? `Keyword match: "${matchedKeyword}"` : "Keyword match";
|
||||
case "semantic_keyword_match":
|
||||
|
|
@ -94,9 +96,9 @@ function describeCause(decision: RoutingDecision): string {
|
|||
case "default_fallback":
|
||||
return "Default model, no route matched";
|
||||
case "classifier_fallback":
|
||||
return "Fallback tier, LLM classifier failed";
|
||||
return "Fallback tier, the classifier failed";
|
||||
case "default_model_fallback":
|
||||
return "Default model, LLM classifier failed";
|
||||
return "Default model, the classifier failed";
|
||||
default:
|
||||
return cause ?? "Unknown";
|
||||
}
|
||||
|
|
@ -145,11 +147,14 @@ export function RoutingDecisionCard({
|
|||
tier_boundaries: tierBoundaries,
|
||||
} = decision;
|
||||
|
||||
// On an override row the score did not decide the tier, so showing it against a
|
||||
// boundary would claim something untrue. Keyed off the cause rather than a marker
|
||||
// On an override row, or one a plugin decided, the score did not choose the tier, so showing
|
||||
// it against a boundary would claim something untrue. Keyed off the cause rather than a marker
|
||||
// inside `signals`, which redaction can remove.
|
||||
const scoreExplanation =
|
||||
score !== undefined && decision.cause !== "reasoning_override" && decision.cause !== "plan_mode"
|
||||
score !== undefined &&
|
||||
decision.cause !== "reasoning_override" &&
|
||||
decision.cause !== "plan_mode" &&
|
||||
decision.cause !== "classifier_plugin"
|
||||
? describeScoreAgainstBoundaries(score, tierBoundaries, tierLabel !== undefined)
|
||||
: null;
|
||||
|
||||
|
|
|
|||
51
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
51
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
|
|
@ -807,6 +807,26 @@ export interface paths {
|
|||
patch?: never;
|
||||
trace?: never;
|
||||
};
|
||||
"/auto_router/classifier_plugins": {
|
||||
parameters: {
|
||||
query?: never;
|
||||
header?: never;
|
||||
path?: never;
|
||||
cookie?: never;
|
||||
};
|
||||
/**
|
||||
* List Classifier Plugins
|
||||
* @description Registered classifier plugin names, for the Admin UI's custom classifier picker.
|
||||
*/
|
||||
get: operations["list_classifier_plugins_auto_router_classifier_plugins_get"];
|
||||
put?: never;
|
||||
post?: never;
|
||||
delete?: never;
|
||||
options?: never;
|
||||
head?: never;
|
||||
patch?: never;
|
||||
trace?: never;
|
||||
};
|
||||
"/auto_router/shadow_eval": {
|
||||
parameters: {
|
||||
query?: never;
|
||||
|
|
@ -23592,6 +23612,17 @@ export interface components {
|
|||
*/
|
||||
timeout_ms: number;
|
||||
};
|
||||
/**
|
||||
* ClassifierPluginsListResponse
|
||||
* @description Names from the proxy config's classifier_plugins registry, for the custom classifier picker.
|
||||
*/
|
||||
ClassifierPluginsListResponse: {
|
||||
/**
|
||||
* Classifier Plugins
|
||||
* @description Registered classifier plugin names an auto-router's classifier_plugin may reference
|
||||
*/
|
||||
classifier_plugins: string[];
|
||||
};
|
||||
/**
|
||||
* CloudZeroExportRequest
|
||||
* @description Request model for CloudZero export operations
|
||||
|
|
@ -37806,6 +37837,26 @@ export interface operations {
|
|||
};
|
||||
};
|
||||
};
|
||||
list_classifier_plugins_auto_router_classifier_plugins_get: {
|
||||
parameters: {
|
||||
query?: never;
|
||||
header?: never;
|
||||
path?: never;
|
||||
cookie?: never;
|
||||
};
|
||||
requestBody?: never;
|
||||
responses: {
|
||||
/** @description Successful Response */
|
||||
200: {
|
||||
headers: {
|
||||
[name: string]: unknown;
|
||||
};
|
||||
content: {
|
||||
"application/json": components["schemas"]["ClassifierPluginsListResponse"];
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
list_shadow_eval_jobs_auto_router_shadow_eval_get: {
|
||||
parameters: {
|
||||
query?: {
|
||||
|
|
|
|||
17
ui/litellm-dashboard/tests/mocks/classifierPlugins.ts
Normal file
17
ui/litellm-dashboard/tests/mocks/classifierPlugins.ts
Normal file
|
|
@ -0,0 +1,17 @@
|
|||
import { vi } from "vitest";
|
||||
|
||||
/**
|
||||
* Stubs the proxy's registered classifier plugin names for any test that renders the auto-router tree.
|
||||
* Exported as a vi.fn so a test can override the query state, which is how the empty-registry and
|
||||
* failed-fetch paths are covered.
|
||||
*/
|
||||
export const REGISTERED_CLASSIFIER_PLUGINS = ["spend-aware", "tier-by-team"];
|
||||
|
||||
export const LOADED_CLASSIFIER_PLUGINS_QUERY = {
|
||||
data: REGISTERED_CLASSIFIER_PLUGINS,
|
||||
isPending: false,
|
||||
isError: false,
|
||||
refetch: vi.fn(),
|
||||
};
|
||||
|
||||
export const useClassifierPlugins = vi.fn(() => LOADED_CLASSIFIER_PLUGINS_QUERY);
|
||||
Loading…
Add table
Reference in a new issue