diff --git a/litellm/__init__.py b/litellm/__init__.py index ae0fee11aeb..3e358a5b275 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -358,6 +358,11 @@ blocked_user_list: Optional[Union[str, List]] = None banned_keywords_list: Optional[Union[str, List]] = None llm_guard_mode: Literal["all", "key-specific", "request-specific"] = "all" guardrail_name_config_map: Dict[str, GuardrailItem] = {} +# Complexity-router classifier plugins declared under the proxy config's top-level +# `classifier_plugins` key, keyed by the name the Admin UI selects and saved models +# reference. Populated at proxy startup; ComplexityRouterConfig resolves string +# classifier_plugin values through it. +classifier_plugin_registry: Dict[str, "ClassifierPlugin"] = {} # mutable-ok: populated at proxy startup include_cost_in_streaming_usage: bool = False reasoning_auto_summary: bool = False ### PROMPTS #### @@ -488,6 +493,8 @@ priority_reservation: Optional[Dict[str, Union[float, "PriorityReservationDict"] # priority_reservation_settings is lazy-loaded via __getattr__ # Only declare for type checking - at runtime __getattr__ handles it if TYPE_CHECKING: + from litellm.types.router import ClassifierPlugin + priority_reservation_settings: Optional["PriorityReservationSettings"] = None diff --git a/litellm/proxy/management_endpoints/auto_router_endpoints.py b/litellm/proxy/management_endpoints/auto_router_endpoints.py index d8ef5305ae9..51a6f9d7210 100644 --- a/litellm/proxy/management_endpoints/auto_router_endpoints.py +++ b/litellm/proxy/management_endpoints/auto_router_endpoints.py @@ -39,6 +39,7 @@ from litellm.types.management_endpoints.auto_router_endpoints import ( AutoRouterCacheStats, AutoRouterRoutingTestRequest, AutoRouterRoutingTestResponse, + ClassifierPluginsListResponse, RequestComplexityRouterConfig, ShadowEvalJobResponse, ShadowEvalResult, @@ -369,6 +370,22 @@ def _summed_agg_row(rows: Sequence[_SessionAggRow]) -> _SessionAggRow: ) +@router.get( + "/auto_router/classifier_plugins", + tags=("auto router",), + dependencies=(Depends(user_api_key_auth),), + response_model=ClassifierPluginsListResponse, +) +async def list_classifier_plugins( + user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)], +) -> ClassifierPluginsListResponse: + """Registered classifier plugin names, for the Admin UI's custom classifier picker.""" + import litellm + + _require_admin_viewer(user_api_key_dict, "list classifier plugins") + return ClassifierPluginsListResponse(classifier_plugins=tuple(sorted(litellm.classifier_plugin_registry))) + + @router.get( "/auto_router/benchmarks", tags=("auto router",), diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index d8fe7fce78f..0dc65257877 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -4089,6 +4089,9 @@ def resolve_classifier_plugin( sync `def classify` passes the runtime_checkable isinstance and would only fail on the first classified request, so reject it here where the error names the config key. """ + registered: Final = litellm.classifier_plugin_registry.get(plugin_path) + if registered is not None: + return registered resolved: Final = get_instance_fn(value=plugin_path, config_file_path=config_file_path) if not isinstance(resolved, ClassifierPlugin) or not inspect.iscoroutinefunction( getattr(resolved, "classify", None) @@ -5489,6 +5492,20 @@ class ProxyConfig: # Load vector stores from config litellm.vector_store_registry.load_vector_stores_from_config(vector_store_registry_config) + ## CLASSIFIER PLUGINS (complexity-router custom classifiers, picked by name in the Admin UI) + classifier_plugins_config: Final = config.get("classifier_plugins", None) + if classifier_plugins_config: + if not isinstance(classifier_plugins_config, dict): + raise TypeError("classifier_plugins must map plugin names to dotted paths") + for plugin_name, plugin_path in classifier_plugins_config.items(): + if not isinstance(plugin_path, str): + raise TypeError(f"classifier_plugins.{plugin_name} must be a dotted-path string") + litellm.classifier_plugin_registry[str(plugin_name)] = resolve_classifier_plugin( + plugin_path=plugin_path, + config_file_path=config_file_path, + source_label=f"classifier_plugins.{plugin_name}", + ) + ## WORKER REGISTRY (Global Control Plane) worker_registry_config: Final = config.get("worker_registry", None) if worker_registry_config: diff --git a/litellm/router_strategy/complexity_router/config.py b/litellm/router_strategy/complexity_router/config.py index 6d43199c948..2610fffa117 100644 --- a/litellm/router_strategy/complexity_router/config.py +++ b/litellm/router_strategy/complexity_router/config.py @@ -546,12 +546,29 @@ class ComplexityRouterConfig(BaseModel): classifier_plugin: ClassifierPlugin | None = Field( default=None, description=( - "Custom classifier deciding the tier; required when classifier_type is 'custom'. In the proxy " - "config, a dotted path to a ClassifierPlugin instance (resolved at startup, like plugins). Its " - "classify(context) receives the request messages and metadata (caller identity included) and " + "Custom classifier deciding the tier; required when classifier_type is 'custom'. A name from " + "the proxy config's top-level classifier_plugins registry (what the Admin UI saves), or in the " + "proxy config a dotted path to a ClassifierPlugin instance (resolved at startup, like plugins). " + "Its classify(context) receives the request messages and metadata (caller identity included) and " "returns the name of the tier to route to, or None to decline and let classifier_fallback decide." ), ) + + @field_validator("classifier_plugin", mode="before") + @classmethod + def _resolve_registered_classifier_plugin(cls, value: object) -> object: + if not isinstance(value, str): + return value + import litellm + + registered: Final = litellm.classifier_plugin_registry.get(value) + if registered is None: + raise ValueError( + f"{value!r} is not a registered classifier plugin; declare it under " + "classifier_plugins in the proxy config" + ) + return registered + classifier_plugin_timeout_ms: int = Field( default=3000, gt=0, diff --git a/litellm/types/management_endpoints/auto_router_endpoints.py b/litellm/types/management_endpoints/auto_router_endpoints.py index 9461297feca..b916028164f 100644 --- a/litellm/types/management_endpoints/auto_router_endpoints.py +++ b/litellm/types/management_endpoints/auto_router_endpoints.py @@ -27,6 +27,14 @@ class RequestComplexityRouterConfig(ComplexityRouterConfig): ) +class ClassifierPluginsListResponse(BaseModel): + """Names from the proxy config's classifier_plugins registry, for the custom classifier picker.""" + + classifier_plugins: tuple[str, ...] = Field( + description="Registered classifier plugin names an auto-router's classifier_plugin may reference", + ) + + class AutoRouterRoutingTestRequest(BaseModel): """A single prompt to classify against a complexity-router config that need not be saved yet.""" diff --git a/tests/test_litellm/proxy/management_endpoints/test_auto_router_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_auto_router_endpoints.py index 77149457e82..0d689401e72 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_auto_router_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_auto_router_endpoints.py @@ -293,6 +293,21 @@ def test_classifier_plugin_is_not_settable_over_http(): _request("what is 2+2", classifier_type="custom", classifier_plugin="my_module.instance") +@pytest.mark.asyncio +async def test_list_classifier_plugins_returns_sorted_registry_names(monkeypatch): + import litellm + from litellm.proxy.management_endpoints.auto_router_endpoints import list_classifier_plugins + + class _Classifier: + async def classify(self, context): + return "SIMPLE" + + monkeypatch.setitem(litellm.classifier_plugin_registry, "tier-by-team", _Classifier()) + monkeypatch.setitem(litellm.classifier_plugin_registry, "spend-aware", _Classifier()) + response = await list_classifier_plugins(user_api_key_dict=ADMIN) + assert response.classifier_plugins == ("spend-aware", "tier-by-team") + + class TestAutoRouterBenchmarks: from litellm.proxy.management_endpoints.auto_router_endpoints import _SessionAggRow diff --git a/tests/test_litellm/proxy/proxy_server/test_proxy_config.py b/tests/test_litellm/proxy/proxy_server/test_proxy_config.py index f31f67c317a..1bcee895ffa 100644 --- a/tests/test_litellm/proxy/proxy_server/test_proxy_config.py +++ b/tests/test_litellm/proxy/proxy_server/test_proxy_config.py @@ -246,6 +246,22 @@ def test_resolve_complexity_router_plugins_rejects_synchronous_classify_method(t ) +def test_resolve_classifier_plugin_prefers_the_registry_over_dotted_import(monkeypatch): + import litellm + from litellm.proxy.proxy_server import resolve_classifier_plugin + + class _Classifier: + async def classify(self, context): + return "SIMPLE" + + instance = _Classifier() + monkeypatch.setitem(litellm.classifier_plugin_registry, "tier-by-team", instance) + resolved = resolve_classifier_plugin( + plugin_path="tier-by-team", config_file_path=None, source_label="classifier_plugins.tier-by-team" + ) + assert resolved is instance + + def test_resolve_complexity_router_plugins_leaves_live_classifier_instance_alone(): class _Classifier: async def classify(self, context): diff --git a/tests/test_litellm/router_strategy/test_complexity_router.py b/tests/test_litellm/router_strategy/test_complexity_router.py index e1e8d9553b3..e831d282927 100644 --- a/tests/test_litellm/router_strategy/test_complexity_router.py +++ b/tests/test_litellm/router_strategy/test_complexity_router.py @@ -4233,6 +4233,52 @@ class TestClassifierPluginConfig: ) +class _RegistryClassifier: + async def classify(self, context): + return "COMPLEX" + + +class TestClassifierPluginRegistry: + """String classifier_plugin values resolve through litellm.classifier_plugin_registry.""" + + def test_registered_name_resolves_to_the_instance(self, monkeypatch): + instance = _RegistryClassifier() + monkeypatch.setitem(litellm.classifier_plugin_registry, "tier-by-team", instance) + config = ComplexityRouterConfig( + tiers={"SIMPLE": "gpt-4o-mini"}, classifier_type="custom", classifier_plugin="tier-by-team" + ) + assert config.classifier_plugin is instance + + def test_unknown_name_is_rejected_at_validation(self): + with pytest.raises(ValidationError, match="not a registered classifier plugin"): + ComplexityRouterConfig( + tiers={"SIMPLE": "gpt-4o-mini"}, classifier_type="custom", classifier_plugin="no-such-plugin" + ) + + def test_live_instance_passes_through_untouched(self): + instance = _RegistryClassifier() + config = ComplexityRouterConfig( + tiers={"SIMPLE": "gpt-4o-mini"}, classifier_type="custom", classifier_plugin=instance + ) + assert config.classifier_plugin is instance + + @pytest.mark.asyncio + async def test_registry_named_plugin_classifies_end_to_end(self, mock_router_instance, monkeypatch): + monkeypatch.setitem(litellm.classifier_plugin_registry, "tier-by-team", _RegistryClassifier()) + router = ComplexityRouter( + model_name="test-complexity-router", + litellm_router_instance=mock_router_instance, + complexity_router_config={ + "tiers": {"SIMPLE": "gpt-4o-mini", "COMPLEX": "claude-sonnet-4-20250514"}, + "classifier_type": "custom", + "classifier_plugin": "tier-by-team", + }, + ) + outcome = await router.aclassify("hello") + assert outcome.cause == "classifier_plugin" + assert outcome.tier == ComplexityTier.COMPLEX + + class TestClassifierPlugin: """classifier_type='custom': an operator hook decides the tier.""" diff --git a/tests/test_litellm/router_utils/test_auto_router_model_naming.py b/tests/test_litellm/router_utils/test_auto_router_model_naming.py index 258ef99c6fb..9d8a4c08882 100644 --- a/tests/test_litellm/router_utils/test_auto_router_model_naming.py +++ b/tests/test_litellm/router_utils/test_auto_router_model_naming.py @@ -138,6 +138,38 @@ def test_validate_rejects_ambiguous_tier_labels(tier_labels, expected_fragment): assert expected_fragment in violation +def test_validate_rejects_an_unregistered_classifier_plugin_name(): + """A stored name the registry does not hold must be refused at write time, not at load.""" + violation = validate_complexity_router_config_write( + complexity_router_config={ + "tiers": VALID_TIERS, + "classifier_type": "custom", + "classifier_plugin": "no-such-plugin", + } + ) + assert violation is not None + assert "complexity_router_config is invalid" in violation + assert "not a registered classifier plugin" in violation + + +def test_validate_accepts_a_registered_classifier_plugin_name(monkeypatch): + import litellm + + class _Classifier: + async def classify(self, context): + return "SIMPLE" + + monkeypatch.setitem(litellm.classifier_plugin_registry, "tier-by-team", _Classifier()) + violation = validate_complexity_router_config_write( + complexity_router_config={ + "tiers": VALID_TIERS, + "classifier_type": "custom", + "classifier_plugin": "tier-by-team", + } + ) + assert violation is None + + @pytest.mark.parametrize( "complexity_router_config", [ diff --git a/ui/litellm-dashboard/src/app/(dashboard)/hooks/autoRouter/useClassifierPlugins.ts b/ui/litellm-dashboard/src/app/(dashboard)/hooks/autoRouter/useClassifierPlugins.ts new file mode 100644 index 00000000000..2472a2628c7 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/hooks/autoRouter/useClassifierPlugins.ts @@ -0,0 +1,13 @@ +import { $api } from "@/lib/http/api"; + +export const useClassifierPlugins = () => + $api.useQuery( + "get", + "/auto_router/classifier_plugins", + {}, + { + // Registration is config-file-only, so the list changes only on a proxy reload. + staleTime: 5 * 60 * 1000, + select: (data) => data.classifier_plugins, + }, + ); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/AutoRouters/autoRouterRows.test.ts b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/AutoRouters/autoRouterRows.test.ts index 9944653b638..3e682ea6c79 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/AutoRouters/autoRouterRows.test.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/AutoRouters/autoRouterRows.test.ts @@ -99,6 +99,23 @@ describe("autoRouterRows", () => { expect(row.typeLabel).toBe("LLM Classifier"); }); + it("labels a router using a custom classifier plugin", () => { + const row = toAutoRouterRow( + { + ...complexityDeployment, + litellm_params: { + ...complexityDeployment.litellm_params, + complexity_router_config: { tiers: {}, classifier_type: "custom", classifier_plugin: "tier-by-team" }, + }, + }, + 0, + ADMIN, + null, + ); + + expect(row.typeLabel).toBe("Custom classifier"); + }); + it("treats a deployment carrying complexity_router_config as complexity even off the canonical model string", () => { expect(isComplexityRouter({ model: "auto_router/legacy", complexity_router_config: { tiers: {} } })).toBe(true); }); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/AutoRouters/autoRouterRows.ts b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/AutoRouters/autoRouterRows.ts index 35172d67e84..0ff56f179cf 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/AutoRouters/autoRouterRows.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/AutoRouters/autoRouterRows.ts @@ -54,8 +54,11 @@ const asStringArray = (value: unknown): string[] => const dedupe = (models: string[]): string[] => Array.from(new Set(models)); -export const complexityTypeLabel = (config: Record): string => - config.classifier_type === "llm" ? "LLM Classifier" : "Heuristic"; +export const complexityTypeLabel = (config: Record): string => { + if (config.classifier_type === "llm") return "LLM Classifier"; + if (config.classifier_type === "custom") return "Custom classifier"; + return "Heuristic"; +}; interface Presentation { typeLabel: string; diff --git a/ui/litellm-dashboard/src/components/add_model/ClassificationMethodConfig.test.tsx b/ui/litellm-dashboard/src/components/add_model/ClassificationMethodConfig.test.tsx new file mode 100644 index 00000000000..5527c0cb714 --- /dev/null +++ b/ui/litellm-dashboard/src/components/add_model/ClassificationMethodConfig.test.tsx @@ -0,0 +1,167 @@ +import { fireEvent, renderWithProviders, screen } from "../../../tests/test-utils"; +import userEvent from "@testing-library/user-event"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { useClassifierPlugins } from "@/app/(dashboard)/hooks/autoRouter/useClassifierPlugins"; +import ClassificationMethodConfig from "./ClassificationMethodConfig"; +import { ComplexityRouterConfigValue } from "./ComplexityRouterConfig"; +import { LOADED_CLASSIFIER_PLUGINS_QUERY } from "../../../tests/mocks/classifierPlugins"; + +vi.mock( + "@/app/(dashboard)/hooks/autoRouter/useComplexityScorerDefaults", + async () => await import("../../../tests/mocks/complexityScorerDefaults"), +); + +vi.mock( + "@/app/(dashboard)/hooks/autoRouter/useClassifierPlugins", + async () => await import("../../../tests/mocks/classifierPlugins"), +); + +const BASE: ComplexityRouterConfigValue = { + tiers: { SIMPLE: ["gpt-4o-mini"], MEDIUM: ["gpt-4o"], COMPLEX: ["o3"], REASONING: ["o3"] }, + classifier_type: "heuristic", +}; + +const CUSTOM: ComplexityRouterConfigValue = { + ...BASE, + classifier_type: "custom", + classifier_plugin: "tier-by-team", + classifier_plugin_timeout_ms: 3000, +}; + +const MODEL_OPTIONS = [{ value: "gpt-4o-mini", label: "gpt-4o-mini" }]; + +const render = (value: ComplexityRouterConfigValue, onChange = vi.fn()) => { + renderWithProviders( + , + ); + return onChange; +}; + +const lastValue = (onChange: ReturnType) => + onChange.mock.calls.at(-1)?.[0] as ComplexityRouterConfigValue | undefined; + +describe("ClassificationMethodConfig classification method radios", () => { + it("offers the custom classifier alongside the heuristic and the LLM classifier", () => { + render(BASE); + + expect(screen.getByRole("radio", { name: /Heuristic/ })).toBeInTheDocument(); + expect(screen.getByRole("radio", { name: /LLM Classifier/ })).toBeInTheDocument(); + expect(screen.getByRole("radio", { name: /Custom classifier/ })).toBeEnabled(); + }); + + // Only the custom branch may carry a plugin: leaving one behind would have the backend reject the + // save outright (classifier_plugin set with a non-custom classifier_type). + it("clears the plugin when the operator switches back to the heuristic", async () => { + const onChange = render(CUSTOM); + + await userEvent.click(screen.getByRole("radio", { name: /Heuristic/ })); + + expect(lastValue(onChange)).toMatchObject({ + classifier_type: "heuristic", + classifier_plugin: undefined, + classifier_plugin_timeout_ms: undefined, + classifier_fallback: undefined, + }); + }); + + it("stamps the default plugin timeout when custom is selected", async () => { + const onChange = render(BASE); + + await userEvent.click(screen.getByRole("radio", { name: /Custom classifier/ })); + + expect(lastValue(onChange)).toMatchObject({ classifier_type: "custom", classifier_plugin_timeout_ms: 3000 }); + }); +}); + +describe("ClassificationMethodConfig custom classifier controls", () => { + it("does not show the plugin controls until custom is the selected method", () => { + render(BASE); + + expect(screen.queryByRole("combobox", { name: "Classifier Plugin" })).not.toBeInTheDocument(); + }); + + it("offers the names the proxy registered", async () => { + render({ ...CUSTOM, classifier_plugin: undefined }); + + fireEvent.mouseDown(screen.getByRole("combobox", { name: "Classifier Plugin" })); + + expect(await screen.findByTitle("tier-by-team")).toBeInTheDocument(); + expect(screen.getByTitle("spend-aware")).toBeInTheDocument(); + }); + + it("records the plugin the operator picks", async () => { + const onChange = render({ ...CUSTOM, classifier_plugin: undefined }); + + fireEvent.mouseDown(screen.getByRole("combobox", { name: "Classifier Plugin" })); + await userEvent.click(await screen.findByTitle("spend-aware")); + + expect(lastValue(onChange)).toMatchObject({ classifier_plugin: "spend-aware", classifier_plugin_timeout_ms: 3000 }); + }); + + it("emits an edited plugin timeout", () => { + const onChange = render(CUSTOM); + + fireEvent.change(screen.getByRole("spinbutton", { name: "Plugin Timeout (ms)" }), { target: { value: "750" } }); + + expect(lastValue(onChange)).toMatchObject({ classifier_plugin_timeout_ms: 750 }); + }); + + // The backend applies classifier_fallback to a plugin exactly as it does to an LLM classifier, so + // gating this picker on the LLM branch would leave the plugin's failure path unconfigurable. + it("offers the fallback picker, which is not LLM-only", () => { + render(CUSTOM); + + expect(screen.getByRole("radio", { name: /Score with the heuristic/ })).toBeInTheDocument(); + expect(screen.getByRole("radio", { name: /Route to the default model/ })).toBeInTheDocument(); + }); + + it("says the score no longer decides the tier under a plugin", () => { + render(CUSTOM); + + expect(screen.getByText(/classifies with your own classifier plugin/)).toBeInTheDocument(); + }); + + it("reports a missing plugin once the form has been submitted", () => { + renderWithProviders( + , + ); + + expect(screen.getByText("A classifier plugin is required")).toBeInTheDocument(); + }); +}); + +describe("ClassificationMethodConfig when the proxy registered no plugins", () => { + const empty = { data: [], isPending: false, isError: false, refetch: vi.fn() }; + const failing = { data: undefined, isPending: false, isError: true, refetch: vi.fn() }; + + afterEach(() => vi.mocked(useClassifierPlugins).mockReturnValue(LOADED_CLASSIFIER_PLUGINS_QUERY)); + + it("disables the custom radio and says how to enable it", () => { + vi.mocked(useClassifierPlugins).mockReturnValue(empty as never); + render(BASE); + + expect(screen.getByRole("radio", { name: /Custom classifier/ })).toBeDisabled(); + expect( + screen.getByText("Declare classifier_plugins in the proxy config to enable custom classifiers"), + ).toBeInTheDocument(); + }); + + // A fetch that never landed leaves the registry unknown, not empty. Blanking the select or greying + // out the radio here would clear an already-configured plugin on the operator's next save. + it("keeps a stored plugin name selectable when the fetch failed", () => { + vi.mocked(useClassifierPlugins).mockReturnValue(failing as never); + render(CUSTOM); + + expect(screen.getByRole("radio", { name: /Custom classifier/ })).toBeEnabled(); + expect(screen.getByRole("combobox", { name: "Classifier Plugin" })).toBeInTheDocument(); + expect(screen.getByTitle("tier-by-team")).toBeInTheDocument(); + expect( + screen.queryByText("Declare classifier_plugins in the proxy config to enable custom classifiers"), + ).not.toBeInTheDocument(); + }); +}); diff --git a/ui/litellm-dashboard/src/components/add_model/ClassificationMethodConfig.tsx b/ui/litellm-dashboard/src/components/add_model/ClassificationMethodConfig.tsx index 03d8fbc2394..aa40c8f3315 100644 --- a/ui/litellm-dashboard/src/components/add_model/ClassificationMethodConfig.tsx +++ b/ui/litellm-dashboard/src/components/add_model/ClassificationMethodConfig.tsx @@ -3,6 +3,7 @@ import { Select as AntdSelect, Card, InputNumber, Radio, Space, Switch, Tooltip, import React from "react"; import ClassifierPromptEditor from "./ClassifierPromptEditor"; import HeuristicScoringConfig from "./HeuristicScoringConfig"; +import { useClassifierPlugins } from "@/app/(dashboard)/hooks/autoRouter/useClassifierPlugins"; import { useComplexityScorerDefaults } from "@/app/(dashboard)/hooks/autoRouter/useComplexityScorerDefaults"; import { ClassifierFallback, @@ -11,6 +12,7 @@ import { DEFAULT_CLASSIFIER_CONTEXT_PER_TURN_CHARS, DEFAULT_CLASSIFIER_CONTEXT_WINDOW_SIZE, DEFAULT_CLASSIFIER_FALLBACK, + DEFAULT_CLASSIFIER_PLUGIN_TIMEOUT_MS, DEFAULT_CLASSIFIER_TIMEOUT_MS, DEFAULT_CLASSIFICATION_RUBRIC, NEW_CLASSIFIER_CLASSIFICATION_RUBRIC, @@ -35,18 +37,29 @@ const CUSTOM_PROMPT_WITH_DEFAULT_MODEL_FALLBACK = "names stay fixed. The scoring below no longer runs at all, since a failed classifier routes to the default " + "model instead:"; +const PLUGIN_WITH_HEURISTIC_FALLBACK = + "This router classifies with your own classifier plugin, so the tier comes from whatever the plugin decides. The " + + "four tier names stay fixed. The scoring below is the heuristic, which now runs only when the plugin fails:"; + +const PLUGIN_WITH_DEFAULT_MODEL_FALLBACK = + "This router classifies with your own classifier plugin, so the tier comes from whatever the plugin decides. The " + + "four tier names stay fixed. The scoring below no longer runs at all, since a failed plugin routes to the default " + + "model instead:"; + /** - * What the scoring breakdown below it actually describes. A custom prompt means the score no longer - * decides the tier, and pairing one with the default-model fallback means the heuristic never runs - * at all, so the panel must not keep implying a score is involved on either router. + * What the scoring breakdown below it actually describes. A custom prompt or a classifier plugin means + * the score no longer decides the tier, and pairing either with the default-model fallback means the + * heuristic never runs at all, so the panel must not keep implying a score is involved. */ const scoringExplanation = (value: ComplexityRouterConfigValue): string => { + const fallsBackToDefaultModel = value.classifier_fallback === "default_model"; + if (value.classifier_type === "custom") { + return fallsBackToDefaultModel ? PLUGIN_WITH_DEFAULT_MODEL_FALLBACK : PLUGIN_WITH_HEURISTIC_FALLBACK; + } const usesCustomPrompt = value.classifier_type === "llm" && Boolean(value.classifier_llm_config?.system_prompt?.trim()); if (!usesCustomPrompt) return DEFAULT_SCORING_EXPLANATION; - return value.classifier_fallback === "default_model" - ? CUSTOM_PROMPT_WITH_DEFAULT_MODEL_FALLBACK - : CUSTOM_PROMPT_WITH_HEURISTIC_FALLBACK; + return fallsBackToDefaultModel ? CUSTOM_PROMPT_WITH_DEFAULT_MODEL_FALLBACK : CUSTOM_PROMPT_WITH_HEURISTIC_FALLBACK; }; /** @@ -105,6 +118,43 @@ const HowClassificationWorks: React.FC<{ value: ComplexityRouterConfigValue }> = ); }; +const ClassifierFallbackPicker: React.FC<{ + fallback: ClassifierFallback | undefined; + onChange: (fallback: ClassifierFallback) => void; + defaultModel: string | undefined; +}> = ({ fallback, onChange, defaultModel }) => ( +
+ + If the classifier fails + + onChange(e.target.value)}> + + + Score with the heuristic{" "} + — right when the classifier grades complexity too + + + + + Route to the default model{defaultModel ? ` (${defaultModel})` : ""}{" "} + — right when your classifier grades something other than complexity + + + + + + + Applies when the classifier call errors, times out, or returns an unparseable response. + +
+); + interface ClassificationMethodConfigProps { value: ComplexityRouterConfigValue; onChange: (value: ComplexityRouterConfigValue) => void; @@ -125,12 +175,21 @@ const ClassificationMethodConfig: React.FC = ({ showValidationErrors = false, defaultModel, }) => { - const hasDefaultModel = Boolean(defaultModel); const classifierModelMissing = showValidationErrors && value.classifier_type === "llm" && !value.classifier_llm_config?.model; + const classifierPluginMissing = + showValidationErrors && value.classifier_type === "custom" && !value.classifier_plugin; const usesCustomPrompt = Boolean(value.classifier_llm_config?.system_prompt?.trim()); const classificationRubric = value.classifier_llm_config?.classification_rubric ?? DEFAULT_CLASSIFICATION_RUBRIC; + const { data: registeredPlugins, isError: pluginsError } = useClassifierPlugins(); + // A failed fetch leaves the registry unknown, not empty: offering the stored name anyway is what keeps + // opening this form on an existing custom router from silently clearing its plugin on the next save. + const registryIsEmpty = !pluginsError && registeredPlugins !== undefined && registeredPlugins.length === 0; + const pluginOptions = Array.from( + new Set([...(registeredPlugins ?? []), ...(value.classifier_plugin ? [value.classifier_plugin] : [])]), + ).map((name) => ({ value: name, label: name })); + const handleClassifierTypeChange = (classifierType: ClassifierType) => { const nextValue: ComplexityRouterConfigValue = { ...value, @@ -153,7 +212,12 @@ const ClassificationMethodConfig: React.FC = ({ : undefined, classifier_context_include_assistant_turns: classifierType === "llm" ? value.classifier_context_include_assistant_turns : undefined, - classifier_fallback: classifierType === "llm" ? value.classifier_fallback : undefined, + classifier_fallback: classifierType === "heuristic" ? undefined : value.classifier_fallback, + classifier_plugin: classifierType === "custom" ? value.classifier_plugin : undefined, + classifier_plugin_timeout_ms: + classifierType === "custom" + ? value.classifier_plugin_timeout_ms ?? DEFAULT_CLASSIFIER_PLUGIN_TIMEOUT_MS + : undefined, }; onChange(nextValue); }; @@ -180,6 +244,18 @@ const ClassificationMethodConfig: React.FC = ({ }); }; + const handleClassifierPluginChange = (classifierPlugin: string) => { + onChange({ + ...value, + classifier_plugin: classifierPlugin, + classifier_plugin_timeout_ms: value.classifier_plugin_timeout_ms ?? DEFAULT_CLASSIFIER_PLUGIN_TIMEOUT_MS, + }); + }; + + const handleClassifierPluginTimeoutChange = (timeoutMs: number | null) => { + onChange({ ...value, classifier_plugin_timeout_ms: timeoutMs ?? DEFAULT_CLASSIFIER_PLUGIN_TIMEOUT_MS }); + }; + const handleClassificationRubricChange = (classificationRubric: ClassificationRubric) => { onChange({ ...value, @@ -245,6 +321,15 @@ const ClassificationMethodConfig: React.FC = ({ LLM Classifier{" "} — use a model to decide the tier (e.g. a small/fast model) + + Custom classifier{" "} + — let a classifier plugin registered in the proxy config decide the tier + + {registryIsEmpty && ( + + Declare classifier_plugins in the proxy config to enable custom classifiers + + )} @@ -321,39 +406,11 @@ const ClassificationMethodConfig: React.FC = ({ classificationRubric={classificationRubric} /> -
- - If the classifier fails - - handleClassifierFallbackChange(e.target.value)} - > - - - Score with the heuristic{" "} - — right when the classifier grades complexity too - - - - - Route to the default model{defaultModel ? ` (${defaultModel})` : ""}{" "} - — right when your prompt grades something other than complexity - - - - - - - Applies when the classifier call errors, times out, or returns an unparseable response. - -
+
Context Window Size @@ -407,6 +464,56 @@ const ClassificationMethodConfig: React.FC = ({
)} + {value.classifier_type === "custom" && ( +
+
+ + Classifier Plugin + + + {classifierPluginMissing && ( + + A classifier plugin is required + + )} + {pluginsError && ( + + The registered plugin names could not be loaded from the proxy. + + )} +
+
+ + Plugin Timeout (ms) + + + + How long the plugin has to classify before it fails and the fallback below takes over. + +
+ +
+ )} + {value.classifier_type === "heuristic" && (
diff --git a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx index 3d4845ecf39..eceb04a7d25 100644 --- a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx +++ b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx @@ -15,6 +15,7 @@ export type { DimensionWeights, TierBoundaries, TokenThresholds }; const { Text } = Typography; export const DEFAULT_CLASSIFIER_TIMEOUT_MS = 3000; +export const DEFAULT_CLASSIFIER_PLUGIN_TIMEOUT_MS = 3000; export const DEFAULT_TIER_DISTANCE_PENALTY = 0.5; export const DEFAULT_CLASSIFIER_CONTEXT_WINDOW_SIZE = 3; export const DEFAULT_CLASSIFIER_CONTEXT_PER_TURN_CHARS = 200; @@ -73,7 +74,7 @@ export interface ClassifierLLMConfig { system_prompt?: string; } -export type ClassifierType = "heuristic" | "llm"; +export type ClassifierType = "heuristic" | "llm" | "custom"; export type ClassifierFallback = "heuristic" | "default_model"; @@ -90,8 +91,8 @@ export type HeuristicScoringRole = "decides" | "fallback_only" | "never"; /** * Whether the heuristic scorer runs on this router at all, which is what gates its knobs. An LLM - * classifier still falls back to the scorer unless the fallback is the default model, so the gate cannot be - * a plain classifier_type check. + * classifier or a classifier plugin still falls back to the scorer unless the fallback is the default + * model, so the gate cannot be a plain classifier_type check. */ export const heuristicScoringRoleFor = ( classifierType: ClassifierType, @@ -115,6 +116,9 @@ export interface ComplexityRouterConfigValue { default_model?: string; classifier_type: ClassifierType; classifier_llm_config?: ClassifierLLMConfig; + /** A name from the proxy config's classifier_plugins registry. Required when classifier_type is "custom". */ + classifier_plugin?: string; + classifier_plugin_timeout_ms?: number; classifier_context_window_size?: number; classifier_context_per_turn_chars?: number; classifier_context_include_assistant_turns?: boolean; diff --git a/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.test.tsx b/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.test.tsx index 5643dafc0f5..ae6f7805869 100644 --- a/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.test.tsx +++ b/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.test.tsx @@ -13,6 +13,11 @@ vi.mock( async () => await import("../../../tests/mocks/complexityScorerDefaults"), ); +vi.mock( + "@/app/(dashboard)/hooks/autoRouter/useClassifierPlugins", + async () => await import("../../../tests/mocks/classifierPlugins"), +); + const ANTHROPIC_PRESET = getPresetByKey("anthropic_family")!; const ANTHROPIC_TIERS = ANTHROPIC_PRESET.complexity_router_config.tiers; @@ -285,6 +290,50 @@ describe("AddAutoRouterTab", () => { }); }); + it("carries a selected classifier plugin and its timeout through to the create payload", async () => { + const user = userEvent.setup(); + vi.mocked(getMissingTiersError).mockReturnValue(null); + + renderWithProviders(); + + await user.type(screen.getByPlaceholderText(/smart_router/i), "plugin-router"); + expandDetailedConfiguration(); + await user.click(screen.getByText("Advanced: Classification Method")); + await user.click(await screen.findByRole("radio", { name: /Custom classifier/ })); + fireEvent.mouseDown(await screen.findByRole("combobox", { name: "Classifier Plugin" })); + await user.click(await screen.findByTitle("tier-by-team")); + + await user.click(screen.getByRole("button", { name: /add auto router/i })); + + await waitFor(() => expect(handleAddAutoRouterSubmit).toHaveBeenCalled()); + expect(vi.mocked(handleAddAutoRouterSubmit).mock.calls.at(-1)?.[0].complexity_router_config).toMatchObject({ + classifier_type: "custom", + classifier_plugin: "tier-by-team", + classifier_plugin_timeout_ms: 3000, + }); + }); + + // The backend rejects classifier_type "custom" with no classifier_plugin, so the form has to say + // what is missing rather than let the save come back as a raw 400. + it("blocks a submit that selects the custom classifier without a plugin", async () => { + const user = userEvent.setup(); + vi.mocked(getMissingTiersError).mockReturnValue(null); + + renderWithProviders(); + + await user.type(screen.getByPlaceholderText(/smart_router/i), "plugin-router"); + expandDetailedConfiguration(); + await user.click(screen.getByText("Advanced: Classification Method")); + await user.click(await screen.findByRole("radio", { name: /Custom classifier/ })); + + await user.click(screen.getByRole("button", { name: /add auto router/i })); + + await waitFor(() => + expect(toast.fromError).toHaveBeenCalledWith("Please select a classifier plugin, or switch back to Heuristic"), + ); + expect(handleAddAutoRouterSubmit).not.toHaveBeenCalled(); + }); + it("carries session affinity turned on through to the create payload", async () => { const user = userEvent.setup(); vi.mocked(getMissingTiersError).mockReturnValue(null); diff --git a/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx b/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx index e35b9581d9c..7008ba89fcb 100644 --- a/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx +++ b/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx @@ -341,6 +341,8 @@ const AddAutoRouterTab: React.FC = ({ tierLabels: complexityRouterConfig.tier_labels, classifierType: complexityRouterConfig.classifier_type, classifierLlmConfig: complexityRouterConfig.classifier_llm_config, + classifierPlugin: complexityRouterConfig.classifier_plugin, + classifierPluginTimeoutMs: complexityRouterConfig.classifier_plugin_timeout_ms, classifierContextWindowSize: complexityRouterConfig.classifier_context_window_size, classifierContextPerTurnChars: complexityRouterConfig.classifier_context_per_turn_chars, classifierContextIncludeAssistantTurns: complexityRouterConfig.classifier_context_include_assistant_turns, @@ -364,7 +366,7 @@ const AddAutoRouterTab: React.FC = ({ }; const submitRecommendedRouter = async (name: string) => { - const { tiers, tierLabels, classifierType, classifierLlmConfig } = complexityRouterConfigParams; + const { tiers, tierLabels, classifierType, classifierLlmConfig, classifierPlugin } = complexityRouterConfigParams; const missingTiersError = getMissingTiersError(tiers); if (missingTiersError) { @@ -386,6 +388,12 @@ const AddAutoRouterTab: React.FC = ({ return; } + if (classifierType === "custom" && !classifierPlugin) { + setShowValidationErrors(true); + toast.fromError("Please select a classifier plugin, or switch back to Heuristic"); + return; + } + const keywordRulesError = getKeywordTierRulesError(keywordTierRules); if (keywordRulesError) { setShowValidationErrors(true); diff --git a/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.test.ts b/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.test.ts index 81d54a7b773..de222739d20 100644 --- a/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.test.ts +++ b/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.test.ts @@ -22,6 +22,8 @@ const baseParams: BuildComplexityRouterConfigParams = { tierLabels: undefined, classifierType: "heuristic", classifierLlmConfig: undefined, + classifierPlugin: undefined, + classifierPluginTimeoutMs: undefined, classifierContextWindowSize: undefined, classifierContextPerTurnChars: undefined, classifierContextIncludeAssistantTurns: undefined, @@ -93,6 +95,86 @@ describe("buildComplexityRouterConfig", () => { expect(config.classifier_llm_config).toBeUndefined(); }); + it("includes the plugin and its timeout only when classifier_type is custom", () => { + const config = buildComplexityRouterConfig({ + ...baseParams, + classifierType: "custom", + classifierPlugin: "tier-by-team", + classifierPluginTimeoutMs: 1500, + }); + + expect(config.classifier_type).toBe("custom"); + expect(config.classifier_plugin).toBe("tier-by-team"); + expect(config.classifier_plugin_timeout_ms).toBe(1500); + expect(config.classifier_llm_config).toBeUndefined(); + }); + + // The backend rejects classifier_plugin alongside a non-custom classifier_type, so a selection + // left on the form after the operator switched away must never reach the wire. + it("omits the plugin and its timeout when classifier_type is heuristic even if they linger in state", () => { + const config = buildComplexityRouterConfig({ + ...baseParams, + classifierPlugin: "tier-by-team", + classifierPluginTimeoutMs: 1500, + }); + + expect(config.classifier_plugin).toBeUndefined(); + expect(config.classifier_plugin_timeout_ms).toBeUndefined(); + }); + + it("omits the plugin and its timeout when classifier_type is llm even if they linger in state", () => { + const config = buildComplexityRouterConfig({ + ...baseParams, + classifierType: "llm", + classifierLlmConfig: { model: "gpt-4o-mini", timeout_ms: 3000 }, + classifierPlugin: "tier-by-team", + classifierPluginTimeoutMs: 1500, + }); + + expect(config.classifier_plugin).toBeUndefined(); + expect(config.classifier_plugin_timeout_ms).toBeUndefined(); + }); + + // The plugin's failure path is configurable exactly like the LLM classifier's, so the fallback + // gate is "not the heuristic" rather than "llm". + it("emits classifier_fallback for a custom classifier as well as an llm one", () => { + const custom = buildComplexityRouterConfig({ + ...baseParams, + classifierType: "custom", + classifierPlugin: "tier-by-team", + classifierFallback: "default_model", + }); + const llm = buildComplexityRouterConfig({ + ...baseParams, + classifierType: "llm", + classifierLlmConfig: { model: "gpt-4o-mini", timeout_ms: 3000 }, + classifierFallback: "default_model", + }); + + expect(custom.classifier_fallback).toBe("default_model"); + expect(llm.classifier_fallback).toBe("default_model"); + }); + + it("omits classifier_fallback for the heuristic, which has nothing to fall back from", () => { + const config = buildComplexityRouterConfig({ ...baseParams, classifierFallback: "default_model" }); + + expect(config.classifier_fallback).toBeUndefined(); + }); + + // heuristicScoringRoleFor keys off the fallback, not the classifier type, so a plugin routing its + // failures to the default model must drop the scorer knobs the same way an LLM classifier does. + it("drops the scorer knobs on a custom classifier that never scores", () => { + const config = buildComplexityRouterConfig({ + ...baseParams, + classifierType: "custom", + classifierPlugin: "tier-by-team", + classifierFallback: "default_model", + tokenThresholds: { simple: 25, complex: 900 }, + }); + + expect(config.token_thresholds).toBeUndefined(); + }); + it("includes classifier_context_window_size and classifier_context_per_turn_chars only when classifier_type is llm", () => { const params: BuildComplexityRouterConfigParams = { ...baseParams, diff --git a/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.ts b/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.ts index 0ab2db8c16f..0b44f60f1d9 100644 --- a/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.ts +++ b/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.ts @@ -75,6 +75,8 @@ export interface BuildComplexityRouterConfigParams { tierLabels: ComplexityTierLabels | undefined; classifierType: ClassifierType; classifierLlmConfig: ClassifierLLMConfig | undefined; + classifierPlugin: string | undefined; + classifierPluginTimeoutMs: number | undefined; classifierContextWindowSize: number | undefined; classifierContextPerTurnChars: number | undefined; classifierContextIncludeAssistantTurns: boolean | undefined; @@ -104,6 +106,8 @@ export interface ComplexityRouterConfigPayload { tier_labels?: ComplexityTierLabels; classifier_type: ClassifierType; classifier_llm_config?: ClassifierLLMConfig; + classifier_plugin?: string; + classifier_plugin_timeout_ms?: number; classifier_context_window_size?: number; classifier_context_per_turn_chars?: number; classifier_context_include_assistant_turns?: boolean; @@ -210,6 +214,8 @@ export const buildComplexityRouterConfig = ({ tierLabels, classifierType, classifierLlmConfig, + classifierPlugin, + classifierPluginTimeoutMs, classifierContextWindowSize, classifierContextPerTurnChars, classifierContextIncludeAssistantTurns, @@ -245,7 +251,11 @@ export const buildComplexityRouterConfig = ({ classifier_type: classifierType, ...(classifierType === "llm" && classifierLlmConfig && { classifier_llm_config: normalizeClassifierLlmConfig(classifierLlmConfig) }), - ...(classifierType === "llm" && classifierFallback !== undefined && { classifier_fallback: classifierFallback }), + ...(classifierType === "custom" && classifierPlugin && { classifier_plugin: classifierPlugin }), + ...(classifierType === "custom" && + classifierPluginTimeoutMs !== undefined && { classifier_plugin_timeout_ms: classifierPluginTimeoutMs }), + ...(classifierType !== "heuristic" && + classifierFallback !== undefined && { classifier_fallback: classifierFallback }), ...(classifierType === "llm" && classifierContextWindowSize !== undefined && { classifier_context_window_size: classifierContextWindowSize, diff --git a/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.test.ts b/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.test.ts index 027e01a9351..a5683be5ae9 100644 --- a/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.test.ts +++ b/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.test.ts @@ -106,6 +106,40 @@ describe("buildUpdatedComplexityRouterConfig", () => { expect(updatedConfig.custom_technical_keywords).toBeUndefined(); }); + it("writes the plugin and its timeout for a custom classifier", () => { + const updatedConfig = buildUpdatedComplexityRouterConfig(storedConfig, { + tiers, + classifier_type: "custom" as const, + classifier_plugin: "tier-by-team", + classifier_plugin_timeout_ms: 1500, + classifier_fallback: "default_model" as const, + }); + + expect(updatedConfig).toMatchObject({ + classifier_type: "custom", + classifier_plugin: "tier-by-team", + classifier_plugin_timeout_ms: 1500, + classifier_fallback: "default_model", + }); + expect(updatedConfig.classifier_llm_config).toBeUndefined(); + }); + + // Both keys are managed, so switching off custom has to remove them: the backend rejects a stored + // classifier_plugin sitting next to a non-custom classifier_type. + it("removes a stored plugin and its timeout when the classifier is no longer custom", () => { + const storedCustom = JSON.stringify({ + ...storedConfigValue, + classifier_type: "custom", + classifier_plugin: "tier-by-team", + classifier_plugin_timeout_ms: 1500, + }); + + const updatedConfig = buildUpdatedComplexityRouterConfig(storedCustom, classifiedTierValue); + + expect(updatedConfig.classifier_plugin).toBeUndefined(); + expect(updatedConfig.classifier_plugin_timeout_ms).toBeUndefined(); + }); + it("preserves a tier configured with more than one model as a pool", () => { const multiModelValue = { ...classifiedTierValue, diff --git a/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.test.tsx b/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.test.tsx index 9702fa85db0..4f7b6057471 100644 --- a/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.test.tsx +++ b/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.test.tsx @@ -10,6 +10,11 @@ vi.mock( async () => await import("../../../tests/mocks/complexityScorerDefaults"), ); +vi.mock( + "@/app/(dashboard)/hooks/autoRouter/useClassifierPlugins", + async () => await import("../../../tests/mocks/classifierPlugins"), +); + const { modelPatchUpdateCall, modelAvailableCall, getAutoRouterClassifierDefaultPromptCall } = vi.hoisted(() => ({ modelPatchUpdateCall: vi.fn().mockResolvedValue({}), modelAvailableCall: vi.fn().mockResolvedValue({ data: [] }), @@ -484,6 +489,93 @@ describe("EditAutoRouterModal deployment affinity", () => { }); }); +describe("EditAutoRouterModal custom classifier plugin", () => { + beforeEach(() => { + modelPatchUpdateCall.mockClear(); + }); + + const STORED_PLUGIN_CONFIG = { + tiers: { SIMPLE: ["gpt-4o-mini"], MEDIUM: ["gpt-4o-mini"], COMPLEX: ["gpt-4o-mini"], REASONING: ["gpt-4o-mini"] }, + classifier_type: "custom", + classifier_plugin: "tier-by-team", + classifier_plugin_timeout_ms: 1500, + classifier_fallback: "heuristic", + }; + + const renderPluginModal = () => + renderWithProviders( + , + ); + + // Every one of these keys is rewritten from form state on save, so a missing hydration line would + // silently wipe the operator's plugin the first time they opened this modal for anything else. + it("preserves a stored plugin, its timeout, and its fallback through an untouched open-and-save", async () => { + const user = userEvent.setup(); + renderPluginModal(); + + await user.click(await screen.findByText("Advanced: Classification Method")); + expect(await screen.findByRole("combobox", { name: "Classifier Plugin" })).toBeInTheDocument(); + expect(screen.getByRole("spinbutton", { name: "Plugin Timeout (ms)" })).toHaveValue("1500"); + + await user.click(screen.getByRole("button", { name: /save changes/i })); + + await waitFor(() => expect(modelPatchUpdateCall).toHaveBeenCalled()); + const config = savedConfig(); + expect(config.classifier_type).toBe("custom"); + expect(config.classifier_plugin).toBe("tier-by-team"); + expect(config.classifier_plugin_timeout_ms).toBe(1500); + expect(config.classifier_fallback).toBe("heuristic"); + }); + + it("persists a switch to another registered plugin", async () => { + const user = userEvent.setup(); + renderPluginModal(); + + await user.click(await screen.findByText("Advanced: Classification Method")); + fireEvent.mouseDown(await screen.findByRole("combobox", { name: "Classifier Plugin" })); + await user.click(await screen.findByTitle("spend-aware")); + await user.click(screen.getByRole("button", { name: /save changes/i })); + + await waitFor(() => expect(modelPatchUpdateCall).toHaveBeenCalled()); + expect(savedConfig().classifier_plugin).toBe("spend-aware"); + }); + + it("blocks a save that leaves the custom classifier with no plugin", async () => { + const user = userEvent.setup(); + renderWithProviders( + , + ); + + await user.click(await screen.findByRole("button", { name: /save changes/i })); + + await waitFor(() => expect(toast.fromError).toHaveBeenCalled()); + expect(modelPatchUpdateCall).not.toHaveBeenCalled(); + }); +}); + describe("EditAutoRouterModal custom classifier prompt and fallback", () => { beforeEach(() => { modelPatchUpdateCall.mockClear(); diff --git a/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.tsx b/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.tsx index 54f992b08f4..c260dae23a1 100644 --- a/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.tsx +++ b/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.tsx @@ -70,6 +70,8 @@ const MANAGED_COMPLEXITY_ROUTER_KEYS = new Set([ "tier_labels", "classifier_type", "classifier_llm_config", + "classifier_plugin", + "classifier_plugin_timeout_ms", "classifier_context_window_size", "classifier_context_per_turn_chars", "classifier_context_include_assistant_turns", @@ -157,7 +159,14 @@ export const buildUpdatedComplexityRouterConfig = ( ...(value.classifier_type === "llm" && value.classifier_llm_config ? { classifier_llm_config: normalizeClassifierLlmConfig(value.classifier_llm_config) } : {}), - ...(value.classifier_type === "llm" && + ...(value.classifier_type === "custom" && value.classifier_plugin + ? { classifier_plugin: value.classifier_plugin } + : {}), + ...(value.classifier_type === "custom" && + value.classifier_plugin_timeout_ms !== undefined && { + classifier_plugin_timeout_ms: value.classifier_plugin_timeout_ms, + }), + ...(value.classifier_type !== "heuristic" && value.classifier_fallback !== undefined && { classifier_fallback: value.classifier_fallback }), ...(value.classifier_type === "llm" && value.classifier_context_window_size !== undefined && { @@ -348,6 +357,12 @@ const EditAutoRouterModal: React.FC = ({ tier_labels: hydrateTierLabels(parsedConfig.tier_labels), classifier_type: parsedConfig.classifier_type || "heuristic", classifier_llm_config: parsedConfig.classifier_llm_config, + classifier_plugin: + typeof parsedConfig.classifier_plugin === "string" ? parsedConfig.classifier_plugin : undefined, + classifier_plugin_timeout_ms: + typeof parsedConfig.classifier_plugin_timeout_ms === "number" + ? parsedConfig.classifier_plugin_timeout_ms + : undefined, classifier_context_window_size: typeof parsedConfig.classifier_context_window_size === "number" ? parsedConfig.classifier_context_window_size @@ -435,7 +450,7 @@ const EditAutoRouterModal: React.FC = ({ const saveValues = async (values: EditAutoRouterFormValues) => { if (isComplexityRouterModel) { - const { tiers, classifier_type, classifier_llm_config } = complexityRouterConfig; + const { tiers, classifier_type, classifier_llm_config, classifier_plugin } = complexityRouterConfig; if (Object.values(tiers).every((models) => models.length === 0)) { setShowValidationErrors(true); toast.fromError("Please select at least one model for a complexity tier"); @@ -446,6 +461,11 @@ const EditAutoRouterModal: React.FC = ({ toast.fromError("Please select a classifier model, or switch back to Heuristic"); return; } + if (classifier_type === "custom" && !classifier_plugin) { + setShowValidationErrors(true); + toast.fromError("Please select a classifier plugin, or switch back to Heuristic"); + return; + } // Same guards the create form applies (add_auto_router_tab.tsx). The backend rejects a // keyword rule with no keyword, and semantic_keyword_matching without an embedding model // or keyword rules (complexity_router/config.py), so without these a save fails as a raw diff --git a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/RoutingDecisionCard.test.tsx b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/RoutingDecisionCard.test.tsx index a2afe0d7a0d..6a0ef7feaa3 100644 --- a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/RoutingDecisionCard.test.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/RoutingDecisionCard.test.tsx @@ -85,6 +85,31 @@ describe("RoutingDecisionCard", () => { expect(screen.queryByText("Score")).not.toBeInTheDocument(); }); + it("names the custom classifier plugin instead of rendering its raw cause string", () => { + render( + , + ); + expect(screen.getByText("Custom classifier plugin")).toBeInTheDocument(); + expect(screen.queryByText("classifier_plugin")).not.toBeInTheDocument(); + }); + + // The plugin chose the tier, so pairing a score with a boundary band would claim the heuristic did. + it("does not claim the score chose the tier on a plugin-decided row", () => { + render(); + + expect(screen.getByText("0.20")).toBeInTheDocument(); + expect(screen.queryByText(/SIMPLE|MEDIUM|COMPLEX|at or above/)).not.toBeInTheDocument(); + }); + it("explains a route that fell back to the default model after the classifier failed", () => { // No tier is recorded on this path, so the card must not show a Tier row: nothing // about the request produced one, the classifier never answered. @@ -99,7 +124,7 @@ describe("RoutingDecisionCard", () => { }} />, ); - expect(screen.getByText("Default model, LLM classifier failed")).toBeInTheDocument(); + expect(screen.getByText("Default model, the classifier failed")).toBeInTheDocument(); expect(screen.queryByText("Tier")).not.toBeInTheDocument(); }); @@ -116,7 +141,7 @@ describe("RoutingDecisionCard", () => { }} />, ); - expect(screen.getByText("Fallback tier, LLM classifier failed")).toBeInTheDocument(); + expect(screen.getByText("Fallback tier, the classifier failed")).toBeInTheDocument(); expect(screen.getByText("SECURITY_REVIEW")).toBeInTheDocument(); }); diff --git a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/RoutingDecisionCard.tsx b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/RoutingDecisionCard.tsx index 0876a539653..a977fd3e2bc 100644 --- a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/RoutingDecisionCard.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/RoutingDecisionCard.tsx @@ -75,6 +75,8 @@ function describeCause(decision: RoutingDecision): string { return `Heuristic, ${tierLabel ?? "REASONING"} override (2 or more reasoning markers)`; case "llm_classifier": return classifierModel ? `LLM classifier (${classifierModel})` : "LLM classifier"; + case "classifier_plugin": + return "Custom classifier plugin"; case "literal_keyword_match": return matchedKeyword ? `Keyword match: "${matchedKeyword}"` : "Keyword match"; case "semantic_keyword_match": @@ -94,9 +96,9 @@ function describeCause(decision: RoutingDecision): string { case "default_fallback": return "Default model, no route matched"; case "classifier_fallback": - return "Fallback tier, LLM classifier failed"; + return "Fallback tier, the classifier failed"; case "default_model_fallback": - return "Default model, LLM classifier failed"; + return "Default model, the classifier failed"; default: return cause ?? "Unknown"; } @@ -145,11 +147,14 @@ export function RoutingDecisionCard({ tier_boundaries: tierBoundaries, } = decision; - // On an override row the score did not decide the tier, so showing it against a - // boundary would claim something untrue. Keyed off the cause rather than a marker + // On an override row, or one a plugin decided, the score did not choose the tier, so showing + // it against a boundary would claim something untrue. Keyed off the cause rather than a marker // inside `signals`, which redaction can remove. const scoreExplanation = - score !== undefined && decision.cause !== "reasoning_override" && decision.cause !== "plan_mode" + score !== undefined && + decision.cause !== "reasoning_override" && + decision.cause !== "plan_mode" && + decision.cause !== "classifier_plugin" ? describeScoreAgainstBoundaries(score, tierBoundaries, tierLabel !== undefined) : null; diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 3a129890267..51069e8c8f7 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -807,6 +807,26 @@ export interface paths { patch?: never; trace?: never; }; + "/auto_router/classifier_plugins": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** + * List Classifier Plugins + * @description Registered classifier plugin names, for the Admin UI's custom classifier picker. + */ + get: operations["list_classifier_plugins_auto_router_classifier_plugins_get"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/auto_router/shadow_eval": { parameters: { query?: never; @@ -23592,6 +23612,17 @@ export interface components { */ timeout_ms: number; }; + /** + * ClassifierPluginsListResponse + * @description Names from the proxy config's classifier_plugins registry, for the custom classifier picker. + */ + ClassifierPluginsListResponse: { + /** + * Classifier Plugins + * @description Registered classifier plugin names an auto-router's classifier_plugin may reference + */ + classifier_plugins: string[]; + }; /** * CloudZeroExportRequest * @description Request model for CloudZero export operations @@ -37806,6 +37837,26 @@ export interface operations { }; }; }; + list_classifier_plugins_auto_router_classifier_plugins_get: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ClassifierPluginsListResponse"]; + }; + }; + }; + }; list_shadow_eval_jobs_auto_router_shadow_eval_get: { parameters: { query?: { diff --git a/ui/litellm-dashboard/tests/mocks/classifierPlugins.ts b/ui/litellm-dashboard/tests/mocks/classifierPlugins.ts new file mode 100644 index 00000000000..64d500c3a69 --- /dev/null +++ b/ui/litellm-dashboard/tests/mocks/classifierPlugins.ts @@ -0,0 +1,17 @@ +import { vi } from "vitest"; + +/** + * Stubs the proxy's registered classifier plugin names for any test that renders the auto-router tree. + * Exported as a vi.fn so a test can override the query state, which is how the empty-registry and + * failed-fetch paths are covered. + */ +export const REGISTERED_CLASSIFIER_PLUGINS = ["spend-aware", "tier-by-team"]; + +export const LOADED_CLASSIFIER_PLUGINS_QUERY = { + data: REGISTERED_CLASSIFIER_PLUGINS, + isPending: false, + isError: false, + refetch: vi.fn(), +}; + +export const useClassifierPlugins = vi.fn(() => LOADED_CLASSIFIER_PLUGINS_QUERY);