mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-07 08:26:10 +00:00
feat(auto-router): reasoning effort for the LLM classifier call
classifier_llm_config gains reasoning_effort, sent on the classifier's own acompletion and mirrored into the logged request body. Classification runs on every request and is short, so a reasoning classifier model is usually worth holding at a cheap level. Setting the effort on the deployment instead moves it for every request that deployment serves, which is wrong whenever the classifier model also serves normal traffic. The level is filtered through the Router's existing target capability filter before either consumer is built, so a level the classifier model refuses reaches neither the provider call nor the spend-log request body. That filter is now public as params_the_target_accepts, since the classifier path is a third caller alongside the two tier-overlay sites, and it already owns the fail-open rules and the guard against capability lookups that authenticate. The dashboard carries the field through both payload builders, since each rebuilds classifier_llm_config from named keys and would otherwise wipe a value set in config.yaml on the next edit.
This commit is contained in:
Tin Chi Lo
parent
7978b9f721
commit
84dd96b2e4
11 changed files with 287 additions and 42 deletions
|
|
@ -11231,8 +11231,8 @@ class Router:
|
|||
return True
|
||||
return supported is None or param in supported
|
||||
|
||||
def _tier_params_the_target_accepts(
|
||||
self, model: str, tier_params: Mapping[str, object], request_kwargs: Mapping[str, object]
|
||||
def params_the_target_accepts(
|
||||
self, model: str, params: Mapping[str, object], request_kwargs: Mapping[str, object]
|
||||
) -> Mapping[str, object]:
|
||||
"""Drop an OpenAI param that no deployment behind ``model`` declares.
|
||||
|
||||
|
|
@ -11269,24 +11269,22 @@ class Router:
|
|||
"""
|
||||
deployments: Final = self.get_model_list(model_name=model) or ()
|
||||
if not deployments:
|
||||
return tier_params
|
||||
allowlisted: Final = self._declared_param_allowlist(tier_params) | self._declared_param_allowlist(
|
||||
request_kwargs
|
||||
)
|
||||
candidates: Final = provider_rejectable_params(tier_params) - self.TIER_PARAMS_NEVER_DROPPED - allowlisted
|
||||
return params
|
||||
allowlisted: Final = self._declared_param_allowlist(params) | self._declared_param_allowlist(request_kwargs)
|
||||
candidates: Final = provider_rejectable_params(params) - self.TIER_PARAMS_NEVER_DROPPED - allowlisted
|
||||
unsupported: Final = frozenset(
|
||||
param
|
||||
for param in candidates
|
||||
if not any(self._deployment_accepts_param(deployment, model, param) for deployment in deployments)
|
||||
)
|
||||
if not unsupported:
|
||||
return tier_params
|
||||
return params
|
||||
verbose_router_logger.warning(
|
||||
"litellm.router.py: dropping tier params %s for model=%s, no deployment behind it declares them",
|
||||
"litellm.router.py: dropping params %s for model=%s, no deployment behind it declares them",
|
||||
", ".join(sorted(unsupported)),
|
||||
model,
|
||||
)
|
||||
return MappingProxyType({key: value for key, value in tier_params.items() if key not in unsupported})
|
||||
return MappingProxyType({key: value for key, value in params.items() if key not in unsupported})
|
||||
|
||||
def get_model_list(
|
||||
self, model_name: str | None = None, team_id: str | None = None
|
||||
|
|
@ -12352,11 +12350,11 @@ class Router:
|
|||
messages = pre_routing_hook_response.messages
|
||||
record_pre_routing_selection(request_kwargs, model)
|
||||
if pre_routing_hook_response.litellm_params:
|
||||
accepted_tier_params: Final = self._tier_params_the_target_accepts(
|
||||
accepted_params: Final = self.params_the_target_accepts(
|
||||
model, pre_routing_hook_response.litellm_params, request_kwargs
|
||||
)
|
||||
self._drop_client_effort_carriers_a_tier_pin_supersedes(request_kwargs, accepted_tier_params)
|
||||
request_kwargs.update(accepted_tier_params)
|
||||
self._drop_client_effort_carriers_a_tier_pin_supersedes(request_kwargs, accepted_params)
|
||||
request_kwargs.update(accepted_params)
|
||||
#########################################################
|
||||
|
||||
# Resolve the strategy and logger AFTER the pre-routing hook, since
|
||||
|
|
@ -12468,11 +12466,11 @@ class Router:
|
|||
messages = pre_routing_hook_response.messages
|
||||
record_pre_routing_selection(request_kwargs, model)
|
||||
if pre_routing_hook_response.litellm_params:
|
||||
accepted_tier_params: Final = self._tier_params_the_target_accepts(
|
||||
accepted_params: Final = self.params_the_target_accepts(
|
||||
model, pre_routing_hook_response.litellm_params, request_kwargs
|
||||
)
|
||||
self._drop_client_effort_carriers_a_tier_pin_supersedes(request_kwargs, accepted_tier_params)
|
||||
request_kwargs.update(accepted_tier_params)
|
||||
self._drop_client_effort_carriers_a_tier_pin_supersedes(request_kwargs, accepted_params)
|
||||
request_kwargs.update(accepted_params)
|
||||
|
||||
# 2. Get healthy deployments
|
||||
healthy_deployments: Final = await self.async_get_healthy_deployments(
|
||||
|
|
|
|||
|
|
@ -1671,11 +1671,22 @@ class ComplexityRouter(CustomLogger):
|
|||
]
|
||||
response_format: Final = classifier_response_format
|
||||
|
||||
classifier_params: Final[Mapping[str, object]] = (
|
||||
MappingProxyType({})
|
||||
if llm_config.reasoning_effort is None
|
||||
else self.litellm_router_instance.params_the_target_accepts(
|
||||
llm_config.model,
|
||||
MappingProxyType({"reasoning_effort": llm_config.reasoning_effort}),
|
||||
MappingProxyType({}),
|
||||
)
|
||||
)
|
||||
|
||||
proxy_server_request: Final = {
|
||||
"body": {
|
||||
"model": llm_config.model,
|
||||
"messages": messages_for_call,
|
||||
"response_format": response_format,
|
||||
**classifier_params,
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -1684,6 +1695,7 @@ class ComplexityRouter(CustomLogger):
|
|||
messages=messages_for_call,
|
||||
response_format=response_format,
|
||||
timeout=llm_config.timeout_ms / 1000,
|
||||
**classifier_params,
|
||||
metadata=metadata,
|
||||
proxy_server_request=proxy_server_request,
|
||||
turn_off_message_logging=turn_off_message_logging,
|
||||
|
|
|
|||
|
|
@ -12,6 +12,7 @@ from typing import Annotated, Final, Literal
|
|||
|
||||
from pydantic import BaseModel, ConfigDict, Field, SkipValidation, field_serializer, field_validator, model_validator
|
||||
|
||||
from litellm.types.llms.openai import REASONING_EFFORT
|
||||
from litellm.types.router import AdaptiveRouterWeights, ClassifierPlugin, RoutingPlugin
|
||||
|
||||
from .tier_predictor import TrainedTierArtifact
|
||||
|
|
@ -436,6 +437,18 @@ class ClassifierLLMConfig(BaseModel):
|
|||
default=3000,
|
||||
description="Timeout budget for the classification call, in milliseconds",
|
||||
)
|
||||
reasoning_effort: REASONING_EFFORT | None = Field(
|
||||
default=None,
|
||||
description=(
|
||||
"reasoning_effort sent on the classifier's own call. Classification is a short, "
|
||||
"latency-sensitive call on every request, so a reasoning classifier model is usually "
|
||||
"worth holding at a cheap level. Setting it on the deployment instead moves effort for "
|
||||
"every request that deployment serves, which is wrong when the classifier model also "
|
||||
"serves normal traffic. A level the target refuses is dropped rather than failing the "
|
||||
"call, so classification still runs. Leave unset to send no effort and inherit whatever "
|
||||
"the deployment is configured with. Ignored by classifier types that call no model."
|
||||
),
|
||||
)
|
||||
classification_rubric: ClassificationRubric | None = Field(
|
||||
default=None,
|
||||
description=(
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ Tests the rule-based complexity scoring and tier assignment logic.
|
|||
|
||||
import asyncio
|
||||
import logging
|
||||
from types import MappingProxyType
|
||||
from typing import Dict, List
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
|
|
@ -1914,6 +1915,50 @@ class TestLLMClassifier:
|
|||
assert outcome.cause == "llm_classifier"
|
||||
assert outcome.tier == ComplexityTier.COMPLEX
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"request_kwargs",
|
||||
[
|
||||
pytest.param({"metadata": {"user_api_key": "sk-abc"}}, id="chat-completions"),
|
||||
pytest.param({"litellm_metadata": {"user_api_key": "sk-abc"}}, id="messages-and-responses"),
|
||||
],
|
||||
)
|
||||
async def test_classifier_reasoning_effort_reaches_the_real_router_for_every_metadata_shape(
|
||||
self, llm_classifier_config, request_kwargs
|
||||
):
|
||||
"""The call must retain an accepted effort through the real Router pipeline.
|
||||
|
||||
The two metadata shapes are the classifier's arrival shapes on chat completions versus
|
||||
Messages and Responses. This drives the same real Router path that resolves deployment
|
||||
support instead of an AsyncMock that would accept every param.
|
||||
"""
|
||||
llm_classifier_config["classifier_llm_config"]["reasoning_effort"] = "low"
|
||||
real_router = Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "haiku-classifier",
|
||||
"litellm_params": {
|
||||
"model": "openai/gpt-4o-mini",
|
||||
"api_key": "sk-classifier",
|
||||
"mock_response": '{"tier": "COMPLEX"}',
|
||||
},
|
||||
}
|
||||
]
|
||||
)
|
||||
router = ComplexityRouter(
|
||||
model_name="test-complexity-router",
|
||||
litellm_router_instance=real_router,
|
||||
complexity_router_config=llm_classifier_config,
|
||||
)
|
||||
|
||||
outcome = await router.aclassify("hi", request_kwargs=request_kwargs)
|
||||
|
||||
assert outcome.cause == "llm_classifier"
|
||||
assert outcome.tier == ComplexityTier.COMPLEX
|
||||
assert real_router.params_the_target_accepts(
|
||||
"haiku-classifier", {"reasoning_effort": "low"}, {}
|
||||
) == {}
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_aclassify_captures_request_body_in_proxy_server_request(
|
||||
self, llm_complexity_router, mock_router_instance
|
||||
|
|
@ -2210,6 +2255,60 @@ class TestLLMClassifier:
|
|||
call_kwargs = mock_router_instance.acompletion.call_args.kwargs
|
||||
assert call_kwargs["metadata"] == {**request_metadata, "internal_call_origin": "autorouter_classifier"}
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_classifier_reasoning_effort_reaches_the_call_and_the_logged_body(
|
||||
self, mock_router_instance, llm_classifier_config
|
||||
):
|
||||
"""A configured classifier reasoning_effort must reach the wire and the spend-log body.
|
||||
|
||||
The target capability filter runs before both the actual call and the logged
|
||||
proxy_server_request body. This makes it impossible for the spend log to claim a level
|
||||
that the provider filter has removed.
|
||||
"""
|
||||
llm_classifier_config["classifier_llm_config"]["reasoning_effort"] = "low"
|
||||
mock_router_instance.params_the_target_accepts.return_value = MappingProxyType({"reasoning_effort": "low"})
|
||||
router = ComplexityRouter(
|
||||
model_name="test-complexity-router",
|
||||
litellm_router_instance=mock_router_instance,
|
||||
complexity_router_config=llm_classifier_config,
|
||||
)
|
||||
mock_router_instance.acompletion = AsyncMock(return_value=_llm_response('{"tier": "SIMPLE"}'))
|
||||
await router.aclassify("hello")
|
||||
call_kwargs = mock_router_instance.acompletion.call_args.kwargs
|
||||
assert call_kwargs["reasoning_effort"] == "low"
|
||||
assert call_kwargs["proxy_server_request"]["body"]["reasoning_effort"] == "low"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_classifier_reasoning_effort_rejected_by_target_is_absent_from_call_and_log(
|
||||
self, mock_router_instance, llm_classifier_config
|
||||
):
|
||||
"""A rejected level must be absent from both consumers of the resolved mapping."""
|
||||
llm_classifier_config["classifier_llm_config"]["reasoning_effort"] = "low"
|
||||
mock_router_instance.params_the_target_accepts.return_value = MappingProxyType({})
|
||||
router = ComplexityRouter(
|
||||
model_name="test-complexity-router",
|
||||
litellm_router_instance=mock_router_instance,
|
||||
complexity_router_config=llm_classifier_config,
|
||||
)
|
||||
mock_router_instance.acompletion = AsyncMock(return_value=_llm_response('{"tier": "SIMPLE"}'))
|
||||
await router.aclassify("hello")
|
||||
call_kwargs = mock_router_instance.acompletion.call_args.kwargs
|
||||
assert "reasoning_effort" not in call_kwargs
|
||||
assert "reasoning_effort" not in call_kwargs["proxy_server_request"]["body"]
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_unset_classifier_reasoning_effort_leaves_the_call_untouched(
|
||||
self, llm_complexity_router, mock_router_instance
|
||||
):
|
||||
"""No configured effort means no effort kwarg and no body key."""
|
||||
mock_router_instance.params_the_target_accepts.return_value = MappingProxyType({})
|
||||
mock_router_instance.acompletion = AsyncMock(return_value=_llm_response('{"tier": "SIMPLE"}'))
|
||||
await llm_complexity_router.aclassify("hello")
|
||||
call_kwargs = mock_router_instance.acompletion.call_args.kwargs
|
||||
mock_router_instance.params_the_target_accepts.assert_not_called()
|
||||
assert "reasoning_effort" not in call_kwargs
|
||||
assert "reasoning_effort" not in call_kwargs["proxy_server_request"]["body"]
|
||||
|
||||
|
||||
class TestRouterPreRoutingAliasOverrides:
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -12021,7 +12021,7 @@ def test_resolved_litellm_models_answers_through_every_channel_a_request_uses(
|
|||
assert set(_resolution_router().resolved_litellm_models(model_name)) == set(expected)
|
||||
|
||||
|
||||
class TestTierParamsTheTargetAccepts:
|
||||
class TestParamsTheTargetAccepts:
|
||||
"""A tier's litellm_params are applied to every request that tier routes, so one the target
|
||||
cannot take raised UnsupportedParamsError before the request left the proxy, turning the whole
|
||||
tier into a 400."""
|
||||
|
|
@ -12041,14 +12041,14 @@ class TestTierParamsTheTargetAccepts:
|
|||
def test_drops_a_param_no_deployment_declares(self):
|
||||
router = self._router("novita/moonshotai/kimi-k3")
|
||||
|
||||
accepted = router._tier_params_the_target_accepts("tiered", {"reasoning_effort": "max"}, {})
|
||||
accepted = router.params_the_target_accepts("tiered", {"reasoning_effort": "max"}, {})
|
||||
|
||||
assert accepted == {}
|
||||
|
||||
def test_keeps_a_param_the_deployment_declares(self):
|
||||
router = self._router("fireworks_ai/kimi-k3")
|
||||
|
||||
accepted = router._tier_params_the_target_accepts("tiered", {"reasoning_effort": "max"}, {})
|
||||
accepted = router.params_the_target_accepts("tiered", {"reasoning_effort": "max"}, {})
|
||||
|
||||
assert accepted == {"reasoning_effort": "max"}
|
||||
|
||||
|
|
@ -12070,7 +12070,7 @@ class TestTierParamsTheTargetAccepts:
|
|||
configuration the request needs while never touching what the provider would reject."""
|
||||
router = self._router("novita/moonshotai/kimi-k3")
|
||||
|
||||
accepted = router._tier_params_the_target_accepts("tiered", {control: value, "reasoning_effort": "max"}, {})
|
||||
accepted = router.params_the_target_accepts("tiered", {control: value, "reasoning_effort": "max"}, {})
|
||||
|
||||
assert accepted == {control: value}
|
||||
|
||||
|
|
@ -12090,7 +12090,7 @@ class TestTierParamsTheTargetAccepts:
|
|||
drop_params or additional_drop_params would silently disable the operator's sanitization."""
|
||||
router = self._router("novita/moonshotai/kimi-k3")
|
||||
|
||||
accepted = router._tier_params_the_target_accepts("tiered", {control: value, "reasoning_effort": "max"}, {})
|
||||
accepted = router.params_the_target_accepts("tiered", {control: value, "reasoning_effort": "max"}, {})
|
||||
|
||||
assert accepted == {control: value}
|
||||
|
||||
|
|
@ -12100,7 +12100,7 @@ class TestTierParamsTheTargetAccepts:
|
|||
both sets and allowlists would never 400 and must not be dropped."""
|
||||
router = self._router("novita/moonshotai/kimi-k3")
|
||||
|
||||
accepted = router._tier_params_the_target_accepts(
|
||||
accepted = router.params_the_target_accepts(
|
||||
"tiered", {"reasoning_effort": "max", "allowed_openai_params": ["reasoning_effort"]}, {}
|
||||
)
|
||||
|
||||
|
|
@ -12109,7 +12109,7 @@ class TestTierParamsTheTargetAccepts:
|
|||
def test_request_allowlist_protects_the_param_it_names(self):
|
||||
router = self._router("novita/moonshotai/kimi-k3")
|
||||
|
||||
accepted = router._tier_params_the_target_accepts(
|
||||
accepted = router.params_the_target_accepts(
|
||||
"tiered", {"reasoning_effort": "max"}, {"allowed_openai_params": ["reasoning_effort"]}
|
||||
)
|
||||
|
||||
|
|
@ -12118,7 +12118,7 @@ class TestTierParamsTheTargetAccepts:
|
|||
def test_allowlist_protects_only_the_params_it_names(self):
|
||||
router = self._router("novita/moonshotai/kimi-k3")
|
||||
|
||||
accepted = router._tier_params_the_target_accepts(
|
||||
accepted = router.params_the_target_accepts(
|
||||
"tiered", {"reasoning_effort": "max", "allowed_openai_params": ["seed"]}, {}
|
||||
)
|
||||
|
||||
|
|
@ -12147,7 +12147,7 @@ class TestTierParamsTheTargetAccepts:
|
|||
today the mismatch fails loudly."""
|
||||
router = self._router("petals/petals-team/StableBeluga2")
|
||||
|
||||
accepted = router._tier_params_the_target_accepts(
|
||||
accepted = router.params_the_target_accepts(
|
||||
"tiered", {"max_completion_tokens": 100, "reasoning_effort": "max"}, {}
|
||||
)
|
||||
|
||||
|
|
@ -12159,7 +12159,7 @@ class TestTierParamsTheTargetAccepts:
|
|||
worse than the error they already get."""
|
||||
router = self._router("ai21/jamba-1.5-mini")
|
||||
|
||||
accepted = router._tier_params_the_target_accepts(
|
||||
accepted = router.params_the_target_accepts(
|
||||
"tiered", {"extra_headers": {"x-tenant": "acme"}, "reasoning_effort": "max"}, {}
|
||||
)
|
||||
|
||||
|
|
@ -12174,7 +12174,7 @@ class TestTierParamsTheTargetAccepts:
|
|||
]
|
||||
)
|
||||
|
||||
accepted = router._tier_params_the_target_accepts("tiered", {"reasoning_effort": "max"}, {})
|
||||
accepted = router.params_the_target_accepts("tiered", {"reasoning_effort": "max"}, {})
|
||||
|
||||
assert accepted == {"reasoning_effort": "max"}
|
||||
|
||||
|
|
@ -12247,7 +12247,7 @@ class TestTierParamsTheTargetAccepts:
|
|||
"""An unresolvable target must never narrow what the request already did."""
|
||||
router = self._router("fireworks_ai/kimi-k3")
|
||||
|
||||
accepted = router._tier_params_the_target_accepts("no-such-group", {"reasoning_effort": "max"}, {})
|
||||
accepted = router.params_the_target_accepts("no-such-group", {"reasoning_effort": "max"}, {})
|
||||
|
||||
assert accepted == {"reasoning_effort": "max"}
|
||||
|
||||
|
|
|
|||
|
|
@ -13,6 +13,7 @@ import ClassifierPromptEditor from "./ClassifierPromptEditor";
|
|||
import CustomTierPromptEditor from "./CustomTierPromptEditor";
|
||||
import { RestrictedSection, restrictedBy } from "./TierRestrictions";
|
||||
import HeuristicScoringConfig from "./HeuristicScoringConfig";
|
||||
import { REASONING_EFFORT_OPTIONS, ReasoningEffort } from "./complexity_router_tiers";
|
||||
import { useComplexityScorerDefaults } from "@/app/(dashboard)/hooks/autoRouter/useComplexityScorerDefaults";
|
||||
import {
|
||||
ClassificationFrequency,
|
||||
|
|
@ -49,6 +50,7 @@ const HEURISTIC_V2_EXPLANATION =
|
|||
"the first tier that meets its trained threshold. It runs locally with no classifier API call.";
|
||||
|
||||
const CLASSIFIER_TIMEOUT_ID = "classifier-timeout-ms";
|
||||
const CLASSIFIER_EFFORT_PROVIDER_DEFAULT = "__provider_default__";
|
||||
const CLASSIFIER_CONTEXT_WINDOW_SIZE_ID = "classifier-context-window-size";
|
||||
const CLASSIFIER_CONTEXT_BUDGET_CHARS_ID = "classifier-context-budget-chars";
|
||||
const HYBRID_BOUNDARY_MARGIN_ID = "hybrid-boundary-margin";
|
||||
|
|
@ -251,6 +253,12 @@ const ClassificationMethodConfig: React.FC<ClassificationMethodConfigProps> = ({
|
|||
const contextBudget = value.classifier_context_budget_chars ?? DEFAULT_CLASSIFIER_CONTEXT_BUDGET_CHARS;
|
||||
const contextBudgetQuotesNothing = contextBudget > 0 && contextBudget < MIN_QUOTED_CONTEXT_TURN_CHARS;
|
||||
const classificationRubric = value.classifier_llm_config?.classification_rubric ?? DEFAULT_CLASSIFICATION_RUBRIC;
|
||||
const classifierEffort = value.classifier_llm_config?.reasoning_effort;
|
||||
// A stored level the option list does not carry (hand-authored, or a level the picker omits) is
|
||||
// listed anyway, so the control renders with its value selected and can be cleared.
|
||||
const classifierEffortOptions = Array.from(
|
||||
new Set<ReasoningEffort>([...REASONING_EFFORT_OPTIONS, ...(classifierEffort ? [classifierEffort] : [])]),
|
||||
);
|
||||
|
||||
const handleClassifierTypeChange = (classifierType: ClassifierType) => {
|
||||
const nextValue: ComplexityRouterConfigValue = {
|
||||
|
|
@ -320,6 +328,18 @@ const ClassificationMethodConfig: React.FC<ClassificationMethodConfigProps> = ({
|
|||
});
|
||||
};
|
||||
|
||||
const handleClassifierReasoningEffortChange = (reasoningEffort: ReasoningEffort | undefined) => {
|
||||
onChange({
|
||||
...value,
|
||||
classifier_llm_config: {
|
||||
...value.classifier_llm_config,
|
||||
model: value.classifier_llm_config?.model ?? "",
|
||||
timeout_ms: value.classifier_llm_config?.timeout_ms ?? DEFAULT_CLASSIFIER_TIMEOUT_MS,
|
||||
reasoning_effort: reasoningEffort,
|
||||
},
|
||||
});
|
||||
};
|
||||
|
||||
const handleClassificationRubricChange = (classificationRubric: ClassificationRubric) => {
|
||||
onChange({
|
||||
...value,
|
||||
|
|
@ -485,15 +505,44 @@ const ClassificationMethodConfig: React.FC<ClassificationMethodConfigProps> = ({
|
|||
<div className="mt-4 space-y-3">
|
||||
<div>
|
||||
<strong className="block mb-1 font-semibold">Classifier Model</strong>
|
||||
<SearchSelect
|
||||
options={modelOptions}
|
||||
value={value.classifier_llm_config?.model ?? ""}
|
||||
onValueChange={handleClassifierModelChange}
|
||||
placeholder="Select the model that will classify request complexity"
|
||||
emptyText="No models found"
|
||||
allowClear={false}
|
||||
className={classifierModelMissing ? "border-destructive" : undefined}
|
||||
/>
|
||||
<div className="flex items-center gap-2">
|
||||
<SearchSelect
|
||||
options={modelOptions}
|
||||
value={value.classifier_llm_config?.model ?? ""}
|
||||
onValueChange={handleClassifierModelChange}
|
||||
placeholder="Select the model that will classify request complexity"
|
||||
emptyText="No models found"
|
||||
allowClear={false}
|
||||
className={`flex-1 ${classifierModelMissing ? "border-destructive" : ""}`}
|
||||
/>
|
||||
{value.classifier_llm_config?.model && (
|
||||
<SimpleTooltip content="Reasoning effort for the classifier call only, so a reasoning classifier model stays cheap without moving effort for the normal traffic that same deployment serves. A level the model does not take is dropped and classification still runs.">
|
||||
<Select
|
||||
value={classifierEffort ?? CLASSIFIER_EFFORT_PROVIDER_DEFAULT}
|
||||
onValueChange={(effort) =>
|
||||
handleClassifierReasoningEffortChange(
|
||||
effort === CLASSIFIER_EFFORT_PROVIDER_DEFAULT ? undefined : (effort as ReasoningEffort),
|
||||
)
|
||||
}
|
||||
>
|
||||
<SelectTrigger
|
||||
aria-label="Classifier reasoning effort"
|
||||
className="w-36 shrink-0 text-muted-foreground"
|
||||
>
|
||||
<SelectValue />
|
||||
</SelectTrigger>
|
||||
<SelectContent>
|
||||
<SelectItem value={CLASSIFIER_EFFORT_PROVIDER_DEFAULT}>Default effort</SelectItem>
|
||||
{classifierEffortOptions.map((effort) => (
|
||||
<SelectItem key={effort} value={effort}>
|
||||
{effort} effort
|
||||
</SelectItem>
|
||||
))}
|
||||
</SelectContent>
|
||||
</Select>
|
||||
</SimpleTooltip>
|
||||
)}
|
||||
</div>
|
||||
{classifierModelMissing && <span className="text-xs text-destructive">A classifier model is required</span>}
|
||||
</div>
|
||||
<div>
|
||||
|
|
|
|||
|
|
@ -170,6 +170,45 @@ describe("ComplexityRouterConfig", () => {
|
|||
expect(screen.queryByText("Context Per-Turn Character Limit")).not.toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("writes and clears classifier reasoning effort", async () => {
|
||||
const onChange = vi.fn();
|
||||
const llmValue: ComplexityRouterConfigValue = {
|
||||
...defaultValue,
|
||||
classifier_type: "llm",
|
||||
classifier_llm_config: { model: "gpt-3.5-turbo", timeout_ms: 3000 },
|
||||
};
|
||||
const first = renderWithProviders(<ComplexityRouterConfig {...baseProps} value={llmValue} onChange={onChange} />);
|
||||
fireEvent.click(screen.getByText("Advanced: Classification Method"));
|
||||
|
||||
const user = userEvent.setup();
|
||||
await user.click(screen.getByRole("combobox", { name: "Classifier reasoning effort" }));
|
||||
await user.click(await screen.findByRole("option", { name: "low effort" }));
|
||||
expect(onChange).toHaveBeenLastCalledWith({
|
||||
...llmValue,
|
||||
classifier_llm_config: { model: "gpt-3.5-turbo", timeout_ms: 3000, reasoning_effort: "low" },
|
||||
});
|
||||
|
||||
onChange.mockClear();
|
||||
first.unmount();
|
||||
renderWithProviders(
|
||||
<ComplexityRouterConfig
|
||||
{...baseProps}
|
||||
value={{
|
||||
...llmValue,
|
||||
classifier_llm_config: { model: "gpt-3.5-turbo", timeout_ms: 3000, reasoning_effort: "low" },
|
||||
}}
|
||||
onChange={onChange}
|
||||
/>,
|
||||
);
|
||||
fireEvent.click(screen.getByText("Advanced: Classification Method"));
|
||||
await user.click(screen.getByRole("combobox", { name: "Classifier reasoning effort" }));
|
||||
await user.click(await screen.findByRole("option", { name: "Default effort" }));
|
||||
expect(onChange).toHaveBeenLastCalledWith({
|
||||
...llmValue,
|
||||
classifier_llm_config: { model: "gpt-3.5-turbo", timeout_ms: 3000, reasoning_effort: undefined },
|
||||
});
|
||||
});
|
||||
|
||||
it("should default the context window and budget when llm is selected", () => {
|
||||
const llmValue: ComplexityRouterConfigValue = {
|
||||
...defaultValue,
|
||||
|
|
|
|||
|
|
@ -126,6 +126,7 @@ export interface ClassifierLLMConfig {
|
|||
timeout_ms: number;
|
||||
classification_rubric?: ClassificationRubric;
|
||||
system_prompt?: string;
|
||||
reasoning_effort?: ReasoningEffort;
|
||||
}
|
||||
|
||||
export type ClassifierType = "heuristic" | "heuristic_v2" | "llm" | "heuristic_first" | "hybrid";
|
||||
|
|
|
|||
|
|
@ -530,6 +530,17 @@ describe("classifier prompt and fallback", () => {
|
|||
expect(config.classifier_llm_config?.system_prompt).toBe("Grade the data sensitivity of the request.");
|
||||
});
|
||||
|
||||
it("carries a classifier reasoning_effort to the wire beside a rubric and beside a custom prompt", () => {
|
||||
// Both payload builders rebuild classifier_llm_config from named keys, so an effort the
|
||||
// operator set in config.yaml is wiped by any dashboard edit unless each one carries it.
|
||||
expect(normalizeClassifierLlmConfig({ model: "m", timeout_ms: 1, reasoning_effort: "low" }).reasoning_effort).toBe(
|
||||
"low",
|
||||
);
|
||||
const withPrompt = { model: "m", timeout_ms: 1, system_prompt: "x", reasoning_effort: "minimal" } as const;
|
||||
expect(normalizeClassifierLlmConfig(withPrompt)).toEqual(withPrompt);
|
||||
expect(normalizeClassifierLlmConfig({ model: "m", timeout_ms: 1 })).not.toHaveProperty("reasoning_effort");
|
||||
});
|
||||
|
||||
it("normalizeClassifierLlmConfig leaves a real prompt untouched and strips an empty one", () => {
|
||||
expect(normalizeClassifierLlmConfig({ model: "m", timeout_ms: 1, system_prompt: "x" })).toEqual({
|
||||
model: "m",
|
||||
|
|
@ -897,6 +908,16 @@ describe("buildComplexityRouterConfig with an edited tier set", () => {
|
|||
expect(build({ classifierType: "heuristic" }).classifier_type).toBe("llm");
|
||||
});
|
||||
|
||||
it("keeps the classifier reasoning_effort, which a custom tier set does not conflict with", () => {
|
||||
// This branch deliberately drops system_prompt and classification_rubric because the backend
|
||||
// rejects those beside tier_definitions. Effort is orthogonal to the tier taxonomy, so
|
||||
// dropping it here would silently lose it on every custom-tier router.
|
||||
const payload = build({
|
||||
classifierLlmConfig: { model: "gpt-4o-mini", timeout_ms: 3000, reasoning_effort: "low" },
|
||||
});
|
||||
expect(payload.classifier_llm_config).toEqual({ model: "gpt-4o-mini", timeout_ms: 3000, reasoning_effort: "low" });
|
||||
});
|
||||
|
||||
it("turns session pinning off rather than leaving a stale true the backend rejects", () => {
|
||||
expect(build({ sessionAffinity: true }).session_affinity).toBe(false);
|
||||
});
|
||||
|
|
|
|||
|
|
@ -57,10 +57,13 @@ export const normalizeClassifierLlmConfig = ({
|
|||
timeout_ms,
|
||||
classification_rubric,
|
||||
system_prompt,
|
||||
}: ClassifierLLMConfig): ClassifierLLMConfig =>
|
||||
system_prompt?.trim()
|
||||
reasoning_effort,
|
||||
}: ClassifierLLMConfig): ClassifierLLMConfig => ({
|
||||
...(system_prompt?.trim()
|
||||
? { model, timeout_ms, system_prompt }
|
||||
: { model, timeout_ms, ...(classification_rubric && { classification_rubric }) };
|
||||
: { model, timeout_ms, ...(classification_rubric && { classification_rubric }) }),
|
||||
...(reasoning_effort && { reasoning_effort }),
|
||||
});
|
||||
|
||||
interface ScorerKnobInputs {
|
||||
classifierType: ClassifierType;
|
||||
|
|
@ -293,11 +296,16 @@ export const customTierWireFields = (
|
|||
tier_definitions: tierDefinitionsFromRows(rows),
|
||||
...(fallback && { fallback_tier: activeTierName(fallback) }),
|
||||
classifier_type: "llm",
|
||||
// Rebuilt from the two fields an edited tier set allows. The backend rejects system_prompt and
|
||||
// Rebuilt from the fields an edited tier set allows. The backend rejects system_prompt and
|
||||
// classification_rubric beside tier_definitions, and both live inside this object rather than at
|
||||
// the top level the omit list covers. The opening instructions ride classification_prompt below.
|
||||
// reasoning_effort is not part of the tier taxonomy, so a custom tier set keeps it.
|
||||
...(classifierLlmConfig && {
|
||||
classifier_llm_config: { model: classifierLlmConfig.model, timeout_ms: classifierLlmConfig.timeout_ms },
|
||||
classifier_llm_config: {
|
||||
model: classifierLlmConfig.model,
|
||||
timeout_ms: classifierLlmConfig.timeout_ms,
|
||||
...(classifierLlmConfig.reasoning_effort && { reasoning_effort: classifierLlmConfig.reasoning_effort }),
|
||||
},
|
||||
}),
|
||||
session_affinity: false,
|
||||
...(classificationPrompt?.trim() && { classification_prompt: classificationPrompt.trim() }),
|
||||
|
|
|
|||
5
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
5
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
|
|
@ -25143,6 +25143,11 @@ export interface components {
|
|||
* @description Model name (from the router's model_list) to call for classification
|
||||
*/
|
||||
model: string;
|
||||
/**
|
||||
* Reasoning Effort
|
||||
* @description reasoning_effort sent on the classifier's own call. Classification is a short, latency-sensitive call on every request, so a reasoning classifier model is usually worth holding at a cheap level. Setting it on the deployment instead moves effort for every request that deployment serves, which is wrong when the classifier model also serves normal traffic. A level the target refuses is dropped rather than failing the call, so classification still runs. Leave unset to send no effort and inherit whatever the deployment is configured with. Ignored by classifier types that call no model.
|
||||
*/
|
||||
reasoning_effort?: ("none" | "minimal" | "low" | "medium" | "high" | "xhigh" | "max") | null;
|
||||
/**
|
||||
* System Prompt
|
||||
* @description Replaces the built-in complexity rubric as the classifier's entire system role. When set, neither the default rubric nor the context-window closing line is appended, so the prompt owns the whole taxonomy and the tier names SIMPLE/MEDIUM/COMPLEX/REASONING become whatever buckets it defines: a prompt that classifies data sensitivity routes on that instead of on difficulty. Two consequences of full replacement. The default rubric's closing paragraph is the classifier's prompt-injection defense, telling it that the caller's quoted system prompt and prior turns are material to judge and never instructions; a replacement that omits it lets a caller ask for a tier and get it. And the heuristic fallback still scores complexity, so a router on some other taxonomy wants classifier_fallback='default_model'. Leave unset for the built-in rubric. Only applies when classifier_type is 'llm'.
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue