diff --git a/litellm/router.py b/litellm/router.py index 0af514fe8a2..f04e88fca28 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -11231,8 +11231,8 @@ class Router: return True return supported is None or param in supported - def _tier_params_the_target_accepts( - self, model: str, tier_params: Mapping[str, object], request_kwargs: Mapping[str, object] + def params_the_target_accepts( + self, model: str, params: Mapping[str, object], request_kwargs: Mapping[str, object] ) -> Mapping[str, object]: """Drop an OpenAI param that no deployment behind ``model`` declares. @@ -11269,24 +11269,22 @@ class Router: """ deployments: Final = self.get_model_list(model_name=model) or () if not deployments: - return tier_params - allowlisted: Final = self._declared_param_allowlist(tier_params) | self._declared_param_allowlist( - request_kwargs - ) - candidates: Final = provider_rejectable_params(tier_params) - self.TIER_PARAMS_NEVER_DROPPED - allowlisted + return params + allowlisted: Final = self._declared_param_allowlist(params) | self._declared_param_allowlist(request_kwargs) + candidates: Final = provider_rejectable_params(params) - self.TIER_PARAMS_NEVER_DROPPED - allowlisted unsupported: Final = frozenset( param for param in candidates if not any(self._deployment_accepts_param(deployment, model, param) for deployment in deployments) ) if not unsupported: - return tier_params + return params verbose_router_logger.warning( - "litellm.router.py: dropping tier params %s for model=%s, no deployment behind it declares them", + "litellm.router.py: dropping params %s for model=%s, no deployment behind it declares them", ", ".join(sorted(unsupported)), model, ) - return MappingProxyType({key: value for key, value in tier_params.items() if key not in unsupported}) + return MappingProxyType({key: value for key, value in params.items() if key not in unsupported}) def get_model_list( self, model_name: str | None = None, team_id: str | None = None @@ -12352,11 +12350,11 @@ class Router: messages = pre_routing_hook_response.messages record_pre_routing_selection(request_kwargs, model) if pre_routing_hook_response.litellm_params: - accepted_tier_params: Final = self._tier_params_the_target_accepts( + accepted_params: Final = self.params_the_target_accepts( model, pre_routing_hook_response.litellm_params, request_kwargs ) - self._drop_client_effort_carriers_a_tier_pin_supersedes(request_kwargs, accepted_tier_params) - request_kwargs.update(accepted_tier_params) + self._drop_client_effort_carriers_a_tier_pin_supersedes(request_kwargs, accepted_params) + request_kwargs.update(accepted_params) ######################################################### # Resolve the strategy and logger AFTER the pre-routing hook, since @@ -12468,11 +12466,11 @@ class Router: messages = pre_routing_hook_response.messages record_pre_routing_selection(request_kwargs, model) if pre_routing_hook_response.litellm_params: - accepted_tier_params: Final = self._tier_params_the_target_accepts( + accepted_params: Final = self.params_the_target_accepts( model, pre_routing_hook_response.litellm_params, request_kwargs ) - self._drop_client_effort_carriers_a_tier_pin_supersedes(request_kwargs, accepted_tier_params) - request_kwargs.update(accepted_tier_params) + self._drop_client_effort_carriers_a_tier_pin_supersedes(request_kwargs, accepted_params) + request_kwargs.update(accepted_params) # 2. Get healthy deployments healthy_deployments: Final = await self.async_get_healthy_deployments( diff --git a/litellm/router_strategy/complexity_router/complexity_router.py b/litellm/router_strategy/complexity_router/complexity_router.py index d205db90607..a67e2b8b442 100644 --- a/litellm/router_strategy/complexity_router/complexity_router.py +++ b/litellm/router_strategy/complexity_router/complexity_router.py @@ -1671,11 +1671,22 @@ class ComplexityRouter(CustomLogger): ] response_format: Final = classifier_response_format + classifier_params: Final[Mapping[str, object]] = ( + MappingProxyType({}) + if llm_config.reasoning_effort is None + else self.litellm_router_instance.params_the_target_accepts( + llm_config.model, + MappingProxyType({"reasoning_effort": llm_config.reasoning_effort}), + MappingProxyType({}), + ) + ) + proxy_server_request: Final = { "body": { "model": llm_config.model, "messages": messages_for_call, "response_format": response_format, + **classifier_params, } } @@ -1684,6 +1695,7 @@ class ComplexityRouter(CustomLogger): messages=messages_for_call, response_format=response_format, timeout=llm_config.timeout_ms / 1000, + **classifier_params, metadata=metadata, proxy_server_request=proxy_server_request, turn_off_message_logging=turn_off_message_logging, diff --git a/litellm/router_strategy/complexity_router/config.py b/litellm/router_strategy/complexity_router/config.py index 0ae0db63fad..a2d1a54442e 100644 --- a/litellm/router_strategy/complexity_router/config.py +++ b/litellm/router_strategy/complexity_router/config.py @@ -12,6 +12,7 @@ from typing import Annotated, Final, Literal from pydantic import BaseModel, ConfigDict, Field, SkipValidation, field_serializer, field_validator, model_validator +from litellm.types.llms.openai import REASONING_EFFORT from litellm.types.router import AdaptiveRouterWeights, ClassifierPlugin, RoutingPlugin from .tier_predictor import TrainedTierArtifact @@ -436,6 +437,18 @@ class ClassifierLLMConfig(BaseModel): default=3000, description="Timeout budget for the classification call, in milliseconds", ) + reasoning_effort: REASONING_EFFORT | None = Field( + default=None, + description=( + "reasoning_effort sent on the classifier's own call. Classification is a short, " + "latency-sensitive call on every request, so a reasoning classifier model is usually " + "worth holding at a cheap level. Setting it on the deployment instead moves effort for " + "every request that deployment serves, which is wrong when the classifier model also " + "serves normal traffic. A level the target refuses is dropped rather than failing the " + "call, so classification still runs. Leave unset to send no effort and inherit whatever " + "the deployment is configured with. Ignored by classifier types that call no model." + ), + ) classification_rubric: ClassificationRubric | None = Field( default=None, description=( diff --git a/tests/test_litellm/router_strategy/test_complexity_router.py b/tests/test_litellm/router_strategy/test_complexity_router.py index 941d78085e4..8889d482c89 100644 --- a/tests/test_litellm/router_strategy/test_complexity_router.py +++ b/tests/test_litellm/router_strategy/test_complexity_router.py @@ -6,6 +6,7 @@ Tests the rule-based complexity scoring and tier assignment logic. import asyncio import logging +from types import MappingProxyType from typing import Dict, List from unittest.mock import AsyncMock, MagicMock, patch @@ -1914,6 +1915,50 @@ class TestLLMClassifier: assert outcome.cause == "llm_classifier" assert outcome.tier == ComplexityTier.COMPLEX + @pytest.mark.asyncio + @pytest.mark.parametrize( + "request_kwargs", + [ + pytest.param({"metadata": {"user_api_key": "sk-abc"}}, id="chat-completions"), + pytest.param({"litellm_metadata": {"user_api_key": "sk-abc"}}, id="messages-and-responses"), + ], + ) + async def test_classifier_reasoning_effort_reaches_the_real_router_for_every_metadata_shape( + self, llm_classifier_config, request_kwargs + ): + """The call must retain an accepted effort through the real Router pipeline. + + The two metadata shapes are the classifier's arrival shapes on chat completions versus + Messages and Responses. This drives the same real Router path that resolves deployment + support instead of an AsyncMock that would accept every param. + """ + llm_classifier_config["classifier_llm_config"]["reasoning_effort"] = "low" + real_router = Router( + model_list=[ + { + "model_name": "haiku-classifier", + "litellm_params": { + "model": "openai/gpt-4o-mini", + "api_key": "sk-classifier", + "mock_response": '{"tier": "COMPLEX"}', + }, + } + ] + ) + router = ComplexityRouter( + model_name="test-complexity-router", + litellm_router_instance=real_router, + complexity_router_config=llm_classifier_config, + ) + + outcome = await router.aclassify("hi", request_kwargs=request_kwargs) + + assert outcome.cause == "llm_classifier" + assert outcome.tier == ComplexityTier.COMPLEX + assert real_router.params_the_target_accepts( + "haiku-classifier", {"reasoning_effort": "low"}, {} + ) == {} + @pytest.mark.asyncio async def test_aclassify_captures_request_body_in_proxy_server_request( self, llm_complexity_router, mock_router_instance @@ -2210,6 +2255,60 @@ class TestLLMClassifier: call_kwargs = mock_router_instance.acompletion.call_args.kwargs assert call_kwargs["metadata"] == {**request_metadata, "internal_call_origin": "autorouter_classifier"} + @pytest.mark.asyncio + async def test_classifier_reasoning_effort_reaches_the_call_and_the_logged_body( + self, mock_router_instance, llm_classifier_config + ): + """A configured classifier reasoning_effort must reach the wire and the spend-log body. + + The target capability filter runs before both the actual call and the logged + proxy_server_request body. This makes it impossible for the spend log to claim a level + that the provider filter has removed. + """ + llm_classifier_config["classifier_llm_config"]["reasoning_effort"] = "low" + mock_router_instance.params_the_target_accepts.return_value = MappingProxyType({"reasoning_effort": "low"}) + router = ComplexityRouter( + model_name="test-complexity-router", + litellm_router_instance=mock_router_instance, + complexity_router_config=llm_classifier_config, + ) + mock_router_instance.acompletion = AsyncMock(return_value=_llm_response('{"tier": "SIMPLE"}')) + await router.aclassify("hello") + call_kwargs = mock_router_instance.acompletion.call_args.kwargs + assert call_kwargs["reasoning_effort"] == "low" + assert call_kwargs["proxy_server_request"]["body"]["reasoning_effort"] == "low" + + @pytest.mark.asyncio + async def test_classifier_reasoning_effort_rejected_by_target_is_absent_from_call_and_log( + self, mock_router_instance, llm_classifier_config + ): + """A rejected level must be absent from both consumers of the resolved mapping.""" + llm_classifier_config["classifier_llm_config"]["reasoning_effort"] = "low" + mock_router_instance.params_the_target_accepts.return_value = MappingProxyType({}) + router = ComplexityRouter( + model_name="test-complexity-router", + litellm_router_instance=mock_router_instance, + complexity_router_config=llm_classifier_config, + ) + mock_router_instance.acompletion = AsyncMock(return_value=_llm_response('{"tier": "SIMPLE"}')) + await router.aclassify("hello") + call_kwargs = mock_router_instance.acompletion.call_args.kwargs + assert "reasoning_effort" not in call_kwargs + assert "reasoning_effort" not in call_kwargs["proxy_server_request"]["body"] + + @pytest.mark.asyncio + async def test_unset_classifier_reasoning_effort_leaves_the_call_untouched( + self, llm_complexity_router, mock_router_instance + ): + """No configured effort means no effort kwarg and no body key.""" + mock_router_instance.params_the_target_accepts.return_value = MappingProxyType({}) + mock_router_instance.acompletion = AsyncMock(return_value=_llm_response('{"tier": "SIMPLE"}')) + await llm_complexity_router.aclassify("hello") + call_kwargs = mock_router_instance.acompletion.call_args.kwargs + mock_router_instance.params_the_target_accepts.assert_not_called() + assert "reasoning_effort" not in call_kwargs + assert "reasoning_effort" not in call_kwargs["proxy_server_request"]["body"] + class TestRouterPreRoutingAliasOverrides: """ diff --git a/tests/test_litellm/test_router.py b/tests/test_litellm/test_router.py index c843a66a1c1..c948bba5b75 100644 --- a/tests/test_litellm/test_router.py +++ b/tests/test_litellm/test_router.py @@ -12021,7 +12021,7 @@ def test_resolved_litellm_models_answers_through_every_channel_a_request_uses( assert set(_resolution_router().resolved_litellm_models(model_name)) == set(expected) -class TestTierParamsTheTargetAccepts: +class TestParamsTheTargetAccepts: """A tier's litellm_params are applied to every request that tier routes, so one the target cannot take raised UnsupportedParamsError before the request left the proxy, turning the whole tier into a 400.""" @@ -12041,14 +12041,14 @@ class TestTierParamsTheTargetAccepts: def test_drops_a_param_no_deployment_declares(self): router = self._router("novita/moonshotai/kimi-k3") - accepted = router._tier_params_the_target_accepts("tiered", {"reasoning_effort": "max"}, {}) + accepted = router.params_the_target_accepts("tiered", {"reasoning_effort": "max"}, {}) assert accepted == {} def test_keeps_a_param_the_deployment_declares(self): router = self._router("fireworks_ai/kimi-k3") - accepted = router._tier_params_the_target_accepts("tiered", {"reasoning_effort": "max"}, {}) + accepted = router.params_the_target_accepts("tiered", {"reasoning_effort": "max"}, {}) assert accepted == {"reasoning_effort": "max"} @@ -12070,7 +12070,7 @@ class TestTierParamsTheTargetAccepts: configuration the request needs while never touching what the provider would reject.""" router = self._router("novita/moonshotai/kimi-k3") - accepted = router._tier_params_the_target_accepts("tiered", {control: value, "reasoning_effort": "max"}, {}) + accepted = router.params_the_target_accepts("tiered", {control: value, "reasoning_effort": "max"}, {}) assert accepted == {control: value} @@ -12090,7 +12090,7 @@ class TestTierParamsTheTargetAccepts: drop_params or additional_drop_params would silently disable the operator's sanitization.""" router = self._router("novita/moonshotai/kimi-k3") - accepted = router._tier_params_the_target_accepts("tiered", {control: value, "reasoning_effort": "max"}, {}) + accepted = router.params_the_target_accepts("tiered", {control: value, "reasoning_effort": "max"}, {}) assert accepted == {control: value} @@ -12100,7 +12100,7 @@ class TestTierParamsTheTargetAccepts: both sets and allowlists would never 400 and must not be dropped.""" router = self._router("novita/moonshotai/kimi-k3") - accepted = router._tier_params_the_target_accepts( + accepted = router.params_the_target_accepts( "tiered", {"reasoning_effort": "max", "allowed_openai_params": ["reasoning_effort"]}, {} ) @@ -12109,7 +12109,7 @@ class TestTierParamsTheTargetAccepts: def test_request_allowlist_protects_the_param_it_names(self): router = self._router("novita/moonshotai/kimi-k3") - accepted = router._tier_params_the_target_accepts( + accepted = router.params_the_target_accepts( "tiered", {"reasoning_effort": "max"}, {"allowed_openai_params": ["reasoning_effort"]} ) @@ -12118,7 +12118,7 @@ class TestTierParamsTheTargetAccepts: def test_allowlist_protects_only_the_params_it_names(self): router = self._router("novita/moonshotai/kimi-k3") - accepted = router._tier_params_the_target_accepts( + accepted = router.params_the_target_accepts( "tiered", {"reasoning_effort": "max", "allowed_openai_params": ["seed"]}, {} ) @@ -12147,7 +12147,7 @@ class TestTierParamsTheTargetAccepts: today the mismatch fails loudly.""" router = self._router("petals/petals-team/StableBeluga2") - accepted = router._tier_params_the_target_accepts( + accepted = router.params_the_target_accepts( "tiered", {"max_completion_tokens": 100, "reasoning_effort": "max"}, {} ) @@ -12159,7 +12159,7 @@ class TestTierParamsTheTargetAccepts: worse than the error they already get.""" router = self._router("ai21/jamba-1.5-mini") - accepted = router._tier_params_the_target_accepts( + accepted = router.params_the_target_accepts( "tiered", {"extra_headers": {"x-tenant": "acme"}, "reasoning_effort": "max"}, {} ) @@ -12174,7 +12174,7 @@ class TestTierParamsTheTargetAccepts: ] ) - accepted = router._tier_params_the_target_accepts("tiered", {"reasoning_effort": "max"}, {}) + accepted = router.params_the_target_accepts("tiered", {"reasoning_effort": "max"}, {}) assert accepted == {"reasoning_effort": "max"} @@ -12247,7 +12247,7 @@ class TestTierParamsTheTargetAccepts: """An unresolvable target must never narrow what the request already did.""" router = self._router("fireworks_ai/kimi-k3") - accepted = router._tier_params_the_target_accepts("no-such-group", {"reasoning_effort": "max"}, {}) + accepted = router.params_the_target_accepts("no-such-group", {"reasoning_effort": "max"}, {}) assert accepted == {"reasoning_effort": "max"} diff --git a/ui/litellm-dashboard/src/components/add_model/ClassificationMethodConfig.tsx b/ui/litellm-dashboard/src/components/add_model/ClassificationMethodConfig.tsx index a29527a20fa..715bcff78a6 100644 --- a/ui/litellm-dashboard/src/components/add_model/ClassificationMethodConfig.tsx +++ b/ui/litellm-dashboard/src/components/add_model/ClassificationMethodConfig.tsx @@ -13,6 +13,7 @@ import ClassifierPromptEditor from "./ClassifierPromptEditor"; import CustomTierPromptEditor from "./CustomTierPromptEditor"; import { RestrictedSection, restrictedBy } from "./TierRestrictions"; import HeuristicScoringConfig from "./HeuristicScoringConfig"; +import { REASONING_EFFORT_OPTIONS, ReasoningEffort } from "./complexity_router_tiers"; import { useComplexityScorerDefaults } from "@/app/(dashboard)/hooks/autoRouter/useComplexityScorerDefaults"; import { ClassificationFrequency, @@ -49,6 +50,7 @@ const HEURISTIC_V2_EXPLANATION = "the first tier that meets its trained threshold. It runs locally with no classifier API call."; const CLASSIFIER_TIMEOUT_ID = "classifier-timeout-ms"; +const CLASSIFIER_EFFORT_PROVIDER_DEFAULT = "__provider_default__"; const CLASSIFIER_CONTEXT_WINDOW_SIZE_ID = "classifier-context-window-size"; const CLASSIFIER_CONTEXT_BUDGET_CHARS_ID = "classifier-context-budget-chars"; const HYBRID_BOUNDARY_MARGIN_ID = "hybrid-boundary-margin"; @@ -251,6 +253,12 @@ const ClassificationMethodConfig: React.FC = ({ const contextBudget = value.classifier_context_budget_chars ?? DEFAULT_CLASSIFIER_CONTEXT_BUDGET_CHARS; const contextBudgetQuotesNothing = contextBudget > 0 && contextBudget < MIN_QUOTED_CONTEXT_TURN_CHARS; const classificationRubric = value.classifier_llm_config?.classification_rubric ?? DEFAULT_CLASSIFICATION_RUBRIC; + const classifierEffort = value.classifier_llm_config?.reasoning_effort; + // A stored level the option list does not carry (hand-authored, or a level the picker omits) is + // listed anyway, so the control renders with its value selected and can be cleared. + const classifierEffortOptions = Array.from( + new Set([...REASONING_EFFORT_OPTIONS, ...(classifierEffort ? [classifierEffort] : [])]), + ); const handleClassifierTypeChange = (classifierType: ClassifierType) => { const nextValue: ComplexityRouterConfigValue = { @@ -320,6 +328,18 @@ const ClassificationMethodConfig: React.FC = ({ }); }; + const handleClassifierReasoningEffortChange = (reasoningEffort: ReasoningEffort | undefined) => { + onChange({ + ...value, + classifier_llm_config: { + ...value.classifier_llm_config, + model: value.classifier_llm_config?.model ?? "", + timeout_ms: value.classifier_llm_config?.timeout_ms ?? DEFAULT_CLASSIFIER_TIMEOUT_MS, + reasoning_effort: reasoningEffort, + }, + }); + }; + const handleClassificationRubricChange = (classificationRubric: ClassificationRubric) => { onChange({ ...value, @@ -485,15 +505,44 @@ const ClassificationMethodConfig: React.FC = ({
Classifier Model - +
+ + {value.classifier_llm_config?.model && ( + + + + )} +
{classifierModelMissing && A classifier model is required}
diff --git a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.test.tsx b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.test.tsx index 349d03356f8..9ffc80ed460 100644 --- a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.test.tsx +++ b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.test.tsx @@ -170,6 +170,45 @@ describe("ComplexityRouterConfig", () => { expect(screen.queryByText("Context Per-Turn Character Limit")).not.toBeInTheDocument(); }); + it("writes and clears classifier reasoning effort", async () => { + const onChange = vi.fn(); + const llmValue: ComplexityRouterConfigValue = { + ...defaultValue, + classifier_type: "llm", + classifier_llm_config: { model: "gpt-3.5-turbo", timeout_ms: 3000 }, + }; + const first = renderWithProviders(); + fireEvent.click(screen.getByText("Advanced: Classification Method")); + + const user = userEvent.setup(); + await user.click(screen.getByRole("combobox", { name: "Classifier reasoning effort" })); + await user.click(await screen.findByRole("option", { name: "low effort" })); + expect(onChange).toHaveBeenLastCalledWith({ + ...llmValue, + classifier_llm_config: { model: "gpt-3.5-turbo", timeout_ms: 3000, reasoning_effort: "low" }, + }); + + onChange.mockClear(); + first.unmount(); + renderWithProviders( + , + ); + fireEvent.click(screen.getByText("Advanced: Classification Method")); + await user.click(screen.getByRole("combobox", { name: "Classifier reasoning effort" })); + await user.click(await screen.findByRole("option", { name: "Default effort" })); + expect(onChange).toHaveBeenLastCalledWith({ + ...llmValue, + classifier_llm_config: { model: "gpt-3.5-turbo", timeout_ms: 3000, reasoning_effort: undefined }, + }); + }); + it("should default the context window and budget when llm is selected", () => { const llmValue: ComplexityRouterConfigValue = { ...defaultValue, diff --git a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx index 73ab4f0abef..67d2ad29f9a 100644 --- a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx +++ b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx @@ -126,6 +126,7 @@ export interface ClassifierLLMConfig { timeout_ms: number; classification_rubric?: ClassificationRubric; system_prompt?: string; + reasoning_effort?: ReasoningEffort; } export type ClassifierType = "heuristic" | "heuristic_v2" | "llm" | "heuristic_first" | "hybrid"; diff --git a/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.test.ts b/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.test.ts index c6370541a45..844470bbe44 100644 --- a/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.test.ts +++ b/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.test.ts @@ -530,6 +530,17 @@ describe("classifier prompt and fallback", () => { expect(config.classifier_llm_config?.system_prompt).toBe("Grade the data sensitivity of the request."); }); + it("carries a classifier reasoning_effort to the wire beside a rubric and beside a custom prompt", () => { + // Both payload builders rebuild classifier_llm_config from named keys, so an effort the + // operator set in config.yaml is wiped by any dashboard edit unless each one carries it. + expect(normalizeClassifierLlmConfig({ model: "m", timeout_ms: 1, reasoning_effort: "low" }).reasoning_effort).toBe( + "low", + ); + const withPrompt = { model: "m", timeout_ms: 1, system_prompt: "x", reasoning_effort: "minimal" } as const; + expect(normalizeClassifierLlmConfig(withPrompt)).toEqual(withPrompt); + expect(normalizeClassifierLlmConfig({ model: "m", timeout_ms: 1 })).not.toHaveProperty("reasoning_effort"); + }); + it("normalizeClassifierLlmConfig leaves a real prompt untouched and strips an empty one", () => { expect(normalizeClassifierLlmConfig({ model: "m", timeout_ms: 1, system_prompt: "x" })).toEqual({ model: "m", @@ -897,6 +908,16 @@ describe("buildComplexityRouterConfig with an edited tier set", () => { expect(build({ classifierType: "heuristic" }).classifier_type).toBe("llm"); }); + it("keeps the classifier reasoning_effort, which a custom tier set does not conflict with", () => { + // This branch deliberately drops system_prompt and classification_rubric because the backend + // rejects those beside tier_definitions. Effort is orthogonal to the tier taxonomy, so + // dropping it here would silently lose it on every custom-tier router. + const payload = build({ + classifierLlmConfig: { model: "gpt-4o-mini", timeout_ms: 3000, reasoning_effort: "low" }, + }); + expect(payload.classifier_llm_config).toEqual({ model: "gpt-4o-mini", timeout_ms: 3000, reasoning_effort: "low" }); + }); + it("turns session pinning off rather than leaving a stale true the backend rejects", () => { expect(build({ sessionAffinity: true }).session_affinity).toBe(false); }); diff --git a/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.ts b/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.ts index 94c5badf6d1..6bcab76fb5c 100644 --- a/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.ts +++ b/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.ts @@ -57,10 +57,13 @@ export const normalizeClassifierLlmConfig = ({ timeout_ms, classification_rubric, system_prompt, -}: ClassifierLLMConfig): ClassifierLLMConfig => - system_prompt?.trim() + reasoning_effort, +}: ClassifierLLMConfig): ClassifierLLMConfig => ({ + ...(system_prompt?.trim() ? { model, timeout_ms, system_prompt } - : { model, timeout_ms, ...(classification_rubric && { classification_rubric }) }; + : { model, timeout_ms, ...(classification_rubric && { classification_rubric }) }), + ...(reasoning_effort && { reasoning_effort }), +}); interface ScorerKnobInputs { classifierType: ClassifierType; @@ -293,11 +296,16 @@ export const customTierWireFields = ( tier_definitions: tierDefinitionsFromRows(rows), ...(fallback && { fallback_tier: activeTierName(fallback) }), classifier_type: "llm", - // Rebuilt from the two fields an edited tier set allows. The backend rejects system_prompt and + // Rebuilt from the fields an edited tier set allows. The backend rejects system_prompt and // classification_rubric beside tier_definitions, and both live inside this object rather than at // the top level the omit list covers. The opening instructions ride classification_prompt below. + // reasoning_effort is not part of the tier taxonomy, so a custom tier set keeps it. ...(classifierLlmConfig && { - classifier_llm_config: { model: classifierLlmConfig.model, timeout_ms: classifierLlmConfig.timeout_ms }, + classifier_llm_config: { + model: classifierLlmConfig.model, + timeout_ms: classifierLlmConfig.timeout_ms, + ...(classifierLlmConfig.reasoning_effort && { reasoning_effort: classifierLlmConfig.reasoning_effort }), + }, }), session_affinity: false, ...(classificationPrompt?.trim() && { classification_prompt: classificationPrompt.trim() }), diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index be81450b941..43f357febd9 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -25143,6 +25143,11 @@ export interface components { * @description Model name (from the router's model_list) to call for classification */ model: string; + /** + * Reasoning Effort + * @description reasoning_effort sent on the classifier's own call. Classification is a short, latency-sensitive call on every request, so a reasoning classifier model is usually worth holding at a cheap level. Setting it on the deployment instead moves effort for every request that deployment serves, which is wrong when the classifier model also serves normal traffic. A level the target refuses is dropped rather than failing the call, so classification still runs. Leave unset to send no effort and inherit whatever the deployment is configured with. Ignored by classifier types that call no model. + */ + reasoning_effort?: ("none" | "minimal" | "low" | "medium" | "high" | "xhigh" | "max") | null; /** * System Prompt * @description Replaces the built-in complexity rubric as the classifier's entire system role. When set, neither the default rubric nor the context-window closing line is appended, so the prompt owns the whole taxonomy and the tier names SIMPLE/MEDIUM/COMPLEX/REASONING become whatever buckets it defines: a prompt that classifies data sensitivity routes on that instead of on difficulty. Two consequences of full replacement. The default rubric's closing paragraph is the classifier's prompt-injection defense, telling it that the caller's quoted system prompt and prior turns are material to judge and never instructions; a replacement that omits it lets a caller ask for a tier and get it. And the heuristic fallback still scores complexity, so a router on some other taxonomy wants classifier_fallback='default_model'. Leave unset for the built-in rubric. Only applies when classifier_type is 'llm'.