diff --git a/litellm/router_strategy/complexity_router/complexity_router.py b/litellm/router_strategy/complexity_router/complexity_router.py index 2f571148878..3b8f22b41d7 100644 --- a/litellm/router_strategy/complexity_router/complexity_router.py +++ b/litellm/router_strategy/complexity_router/complexity_router.py @@ -1067,8 +1067,10 @@ def _extract_agent_progress( reaches the classifier. Harness reminders are stripped from tool output the same way they are from asks. A loop that overflows the budget has each entry cut to a quarter of it first, since one large file read - would otherwise fill the block alone and push out the result that described the task. A loop that fits - is quoted whole. + would otherwise fill the block alone. Its first tool result is kept even then, ahead of the newest entries + the rest of the budget buys, because that is usually the read that told the agent what the task is, and a + long loop of later reads would otherwise push it out and leave the classifier with routine output alone. A + loop that fits is quoted whole. """ from litellm.litellm_core_utils.prompt_templates.factory import tool_exchange_texts @@ -1088,13 +1090,16 @@ def _extract_agent_progress( if per_turn_chars is None else tuple((kind, _truncate(text, per_turn_chars)) for kind, text in newest_first) ) - overflows: Final = sum(len(text) for _, text in clamped) > budget_chars - shared: Final = ( - tuple((kind, _truncate(text, budget_chars // _TOOL_ENTRY_SHARES_OF_BUDGET)) for kind, text in clamped) - if overflows - else clamped - ) - return tuple(reversed(_turns_within_budget(shared, budget_chars))) + if sum(len(text) for _, text in clamped) <= budget_chars: + return tuple(reversed(clamped)) + share: Final = budget_chars // _TOOL_ENTRY_SHARES_OF_BUDGET + oldest_first: Final = tuple((kind, _truncate(text, share)) for kind, text in reversed(clamped)) + first_result: Final = next((index for index, (kind, _) in enumerate(oldest_first) if kind == "tool result"), None) + if first_result is None or share < _MIN_QUOTED_TURN_CHARS: + return tuple(reversed(_turns_within_budget(tuple(reversed(oldest_first)), budget_chars))) + anchor: Final = oldest_first[first_result] + newer: Final = tuple(reversed(oldest_first[first_result + 1 :])) + return (anchor, *reversed(_turns_within_budget(newer, budget_chars - len(anchor[1])))) def _decision_is_pinnable(decision: StandardLoggingRoutingDecision | None) -> bool: @@ -2116,8 +2121,8 @@ class ComplexityRouter(CustomLogger): custom tier set, and classifier_fallback otherwise decides between the heuristic scorer and default_model. The outcome's `cause` reports which path actually ran. - A user_turn tool turn (`tool_turn`) trips a circuit breaker of its own: its failure only keeps the held - tier, so it must not switch the classifier off for every other session's new asks. + A tool turn (`tool_turn`) trips a circuit breaker of its own, since its classification carries the loop's + tool output and is the slower call: its timeouts must not switch the classifier off for new asks. """ breaker: Final = self._tool_turn_circuit_breaker if tool_turn else self._classifier_circuit_breaker if self.config.classifier_type == "heuristic_v2": @@ -3793,11 +3798,12 @@ class ComplexityRouter(CustomLogger): """Whether no configured tier outranks the plan-mode floor, i.e. the classifier's answer could never rise above it and classification would be pure spend.""" floor: Final = self._resolve_plan_mode_floor() - if floor is None: - return False + return floor is not None and self._is_top_configured_tier(floor) + + def _is_top_configured_tier(self, tier: ComplexityTier | str) -> bool: configured: Final = frozenset(self.config.tiers) names: Final = self.config.tier_names() - return all(name not in configured for name in names[self._active_tier_severity(floor) + 1 :]) + return all(name not in configured for name in names[self._active_tier_severity(tier) + 1 :]) def _matched_escalation_keyword(self, user_message: str) -> str | None: """The escalation keyword the prompt contains, or None when escalation is off. @@ -4847,7 +4853,8 @@ class ComplexityRouter(CustomLogger): classified on the work done so far, and only a higher tier replaces the pin, so a loop moves up once a tool result shows harder work and does not drop back mid-task. The tier is compared before routing, so a turn that stays on the pin leaves no routing state behind, and it gets the classification back so the held turn - still reports the classifier call it paid for. + still reports the classifier call it paid for. A pin no classification could replace, the top configured tier + or the tier a keyword_tier_rules match on the ask decides before any classifier runs, is kept without one. """ if ( self.config.classification_mode != "user_turn" @@ -4860,7 +4867,12 @@ class ComplexityRouter(CustomLogger): ask, system_prompt = _extract_current_ask_and_system_prompt( resolved_messages, self._reminder_markers_for_request(request_kwargs) ) - if held_tier is None or ask is None: + if ( + held_tier is None + or ask is None + or self._is_top_configured_tier(held_tier) + or await self._resolve_keyword_tier_override(ask, request_kwargs) is not None + ): return None classification: Final = await self.aclassify( ask, system_prompt, request_kwargs, resolved_messages, raw_messages=messages, tool_turn=True @@ -5166,7 +5178,15 @@ class ComplexityRouter(CustomLogger): if housekeeping_tier is not None else classification if classification is not None - else await self.aclassify(ask, system_prompt, request_kwargs, resolved_messages, raw_messages=messages) + else await self.aclassify( + ask, + system_prompt, + request_kwargs, + resolved_messages, + raw_messages=messages, + tool_turn=self.config.classifier_context_include_tool_turns + and not _newest_turn_is_human_ask(resolved_messages, marker_pairs), + ) ) tier, score, signals = outcome.tier, outcome.score, outcome.signals classified_tier: Final = tier diff --git a/litellm/router_strategy/complexity_router/config.py b/litellm/router_strategy/complexity_router/config.py index 45282832e19..cafd1ac2b50 100644 --- a/litellm/router_strategy/complexity_router/config.py +++ b/litellm/router_strategy/complexity_router/config.py @@ -1377,13 +1377,16 @@ class ComplexityRouterConfig(LiteLLMBaseModel): "is a floor for its tool turns: each one is classified, and only a higher tier moves the " "session up and replaces the pin, so a loop never drops back mid-task, at the cost of a " "classifier call per tool turn, which the turn waits on, up to the classifier timeout, before it " - "routes. A tool turn whose classifier call fails keeps the held tier, and with the circuit breaker " - "on it opens a breaker of its own, so later tool turns replay the pin at once while new asks keep " - "classifying. Under heuristic_first and hybrid a turn with tool traffic " + "routes. That call is skipped when nothing could outrank the pin: the top configured tier, or an " + "ask a keyword_tier_rules match decides. A tool turn whose classifier call fails keeps the held " + "tier. With the circuit breaker on, tool turns in either mode trip a breaker of their own, so a " + "timeout on one opens it for the worker's tool turns while new asks keep classifying. Under " + "heuristic_first and hybrid a turn with tool traffic " "always goes to the classifier model, since the local scorer reads only the ask. The block is bounded " "on its own by classifier_context_budget_chars and classifier_context_per_turn_chars, " "newest first, and once a loop overflows that budget each entry is cut to a quarter of it, so " - "one large file read cannot push out the rest. It is not bounded by " + "one large file read cannot push out the rest, and the first tool result after the ask, usually " + "the read that described the task, is kept ahead of the newest ones. It is not bounded by " "classifier_context_window_size, since one ask can run dozens " "of tool calls; setting the window size to 0 still turns all context off. Tool output " "such as file contents is sent to the classifier model, which may be a different " diff --git a/tests/unit/router_strategy/test_complexity_router.py b/tests/unit/router_strategy/test_complexity_router.py index 68ea968a726..21161f2f086 100644 --- a/tests/unit/router_strategy/test_complexity_router.py +++ b/tests/unit/router_strategy/test_complexity_router.py @@ -11733,6 +11733,42 @@ class TestAgentProgressReachesClassifier: assert response.model == "strong" assert "balance = 0" in client.states[-1] + @pytest.mark.asyncio + @pytest.mark.parametrize("reads,read_chars", [(4, 3000), (6, 1500), (20, 400)]) + async def test_long_loop_keeps_the_read_that_described_the_task(self, reads: int, read_chars: int) -> None: + """Every later read in a long loop is routine on its own, so once they overflow the budget the classifier + must still see the task file the loop opened with, or every_request drops the loop to the cheap tier.""" + client: Final = _TaskAwareJevClient() + router: Final = self._jev_router(client, classifier_context_include_tool_turns=True) + later_reads: Final = tuple( + turn + for i in range(reads) + for turn in ( + { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": f"call_read_{i}", + "type": "function", + "function": {"name": "read_file", "arguments": f'{{"path": "src/module_{i}.py"}}'}, + } + ], + }, + {"role": "tool", "tool_call_id": f"call_read_{i}", "content": f"# module {i}\n" + "x = 1\n" * (read_chars // 6)}, + ) + ) + + response: Final = await router.async_pre_routing_hook( + model="test-router", + request_kwargs={}, + messages=[*_tool_loop_continuation("chat_completions"), *later_reads], + ) + + assert response is not None + assert response.model == "strong" + assert f"# module {reads - 1}" in client.states[-1] + @pytest.mark.asyncio async def test_zero_context_window_turns_tool_context_off(self) -> None: client: Final = _TaskAwareJevClient() @@ -11859,6 +11895,59 @@ class TestAgentProgressReachesClassifier: assert decisions[1] is not None and decisions[1].get("classifier_failure_reason") == "timeout" assert decisions[3] is not None and decisions[3].get("classifier_failure_reason") == "circuit_open" + @pytest.mark.asyncio + async def test_every_request_tool_turn_timeout_leaves_new_asks_classifying(self) -> None: + """A long loop's tool turn is the slow classification, so its timeout must not switch the classifier off + for another session's new ask, while the loop's next tool turn skips the call that just timed out.""" + client: Final = _ScriptedJevClient(("TIMEOUT", "COMPLEX")) + router: Final = self._jev_router( + client, classification_mode="every_request", classifier_context_include_tool_turns=True + ) + loop: Final = _tool_loop_continuation("chat_completions") + new_ask: Final = [{"role": "user", "content": "Design a lock-free MPMC queue and prove linearizability"}] + + responses: Final = [ + await router.async_pre_routing_hook(model="test-router", request_kwargs={}, messages=turn) + for turn in (loop, new_ask, [*loop, *_RUN_TESTS_TURNS]) + ] + + assert len(client.states) == 2 + assert responses[1] is not None and responses[1].model == "strong" + decisions: Final = [response.routing_decision if response is not None else None for response in responses] + assert decisions[0] is not None and decisions[0].get("classifier_failure_reason") == "timeout" + assert decisions[2] is not None and decisions[2].get("classifier_failure_reason") == "circuit_open" + + @pytest.mark.asyncio + @pytest.mark.parametrize( + "overrides,ask_tier,held_model", + [ + pytest.param({}, ("REASONING",), "top", id="held-top-tier"), + pytest.param( + {"keyword_tier_rules": [{"keywords": ["TASK.md"], "tier": "MEDIUM"}]}, (), "mid", id="keyword-rule" + ), + ], + ) + async def test_user_turn_skips_the_classifier_when_nothing_could_replace_the_pin( + self, overrides: dict[str, object], ask_tier: tuple[str, ...], held_model: str + ) -> None: + """A tool turn whose classification could never move the session, because it holds the top tier or a + keyword rule decides its ask before any classifier runs, must route on the pin without paying for a call.""" + client: Final = _ScriptedJevClient(ask_tier) + router: Final = self._jev_router( + client, classification_mode="user_turn", classifier_context_include_tool_turns=True, **overrides + ) + loop: Final = _tool_loop_continuation("chat_completions") + + responses: Final = [ + await router.async_pre_routing_hook( + model="test-router", request_kwargs={"metadata": {"session_id": "agent-loop"}}, messages=turn + ) + for turn in (loop[:1], loop, [*loop, *_RUN_TESTS_TURNS]) + ] + + assert [response.model for response in responses if response is not None] == [held_model] * 3 + assert len(client.states) == len(ask_tier) + @pytest.mark.asyncio async def test_user_turn_escalation_never_samples_below_the_held_tier(self) -> None: """An adaptive router scoring every model must not let an escalating tool turn land below the tier the diff --git a/ui/litellm-dashboard/src/components/add_model/ClassificationMethodConfig.tsx b/ui/litellm-dashboard/src/components/add_model/ClassificationMethodConfig.tsx index b47f10ee426..75751cccbf5 100644 --- a/ui/litellm-dashboard/src/components/add_model/ClassificationMethodConfig.tsx +++ b/ui/litellm-dashboard/src/components/add_model/ClassificationMethodConfig.tsx @@ -24,7 +24,7 @@ import { getHeuristicV2SuccessThresholdError } from "./build_complexity_router_c import ClassifierPluginTimeoutField from "./ClassifierPluginTimeoutField"; import ClassifierTypeRadios from "./ClassifierTypeRadios"; import { ClassifierMenu, ClassifierOption } from "./AutoRouterClassifierTabs"; -import { isHeuristicChain } from "./classifier_types"; +import { isHeuristicChain, readsToolTurns } from "./classifier_types"; import HeuristicKeywordOverrides from "./HeuristicKeywordOverrides"; import type { ReasoningEffort } from "./complexity_router_tiers"; import { useComplexityScorerDefaults } from "@/app/(dashboard)/hooks/autoRouter/useComplexityScorerDefaults"; @@ -49,6 +49,7 @@ import { DEFAULT_HYBRID_BOUNDARY_MARGIN, HEURISTIC_FIRST_MAX_TIER_KEYS, effectiveClassifierType, + classificationFrequency, } from "./ComplexityRouterConfig"; const DEFAULT_SCORING_EXPLANATION = @@ -361,6 +362,13 @@ const ClassificationMethodConfig: React.FC = ({ }); }; + const handleClassifierContextIncludeToolTurnsChange = (includeToolTurns: boolean) => { + onChange({ + ...value, + classifier_context_include_tool_turns: includeToolTurns, + }); + }; + return ( <> {!advancedOnly && showSelection && ( @@ -735,6 +743,26 @@ const ClassificationMethodConfig: React.FC = ({ last N user turns. + {readsToolTurns(classifierType) && classificationFrequency(value) !== "session" && ( +
+
+ + Include Tool Turns + + + +
+ + Let the classifier read the tool calls and results that followed the current ask, so an agent loop is + classified on what its tools turned up, such as the file that describes the task. + +
+ )} )} diff --git a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.integration.test.tsx b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.integration.test.tsx index acbece64904..3d321974e74 100644 --- a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.integration.test.tsx +++ b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.integration.test.tsx @@ -445,6 +445,43 @@ describe("ComplexityRouterConfig", () => { ); }); + it("should turn tool turns on from its switch when classifier_type is llm", async () => { + const onChange = vi.fn(); + const llmValue: ComplexityRouterConfigValue = { + ...defaultValue, + classifier_type: "llm", + classifier_llm_config: { model: "gpt-3.5-turbo", timeout_ms: 3000 }, + }; + renderWithProviders(); + + openAutoRouterAdvanced("Classification Method"); + const toolTurns = screen.getByRole("switch", { name: "Include Tool Turns" }); + expect(toolTurns).not.toBeChecked(); + fireEvent.click(toolTurns); + + expect(onChange).toHaveBeenCalledWith(expect.objectContaining({ classifier_context_include_tool_turns: true })); + }); + + it("should hide the tool-turns switch when classifier_type is heuristic", async () => { + renderWithProviders(); + openAutoRouterAdvanced("Classification Method"); + expect(screen.queryByRole("switch", { name: "Include Tool Turns" })).not.toBeInTheDocument(); + }); + + it("should hide the tool-turns switch when session affinity is on", async () => { + const pinnedValue: ComplexityRouterConfigValue = { + ...defaultValue, + classifier_type: "llm", + classifier_llm_config: { model: "gpt-3.5-turbo", timeout_ms: 3000 }, + session_affinity: true, + classifier_context_include_tool_turns: true, + }; + renderWithProviders(); + openAutoRouterAdvanced("Classification Method"); + expect(screen.getByRole("switch", { name: "Include Assistant Turns" })).toBeInTheDocument(); + expect(screen.queryByRole("switch", { name: "Include Tool Turns" })).not.toBeInTheDocument(); + }); + it("should hide classifier context fields when classifier_type is heuristic", async () => { renderWithProviders(); openAutoRouterAdvanced("Classification Method"); diff --git a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx index ac13af14a6b..6cdd4efad6a 100644 --- a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx +++ b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx @@ -382,6 +382,7 @@ export interface ComplexityRouterConfigValue { classifier_context_budget_chars?: number; classifier_context_per_turn_chars?: number; classifier_context_include_assistant_turns?: boolean; + classifier_context_include_tool_turns?: boolean; classifier_fallback?: ClassifierFallback; /** Classification instructions only; the router appends derived tier bullets after them. */ classification_prompt?: string; diff --git a/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.test.ts b/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.test.ts index b25baa0d1e4..5cdb6b75a28 100644 --- a/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.test.ts +++ b/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.test.ts @@ -780,6 +780,82 @@ describe("buildComplexityRouterConfig assistant turns", () => { }); }); +describe("buildComplexityRouterConfig tool turns", () => { + const llmParams: BuildComplexityRouterConfigParams = { + ...baseParams, + classifierType: "llm", + classifierLlmConfig: { model: "gpt-4o-mini", timeout_ms: 3000 }, + }; + + it.each(["llm", "heuristic_first", "hybrid"] as const)("emits the field when %s classifies", (classifierType) => { + const config = buildComplexityRouterConfig({ + ...llmParams, + classifierType, + classifierContextIncludeToolTurns: true, + }); + expect(config.classifier_context_include_tool_turns).toBe(true); + }); + + it("emits it for the decision model under its oss_classifier wire name", () => { + const config = buildComplexityRouterConfig({ + ...baseParams, + classifierType: "jev", + classifierContextIncludeToolTurns: true, + }); + expect(config.classifier_type).toBe("oss_classifier"); + expect(config.classifier_context_include_tool_turns).toBe(true); + }); + + it("emits the switch turned off", () => { + const config = buildComplexityRouterConfig({ ...llmParams, classifierContextIncludeToolTurns: false }); + expect(config.classifier_context_include_tool_turns).toBe(false); + }); + + it.each(["heuristic", "heuristic_v2", "capability", "llm_v2", "custom"] as const)( + "omits it for %s, which the backend rejects it beside", + (classifierType) => { + const config = buildComplexityRouterConfig({ + ...llmParams, + classifierType, + classifierContextIncludeToolTurns: true, + }); + expect(config.classifier_context_include_tool_turns).toBeUndefined(); + }, + ); + + it("omits it with session affinity on, which the backend rejects it beside", () => { + const config = buildComplexityRouterConfig({ + ...llmParams, + sessionAffinity: true, + classifierContextIncludeToolTurns: true, + }); + expect(config.session_affinity).toBe(true); + expect(config.classifier_context_include_tool_turns).toBeUndefined(); + }); + + it("emits it under an edited tier set, whose save turns a lingering session affinity off", () => { + const pinnedCustomParams: BuildComplexityRouterConfigParams = { + ...llmParams, + sessionAffinity: true, + classifierContextIncludeToolTurns: true, + customTierSet: { + tiers: [ + { id: "quick", name: "QUICK", definition: "Short answers", models: ["fast"] }, + { id: "review", name: "REVIEW", definition: "Deep review", models: ["strong"] }, + ], + fallback_tier_id: "quick", + }, + }; + const config = buildComplexityRouterConfig(pinnedCustomParams); + expect(config.session_affinity).toBe(false); + expect(config.classifier_context_include_tool_turns).toBe(true); + }); + + it("omits it when unset, leaving the backend default", () => { + expect(buildComplexityRouterConfig(llmParams).classifier_context_include_tool_turns).toBeUndefined(); + }); +}); + describe("classifier prompt and fallback", () => { const llmParams: BuildComplexityRouterConfigParams = { ...baseParams, diff --git a/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.ts b/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.ts index 511a1653aae..8a719fd659f 100644 --- a/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.ts +++ b/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.ts @@ -5,7 +5,7 @@ import { type FuseSettings, } from "./forecast_classifier_config"; import type { ModelGroup } from "../llm_calls/fetch_models"; -import { isHeuristicChain, type LocalHeuristic } from "./classifier_types"; +import { isHeuristicChain, readsToolTurns, type LocalHeuristic } from "./classifier_types"; import { KeywordTierRule } from "./KeywordTierRules"; import { type JevClassifierConfig, @@ -127,6 +127,7 @@ export interface StoredComplexityRouterConfig { classifier_context_budget_chars?: unknown; classifier_context_per_turn_chars?: unknown; classifier_context_include_assistant_turns?: unknown; + classifier_context_include_tool_turns?: unknown; classifier_fallback?: unknown; classification_mode?: unknown; tier_boundaries?: unknown; @@ -182,6 +183,7 @@ export interface BuildComplexityRouterConfigParams { classifierContextBudgetChars: number | undefined; classifierContextPerTurnChars?: number; classifierContextIncludeAssistantTurns: boolean | undefined; + classifierContextIncludeToolTurns?: boolean; classifierFallback: ClassifierFallback | undefined; classificationPrompt: string | undefined; classificationExamples: string | undefined; @@ -266,6 +268,7 @@ export interface ComplexityRouterConfigPayload { classifier_context_budget_chars?: number; classifier_context_per_turn_chars?: number; classifier_context_include_assistant_turns?: boolean; + classifier_context_include_tool_turns?: boolean; classifier_fallback?: ClassifierFallback; classification_prompt?: string; classification_examples?: string; @@ -567,6 +570,19 @@ export const hydratePlanModeMinTier = ( return tierRowByName(customTierSet.tiers, stored)?.id; }; +const toolTurnWireFields = ( + effectiveType: ClassifierType, + { + classifierContextIncludeToolTurns, + sessionAffinity, + customTierSet, + }: Pick, +): Partial => { + const pinsSession = sessionAffinity && !customTierSet; + if (!readsToolTurns(effectiveType) || pinsSession || classifierContextIncludeToolTurns === undefined) return {}; + return { classifier_context_include_tool_turns: classifierContextIncludeToolTurns }; +}; + const classifierWireFields = ( effectiveType: ClassifierType, { @@ -640,6 +656,7 @@ export const buildComplexityRouterConfig = ({ classifierContextBudgetChars, classifierContextPerTurnChars, classifierContextIncludeAssistantTurns, + classifierContextIncludeToolTurns, classifierFallback, classificationPrompt, classificationExamples, @@ -768,6 +785,7 @@ export const buildComplexityRouterConfig = ({ heuristic_v2_success_threshold: heuristicV2SuccessThreshold, }), ...classifierWireFields(effectiveType, classifierInputs), + ...toolTurnWireFields(effectiveType, { classifierContextIncludeToolTurns, sessionAffinity, customTierSet }), ...(effectiveType === "capability" && capabilityClassifierConfig && { capability_classifier_config: capabilityClassifierConfig }), ...(effectiveType === "llm_v2" && { llm_v2_config: llmV2Config }), diff --git a/ui/litellm-dashboard/src/components/add_model/classifier_type_transition.test.ts b/ui/litellm-dashboard/src/components/add_model/classifier_type_transition.test.ts index a26b39c2980..bd96f865fa7 100644 --- a/ui/litellm-dashboard/src/components/add_model/classifier_type_transition.test.ts +++ b/ui/litellm-dashboard/src/components/add_model/classifier_type_transition.test.ts @@ -99,6 +99,20 @@ describe("transitionClassifierType", () => { expect(result.llm_v2_config).toBeUndefined(); }); + it.each([ + ["jev", true], + ["heuristic_first", true], + ["hybrid", true], + ["heuristic", undefined], + ["heuristic_v2", undefined], + ["capability", undefined], + ["llm_v2", undefined], + ["custom", undefined], + ] as const)("carries tool turns into %s only when it can read them", (target, expected) => { + const result = transitionClassifierType({ ...standard, classifier_context_include_tool_turns: true }, target); + expect(result.classifier_context_include_tool_turns).toBe(expected); + }); + it("clears the inactive non-reasoning pool and plan floor when switching to local classification", () => { const initial: ComplexityRouterConfigValue = { ...standard, diff --git a/ui/litellm-dashboard/src/components/add_model/classifier_type_transition.ts b/ui/litellm-dashboard/src/components/add_model/classifier_type_transition.ts index ba758eac471..2db01d402c8 100644 --- a/ui/litellm-dashboard/src/components/add_model/classifier_type_transition.ts +++ b/ui/litellm-dashboard/src/components/add_model/classifier_type_transition.ts @@ -9,10 +9,12 @@ import { NEW_CLASSIFIER_CLASSIFICATION_RUBRIC, usesLlmClassifier, usesClassifierContext, + effectiveClassifierType, } from "./ComplexityRouterConfig"; import { defaultJevClassifierConfig } from "./jev_classifier_config"; import { isForecastClassifier, prepareForecastClassifier } from "./forecast_classifier_config"; import { nonReasoningTierFields } from "./nonReasoningTierFields"; +import { readsToolTurns } from "./classifier_types"; export const transitionClassifierType = ( value: ComplexityRouterConfigValue, @@ -46,6 +48,11 @@ export const transitionClassifierType = ( classifier_context_include_assistant_turns: usesClassifierContext(classifierType) ? value.classifier_context_include_assistant_turns : undefined, + classifier_context_include_tool_turns: readsToolTurns( + effectiveClassifierType({ custom_tier_set: value.custom_tier_set, classifier_type: classifierType }), + ) + ? value.classifier_context_include_tool_turns + : undefined, classifier_fallback: usesClassifierContext(classifierType) ? value.classifier_fallback : undefined, heuristic_first_max_tier: classifierType === "heuristic_first" diff --git a/ui/litellm-dashboard/src/components/add_model/classifier_types.ts b/ui/litellm-dashboard/src/components/add_model/classifier_types.ts index c2c3bb1a0c3..ca32d26caba 100644 --- a/ui/litellm-dashboard/src/components/add_model/classifier_types.ts +++ b/ui/litellm-dashboard/src/components/add_model/classifier_types.ts @@ -19,3 +19,6 @@ export const usesLlmClassifier = (classifierType: ClassifierType): boolean => export const usesClassifierContext = (classifierType: ClassifierType): boolean => classifierType === "jev" || usesLlmClassifier(classifierType); + +export const readsToolTurns = (classifierType: ClassifierType): boolean => + (["llm", "jev", "heuristic_first", "hybrid"] as const).some((type) => type === classifierType); diff --git a/ui/litellm-dashboard/src/components/add_model/complexity_router_builder_params.ts b/ui/litellm-dashboard/src/components/add_model/complexity_router_builder_params.ts index aed6253d89b..7bee8560762 100644 --- a/ui/litellm-dashboard/src/components/add_model/complexity_router_builder_params.ts +++ b/ui/litellm-dashboard/src/components/add_model/complexity_router_builder_params.ts @@ -40,6 +40,7 @@ export const builderParamsFromValue = ( classifierContextBudgetChars: value.classifier_context_budget_chars, classifierContextPerTurnChars: value.classifier_context_per_turn_chars, classifierContextIncludeAssistantTurns: value.classifier_context_include_assistant_turns, + classifierContextIncludeToolTurns: value.classifier_context_include_tool_turns, classifierFallback: value.classifier_fallback, sessionAffinity: value.session_affinity ?? DEFAULT_SESSION_AFFINITY, sessionAffinityTtlSeconds: value.session_affinity_ttl_seconds, diff --git a/ui/litellm-dashboard/src/components/edit_auto_router/build_updated_complexity_router_config.test.ts b/ui/litellm-dashboard/src/components/edit_auto_router/build_updated_complexity_router_config.test.ts index 6238e30ce07..daccf49f7bb 100644 --- a/ui/litellm-dashboard/src/components/edit_auto_router/build_updated_complexity_router_config.test.ts +++ b/ui/litellm-dashboard/src/components/edit_auto_router/build_updated_complexity_router_config.test.ts @@ -592,6 +592,52 @@ describe("buildUpdatedComplexityRouterConfig assistant turns", () => { }); }); +const STORED_TOOL_TURNS = { + tiers: { SIMPLE: ["gpt-4o-mini"], MEDIUM: [], COMPLEX: [], REASONING: [] }, + classifier_type: "llm" as const, + classifier_llm_config: { model: "gpt-4o-mini", timeout_ms: 3000 }, + classifier_context_include_tool_turns: true, +}; + +describe("buildUpdatedComplexityRouterConfig tool turns", () => { + const hydrated = hydrateComplexityRouterConfig(STORED_TOOL_TURNS, undefined); + + it("keeps a stored true through an unrelated edit", () => { + const result = buildUpdatedComplexityRouterConfig(STORED_TOOL_TURNS, { + ...hydrated, + classifier_context_window_size: 5, + }); + expect(result.classifier_context_window_size).toBe(5); + expect(result.classifier_context_include_tool_turns).toBe(true); + }); + + it("persists turning tool turns back off", () => { + const result = buildUpdatedComplexityRouterConfig(STORED_TOOL_TURNS, { + ...hydrated, + classifier_context_include_tool_turns: false, + }); + expect(result.classifier_context_include_tool_turns).toBe(false); + }); + + it("drops a stored true when the edit switches the classifier to heuristic", () => { + const result = buildUpdatedComplexityRouterConfig(STORED_TOOL_TURNS, { + ...hydrated, + classifier_type: "heuristic", + }); + expect(result.classifier_type).toBe("heuristic"); + expect(result).not.toHaveProperty("classifier_context_include_tool_turns"); + }); + + it("drops a stored true when the edit turns session affinity on", () => { + const result = buildUpdatedComplexityRouterConfig(STORED_TOOL_TURNS, { + ...hydrated, + session_affinity: true, + }); + expect(result.session_affinity).toBe(true); + expect(result).not.toHaveProperty("classifier_context_include_tool_turns"); + }); +}); + describe("buildUpdatedComplexityRouterConfig session affinity", () => { it("writes session_affinity=false when the toggle is off", () => { const result = buildUpdatedComplexityRouterConfig(STORED, { ...FORM_VALUE, session_affinity: false }); @@ -974,11 +1020,13 @@ describe("managed keys survive an untouched open-and-save", () => { ]); // The stall keys are rejected beside the session pinning and user-turn classification this - // fixture sets, so they get their own round trip below rather than widening this one. + // fixture sets, and tool turns beside the session pinning, so they get their own round trips below + // rather than widening this one. const KEYS_ANOTHER_CLASSIFICATION_FREQUENCY_OWNS = new Set([ "stall_escalation_enabled", "stall_escalation_window", "stall_escalation_repeat_threshold", + "classifier_context_include_tool_turns", ]); // The opt-in fifth tier requires the LLM classifier, which this heuristic_first fixture is not, @@ -1070,6 +1118,17 @@ describe("managed keys survive an untouched open-and-save", () => { expect(saved.stall_escalation_repeat_threshold).toBe(4); }); + it("carries tool turns through their own round trip", () => { + const stored: Record = { + ...STORED_ALL_MANAGED, + session_affinity: false, + classifier_context_include_tool_turns: true, + }; + const saved = buildUpdatedComplexityRouterConfig(stored, hydrateComplexityRouterConfig(stored, undefined)); + + expect(saved.classifier_context_include_tool_turns).toBe(true); + }); + it("leaves the stall keys out of a saved config that never had them on", () => { const stored: Record = { ...STORED_ALL_MANAGED, diff --git a/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.tsx b/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.tsx index 6b437ca85df..44ba1df0ef4 100644 --- a/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.tsx +++ b/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.tsx @@ -110,6 +110,7 @@ export const MANAGED_COMPLEXITY_ROUTER_KEYS = new Set([ "classifier_context_window_size", "classifier_context_budget_chars", "classifier_context_include_assistant_turns", + "classifier_context_include_tool_turns", "classifier_fallback", "classification_prompt", "classification_examples", diff --git a/ui/litellm-dashboard/src/components/edit_auto_router/hydrate_complexity_router_config.ts b/ui/litellm-dashboard/src/components/edit_auto_router/hydrate_complexity_router_config.ts index 75fd6e15bc1..52c650c8b20 100644 --- a/ui/litellm-dashboard/src/components/edit_auto_router/hydrate_complexity_router_config.ts +++ b/ui/litellm-dashboard/src/components/edit_auto_router/hydrate_complexity_router_config.ts @@ -99,6 +99,10 @@ export const hydrateComplexityRouterConfig = ( typeof parsedConfig.classifier_context_include_assistant_turns === "boolean" ? parsedConfig.classifier_context_include_assistant_turns : undefined, + classifier_context_include_tool_turns: + typeof parsedConfig.classifier_context_include_tool_turns === "boolean" + ? parsedConfig.classifier_context_include_tool_turns + : undefined, classifier_fallback: parsedConfig.classifier_fallback === "default_model" || parsedConfig.classifier_fallback === "heuristic" ? parsedConfig.classifier_fallback diff --git a/ui/litellm-dashboard/src/lib/autorouter_presets.ts b/ui/litellm-dashboard/src/lib/autorouter_presets.ts index fd400a8163a..9f212fce42f 100644 --- a/ui/litellm-dashboard/src/lib/autorouter_presets.ts +++ b/ui/litellm-dashboard/src/lib/autorouter_presets.ts @@ -353,6 +353,7 @@ export const buildPresetPrefill = ( classifier_context_budget_chars: config.classifier_context_budget_chars, classifier_context_per_turn_chars: config.classifier_context_per_turn_chars, classifier_context_include_assistant_turns: config.classifier_context_include_assistant_turns, + classifier_context_include_tool_turns: config.classifier_context_include_tool_turns, classification_mode: config.classification_mode ?? DEFAULT_CLASSIFICATION_MODE, session_affinity: config.session_affinity ?? DEFAULT_SESSION_AFFINITY, session_affinity_ttl_seconds: config.session_affinity_ttl_seconds, diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 092a1f55dfd..f1dd3e01e99 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -42470,7 +42470,7 @@ export interface components { classifier_context_include_assistant_turns: boolean; /** * Classifier Context Include Tool Turns - * @description Send the tool calls and tool results that followed the current ask to the classifier as the work done on it so far, so an agent loop's continuation turn is classified on what its tools turned up rather than on the ask alone: 'What is in TASK.md?' stays SIMPLE until the file it reads describes a complex task, and the turns after that classify as that task. With classification_mode='every_request' every turn of the loop is classified this way. With classification_mode='user_turn' the tier pinned for the ask is a floor for its tool turns: each one is classified, and only a higher tier moves the session up and replaces the pin, so a loop never drops back mid-task, at the cost of a classifier call per tool turn, which the turn waits on, up to the classifier timeout, before it routes. A tool turn whose classifier call fails keeps the held tier, and with the circuit breaker on it opens a breaker of its own, so later tool turns replay the pin at once while new asks keep classifying. Under heuristic_first and hybrid a turn with tool traffic always goes to the classifier model, since the local scorer reads only the ask. The block is bounded on its own by classifier_context_budget_chars and classifier_context_per_turn_chars, newest first, and once a loop overflows that budget each entry is cut to a quarter of it, so one large file read cannot push out the rest. It is not bounded by classifier_context_window_size, since one ask can run dozens of tool calls; setting the window size to 0 still turns all context off. Tool output such as file contents is sent to the classifier model, which may be a different deployment or provider than the routed completion model. Off by default because enabling it shifts tier decisions, and therefore spend, for an already-deployed router. Requires classifier_type llm, oss_classifier, heuristic_first, or hybrid, the ones that send the conversation to a classifier model, and cannot be combined with session_affinity, which replays its pin without classifying. + * @description Send the tool calls and tool results that followed the current ask to the classifier as the work done on it so far, so an agent loop's continuation turn is classified on what its tools turned up rather than on the ask alone: 'What is in TASK.md?' stays SIMPLE until the file it reads describes a complex task, and the turns after that classify as that task. With classification_mode='every_request' every turn of the loop is classified this way. With classification_mode='user_turn' the tier pinned for the ask is a floor for its tool turns: each one is classified, and only a higher tier moves the session up and replaces the pin, so a loop never drops back mid-task, at the cost of a classifier call per tool turn, which the turn waits on, up to the classifier timeout, before it routes. That call is skipped when nothing could outrank the pin: the top configured tier, or an ask a keyword_tier_rules match decides. A tool turn whose classifier call fails keeps the held tier. With the circuit breaker on, tool turns in either mode trip a breaker of their own, so a timeout on one opens it for the worker's tool turns while new asks keep classifying. Under heuristic_first and hybrid a turn with tool traffic always goes to the classifier model, since the local scorer reads only the ask. The block is bounded on its own by classifier_context_budget_chars and classifier_context_per_turn_chars, newest first, and once a loop overflows that budget each entry is cut to a quarter of it, so one large file read cannot push out the rest, and the first tool result after the ask, usually the read that described the task, is kept ahead of the newest ones. It is not bounded by classifier_context_window_size, since one ask can run dozens of tool calls; setting the window size to 0 still turns all context off. Tool output such as file contents is sent to the classifier model, which may be a different deployment or provider than the routed completion model. Off by default because enabling it shifts tier decisions, and therefore spend, for an already-deployed router. Requires classifier_type llm, oss_classifier, heuristic_first, or hybrid, the ones that send the conversation to a classifier model, and cannot be combined with session_affinity, which replays its pin without classifying. * @default false */ classifier_context_include_tool_turns: boolean;