From dc9bbb762fb99f7351703b0dc6716ca05c41a56b Mon Sep 17 00:00:00 2001 From: moe-berri Date: Tue, 17 Nov 2026 11:16:10 +0800 Subject: [PATCH] fix(complexity_router): classify short asks in long conversations (#52183) * fix(complexity_router): classify short asks in long conversations * fix(complexity_router): validate active context limits without recursion * test(rag): wait for keepalive frames before releasing retrieval * test(ui): control availability debounce in context-limit regressions --- .../complexity_router/README.md | 18 +++ .../complexity_router/complexity_router.py | 26 ++++- .../complexity_router/config.py | 16 +++ .../add_model/AutoRouterRoutingTest.tsx | 10 +- .../add_model/ClassificationMethodConfig.tsx | 8 ++ ...omplexityRouterConfig.integration.test.tsx | 40 +++++++ .../add_model/ComplexityRouterConfig.tsx | 1 + .../HeuristicFirstContextLimitField.tsx | 44 ++++++++ .../add_auto_router_tab.integration.test.tsx | 36 ++++++ .../add_model/add_auto_router_tab.tsx | 2 + .../build_complexity_router_config.test.ts | 26 ++++- .../build_complexity_router_config.ts | 51 ++++++++- .../classifier_type_transition.test.ts | 10 ++ .../add_model/classifier_type_transition.ts | 2 + .../complexity_router_builder_params.ts | 1 + .../src/components/add_model/tier_rows.ts | 7 +- ...d_updated_complexity_router_config.test.ts | 20 ++++ ...dit_auto_router_modal.integration.test.tsx | 56 +++++++--- .../edit_auto_router_modal.tsx | 4 + .../hydrate_complexity_router_config.ts | 4 + .../src/i18n/locales/en.json | 9 +- .../src/lib/http/schema.d.ts | 5 + .../tests/autoRouterSetup.ts | 14 ++- .../proxy/rag_endpoints/test_rag_endpoints.py | 18 ++- .../router_strategy/test_complexity_router.py | 103 ++++++++++++++++++ 25 files changed, 503 insertions(+), 28 deletions(-) create mode 100644 litellm/ui/litellm-dashboard/src/components/add_model/HeuristicFirstContextLimitField.tsx diff --git a/litellm/router_strategy/complexity_router/README.md b/litellm/router_strategy/complexity_router/README.md index 394052c1dd4..000915ea727 100644 --- a/litellm/router_strategy/complexity_router/README.md +++ b/litellm/router_strategy/complexity_router/README.md @@ -591,6 +591,24 @@ With the default heuristic v1, a request short-circuits, meaning it routes on th two things hold: the scorer landed at or below `heuristic_first_max_tier`, and it produced at least one signal. Everything else goes to the classifier, which then decides as it normally would. +For coding agents with long conversations, set `heuristic_first_max_context_tokens: 8000` alongside +`heuristic_first_max_tier`. Above that estimated size, even a short follow-up such as "why did that +fail?" goes to the classifier. At or below the limit, the usual local-scoring rules apply. Omit the +setting or use `null` to keep the existing behavior. The value must be a positive integer and is only +accepted with `classifier_type: heuristic_first`. + +The estimate counts message text and tool-result bodies, including structured tool results, at +roughly four characters per token. It excludes images and tool-call arguments. The classifier gets +the same size estimate with its existing recent-turn context, not the full tool transcript. The +limit requires a classifier decision, not a higher tier, and classifier failures still use +`classifier_fallback`. + +Existing pins take precedence. With `classification_mode: user_turn`, tool continuations reuse the +held tier and the next human ask checks the limit again. With `session_affinity: true`, the session +pin also takes precedence on new asks. In the dashboard, set **Max conversation tokens before judge** +under the heuristic-first classification settings. Clear the field to disable the limit. Confirm +`llm_classifier` in the routing decision for a new ask above the limit. + The signal requirement is what keeps this from quietly routing everything to your cheapest model. A prompt where no dimension fires scores exactly 0.0, which is below `simple_medium`, so the score to tier mapping calls it SIMPLE by default rather than by evidence. Around half of general traffic diff --git a/litellm/router_strategy/complexity_router/complexity_router.py b/litellm/router_strategy/complexity_router/complexity_router.py index 79dfd64f3e8..51ab2256093 100644 --- a/litellm/router_strategy/complexity_router/complexity_router.py +++ b/litellm/router_strategy/complexity_router/complexity_router.py @@ -22,6 +22,7 @@ import json import random import re import time +from collections import deque from collections.abc import Awaitable, Callable, Iterator, Mapping, Sequence from dataclasses import dataclass from functools import partial @@ -554,6 +555,27 @@ def _message_text(content: object) -> str: return content if isinstance(content, str) else "" +def _conversation_text_lengths(content: object) -> Iterator[int]: + pending: Final[deque[object]] = deque((content,)) # mutable-ok: explicit stack avoids recursive content traversal + while pending: + part = pending.pop() + if isinstance(part, str): + yield len(part) + elif isinstance(part, list): + parts = cast(Sequence[object], part) # cast-ok: every list element is an object + pending.extend(parts) + elif isinstance(part, Mapping): + block = cast(Mapping[object, object], part) # cast-ok: mapping keys and values are objects + if block.get("type") == "text": + pending.append(block.get("text")) + elif block.get("type") == "tool_result": + pending.append(block.get("content")) + + +def _estimated_conversation_tokens(messages: Sequence[Mapping[str, object]] | None) -> int: + return sum(sum(_conversation_text_lengths(message.get("content"))) // 4 for message in messages or ()) + + def _reminder_block_spans(lowered: str, open_marker: str, close_marker: str) -> Iterator[tuple[int, int]]: """Span of each complete reminder block for one marker pair, left to right. @@ -2206,10 +2228,12 @@ class ComplexityRouter(CustomLogger): """ scored: Final = self._classify_locally(prompt, system_prompt) threshold: Final = self.config.heuristic_first_max_tier + context_limit: Final = self.config.heuristic_first_max_context_tokens decided_cheaply: Final = ( threshold is not None and self._local_outcome_is_decided(scored) and not self._classifier_image_parts(messages) + and (context_limit is None or _estimated_conversation_tokens(messages) <= context_limit) and self._active_tier_severity(scored.tier) <= self._active_tier_severity(threshold) ) if decided_cheaply: @@ -3091,7 +3115,7 @@ class ComplexityRouter(CustomLogger): else () ) - cumulative_tokens: Final = sum(len(_message_text(msg.get("content"))) // 4 for msg in messages or ()) + cumulative_tokens: Final = _estimated_conversation_tokens(messages) trajectory_block: Final = ( (f"\nConversation so far: ~{cumulative_tokens} tokens across the request",) if has_prior_conversation diff --git a/litellm/router_strategy/complexity_router/config.py b/litellm/router_strategy/complexity_router/config.py index e42fa6e426b..0d958c5be5f 100644 --- a/litellm/router_strategy/complexity_router/config.py +++ b/litellm/router_strategy/complexity_router/config.py @@ -1265,6 +1265,16 @@ class ComplexityRouterConfig(LiteLLMBaseModel): "may not name the highest one, since that would make the LLM classifier unreachable." ), ) + heuristic_first_max_context_tokens: int | None = Field( + default=None, + gt=0, + strict=True, + description=( + "Optional conversation-size limit for heuristic_first. Above this estimate, call the LLM classifier " + "even for a short current ask. Counts text and tool-result bodies at roughly four characters per token. " + "Unset preserves the local shortcut. Existing session and user-turn pins still take precedence." + ), + ) hybrid_boundary_margin: float | None = Field( default=None, ge=0, @@ -1950,6 +1960,12 @@ class ComplexityRouterConfig(LiteLLMBaseModel): ) return self + @model_validator(mode="after") + def _validate_heuristic_first_context_limit(self) -> "ComplexityRouterConfig": + if self.heuristic_first_max_context_tokens is not None and self.classifier_type != "heuristic_first": + raise ValueError("heuristic_first_max_context_tokens requires classifier_type 'heuristic_first'") + return self + @field_validator("heuristic_first_max_tier", mode="before") @classmethod def _coerce_heuristic_first_max_tier(cls, value: object) -> object: diff --git a/litellm/ui/litellm-dashboard/src/components/add_model/AutoRouterRoutingTest.tsx b/litellm/ui/litellm-dashboard/src/components/add_model/AutoRouterRoutingTest.tsx index a56f1d4fb60..fec22789719 100644 --- a/litellm/ui/litellm-dashboard/src/components/add_model/AutoRouterRoutingTest.tsx +++ b/litellm/ui/litellm-dashboard/src/components/add_model/AutoRouterRoutingTest.tsx @@ -6,7 +6,11 @@ import { Button } from "@/components/ui/button"; import { Textarea } from "@/components/ui/textarea"; import RoutingDecisionCard from "@/components/logs/detail/RoutingDecisionCard"; import { AutoRouterRoutingTestResult, testAutoRouterRouting } from "../networking"; -import { ComplexityRouterConfigPayload, getHeuristicV2SuccessThresholdError } from "./build_complexity_router_config"; +import { + ComplexityRouterConfigPayload, + getHeuristicV2SuccessThresholdError, + getHeuristicFirstContextLimitError, +} from "./build_complexity_router_config"; import { buildAutoRouterRoutingTestRequest } from "./build_auto_router_routing_test_request"; interface AutoRouterRoutingTestProps { @@ -33,7 +37,9 @@ const AutoRouterRoutingTest: React.FC = ({ const { t } = useTranslation(); const [prompt, setPrompt] = React.useState(""); const [state, setState] = React.useState({ status: "idle" }); - const configError = getHeuristicV2SuccessThresholdError(config.heuristic_v2_success_threshold); + const configError = + getHeuristicV2SuccessThresholdError(config.heuristic_v2_success_threshold) ?? + getHeuristicFirstContextLimitError(config.heuristic_first_max_context_tokens); const send = async () => { if (configError) return; diff --git a/litellm/ui/litellm-dashboard/src/components/add_model/ClassificationMethodConfig.tsx b/litellm/ui/litellm-dashboard/src/components/add_model/ClassificationMethodConfig.tsx index 6f514785bb9..54137773983 100644 --- a/litellm/ui/litellm-dashboard/src/components/add_model/ClassificationMethodConfig.tsx +++ b/litellm/ui/litellm-dashboard/src/components/add_model/ClassificationMethodConfig.tsx @@ -1,3 +1,4 @@ +import HeuristicFirstContextLimitField from "./HeuristicFirstContextLimitField"; import ClassifierPrimarySettings from "./ClassifierPrimarySettings"; import { AutoRouterAllowanceNote } from "./AutoRouterAvailability"; import { transitionClassifierType } from "./classifier_type_transition"; @@ -519,6 +520,13 @@ const ClassificationMethodConfig: React.FC = ({ )} + {showSelection && classifierType === "heuristic_first" && ( + onChange({ ...value, heuristic_first_max_context_tokens: limit })} + /> + )} + {showSelection && classifierType === "hybrid" && (