fix(complexity_router): classify short asks in long conversations (#52183)

* fix(complexity_router): classify short asks in long conversations

* fix(complexity_router): validate active context limits without recursion

* test(rag): wait for keepalive frames before releasing retrieval

* test(ui): control availability debounce in context-limit regressions
This commit is contained in:
moe-berri 2026-11-17 11:16:10 +08:00 • committed by GitHub
parent d2664e6b64
commit dc9bbb762f
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
25 changed files with 503 additions and 28 deletions

View file

@ -591,6 +591,24 @@ With the default heuristic v1, a request short-circuits, meaning it routes on th
two things hold: the scorer landed at or below `heuristic_first_max_tier`, and it produced at least
one signal. Everything else goes to the classifier, which then decides as it normally would.
For coding agents with long conversations, set `heuristic_first_max_context_tokens: 8000` alongside
`heuristic_first_max_tier`. Above that estimated size, even a short follow-up such as "why did that
fail?" goes to the classifier. At or below the limit, the usual local-scoring rules apply. Omit the
setting or use `null` to keep the existing behavior. The value must be a positive integer and is only
accepted with `classifier_type: heuristic_first`.
The estimate counts message text and tool-result bodies, including structured tool results, at
roughly four characters per token. It excludes images and tool-call arguments. The classifier gets
the same size estimate with its existing recent-turn context, not the full tool transcript. The
limit requires a classifier decision, not a higher tier, and classifier failures still use
`classifier_fallback`.
Existing pins take precedence. With `classification_mode: user_turn`, tool continuations reuse the
held tier and the next human ask checks the limit again. With `session_affinity: true`, the session
pin also takes precedence on new asks. In the dashboard, set **Max conversation tokens before judge**
under the heuristic-first classification settings. Clear the field to disable the limit. Confirm
`llm_classifier` in the routing decision for a new ask above the limit.
The signal requirement is what keeps this from quietly routing everything to your cheapest model.
A prompt where no dimension fires scores exactly 0.0, which is below `simple_medium`, so the score
to tier mapping calls it SIMPLE by default rather than by evidence. Around half of general traffic

View file

@ -22,6 +22,7 @@ import json
import random
import re
import time
from collections import deque
from collections.abc import Awaitable, Callable, Iterator, Mapping, Sequence
from dataclasses import dataclass
from functools import partial
@ -554,6 +555,27 @@ def _message_text(content: object) -> str:
return content if isinstance(content, str) else ""
def _conversation_text_lengths(content: object) -> Iterator[int]:
pending: Final[deque[object]] = deque((content,)) # mutable-ok: explicit stack avoids recursive content traversal
while pending:
part = pending.pop()
if isinstance(part, str):
yield len(part)
elif isinstance(part, list):
parts = cast(Sequence[object], part) # cast-ok: every list element is an object
pending.extend(parts)
elif isinstance(part, Mapping):
block = cast(Mapping[object, object], part) # cast-ok: mapping keys and values are objects
if block.get("type") == "text":
pending.append(block.get("text"))
elif block.get("type") == "tool_result":
pending.append(block.get("content"))
def _estimated_conversation_tokens(messages: Sequence[Mapping[str, object]] | None) -> int:
return sum(sum(_conversation_text_lengths(message.get("content"))) // 4 for message in messages or ())
def _reminder_block_spans(lowered: str, open_marker: str, close_marker: str) -> Iterator[tuple[int, int]]:
"""Span of each complete reminder block for one marker pair, left to right.
@ -2206,10 +2228,12 @@ class ComplexityRouter(CustomLogger):
"""
scored: Final = self._classify_locally(prompt, system_prompt)
threshold: Final = self.config.heuristic_first_max_tier
context_limit: Final = self.config.heuristic_first_max_context_tokens
decided_cheaply: Final = (
threshold is not None
and self._local_outcome_is_decided(scored)
and not self._classifier_image_parts(messages)
and (context_limit is None or _estimated_conversation_tokens(messages) <= context_limit)
and self._active_tier_severity(scored.tier) <= self._active_tier_severity(threshold)
)
if decided_cheaply:
@ -3091,7 +3115,7 @@ class ComplexityRouter(CustomLogger):
else ()
)
cumulative_tokens: Final = sum(len(_message_text(msg.get("content"))) // 4 for msg in messages or ())
cumulative_tokens: Final = _estimated_conversation_tokens(messages)
trajectory_block: Final = (
(f"\nConversation so far: ~{cumulative_tokens} tokens across the request",)
if has_prior_conversation

View file

@ -1265,6 +1265,16 @@ class ComplexityRouterConfig(LiteLLMBaseModel):
"may not name the highest one, since that would make the LLM classifier unreachable."
),
)
heuristic_first_max_context_tokens: int | None = Field(
default=None,
gt=0,
strict=True,
description=(
"Optional conversation-size limit for heuristic_first. Above this estimate, call the LLM classifier "
"even for a short current ask. Counts text and tool-result bodies at roughly four characters per token. "
"Unset preserves the local shortcut. Existing session and user-turn pins still take precedence."
),
)
hybrid_boundary_margin: float | None = Field(
default=None,
ge=0,
@ -1950,6 +1960,12 @@ class ComplexityRouterConfig(LiteLLMBaseModel):
)
return self
@model_validator(mode="after")
def _validate_heuristic_first_context_limit(self) -> "ComplexityRouterConfig":
if self.heuristic_first_max_context_tokens is not None and self.classifier_type != "heuristic_first":
raise ValueError("heuristic_first_max_context_tokens requires classifier_type 'heuristic_first'")
return self
@field_validator("heuristic_first_max_tier", mode="before")
@classmethod
def _coerce_heuristic_first_max_tier(cls, value: object) -> object:

View file

@ -6,7 +6,11 @@ import { Button } from "@/components/ui/button";
import { Textarea } from "@/components/ui/textarea";
import RoutingDecisionCard from "@/components/logs/detail/RoutingDecisionCard";
import { AutoRouterRoutingTestResult, testAutoRouterRouting } from "../networking";
import { ComplexityRouterConfigPayload, getHeuristicV2SuccessThresholdError } from "./build_complexity_router_config";
import {
ComplexityRouterConfigPayload,
getHeuristicV2SuccessThresholdError,
getHeuristicFirstContextLimitError,
} from "./build_complexity_router_config";
import { buildAutoRouterRoutingTestRequest } from "./build_auto_router_routing_test_request";
interface AutoRouterRoutingTestProps {
@ -33,7 +37,9 @@ const AutoRouterRoutingTest: React.FC<AutoRouterRoutingTestProps> = ({
const { t } = useTranslation();
const [prompt, setPrompt] = React.useState<string>("");
const [state, setState] = React.useState<TestState>({ status: "idle" });
const configError = getHeuristicV2SuccessThresholdError(config.heuristic_v2_success_threshold);
const configError =
getHeuristicV2SuccessThresholdError(config.heuristic_v2_success_threshold) ??
getHeuristicFirstContextLimitError(config.heuristic_first_max_context_tokens);
const send = async () => {
if (configError) return;

View file

@ -1,3 +1,4 @@
import HeuristicFirstContextLimitField from "./HeuristicFirstContextLimitField";
import ClassifierPrimarySettings from "./ClassifierPrimarySettings";
import { AutoRouterAllowanceNote } from "./AutoRouterAvailability";
import { transitionClassifierType } from "./classifier_type_transition";
@ -519,6 +520,13 @@ const ClassificationMethodConfig: React.FC<ClassificationMethodConfigProps> = ({
</div>
)}
{showSelection && classifierType === "heuristic_first" && (
<HeuristicFirstContextLimitField
value={value.heuristic_first_max_context_tokens}
onChange={(limit) => onChange({ ...value, heuristic_first_max_context_tokens: limit })}
/>
)}
{showSelection && classifierType === "hybrid" && (
<div className="mt-4 space-y-2">
<Label htmlFor={HYBRID_BOUNDARY_MARGIN_ID} className="block font-semibold">

View file

@ -1924,3 +1924,43 @@ describe("classifier vision settings", () => {
expect(screen.queryByText("Use images for classification")).not.toBeInTheDocument();
});
});
describe("heuristic-first conversation limit", () => {
it("shows saved limits and carries edits, invalid drafts, and clearing into form state", () => {
const initial: ComplexityRouterConfigValue = {
...defaultValue,
classifier_type: "heuristic_first",
heuristic_first_max_tier: "SIMPLE",
classifier_llm_config: { model: "gpt-4" },
heuristic_first_max_context_tokens: 8000,
};
const onChange = vi.fn();
function Editor() {
const [value, setValue] = React.useState(initial);
return (
<ComplexityRouterConfig
{...baseProps}
value={value}
onChange={(next) => {
setValue(next);
onChange(next);
}}
/>
);
}
renderWithProviders(<Editor />);
fireEvent.click(screen.getByRole("button", { name: /^Advanced settings/ }));
const input = screen.getByRole("textbox", { name: "Max conversation tokens before judge" });
expect(input).toHaveValue("8000");
fireEvent.change(input, { target: { value: "12000" } });
expect(onChange).toHaveBeenLastCalledWith(expect.objectContaining({ heuristic_first_max_context_tokens: 12000 }));
fireEvent.change(input, { target: { value: "invalid" } });
expect(screen.getByText("Conversation limit must be a positive whole number")).toBeInTheDocument();
expect(onChange).toHaveBeenLastCalledWith(expect.objectContaining({ heuristic_first_max_context_tokens: NaN }));
fireEvent.change(input, { target: { value: "" } });
expect(onChange).toHaveBeenLastCalledWith(
expect.objectContaining({ heuristic_first_max_context_tokens: undefined }),
);
expect(screen.queryByText("Conversation limit must be a positive whole number")).not.toBeInTheDocument();
});
});

View file

@ -407,6 +407,7 @@ export interface ComplexityRouterConfigValue {
classification_examples?: string;
/** Highest tier the scorer may decide alone under heuristic_first. Required by that type, rejected by the others. */
heuristic_first_max_tier?: string;
heuristic_first_max_context_tokens?: number;
/** How near a tier boundary a score may land before hybrid defers to the classifier. Required by that type, rejected by the others. */
hybrid_boundary_margin?: number;
classification_mode?: ClassificationMode;

View file

@ -0,0 +1,44 @@
import React from "react";
import { useTranslation } from "react-i18next";
import { Input } from "@/components/ui/input";
import { Label } from "@/components/ui/label";
import { getHeuristicFirstContextLimitError } from "./build_complexity_router_config";
type Props = { value: number | undefined; onChange: (value: number | undefined) => void };
export default function HeuristicFirstContextLimitField({ value, onChange }: Props) {
const { t } = useTranslation();
const [draft, setDraft] = React.useState<string | null>(null);
const error = getHeuristicFirstContextLimitError(value);
const displayDraft = draft !== null && Object.is(value, draft.trim() === "" ? undefined : Number(draft));
const handleChange = (event: React.ChangeEvent<HTMLInputElement>) => {
const raw = event.target.value;
setDraft(raw);
onChange(raw.trim() === "" ? undefined : Number(raw));
};
return (
<div className="mt-4 space-y-2">
<Label htmlFor="auto-router-context-limit" className="block font-semibold">
{t("addModel.classificationMethodConfig.contextLimit.label")}
</Label>
<Input
id="auto-router-context-limit"
type="text"
inputMode="numeric"
placeholder="8000"
value={displayDraft ? draft : value?.toString() ?? ""}
onChange={handleChange}
aria-invalid={Boolean(error)}
aria-describedby={`auto-router-context-limit-help${error ? " auto-router-context-limit-error" : ""}`}
/>
<p id="auto-router-context-limit-help" className="text-sm text-muted-foreground">
{t("addModel.classificationMethodConfig.contextLimit.help")}
</p>
{error && (
<p id="auto-router-context-limit-error" className="text-sm text-destructive" role="alert">
{error}
</p>
)}
</div>
);
}

View file

@ -4,6 +4,7 @@ import {
openAutoRouterAdvanced,
selectAutoRouterOption,
selectAutoRouterApproach,
withSettledRouterAvailability,
} from "../../../tests/autoRouterSetup";
import {
renderWithProviders,
@ -1380,6 +1381,41 @@ describe("AddAutoRouterTab", () => {
);
});
it("ignores a hidden conversation limit after customizing tiers and validates it again on restore", async () => {
mockFetchAvailableModels.mockResolvedValue(ALL_FAMILY_MODELS);
renderAdvanced(<Harness />);
await waitForAutomaticTiers();
await userEvent.click(screen.getByRole("radio", { name: "LLM" }));
await selectAutoRouterOption("Judge model", ALL_FAMILY_MODELS[0].model_group);
openAutoRouterAdvanced("Classification Method");
await selectAutoRouterOption("Local checks before the judge", "Heuristic first");
fireEvent.change(screen.getByRole("textbox", { name: "Max conversation tokens before judge" }), {
target: { value: "invalid" },
});
expect(screen.getByRole("button", { name: "Add Auto Router" })).toBeDisabled();
await userEvent.click(screen.getByRole("button", { name: "Edit tiers" }));
await withSettledRouterAvailability(() => {
fireEvent.change(screen.getByLabelText("Definition for tier 1"), { target: { value: "Short requests" } });
});
expect(screen.queryByRole("textbox", { name: "Max conversation tokens before judge" })).not.toBeInTheDocument();
expect(screen.getByRole("button", { name: "Add Auto Router" })).toBeEnabled();
await userEvent.click(screen.getByRole("button", { name: "Restore defaults" }));
expect(screen.getByRole("textbox", { name: "Max conversation tokens before judge" })).toHaveAttribute(
"aria-invalid",
"true",
);
expect(screen.getByRole("button", { name: "Add Auto Router" })).toBeDisabled();
await withSettledRouterAvailability(() => {
fireEvent.change(screen.getByLabelText("Definition for tier 1"), { target: { value: "Short requests" } });
});
expect(screen.getByRole("button", { name: "Add Auto Router" })).toBeEnabled();
await userEvent.click(screen.getByRole("button", { name: "Add Auto Router" }));
await waitFor(() => expect(handleAddAutoRouterSubmit).toHaveBeenCalledOnce());
const config = vi.mocked(handleAddAutoRouterSubmit).mock.calls.at(-1)?.[0].complexity_router_config;
expect(config?.classifier_type).toBe("llm");
expect(config).not.toHaveProperty("heuristic_first_max_context_tokens");
});
it("preserves classifier tuning when choosing models automatically", async () => {
const user = userEvent.setup();
mockFetchAvailableModels.mockResolvedValue(ALL_FAMILY_MODELS);

View file

@ -50,6 +50,7 @@ import {
getKeywordTierRulesError,
getClassifierModelError,
getHeuristicV2SuccessThresholdError,
getHeuristicFirstConfigError,
getReminderMarkersError,
getClassifierPluginTimeoutError,
getClassifierReasoningEffortError,
@ -137,6 +138,7 @@ export const getSubmitBlockedReason = (
: null) ??
getClassifierModelError(config) ??
getHeuristicV2SuccessThresholdError(config.heuristic_v2_success_threshold) ??
getHeuristicFirstConfigError(config) ??
getReminderMarkersError(config.reminder_markers) ??
getClassifierPluginTimeoutError(config.classifier_type, config.classifier_plugin_timeout_ms) ??
(heuristicScoringRole(config) === "decides" ? customDimensionsError(config.custom_dimensions) : null) ??

View file

@ -6,6 +6,7 @@ import {
getKeywordTierRulesError,
getClassifierModelError,
getHeuristicV2SuccessThresholdError,
getHeuristicFirstContextLimitError,
getReminderMarkersError,
getClassifierPluginTimeoutError,
getClassifierReasoningEffortError,
@ -1217,6 +1218,23 @@ describe("heuristic_first", () => {
classifierFallback: "default_model",
};
it("persists the conversation limit only for heuristic-first routing", () => {
const params = { ...heuristicFirstParams, heuristicFirstMaxContextTokens: 8000 };
expect(buildComplexityRouterConfig(params).heuristic_first_max_context_tokens).toBe(8000);
expect(buildComplexityRouterConfig(heuristicFirstParams).heuristic_first_max_context_tokens).toBeUndefined();
expect(
buildComplexityRouterConfig({ ...params, classifierType: "llm" }).heuristic_first_max_context_tokens,
).toBeUndefined();
});
it.each([0, -1, 1.5, NaN, Infinity])("rejects invalid conversation limit %s", (limit) => {
expect(getHeuristicFirstContextLimitError(limit)).toBe("Conversation limit must be a positive whole number");
});
it.each([undefined, 1, 8000])("accepts optional conversation limit %s", (limit) => {
expect(getHeuristicFirstContextLimitError(limit)).toBeNull();
});
it("emits heuristic_first_max_tier", () => {
const config = buildComplexityRouterConfig(heuristicFirstParams);
expect(config.classifier_type).toBe("heuristic_first");
@ -1419,6 +1437,7 @@ describe("buildComplexityRouterConfig with an edited tier set", () => {
customDimensions: [{ id: "row-1", name: "sqlDdl", weight: 0.4, keywords: ["orbitmesh"] }],
reasoningOverrideMinScore: 0.5,
heuristicFirstMaxTier: "SIMPLE",
heuristicFirstMaxContextTokens: 8000,
hybridBoundaryMargin: 0.03,
localHeuristic: "heuristic",
customTechnicalKeywords: ["kubernetes"],
@ -1427,7 +1446,12 @@ describe("buildComplexityRouterConfig with an edited tier set", () => {
stallEscalationRepeatThreshold: 3,
};
// custom_dimensions only ever ship when the scorer decides, so "llm" cannot prove it emits.
const emittingType = ["heuristic_first_max_tier", "custom_dimensions", "local_heuristic"].includes(key)
const emittingType = [
"heuristic_first_max_tier",
"heuristic_first_max_context_tokens",
"custom_dimensions",
"local_heuristic",
].includes(key)
? "heuristic_first"
: "llm";
const typeForKey = key === "hybrid_boundary_margin" ? "hybrid" : emittingType;

View file

@ -115,6 +115,7 @@ export interface StoredComplexityRouterConfig {
classification_prompt?: unknown;
classification_examples?: unknown;
heuristic_first_max_tier?: unknown;
heuristic_first_max_context_tokens?: unknown;
hybrid_boundary_margin?: unknown;
tier_labels?: unknown;
classifier_type?: ClassifierType | "oss_classifier";
@ -188,6 +189,7 @@ export interface BuildComplexityRouterConfigParams {
classificationPrompt: string | undefined;
classificationExamples: string | undefined;
heuristicFirstMaxTier: string | undefined;
heuristicFirstMaxContextTokens?: number;
hybridBoundaryMargin?: number;
classificationMode: ClassificationMode | undefined;
sessionAffinity: boolean;
@ -275,6 +277,7 @@ export interface ComplexityRouterConfigPayload {
classification_prompt?: string;
classification_examples?: string;
heuristic_first_max_tier?: string;
heuristic_first_max_context_tokens?: number;
hybrid_boundary_margin?: number;
classification_mode: ClassificationMode;
session_affinity: boolean;
@ -397,6 +400,18 @@ export const getKeywordTierRulesError = (
return t("addModel.buildComplexityRouterConfig.keywordRulesOrphaned", { rules: orphaned.join(", ") });
};
export const getHeuristicFirstContextLimitError = (limit: number | undefined): string | null => {
if (limit === undefined) return null;
return Number.isSafeInteger(limit) && limit > 0
? null
: i18n.t("addModel.buildComplexityRouterConfig.contextLimitPositiveInteger");
};
export const getHeuristicFirstConfigError = (config: ComplexityRouterConfigValue): string | null =>
effectiveClassifierType(config) === "heuristic_first"
? getHeuristicFirstContextLimitError(config.heuristic_first_max_context_tokens)
: null;
export const getHeuristicV2SuccessThresholdError = (
threshold: number | undefined,
t: TFunction = i18n.t.bind(i18n),
@ -595,12 +610,36 @@ export const hydratePlanModeMinTier = (
return tierRowByName(customTierSet.tiers, stored)?.id;
};
const localClassifierWireFields = (
effectiveType: ClassifierType,
{
heuristicFirstMaxTier,
heuristicFirstMaxContextTokens,
hybridBoundaryMargin,
}: Pick<
BuildComplexityRouterConfigParams,
"heuristicFirstMaxTier" | "heuristicFirstMaxContextTokens" | "hybridBoundaryMargin"
>,
): Partial<ComplexityRouterConfigPayload> => ({
...(effectiveType === "heuristic_first" &&
heuristicFirstMaxTier?.trim() && {
heuristic_first_max_tier: heuristicFirstMaxTier,
}),
...(effectiveType === "heuristic_first" &&
heuristicFirstMaxContextTokens !== undefined && {
heuristic_first_max_context_tokens: heuristicFirstMaxContextTokens,
}),
...(effectiveType === "hybrid" &&
hybridBoundaryMargin !== undefined && { hybrid_boundary_margin: hybridBoundaryMargin }),
});
const classifierWireFields = (
effectiveType: ClassifierType,
{
classifierLlmConfig,
classifierFallback,
heuristicFirstMaxTier,
heuristicFirstMaxContextTokens,
hybridBoundaryMargin,
classifierContextWindowSize,
classifierContextBudgetChars,
@ -611,6 +650,7 @@ const classifierWireFields = (
| "classifierLlmConfig"
| "classifierFallback"
| "heuristicFirstMaxTier"
| "heuristicFirstMaxContextTokens"
| "hybridBoundaryMargin"
| "classifierContextWindowSize"
| "classifierContextBudgetChars"
@ -627,10 +667,11 @@ const classifierWireFields = (
: normalizeClassifierLlmConfig(classifierLlmConfig),
}),
...(supportsFallback && classifierFallback !== undefined && { classifier_fallback: classifierFallback }),
...(effectiveType === "heuristic_first" &&
heuristicFirstMaxTier?.trim() && { heuristic_first_max_tier: heuristicFirstMaxTier }),
...(effectiveType === "hybrid" &&
hybridBoundaryMargin !== undefined && { hybrid_boundary_margin: hybridBoundaryMargin }),
...localClassifierWireFields(effectiveType, {
heuristicFirstMaxTier,
heuristicFirstMaxContextTokens,
hybridBoundaryMargin,
}),
...(usesClassifierContext(effectiveType) &&
classifierContextWindowSize !== undefined && {
classifier_context_window_size: classifierContextWindowSize,
@ -672,6 +713,7 @@ export const buildComplexityRouterConfig = ({
classificationPrompt,
classificationExamples,
heuristicFirstMaxTier,
heuristicFirstMaxContextTokens,
hybridBoundaryMargin,
classificationMode,
sessionAffinity,
@ -739,6 +781,7 @@ export const buildComplexityRouterConfig = ({
classifierLlmConfig,
classifierFallback,
heuristicFirstMaxTier,
heuristicFirstMaxContextTokens,
hybridBoundaryMargin,
classifierContextWindowSize,
classifierContextBudgetChars,

View file

@ -118,3 +118,13 @@ describe("transitionClassifierType", () => {
expect(result.tiers.SIMPLE).toEqual(["efficient"]);
});
});
it("clears the conversation limit when leaving heuristic-first", () => {
const initial = {
...standard,
classifier_type: "heuristic_first" as const,
heuristic_first_max_context_tokens: 8000,
};
expect(transitionClassifierType(initial, "heuristic_first").heuristic_first_max_context_tokens).toBe(8000);
expect(transitionClassifierType(initial, "llm").heuristic_first_max_context_tokens).toBeUndefined();
});

View file

@ -51,6 +51,8 @@ export const transitionClassifierType = (
classifierType === "heuristic_first"
? value.heuristic_first_max_tier ?? DEFAULT_HEURISTIC_FIRST_MAX_TIER
: undefined,
heuristic_first_max_context_tokens:
classifierType === "heuristic_first" ? value.heuristic_first_max_context_tokens : undefined,
hybrid_boundary_margin:
classifierType === "hybrid" ? value.hybrid_boundary_margin ?? DEFAULT_HYBRID_BOUNDARY_MARGIN : undefined,
...nonReasoningTierFields(classifierType, value),

View file

@ -26,6 +26,7 @@ export const builderParamsFromValue = (
classificationPrompt: value.classification_prompt,
classificationExamples: value.classification_examples,
heuristicFirstMaxTier: value.heuristic_first_max_tier,
heuristicFirstMaxContextTokens: value.heuristic_first_max_context_tokens,
hybridBoundaryMargin: value.hybrid_boundary_margin,
classificationMode: value.classification_mode,
tierLabels: value.tier_labels,

View file

@ -145,7 +145,12 @@ export const CUSTOM_TIER_RESTRICTIONS = {
reason: "addModel.tierRows.restrictions.sessionAffinity",
},
heuristicClassifier: {
omit: ["heuristic_first_max_tier", "hybrid_boundary_margin", "local_heuristic"],
omit: [
"heuristic_first_max_tier",
"heuristic_first_max_context_tokens",
"hybrid_boundary_margin",
"local_heuristic",
],
reason: "addModel.tierRows.restrictions.heuristicClassifier",
},
heuristicScoring: {

View file

@ -921,6 +921,7 @@ describe("managed keys survive an untouched open-and-save", () => {
local_heuristic: "heuristic",
heuristic_v2_success_threshold: 0.89,
heuristic_first_max_tier: "SIMPLE",
heuristic_first_max_context_tokens: 8000,
classifier_llm_config: { model: "gpt-4o-mini", timeout_ms: 3000, reasoning_effort: "low" },
classifier_context_window_size: 5,
classifier_context_budget_chars: 4000,
@ -1306,3 +1307,22 @@ describe("untouched save round trip", () => {
expect(saved.reminder_markers).toEqual([{ open: "<other>", close: "</other>" }]);
});
});
it("edits and clears the stored heuristic-first conversation limit", () => {
const stored = {
tiers: { SIMPLE: ["simple"], MEDIUM: ["medium"] },
classifier_type: "heuristic_first",
heuristic_first_max_tier: "SIMPLE",
classifier_llm_config: { model: "judge" },
heuristic_first_max_context_tokens: 8000,
};
const hydrated = hydrateComplexityRouterConfig(stored, undefined);
expect(hydrated.heuristic_first_max_context_tokens).toBe(8000);
expect(
buildUpdatedComplexityRouterConfig(stored, { ...hydrated, heuristic_first_max_context_tokens: 12000 })
.heuristic_first_max_context_tokens,
).toBe(12000);
expect(
buildUpdatedComplexityRouterConfig(stored, { ...hydrated, heuristic_first_max_context_tokens: undefined }),
).not.toHaveProperty("heuristic_first_max_context_tokens");
});

View file

@ -2,7 +2,11 @@ vi.mock("@/app/(dashboard)/hooks/models/useModelCostMap", () => ({
useModelCostMap: () => ({ data: { "typesafe/jev-latest": { litellm_provider: "typesafe", mode: "evaluation" } } }),
}));
import { openAutoRouterAdvanced, selectAutoRouterOption } from "../../../tests/autoRouterSetup";
import {
openAutoRouterAdvanced,
selectAutoRouterOption,
withSettledRouterAvailability,
} from "../../../tests/autoRouterSetup";
import userEvent from "@testing-library/user-event";
import { describe, expect, it, vi } from "vitest";
@ -90,17 +94,6 @@ const savedConfig = () => {
return payload?.litellm_params?.complexity_router_config;
};
const renderWithSettledAvailability = async (renderModal: () => unknown) => {
vi.useFakeTimers();
try {
renderModal();
await act(() => vi.advanceTimersByTimeAsync(300));
await act(() => vi.runOnlyPendingTimersAsync());
} finally {
vi.useRealTimers();
}
};
describe("EditAutoRouterModal keyword matching", () => {
beforeEach(() => {
modelPatchUpdateCall.mockClear();
@ -221,6 +214,41 @@ describe("EditAutoRouterModal keyword matching", () => {
else expect(savedConfig().heuristic_v2_success_threshold).toBe(0);
});
it("saves custom tiers after an invalid conversation limit becomes hidden", async () => {
testQueryClient.clear();
vi.mocked(apiClient.post).mockResolvedValue({ allowances: [], error: null });
renderModal({
modelData: {
...MODEL_DATA,
litellm_params: {
...MODEL_DATA.litellm_params,
complexity_router_config: {
...STORED_CONFIG,
classifier_type: "heuristic_first",
heuristic_first_max_tier: "SIMPLE",
classifier_llm_config: { model: "gpt-4o-mini" },
heuristic_first_max_context_tokens: 8000,
},
},
},
});
openAutoRouterAdvanced("Classification Method");
fireEvent.change(screen.getByRole("textbox", { name: "Max conversation tokens before judge" }), {
target: { value: "invalid" },
});
expect(screen.getByRole("button", { name: "Save Changes" })).toBeDisabled();
await userEvent.click(screen.getByRole("button", { name: "Edit tiers" }));
await withSettledRouterAvailability(() => {
fireEvent.change(screen.getByLabelText("Definition for tier 1"), { target: { value: "Short requests" } });
});
expect(screen.queryByRole("textbox", { name: "Max conversation tokens before judge" })).not.toBeInTheDocument();
expect(screen.getByRole("button", { name: "Save Changes" })).toBeEnabled();
await userEvent.click(screen.getByRole("button", { name: "Save Changes" }));
await waitFor(() => expect(modelPatchUpdateCall).toHaveBeenCalledOnce());
expect(savedConfig().classifier_type).toBe("llm");
expect(savedConfig()).not.toHaveProperty("heuristic_first_max_context_tokens");
});
it("saves a legacy heuristic-first router with a v2 chain and an edited success threshold", async () => {
renderModal({
modelData: {
@ -1470,7 +1498,7 @@ describe("EditAutoRouterModal prompt compression", () => {
{ auto_router_model_compression: "model-compressor" },
])("should preserve the exact stored compression fields through an untouched save: %j", async (stored) => {
const user = userEvent.setup();
await renderWithSettledAvailability(() => renderWithStoredCompression(stored));
await withSettledRouterAvailability(() => renderWithStoredCompression(stored));
await waitFor(() => expect(screen.getByRole("button", { name: /save changes/i })).toBeEnabled());
await user.click(await screen.findByRole("button", { name: /save changes/i }));
@ -1547,7 +1575,7 @@ describe("EditAutoRouterModal classifier vision", () => {
it("hydrates and keeps a stored vision setting through an untouched save", async () => {
const user = userEvent.setup();
await renderWithSettledAvailability(renderModal);
await withSettledRouterAvailability(renderModal);
openAutoRouterAdvanced("Classification Method");
expect(screen.getByRole("switch", { name: "Use images for classification" })).toBeChecked();

View file

@ -43,6 +43,7 @@ import {
buildComplexityRouterConfig,
getClassifierModelError,
getHeuristicV2SuccessThresholdError,
getHeuristicFirstConfigError,
getReminderMarkersError,
getClassifierPluginTimeoutError,
getClassifierReasoningEffortError,
@ -116,6 +117,7 @@ export const MANAGED_COMPLEXITY_ROUTER_KEYS = new Set([
"classification_prompt",
"classification_examples",
"heuristic_first_max_tier",
"heuristic_first_max_context_tokens",
"hybrid_boundary_margin",
"heuristic_v2_success_threshold",
"classification_mode",
@ -344,6 +346,7 @@ const ExistingEditAutoRouterModal: React.FC<EditAutoRouterModalProps> = ({
getKeywordTierRulesError(keywordTierRules, activeTierRows(complexityRouterConfig)) ??
getClassifierModelError(complexityRouterConfig) ??
getHeuristicV2SuccessThresholdError(complexityRouterConfig.heuristic_v2_success_threshold) ??
getHeuristicFirstConfigError(complexityRouterConfig) ??
getForecastConfigError(complexityRouterConfig) ??
(heuristicScoringRole(complexityRouterConfig) === "decides"
? customDimensionsError(complexityRouterConfig.custom_dimensions)
@ -484,6 +487,7 @@ const ExistingEditAutoRouterModal: React.FC<EditAutoRouterModalProps> = ({
const classifierError =
getClassifierModelError(complexityRouterConfig) ??
getHeuristicV2SuccessThresholdError(complexityRouterConfig.heuristic_v2_success_threshold) ??
getHeuristicFirstConfigError(complexityRouterConfig) ??
getReminderMarkersError(complexityRouterConfig.reminder_markers) ??
getClassifierPluginTimeoutError(
complexityRouterConfig.classifier_type,

View file

@ -115,6 +115,10 @@ export const hydrateComplexityRouterConfig = (
typeof parsedConfig.heuristic_first_max_tier === "string" && parsedConfig.heuristic_first_max_tier.trim() !== ""
? parsedConfig.heuristic_first_max_tier
: undefined,
heuristic_first_max_context_tokens:
typeof parsedConfig.heuristic_first_max_context_tokens === "number"
? parsedConfig.heuristic_first_max_context_tokens
: undefined,
hybrid_boundary_margin:
typeof parsedConfig.hybrid_boundary_margin === "number" ? parsedConfig.hybrid_boundary_margin : undefined,
classification_mode:

View file

@ -17809,7 +17809,11 @@
"empty": "Type to add a keyword"
},
"scoringRulesAllowance": "Custom scoring rules",
"keywordOverridesTitle": "Heuristic Keyword Overrides"
"keywordOverridesTitle": "Heuristic Keyword Overrides",
"contextLimit": {
"label": "Max conversation tokens before judge",
"help": "Above this estimated size, ask the judge even for short follow-ups. Counts text and tool results. Existing session and tool-continuation pins still apply. Leave blank to disable."
}
},
"autoRouterClassifierTabs": {
"option": {
@ -18391,7 +18395,8 @@
"selectClassifierOrHeuristic": "Please select a classifier model, or switch back to Heuristic",
"reasoningEffortUnsupported": "{{effort}} reasoning effort is not supported by every deployment in {{model}}. Choose Default or a supported value.",
"selectEmbeddingModel": "Select an embedding model to use semantic keyword matching",
"addKeywordRuleForSemantic": "Add at least one keyword tier rule to use semantic keyword matching"
"addKeywordRuleForSemantic": "Add at least one keyword tier rule to use semantic keyword matching",
"contextLimitPositiveInteger": "Conversation limit must be a positive whole number"
},
"adaptiveRoutingConfig": {
"enable": "Enable adaptive bandit selection",

View file

@ -45852,6 +45852,11 @@ export interface components {
* @description Tier routed to when the LLM classifier fails (timeout, provider error, or an unparseable reply). Required with tier_definitions and must name a defined tier; the heuristic scorer cannot produce custom tiers, so this replaces the heuristic fallback for custom tier sets.
*/
fallback_tier?: string | null;
/**
* Heuristic First Max Context Tokens
* @description Optional conversation-size limit for heuristic_first. Above this estimate, call the LLM classifier even for a short current ask. Counts text and tool-result bodies at roughly four characters per token. Unset preserves the local shortcut. Existing session and user-turn pins still take precedence.
*/
heuristic_first_max_context_tokens?: number | null;
/**
* Heuristic First Max Tier
* @description The highest tier the local scorer may decide on its own; required when classifier_type is 'heuristic_first' and rejected otherwise. A request whose heuristic tier is at or below this one skips the LLM classifier and routes straight to that heuristic tier, so the classifier call is only paid for on traffic the scorer could not place cheaply. The scorer must also have produced at least one signal: a prompt where no dimension fired scores 0.0 and would otherwise land SIMPLE by default rather than by evidence. With local_heuristic 'heuristic_v2', the predicted tier must meet its success threshold. Names a built-in tier, and may not name the highest one, since that would make the LLM classifier unreachable.

View file

@ -1,5 +1,6 @@
import userEvent from "@testing-library/user-event";
import { chooseSelectOption, fireEvent, screen } from "./test-utils";
import { vi } from "vitest";
import { act, chooseSelectOption, fireEvent, screen } from "./test-utils";
const groups: Record<string, string> = {
"Heuristic Keyword Overrides": "Heuristic tuning",
@ -38,3 +39,14 @@ export const selectAutoRouterApproach = async (option: string) => {
await userEvent.click(screen.getByRole("radio", { name: "LLM" }));
await selectAutoRouterOption("Routing approach", option);
};
export const withSettledRouterAvailability = async (changeForm: () => void) => {
vi.useFakeTimers();
try {
changeForm();
await act(() => vi.advanceTimersByTimeAsync(300));
await act(() => vi.runOnlyPendingTimersAsync());
} finally {
vi.useRealTimers();
}
};

View file

@ -7,6 +7,7 @@ import io
import json
from collections.abc import Iterator, Mapping, Sequence
from dataclasses import dataclass
from itertools import count
from types import MappingProxyType
from typing import Final
from unittest.mock import AsyncMock, MagicMock, patch
@ -15,6 +16,7 @@ import httpx
import pytest
import respx
from fastapi.testclient import TestClient
from starlette.types import Message, Receive, Scope, Send
import litellm
from litellm import Router
@ -2231,9 +2233,21 @@ def test_rag_query_stream_pings_while_retrieval_is_still_running(client_internal
import litellm as litellm_module
monkeypatch.setattr(litellm_module, "sse_keepalive_ping_interval_seconds", 0.05)
retrieval_released: Final = asyncio.Event()
ping_count: Final = count(1)
async def observe_pings(scope: Scope, receive: Receive, send: Send) -> None:
async def observe_send(message: Message) -> None:
await send(message)
if message["type"] == "http.response.body":
for _ in range(message.get("body", b"").count(b": ping\n\n")):
if next(ping_count) == 3:
retrieval_released.set()
await client_internal_user.app(scope, receive, observe_send)
async def slow_aquery(**kwargs):
await asyncio.sleep(0.3)
await asyncio.wait_for(retrieval_released.wait(), timeout=10)
return await litellm_module.acompletion(
model="gpt-4o-mini",
messages=[{"role": "user", "content": "What is the codename?"}],
@ -2252,7 +2266,7 @@ def test_rag_query_stream_pings_while_retrieval_is_still_running(client_internal
"litellm.proxy.proxy_server.prisma_client", None
), # test-quality-ok: proxy module global, no injection seam
):
response = client_internal_user.post(
response = TestClient(observe_pings).post(
"/v1/rag/query",
json={
"model": "gpt-4o-mini",

View file

@ -14513,6 +14513,109 @@ class TestHeuristicFirstConfig:
assert ComplexityRouterConfig(tiers=dict(HEURISTIC_FIRST_TIERS)).uses_llm_classifier is False
@pytest.mark.parametrize("limit", (0, -1, 1.5, True))
def test_heuristic_context_limit_rejects_invalid_values(limit: object) -> None:
with pytest.raises(ValidationError, match="heuristic_first_max_context_tokens"):
_heuristic_first_router(MagicMock(), heuristic_first_max_context_tokens=limit)
def test_heuristic_context_limit_requires_heuristic_first() -> None:
with pytest.raises(ValidationError, match="heuristic_first_max_context_tokens"):
ComplexityRouterConfig(tiers=dict(HEURISTIC_FIRST_TIERS), heuristic_first_max_context_tokens=8000)
@pytest.mark.asyncio
@pytest.mark.parametrize(
"history_content",
(
"trace " * 6000,
[{"type": "text", "text": "trace " * 6000}],
[{"type": "tool_result", "tool_use_id": "call_1", "content": "trace " * 6000}],
[{"type": "tool_result", "tool_use_id": "call_1", "content": [{"type": "text", "text": "trace " * 6000}]}],
),
ids=("plain", "text-parts", "tool-result", "nested-tool-result"),
)
async def test_short_followup_in_long_conversation_reaches_classifier(
mock_router_instance: MagicMock, history_content: object
) -> None:
mock_router_instance.acompletion = AsyncMock(return_value=_llm_response('{"tier": "COMPLEX"}'))
router: Final = _heuristic_first_router(mock_router_instance, heuristic_first_max_context_tokens=8000)
messages: Final[list[dict[str, object]]] = [
{"role": "user", "content": "debug this failure"},
{"role": "user", "content": history_content},
{"role": "user", "content": "why did that fail?"},
]
result: Final = await router.async_pre_routing_hook(
model="test-complexity-router", messages=messages, request_kwargs={}
)
assert result is not None
assert result.model == HEURISTIC_FIRST_TIERS["COMPLEX"]
assert result.routing_decision["cause"] == "llm_classifier"
mock_router_instance.acompletion.assert_awaited_once()
classifier_messages: Final = mock_router_instance.acompletion.call_args.kwargs["messages"]
assert "Conversation so far: ~9008 tokens" in str(classifier_messages)
@pytest.mark.asyncio
@pytest.mark.parametrize(
"limit, history_chars, cause",
(
(None, 40000, "heuristic_first_short_circuit"),
(10, 35, "heuristic_first_short_circuit"),
(10, 36, "heuristic_first_short_circuit"),
(10, 40, "llm_classifier"),
),
)
async def test_heuristic_context_limit_boundary_and_disabled(
mock_router_instance: MagicMock, limit: int | None, history_chars: int, cause: str
) -> None:
mock_router_instance.acompletion = AsyncMock(return_value=_llm_response('{"tier": "COMPLEX"}'))
router: Final = _heuristic_first_router(mock_router_instance, heuristic_first_max_context_tokens=limit)
messages: Final = [{"role": "system", "content": "x" * history_chars}, {"role": "user", "content": "hi!!"}]
outcome: Final = await router.aclassify("hi!!", messages=messages)
assert outcome.cause == cause
assert mock_router_instance.acompletion.await_count == int(cause == "llm_classifier")
@pytest.mark.asyncio
async def test_long_context_preserves_tool_pin_then_classifies_next_human_ask(mock_router_instance: MagicMock) -> None:
mock_router_instance.health.return_value.session_pins = memory_pins()
mock_router_instance.acompletion = AsyncMock(return_value=_llm_response('{"tier": "COMPLEX"}'))
router: Final = _heuristic_first_router(
mock_router_instance,
heuristic_first_max_context_tokens=8000,
classification_mode="user_turn",
session_affinity=False,
)
first: Final[list[dict[str, object]]] = [{"role": "user", "content": "hi"}]
continuation: Final[list[dict[str, object]]] = [
*first,
{
"role": "assistant",
"content": None,
"tool_calls": [{"id": "call_1", "type": "function", "function": {"name": "read", "arguments": "{}"}}],
},
{"role": "tool", "tool_call_id": "call_1", "content": "trace " * 6000},
]
for messages, cause, model in (
(first, "heuristic_first_short_circuit", HEURISTIC_FIRST_TIERS["SIMPLE"]),
(continuation, "user_turn_continuation", HEURISTIC_FIRST_TIERS["SIMPLE"]),
(
[*continuation, {"role": "user", "content": "why did that fail?"}],
"llm_classifier",
HEURISTIC_FIRST_TIERS["COMPLEX"],
),
):
result: Final = await router.async_pre_routing_hook(
model="test-complexity-router",
messages=messages,
request_kwargs={"metadata": {"session_id": "context-test"}},
)
assert result is not None
assert (result.model, result.routing_decision["cause"]) == (model, cause)
mock_router_instance.acompletion.assert_awaited_once()
class TestHeuristicFirst:
"""Behavior of the heuristic-first chain: when the classifier call is skipped, and when it is not."""