mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-17 09:34:48 +00:00
fix(complexity_router): classify short asks in long conversations (#52183)
* fix(complexity_router): classify short asks in long conversations * fix(complexity_router): validate active context limits without recursion * test(rag): wait for keepalive frames before releasing retrieval * test(ui): control availability debounce in context-limit regressions
This commit is contained in:
parent
d2664e6b64
commit
dc9bbb762f
25 changed files with 503 additions and 28 deletions
|
|
@ -591,6 +591,24 @@ With the default heuristic v1, a request short-circuits, meaning it routes on th
|
|||
two things hold: the scorer landed at or below `heuristic_first_max_tier`, and it produced at least
|
||||
one signal. Everything else goes to the classifier, which then decides as it normally would.
|
||||
|
||||
For coding agents with long conversations, set `heuristic_first_max_context_tokens: 8000` alongside
|
||||
`heuristic_first_max_tier`. Above that estimated size, even a short follow-up such as "why did that
|
||||
fail?" goes to the classifier. At or below the limit, the usual local-scoring rules apply. Omit the
|
||||
setting or use `null` to keep the existing behavior. The value must be a positive integer and is only
|
||||
accepted with `classifier_type: heuristic_first`.
|
||||
|
||||
The estimate counts message text and tool-result bodies, including structured tool results, at
|
||||
roughly four characters per token. It excludes images and tool-call arguments. The classifier gets
|
||||
the same size estimate with its existing recent-turn context, not the full tool transcript. The
|
||||
limit requires a classifier decision, not a higher tier, and classifier failures still use
|
||||
`classifier_fallback`.
|
||||
|
||||
Existing pins take precedence. With `classification_mode: user_turn`, tool continuations reuse the
|
||||
held tier and the next human ask checks the limit again. With `session_affinity: true`, the session
|
||||
pin also takes precedence on new asks. In the dashboard, set **Max conversation tokens before judge**
|
||||
under the heuristic-first classification settings. Clear the field to disable the limit. Confirm
|
||||
`llm_classifier` in the routing decision for a new ask above the limit.
|
||||
|
||||
The signal requirement is what keeps this from quietly routing everything to your cheapest model.
|
||||
A prompt where no dimension fires scores exactly 0.0, which is below `simple_medium`, so the score
|
||||
to tier mapping calls it SIMPLE by default rather than by evidence. Around half of general traffic
|
||||
|
|
|
|||
|
|
@ -22,6 +22,7 @@ import json
|
|||
import random
|
||||
import re
|
||||
import time
|
||||
from collections import deque
|
||||
from collections.abc import Awaitable, Callable, Iterator, Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from functools import partial
|
||||
|
|
@ -554,6 +555,27 @@ def _message_text(content: object) -> str:
|
|||
return content if isinstance(content, str) else ""
|
||||
|
||||
|
||||
def _conversation_text_lengths(content: object) -> Iterator[int]:
|
||||
pending: Final[deque[object]] = deque((content,)) # mutable-ok: explicit stack avoids recursive content traversal
|
||||
while pending:
|
||||
part = pending.pop()
|
||||
if isinstance(part, str):
|
||||
yield len(part)
|
||||
elif isinstance(part, list):
|
||||
parts = cast(Sequence[object], part) # cast-ok: every list element is an object
|
||||
pending.extend(parts)
|
||||
elif isinstance(part, Mapping):
|
||||
block = cast(Mapping[object, object], part) # cast-ok: mapping keys and values are objects
|
||||
if block.get("type") == "text":
|
||||
pending.append(block.get("text"))
|
||||
elif block.get("type") == "tool_result":
|
||||
pending.append(block.get("content"))
|
||||
|
||||
|
||||
def _estimated_conversation_tokens(messages: Sequence[Mapping[str, object]] | None) -> int:
|
||||
return sum(sum(_conversation_text_lengths(message.get("content"))) // 4 for message in messages or ())
|
||||
|
||||
|
||||
def _reminder_block_spans(lowered: str, open_marker: str, close_marker: str) -> Iterator[tuple[int, int]]:
|
||||
"""Span of each complete reminder block for one marker pair, left to right.
|
||||
|
||||
|
|
@ -2206,10 +2228,12 @@ class ComplexityRouter(CustomLogger):
|
|||
"""
|
||||
scored: Final = self._classify_locally(prompt, system_prompt)
|
||||
threshold: Final = self.config.heuristic_first_max_tier
|
||||
context_limit: Final = self.config.heuristic_first_max_context_tokens
|
||||
decided_cheaply: Final = (
|
||||
threshold is not None
|
||||
and self._local_outcome_is_decided(scored)
|
||||
and not self._classifier_image_parts(messages)
|
||||
and (context_limit is None or _estimated_conversation_tokens(messages) <= context_limit)
|
||||
and self._active_tier_severity(scored.tier) <= self._active_tier_severity(threshold)
|
||||
)
|
||||
if decided_cheaply:
|
||||
|
|
@ -3091,7 +3115,7 @@ class ComplexityRouter(CustomLogger):
|
|||
else ()
|
||||
)
|
||||
|
||||
cumulative_tokens: Final = sum(len(_message_text(msg.get("content"))) // 4 for msg in messages or ())
|
||||
cumulative_tokens: Final = _estimated_conversation_tokens(messages)
|
||||
trajectory_block: Final = (
|
||||
(f"\nConversation so far: ~{cumulative_tokens} tokens across the request",)
|
||||
if has_prior_conversation
|
||||
|
|
|
|||
|
|
@ -1265,6 +1265,16 @@ class ComplexityRouterConfig(LiteLLMBaseModel):
|
|||
"may not name the highest one, since that would make the LLM classifier unreachable."
|
||||
),
|
||||
)
|
||||
heuristic_first_max_context_tokens: int | None = Field(
|
||||
default=None,
|
||||
gt=0,
|
||||
strict=True,
|
||||
description=(
|
||||
"Optional conversation-size limit for heuristic_first. Above this estimate, call the LLM classifier "
|
||||
"even for a short current ask. Counts text and tool-result bodies at roughly four characters per token. "
|
||||
"Unset preserves the local shortcut. Existing session and user-turn pins still take precedence."
|
||||
),
|
||||
)
|
||||
hybrid_boundary_margin: float | None = Field(
|
||||
default=None,
|
||||
ge=0,
|
||||
|
|
@ -1950,6 +1960,12 @@ class ComplexityRouterConfig(LiteLLMBaseModel):
|
|||
)
|
||||
return self
|
||||
|
||||
@model_validator(mode="after")
|
||||
def _validate_heuristic_first_context_limit(self) -> "ComplexityRouterConfig":
|
||||
if self.heuristic_first_max_context_tokens is not None and self.classifier_type != "heuristic_first":
|
||||
raise ValueError("heuristic_first_max_context_tokens requires classifier_type 'heuristic_first'")
|
||||
return self
|
||||
|
||||
@field_validator("heuristic_first_max_tier", mode="before")
|
||||
@classmethod
|
||||
def _coerce_heuristic_first_max_tier(cls, value: object) -> object:
|
||||
|
|
|
|||
|
|
@ -6,7 +6,11 @@ import { Button } from "@/components/ui/button";
|
|||
import { Textarea } from "@/components/ui/textarea";
|
||||
import RoutingDecisionCard from "@/components/logs/detail/RoutingDecisionCard";
|
||||
import { AutoRouterRoutingTestResult, testAutoRouterRouting } from "../networking";
|
||||
import { ComplexityRouterConfigPayload, getHeuristicV2SuccessThresholdError } from "./build_complexity_router_config";
|
||||
import {
|
||||
ComplexityRouterConfigPayload,
|
||||
getHeuristicV2SuccessThresholdError,
|
||||
getHeuristicFirstContextLimitError,
|
||||
} from "./build_complexity_router_config";
|
||||
import { buildAutoRouterRoutingTestRequest } from "./build_auto_router_routing_test_request";
|
||||
|
||||
interface AutoRouterRoutingTestProps {
|
||||
|
|
@ -33,7 +37,9 @@ const AutoRouterRoutingTest: React.FC<AutoRouterRoutingTestProps> = ({
|
|||
const { t } = useTranslation();
|
||||
const [prompt, setPrompt] = React.useState<string>("");
|
||||
const [state, setState] = React.useState<TestState>({ status: "idle" });
|
||||
const configError = getHeuristicV2SuccessThresholdError(config.heuristic_v2_success_threshold);
|
||||
const configError =
|
||||
getHeuristicV2SuccessThresholdError(config.heuristic_v2_success_threshold) ??
|
||||
getHeuristicFirstContextLimitError(config.heuristic_first_max_context_tokens);
|
||||
|
||||
const send = async () => {
|
||||
if (configError) return;
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
import HeuristicFirstContextLimitField from "./HeuristicFirstContextLimitField";
|
||||
import ClassifierPrimarySettings from "./ClassifierPrimarySettings";
|
||||
import { AutoRouterAllowanceNote } from "./AutoRouterAvailability";
|
||||
import { transitionClassifierType } from "./classifier_type_transition";
|
||||
|
|
@ -519,6 +520,13 @@ const ClassificationMethodConfig: React.FC<ClassificationMethodConfigProps> = ({
|
|||
</div>
|
||||
)}
|
||||
|
||||
{showSelection && classifierType === "heuristic_first" && (
|
||||
<HeuristicFirstContextLimitField
|
||||
value={value.heuristic_first_max_context_tokens}
|
||||
onChange={(limit) => onChange({ ...value, heuristic_first_max_context_tokens: limit })}
|
||||
/>
|
||||
)}
|
||||
|
||||
{showSelection && classifierType === "hybrid" && (
|
||||
<div className="mt-4 space-y-2">
|
||||
<Label htmlFor={HYBRID_BOUNDARY_MARGIN_ID} className="block font-semibold">
|
||||
|
|
|
|||
|
|
@ -1924,3 +1924,43 @@ describe("classifier vision settings", () => {
|
|||
expect(screen.queryByText("Use images for classification")).not.toBeInTheDocument();
|
||||
});
|
||||
});
|
||||
|
||||
describe("heuristic-first conversation limit", () => {
|
||||
it("shows saved limits and carries edits, invalid drafts, and clearing into form state", () => {
|
||||
const initial: ComplexityRouterConfigValue = {
|
||||
...defaultValue,
|
||||
classifier_type: "heuristic_first",
|
||||
heuristic_first_max_tier: "SIMPLE",
|
||||
classifier_llm_config: { model: "gpt-4" },
|
||||
heuristic_first_max_context_tokens: 8000,
|
||||
};
|
||||
const onChange = vi.fn();
|
||||
function Editor() {
|
||||
const [value, setValue] = React.useState(initial);
|
||||
return (
|
||||
<ComplexityRouterConfig
|
||||
{...baseProps}
|
||||
value={value}
|
||||
onChange={(next) => {
|
||||
setValue(next);
|
||||
onChange(next);
|
||||
}}
|
||||
/>
|
||||
);
|
||||
}
|
||||
renderWithProviders(<Editor />);
|
||||
fireEvent.click(screen.getByRole("button", { name: /^Advanced settings/ }));
|
||||
const input = screen.getByRole("textbox", { name: "Max conversation tokens before judge" });
|
||||
expect(input).toHaveValue("8000");
|
||||
fireEvent.change(input, { target: { value: "12000" } });
|
||||
expect(onChange).toHaveBeenLastCalledWith(expect.objectContaining({ heuristic_first_max_context_tokens: 12000 }));
|
||||
fireEvent.change(input, { target: { value: "invalid" } });
|
||||
expect(screen.getByText("Conversation limit must be a positive whole number")).toBeInTheDocument();
|
||||
expect(onChange).toHaveBeenLastCalledWith(expect.objectContaining({ heuristic_first_max_context_tokens: NaN }));
|
||||
fireEvent.change(input, { target: { value: "" } });
|
||||
expect(onChange).toHaveBeenLastCalledWith(
|
||||
expect.objectContaining({ heuristic_first_max_context_tokens: undefined }),
|
||||
);
|
||||
expect(screen.queryByText("Conversation limit must be a positive whole number")).not.toBeInTheDocument();
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -407,6 +407,7 @@ export interface ComplexityRouterConfigValue {
|
|||
classification_examples?: string;
|
||||
/** Highest tier the scorer may decide alone under heuristic_first. Required by that type, rejected by the others. */
|
||||
heuristic_first_max_tier?: string;
|
||||
heuristic_first_max_context_tokens?: number;
|
||||
/** How near a tier boundary a score may land before hybrid defers to the classifier. Required by that type, rejected by the others. */
|
||||
hybrid_boundary_margin?: number;
|
||||
classification_mode?: ClassificationMode;
|
||||
|
|
|
|||
|
|
@ -0,0 +1,44 @@
|
|||
import React from "react";
|
||||
import { useTranslation } from "react-i18next";
|
||||
import { Input } from "@/components/ui/input";
|
||||
import { Label } from "@/components/ui/label";
|
||||
import { getHeuristicFirstContextLimitError } from "./build_complexity_router_config";
|
||||
|
||||
type Props = { value: number | undefined; onChange: (value: number | undefined) => void };
|
||||
|
||||
export default function HeuristicFirstContextLimitField({ value, onChange }: Props) {
|
||||
const { t } = useTranslation();
|
||||
const [draft, setDraft] = React.useState<string | null>(null);
|
||||
const error = getHeuristicFirstContextLimitError(value);
|
||||
const displayDraft = draft !== null && Object.is(value, draft.trim() === "" ? undefined : Number(draft));
|
||||
const handleChange = (event: React.ChangeEvent<HTMLInputElement>) => {
|
||||
const raw = event.target.value;
|
||||
setDraft(raw);
|
||||
onChange(raw.trim() === "" ? undefined : Number(raw));
|
||||
};
|
||||
return (
|
||||
<div className="mt-4 space-y-2">
|
||||
<Label htmlFor="auto-router-context-limit" className="block font-semibold">
|
||||
{t("addModel.classificationMethodConfig.contextLimit.label")}
|
||||
</Label>
|
||||
<Input
|
||||
id="auto-router-context-limit"
|
||||
type="text"
|
||||
inputMode="numeric"
|
||||
placeholder="8000"
|
||||
value={displayDraft ? draft : value?.toString() ?? ""}
|
||||
onChange={handleChange}
|
||||
aria-invalid={Boolean(error)}
|
||||
aria-describedby={`auto-router-context-limit-help${error ? " auto-router-context-limit-error" : ""}`}
|
||||
/>
|
||||
<p id="auto-router-context-limit-help" className="text-sm text-muted-foreground">
|
||||
{t("addModel.classificationMethodConfig.contextLimit.help")}
|
||||
</p>
|
||||
{error && (
|
||||
<p id="auto-router-context-limit-error" className="text-sm text-destructive" role="alert">
|
||||
{error}
|
||||
</p>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
|
@ -4,6 +4,7 @@ import {
|
|||
openAutoRouterAdvanced,
|
||||
selectAutoRouterOption,
|
||||
selectAutoRouterApproach,
|
||||
withSettledRouterAvailability,
|
||||
} from "../../../tests/autoRouterSetup";
|
||||
import {
|
||||
renderWithProviders,
|
||||
|
|
@ -1380,6 +1381,41 @@ describe("AddAutoRouterTab", () => {
|
|||
);
|
||||
});
|
||||
|
||||
it("ignores a hidden conversation limit after customizing tiers and validates it again on restore", async () => {
|
||||
mockFetchAvailableModels.mockResolvedValue(ALL_FAMILY_MODELS);
|
||||
renderAdvanced(<Harness />);
|
||||
await waitForAutomaticTiers();
|
||||
await userEvent.click(screen.getByRole("radio", { name: "LLM" }));
|
||||
await selectAutoRouterOption("Judge model", ALL_FAMILY_MODELS[0].model_group);
|
||||
openAutoRouterAdvanced("Classification Method");
|
||||
await selectAutoRouterOption("Local checks before the judge", "Heuristic first");
|
||||
fireEvent.change(screen.getByRole("textbox", { name: "Max conversation tokens before judge" }), {
|
||||
target: { value: "invalid" },
|
||||
});
|
||||
expect(screen.getByRole("button", { name: "Add Auto Router" })).toBeDisabled();
|
||||
await userEvent.click(screen.getByRole("button", { name: "Edit tiers" }));
|
||||
await withSettledRouterAvailability(() => {
|
||||
fireEvent.change(screen.getByLabelText("Definition for tier 1"), { target: { value: "Short requests" } });
|
||||
});
|
||||
expect(screen.queryByRole("textbox", { name: "Max conversation tokens before judge" })).not.toBeInTheDocument();
|
||||
expect(screen.getByRole("button", { name: "Add Auto Router" })).toBeEnabled();
|
||||
await userEvent.click(screen.getByRole("button", { name: "Restore defaults" }));
|
||||
expect(screen.getByRole("textbox", { name: "Max conversation tokens before judge" })).toHaveAttribute(
|
||||
"aria-invalid",
|
||||
"true",
|
||||
);
|
||||
expect(screen.getByRole("button", { name: "Add Auto Router" })).toBeDisabled();
|
||||
await withSettledRouterAvailability(() => {
|
||||
fireEvent.change(screen.getByLabelText("Definition for tier 1"), { target: { value: "Short requests" } });
|
||||
});
|
||||
expect(screen.getByRole("button", { name: "Add Auto Router" })).toBeEnabled();
|
||||
await userEvent.click(screen.getByRole("button", { name: "Add Auto Router" }));
|
||||
await waitFor(() => expect(handleAddAutoRouterSubmit).toHaveBeenCalledOnce());
|
||||
const config = vi.mocked(handleAddAutoRouterSubmit).mock.calls.at(-1)?.[0].complexity_router_config;
|
||||
expect(config?.classifier_type).toBe("llm");
|
||||
expect(config).not.toHaveProperty("heuristic_first_max_context_tokens");
|
||||
});
|
||||
|
||||
it("preserves classifier tuning when choosing models automatically", async () => {
|
||||
const user = userEvent.setup();
|
||||
mockFetchAvailableModels.mockResolvedValue(ALL_FAMILY_MODELS);
|
||||
|
|
|
|||
|
|
@ -50,6 +50,7 @@ import {
|
|||
getKeywordTierRulesError,
|
||||
getClassifierModelError,
|
||||
getHeuristicV2SuccessThresholdError,
|
||||
getHeuristicFirstConfigError,
|
||||
getReminderMarkersError,
|
||||
getClassifierPluginTimeoutError,
|
||||
getClassifierReasoningEffortError,
|
||||
|
|
@ -137,6 +138,7 @@ export const getSubmitBlockedReason = (
|
|||
: null) ??
|
||||
getClassifierModelError(config) ??
|
||||
getHeuristicV2SuccessThresholdError(config.heuristic_v2_success_threshold) ??
|
||||
getHeuristicFirstConfigError(config) ??
|
||||
getReminderMarkersError(config.reminder_markers) ??
|
||||
getClassifierPluginTimeoutError(config.classifier_type, config.classifier_plugin_timeout_ms) ??
|
||||
(heuristicScoringRole(config) === "decides" ? customDimensionsError(config.custom_dimensions) : null) ??
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ import {
|
|||
getKeywordTierRulesError,
|
||||
getClassifierModelError,
|
||||
getHeuristicV2SuccessThresholdError,
|
||||
getHeuristicFirstContextLimitError,
|
||||
getReminderMarkersError,
|
||||
getClassifierPluginTimeoutError,
|
||||
getClassifierReasoningEffortError,
|
||||
|
|
@ -1217,6 +1218,23 @@ describe("heuristic_first", () => {
|
|||
classifierFallback: "default_model",
|
||||
};
|
||||
|
||||
it("persists the conversation limit only for heuristic-first routing", () => {
|
||||
const params = { ...heuristicFirstParams, heuristicFirstMaxContextTokens: 8000 };
|
||||
expect(buildComplexityRouterConfig(params).heuristic_first_max_context_tokens).toBe(8000);
|
||||
expect(buildComplexityRouterConfig(heuristicFirstParams).heuristic_first_max_context_tokens).toBeUndefined();
|
||||
expect(
|
||||
buildComplexityRouterConfig({ ...params, classifierType: "llm" }).heuristic_first_max_context_tokens,
|
||||
).toBeUndefined();
|
||||
});
|
||||
|
||||
it.each([0, -1, 1.5, NaN, Infinity])("rejects invalid conversation limit %s", (limit) => {
|
||||
expect(getHeuristicFirstContextLimitError(limit)).toBe("Conversation limit must be a positive whole number");
|
||||
});
|
||||
|
||||
it.each([undefined, 1, 8000])("accepts optional conversation limit %s", (limit) => {
|
||||
expect(getHeuristicFirstContextLimitError(limit)).toBeNull();
|
||||
});
|
||||
|
||||
it("emits heuristic_first_max_tier", () => {
|
||||
const config = buildComplexityRouterConfig(heuristicFirstParams);
|
||||
expect(config.classifier_type).toBe("heuristic_first");
|
||||
|
|
@ -1419,6 +1437,7 @@ describe("buildComplexityRouterConfig with an edited tier set", () => {
|
|||
customDimensions: [{ id: "row-1", name: "sqlDdl", weight: 0.4, keywords: ["orbitmesh"] }],
|
||||
reasoningOverrideMinScore: 0.5,
|
||||
heuristicFirstMaxTier: "SIMPLE",
|
||||
heuristicFirstMaxContextTokens: 8000,
|
||||
hybridBoundaryMargin: 0.03,
|
||||
localHeuristic: "heuristic",
|
||||
customTechnicalKeywords: ["kubernetes"],
|
||||
|
|
@ -1427,7 +1446,12 @@ describe("buildComplexityRouterConfig with an edited tier set", () => {
|
|||
stallEscalationRepeatThreshold: 3,
|
||||
};
|
||||
// custom_dimensions only ever ship when the scorer decides, so "llm" cannot prove it emits.
|
||||
const emittingType = ["heuristic_first_max_tier", "custom_dimensions", "local_heuristic"].includes(key)
|
||||
const emittingType = [
|
||||
"heuristic_first_max_tier",
|
||||
"heuristic_first_max_context_tokens",
|
||||
"custom_dimensions",
|
||||
"local_heuristic",
|
||||
].includes(key)
|
||||
? "heuristic_first"
|
||||
: "llm";
|
||||
const typeForKey = key === "hybrid_boundary_margin" ? "hybrid" : emittingType;
|
||||
|
|
|
|||
|
|
@ -115,6 +115,7 @@ export interface StoredComplexityRouterConfig {
|
|||
classification_prompt?: unknown;
|
||||
classification_examples?: unknown;
|
||||
heuristic_first_max_tier?: unknown;
|
||||
heuristic_first_max_context_tokens?: unknown;
|
||||
hybrid_boundary_margin?: unknown;
|
||||
tier_labels?: unknown;
|
||||
classifier_type?: ClassifierType | "oss_classifier";
|
||||
|
|
@ -188,6 +189,7 @@ export interface BuildComplexityRouterConfigParams {
|
|||
classificationPrompt: string | undefined;
|
||||
classificationExamples: string | undefined;
|
||||
heuristicFirstMaxTier: string | undefined;
|
||||
heuristicFirstMaxContextTokens?: number;
|
||||
hybridBoundaryMargin?: number;
|
||||
classificationMode: ClassificationMode | undefined;
|
||||
sessionAffinity: boolean;
|
||||
|
|
@ -275,6 +277,7 @@ export interface ComplexityRouterConfigPayload {
|
|||
classification_prompt?: string;
|
||||
classification_examples?: string;
|
||||
heuristic_first_max_tier?: string;
|
||||
heuristic_first_max_context_tokens?: number;
|
||||
hybrid_boundary_margin?: number;
|
||||
classification_mode: ClassificationMode;
|
||||
session_affinity: boolean;
|
||||
|
|
@ -397,6 +400,18 @@ export const getKeywordTierRulesError = (
|
|||
return t("addModel.buildComplexityRouterConfig.keywordRulesOrphaned", { rules: orphaned.join(", ") });
|
||||
};
|
||||
|
||||
export const getHeuristicFirstContextLimitError = (limit: number | undefined): string | null => {
|
||||
if (limit === undefined) return null;
|
||||
return Number.isSafeInteger(limit) && limit > 0
|
||||
? null
|
||||
: i18n.t("addModel.buildComplexityRouterConfig.contextLimitPositiveInteger");
|
||||
};
|
||||
|
||||
export const getHeuristicFirstConfigError = (config: ComplexityRouterConfigValue): string | null =>
|
||||
effectiveClassifierType(config) === "heuristic_first"
|
||||
? getHeuristicFirstContextLimitError(config.heuristic_first_max_context_tokens)
|
||||
: null;
|
||||
|
||||
export const getHeuristicV2SuccessThresholdError = (
|
||||
threshold: number | undefined,
|
||||
t: TFunction = i18n.t.bind(i18n),
|
||||
|
|
@ -595,12 +610,36 @@ export const hydratePlanModeMinTier = (
|
|||
return tierRowByName(customTierSet.tiers, stored)?.id;
|
||||
};
|
||||
|
||||
const localClassifierWireFields = (
|
||||
effectiveType: ClassifierType,
|
||||
{
|
||||
heuristicFirstMaxTier,
|
||||
heuristicFirstMaxContextTokens,
|
||||
hybridBoundaryMargin,
|
||||
}: Pick<
|
||||
BuildComplexityRouterConfigParams,
|
||||
"heuristicFirstMaxTier" | "heuristicFirstMaxContextTokens" | "hybridBoundaryMargin"
|
||||
>,
|
||||
): Partial<ComplexityRouterConfigPayload> => ({
|
||||
...(effectiveType === "heuristic_first" &&
|
||||
heuristicFirstMaxTier?.trim() && {
|
||||
heuristic_first_max_tier: heuristicFirstMaxTier,
|
||||
}),
|
||||
...(effectiveType === "heuristic_first" &&
|
||||
heuristicFirstMaxContextTokens !== undefined && {
|
||||
heuristic_first_max_context_tokens: heuristicFirstMaxContextTokens,
|
||||
}),
|
||||
...(effectiveType === "hybrid" &&
|
||||
hybridBoundaryMargin !== undefined && { hybrid_boundary_margin: hybridBoundaryMargin }),
|
||||
});
|
||||
|
||||
const classifierWireFields = (
|
||||
effectiveType: ClassifierType,
|
||||
{
|
||||
classifierLlmConfig,
|
||||
classifierFallback,
|
||||
heuristicFirstMaxTier,
|
||||
heuristicFirstMaxContextTokens,
|
||||
hybridBoundaryMargin,
|
||||
classifierContextWindowSize,
|
||||
classifierContextBudgetChars,
|
||||
|
|
@ -611,6 +650,7 @@ const classifierWireFields = (
|
|||
| "classifierLlmConfig"
|
||||
| "classifierFallback"
|
||||
| "heuristicFirstMaxTier"
|
||||
| "heuristicFirstMaxContextTokens"
|
||||
| "hybridBoundaryMargin"
|
||||
| "classifierContextWindowSize"
|
||||
| "classifierContextBudgetChars"
|
||||
|
|
@ -627,10 +667,11 @@ const classifierWireFields = (
|
|||
: normalizeClassifierLlmConfig(classifierLlmConfig),
|
||||
}),
|
||||
...(supportsFallback && classifierFallback !== undefined && { classifier_fallback: classifierFallback }),
|
||||
...(effectiveType === "heuristic_first" &&
|
||||
heuristicFirstMaxTier?.trim() && { heuristic_first_max_tier: heuristicFirstMaxTier }),
|
||||
...(effectiveType === "hybrid" &&
|
||||
hybridBoundaryMargin !== undefined && { hybrid_boundary_margin: hybridBoundaryMargin }),
|
||||
...localClassifierWireFields(effectiveType, {
|
||||
heuristicFirstMaxTier,
|
||||
heuristicFirstMaxContextTokens,
|
||||
hybridBoundaryMargin,
|
||||
}),
|
||||
...(usesClassifierContext(effectiveType) &&
|
||||
classifierContextWindowSize !== undefined && {
|
||||
classifier_context_window_size: classifierContextWindowSize,
|
||||
|
|
@ -672,6 +713,7 @@ export const buildComplexityRouterConfig = ({
|
|||
classificationPrompt,
|
||||
classificationExamples,
|
||||
heuristicFirstMaxTier,
|
||||
heuristicFirstMaxContextTokens,
|
||||
hybridBoundaryMargin,
|
||||
classificationMode,
|
||||
sessionAffinity,
|
||||
|
|
@ -739,6 +781,7 @@ export const buildComplexityRouterConfig = ({
|
|||
classifierLlmConfig,
|
||||
classifierFallback,
|
||||
heuristicFirstMaxTier,
|
||||
heuristicFirstMaxContextTokens,
|
||||
hybridBoundaryMargin,
|
||||
classifierContextWindowSize,
|
||||
classifierContextBudgetChars,
|
||||
|
|
|
|||
|
|
@ -118,3 +118,13 @@ describe("transitionClassifierType", () => {
|
|||
expect(result.tiers.SIMPLE).toEqual(["efficient"]);
|
||||
});
|
||||
});
|
||||
|
||||
it("clears the conversation limit when leaving heuristic-first", () => {
|
||||
const initial = {
|
||||
...standard,
|
||||
classifier_type: "heuristic_first" as const,
|
||||
heuristic_first_max_context_tokens: 8000,
|
||||
};
|
||||
expect(transitionClassifierType(initial, "heuristic_first").heuristic_first_max_context_tokens).toBe(8000);
|
||||
expect(transitionClassifierType(initial, "llm").heuristic_first_max_context_tokens).toBeUndefined();
|
||||
});
|
||||
|
|
|
|||
|
|
@ -51,6 +51,8 @@ export const transitionClassifierType = (
|
|||
classifierType === "heuristic_first"
|
||||
? value.heuristic_first_max_tier ?? DEFAULT_HEURISTIC_FIRST_MAX_TIER
|
||||
: undefined,
|
||||
heuristic_first_max_context_tokens:
|
||||
classifierType === "heuristic_first" ? value.heuristic_first_max_context_tokens : undefined,
|
||||
hybrid_boundary_margin:
|
||||
classifierType === "hybrid" ? value.hybrid_boundary_margin ?? DEFAULT_HYBRID_BOUNDARY_MARGIN : undefined,
|
||||
...nonReasoningTierFields(classifierType, value),
|
||||
|
|
|
|||
|
|
@ -26,6 +26,7 @@ export const builderParamsFromValue = (
|
|||
classificationPrompt: value.classification_prompt,
|
||||
classificationExamples: value.classification_examples,
|
||||
heuristicFirstMaxTier: value.heuristic_first_max_tier,
|
||||
heuristicFirstMaxContextTokens: value.heuristic_first_max_context_tokens,
|
||||
hybridBoundaryMargin: value.hybrid_boundary_margin,
|
||||
classificationMode: value.classification_mode,
|
||||
tierLabels: value.tier_labels,
|
||||
|
|
|
|||
|
|
@ -145,7 +145,12 @@ export const CUSTOM_TIER_RESTRICTIONS = {
|
|||
reason: "addModel.tierRows.restrictions.sessionAffinity",
|
||||
},
|
||||
heuristicClassifier: {
|
||||
omit: ["heuristic_first_max_tier", "hybrid_boundary_margin", "local_heuristic"],
|
||||
omit: [
|
||||
"heuristic_first_max_tier",
|
||||
"heuristic_first_max_context_tokens",
|
||||
"hybrid_boundary_margin",
|
||||
"local_heuristic",
|
||||
],
|
||||
reason: "addModel.tierRows.restrictions.heuristicClassifier",
|
||||
},
|
||||
heuristicScoring: {
|
||||
|
|
|
|||
|
|
@ -921,6 +921,7 @@ describe("managed keys survive an untouched open-and-save", () => {
|
|||
local_heuristic: "heuristic",
|
||||
heuristic_v2_success_threshold: 0.89,
|
||||
heuristic_first_max_tier: "SIMPLE",
|
||||
heuristic_first_max_context_tokens: 8000,
|
||||
classifier_llm_config: { model: "gpt-4o-mini", timeout_ms: 3000, reasoning_effort: "low" },
|
||||
classifier_context_window_size: 5,
|
||||
classifier_context_budget_chars: 4000,
|
||||
|
|
@ -1306,3 +1307,22 @@ describe("untouched save round trip", () => {
|
|||
expect(saved.reminder_markers).toEqual([{ open: "<other>", close: "</other>" }]);
|
||||
});
|
||||
});
|
||||
|
||||
it("edits and clears the stored heuristic-first conversation limit", () => {
|
||||
const stored = {
|
||||
tiers: { SIMPLE: ["simple"], MEDIUM: ["medium"] },
|
||||
classifier_type: "heuristic_first",
|
||||
heuristic_first_max_tier: "SIMPLE",
|
||||
classifier_llm_config: { model: "judge" },
|
||||
heuristic_first_max_context_tokens: 8000,
|
||||
};
|
||||
const hydrated = hydrateComplexityRouterConfig(stored, undefined);
|
||||
expect(hydrated.heuristic_first_max_context_tokens).toBe(8000);
|
||||
expect(
|
||||
buildUpdatedComplexityRouterConfig(stored, { ...hydrated, heuristic_first_max_context_tokens: 12000 })
|
||||
.heuristic_first_max_context_tokens,
|
||||
).toBe(12000);
|
||||
expect(
|
||||
buildUpdatedComplexityRouterConfig(stored, { ...hydrated, heuristic_first_max_context_tokens: undefined }),
|
||||
).not.toHaveProperty("heuristic_first_max_context_tokens");
|
||||
});
|
||||
|
|
|
|||
|
|
@ -2,7 +2,11 @@ vi.mock("@/app/(dashboard)/hooks/models/useModelCostMap", () => ({
|
|||
useModelCostMap: () => ({ data: { "typesafe/jev-latest": { litellm_provider: "typesafe", mode: "evaluation" } } }),
|
||||
}));
|
||||
|
||||
import { openAutoRouterAdvanced, selectAutoRouterOption } from "../../../tests/autoRouterSetup";
|
||||
import {
|
||||
openAutoRouterAdvanced,
|
||||
selectAutoRouterOption,
|
||||
withSettledRouterAvailability,
|
||||
} from "../../../tests/autoRouterSetup";
|
||||
import userEvent from "@testing-library/user-event";
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
|
||||
|
|
@ -90,17 +94,6 @@ const savedConfig = () => {
|
|||
return payload?.litellm_params?.complexity_router_config;
|
||||
};
|
||||
|
||||
const renderWithSettledAvailability = async (renderModal: () => unknown) => {
|
||||
vi.useFakeTimers();
|
||||
try {
|
||||
renderModal();
|
||||
await act(() => vi.advanceTimersByTimeAsync(300));
|
||||
await act(() => vi.runOnlyPendingTimersAsync());
|
||||
} finally {
|
||||
vi.useRealTimers();
|
||||
}
|
||||
};
|
||||
|
||||
describe("EditAutoRouterModal keyword matching", () => {
|
||||
beforeEach(() => {
|
||||
modelPatchUpdateCall.mockClear();
|
||||
|
|
@ -221,6 +214,41 @@ describe("EditAutoRouterModal keyword matching", () => {
|
|||
else expect(savedConfig().heuristic_v2_success_threshold).toBe(0);
|
||||
});
|
||||
|
||||
it("saves custom tiers after an invalid conversation limit becomes hidden", async () => {
|
||||
testQueryClient.clear();
|
||||
vi.mocked(apiClient.post).mockResolvedValue({ allowances: [], error: null });
|
||||
renderModal({
|
||||
modelData: {
|
||||
...MODEL_DATA,
|
||||
litellm_params: {
|
||||
...MODEL_DATA.litellm_params,
|
||||
complexity_router_config: {
|
||||
...STORED_CONFIG,
|
||||
classifier_type: "heuristic_first",
|
||||
heuristic_first_max_tier: "SIMPLE",
|
||||
classifier_llm_config: { model: "gpt-4o-mini" },
|
||||
heuristic_first_max_context_tokens: 8000,
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
openAutoRouterAdvanced("Classification Method");
|
||||
fireEvent.change(screen.getByRole("textbox", { name: "Max conversation tokens before judge" }), {
|
||||
target: { value: "invalid" },
|
||||
});
|
||||
expect(screen.getByRole("button", { name: "Save Changes" })).toBeDisabled();
|
||||
await userEvent.click(screen.getByRole("button", { name: "Edit tiers" }));
|
||||
await withSettledRouterAvailability(() => {
|
||||
fireEvent.change(screen.getByLabelText("Definition for tier 1"), { target: { value: "Short requests" } });
|
||||
});
|
||||
expect(screen.queryByRole("textbox", { name: "Max conversation tokens before judge" })).not.toBeInTheDocument();
|
||||
expect(screen.getByRole("button", { name: "Save Changes" })).toBeEnabled();
|
||||
await userEvent.click(screen.getByRole("button", { name: "Save Changes" }));
|
||||
await waitFor(() => expect(modelPatchUpdateCall).toHaveBeenCalledOnce());
|
||||
expect(savedConfig().classifier_type).toBe("llm");
|
||||
expect(savedConfig()).not.toHaveProperty("heuristic_first_max_context_tokens");
|
||||
});
|
||||
|
||||
it("saves a legacy heuristic-first router with a v2 chain and an edited success threshold", async () => {
|
||||
renderModal({
|
||||
modelData: {
|
||||
|
|
@ -1470,7 +1498,7 @@ describe("EditAutoRouterModal prompt compression", () => {
|
|||
{ auto_router_model_compression: "model-compressor" },
|
||||
])("should preserve the exact stored compression fields through an untouched save: %j", async (stored) => {
|
||||
const user = userEvent.setup();
|
||||
await renderWithSettledAvailability(() => renderWithStoredCompression(stored));
|
||||
await withSettledRouterAvailability(() => renderWithStoredCompression(stored));
|
||||
|
||||
await waitFor(() => expect(screen.getByRole("button", { name: /save changes/i })).toBeEnabled());
|
||||
await user.click(await screen.findByRole("button", { name: /save changes/i }));
|
||||
|
|
@ -1547,7 +1575,7 @@ describe("EditAutoRouterModal classifier vision", () => {
|
|||
|
||||
it("hydrates and keeps a stored vision setting through an untouched save", async () => {
|
||||
const user = userEvent.setup();
|
||||
await renderWithSettledAvailability(renderModal);
|
||||
await withSettledRouterAvailability(renderModal);
|
||||
|
||||
openAutoRouterAdvanced("Classification Method");
|
||||
expect(screen.getByRole("switch", { name: "Use images for classification" })).toBeChecked();
|
||||
|
|
|
|||
|
|
@ -43,6 +43,7 @@ import {
|
|||
buildComplexityRouterConfig,
|
||||
getClassifierModelError,
|
||||
getHeuristicV2SuccessThresholdError,
|
||||
getHeuristicFirstConfigError,
|
||||
getReminderMarkersError,
|
||||
getClassifierPluginTimeoutError,
|
||||
getClassifierReasoningEffortError,
|
||||
|
|
@ -116,6 +117,7 @@ export const MANAGED_COMPLEXITY_ROUTER_KEYS = new Set([
|
|||
"classification_prompt",
|
||||
"classification_examples",
|
||||
"heuristic_first_max_tier",
|
||||
"heuristic_first_max_context_tokens",
|
||||
"hybrid_boundary_margin",
|
||||
"heuristic_v2_success_threshold",
|
||||
"classification_mode",
|
||||
|
|
@ -344,6 +346,7 @@ const ExistingEditAutoRouterModal: React.FC<EditAutoRouterModalProps> = ({
|
|||
getKeywordTierRulesError(keywordTierRules, activeTierRows(complexityRouterConfig)) ??
|
||||
getClassifierModelError(complexityRouterConfig) ??
|
||||
getHeuristicV2SuccessThresholdError(complexityRouterConfig.heuristic_v2_success_threshold) ??
|
||||
getHeuristicFirstConfigError(complexityRouterConfig) ??
|
||||
getForecastConfigError(complexityRouterConfig) ??
|
||||
(heuristicScoringRole(complexityRouterConfig) === "decides"
|
||||
? customDimensionsError(complexityRouterConfig.custom_dimensions)
|
||||
|
|
@ -484,6 +487,7 @@ const ExistingEditAutoRouterModal: React.FC<EditAutoRouterModalProps> = ({
|
|||
const classifierError =
|
||||
getClassifierModelError(complexityRouterConfig) ??
|
||||
getHeuristicV2SuccessThresholdError(complexityRouterConfig.heuristic_v2_success_threshold) ??
|
||||
getHeuristicFirstConfigError(complexityRouterConfig) ??
|
||||
getReminderMarkersError(complexityRouterConfig.reminder_markers) ??
|
||||
getClassifierPluginTimeoutError(
|
||||
complexityRouterConfig.classifier_type,
|
||||
|
|
|
|||
|
|
@ -115,6 +115,10 @@ export const hydrateComplexityRouterConfig = (
|
|||
typeof parsedConfig.heuristic_first_max_tier === "string" && parsedConfig.heuristic_first_max_tier.trim() !== ""
|
||||
? parsedConfig.heuristic_first_max_tier
|
||||
: undefined,
|
||||
heuristic_first_max_context_tokens:
|
||||
typeof parsedConfig.heuristic_first_max_context_tokens === "number"
|
||||
? parsedConfig.heuristic_first_max_context_tokens
|
||||
: undefined,
|
||||
hybrid_boundary_margin:
|
||||
typeof parsedConfig.hybrid_boundary_margin === "number" ? parsedConfig.hybrid_boundary_margin : undefined,
|
||||
classification_mode:
|
||||
|
|
|
|||
|
|
@ -17809,7 +17809,11 @@
|
|||
"empty": "Type to add a keyword"
|
||||
},
|
||||
"scoringRulesAllowance": "Custom scoring rules",
|
||||
"keywordOverridesTitle": "Heuristic Keyword Overrides"
|
||||
"keywordOverridesTitle": "Heuristic Keyword Overrides",
|
||||
"contextLimit": {
|
||||
"label": "Max conversation tokens before judge",
|
||||
"help": "Above this estimated size, ask the judge even for short follow-ups. Counts text and tool results. Existing session and tool-continuation pins still apply. Leave blank to disable."
|
||||
}
|
||||
},
|
||||
"autoRouterClassifierTabs": {
|
||||
"option": {
|
||||
|
|
@ -18391,7 +18395,8 @@
|
|||
"selectClassifierOrHeuristic": "Please select a classifier model, or switch back to Heuristic",
|
||||
"reasoningEffortUnsupported": "{{effort}} reasoning effort is not supported by every deployment in {{model}}. Choose Default or a supported value.",
|
||||
"selectEmbeddingModel": "Select an embedding model to use semantic keyword matching",
|
||||
"addKeywordRuleForSemantic": "Add at least one keyword tier rule to use semantic keyword matching"
|
||||
"addKeywordRuleForSemantic": "Add at least one keyword tier rule to use semantic keyword matching",
|
||||
"contextLimitPositiveInteger": "Conversation limit must be a positive whole number"
|
||||
},
|
||||
"adaptiveRoutingConfig": {
|
||||
"enable": "Enable adaptive bandit selection",
|
||||
|
|
|
|||
5
litellm/ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
5
litellm/ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
|
|
@ -45852,6 +45852,11 @@ export interface components {
|
|||
* @description Tier routed to when the LLM classifier fails (timeout, provider error, or an unparseable reply). Required with tier_definitions and must name a defined tier; the heuristic scorer cannot produce custom tiers, so this replaces the heuristic fallback for custom tier sets.
|
||||
*/
|
||||
fallback_tier?: string | null;
|
||||
/**
|
||||
* Heuristic First Max Context Tokens
|
||||
* @description Optional conversation-size limit for heuristic_first. Above this estimate, call the LLM classifier even for a short current ask. Counts text and tool-result bodies at roughly four characters per token. Unset preserves the local shortcut. Existing session and user-turn pins still take precedence.
|
||||
*/
|
||||
heuristic_first_max_context_tokens?: number | null;
|
||||
/**
|
||||
* Heuristic First Max Tier
|
||||
* @description The highest tier the local scorer may decide on its own; required when classifier_type is 'heuristic_first' and rejected otherwise. A request whose heuristic tier is at or below this one skips the LLM classifier and routes straight to that heuristic tier, so the classifier call is only paid for on traffic the scorer could not place cheaply. The scorer must also have produced at least one signal: a prompt where no dimension fired scores 0.0 and would otherwise land SIMPLE by default rather than by evidence. With local_heuristic 'heuristic_v2', the predicted tier must meet its success threshold. Names a built-in tier, and may not name the highest one, since that would make the LLM classifier unreachable.
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
import userEvent from "@testing-library/user-event";
|
||||
import { chooseSelectOption, fireEvent, screen } from "./test-utils";
|
||||
import { vi } from "vitest";
|
||||
import { act, chooseSelectOption, fireEvent, screen } from "./test-utils";
|
||||
|
||||
const groups: Record<string, string> = {
|
||||
"Heuristic Keyword Overrides": "Heuristic tuning",
|
||||
|
|
@ -38,3 +39,14 @@ export const selectAutoRouterApproach = async (option: string) => {
|
|||
await userEvent.click(screen.getByRole("radio", { name: "LLM" }));
|
||||
await selectAutoRouterOption("Routing approach", option);
|
||||
};
|
||||
|
||||
export const withSettledRouterAvailability = async (changeForm: () => void) => {
|
||||
vi.useFakeTimers();
|
||||
try {
|
||||
changeForm();
|
||||
await act(() => vi.advanceTimersByTimeAsync(300));
|
||||
await act(() => vi.runOnlyPendingTimersAsync());
|
||||
} finally {
|
||||
vi.useRealTimers();
|
||||
}
|
||||
};
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ import io
|
|||
import json
|
||||
from collections.abc import Iterator, Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from itertools import count
|
||||
from types import MappingProxyType
|
||||
from typing import Final
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
|
@ -15,6 +16,7 @@ import httpx
|
|||
import pytest
|
||||
import respx
|
||||
from fastapi.testclient import TestClient
|
||||
from starlette.types import Message, Receive, Scope, Send
|
||||
|
||||
import litellm
|
||||
from litellm import Router
|
||||
|
|
@ -2231,9 +2233,21 @@ def test_rag_query_stream_pings_while_retrieval_is_still_running(client_internal
|
|||
import litellm as litellm_module
|
||||
|
||||
monkeypatch.setattr(litellm_module, "sse_keepalive_ping_interval_seconds", 0.05)
|
||||
retrieval_released: Final = asyncio.Event()
|
||||
ping_count: Final = count(1)
|
||||
|
||||
async def observe_pings(scope: Scope, receive: Receive, send: Send) -> None:
|
||||
async def observe_send(message: Message) -> None:
|
||||
await send(message)
|
||||
if message["type"] == "http.response.body":
|
||||
for _ in range(message.get("body", b"").count(b": ping\n\n")):
|
||||
if next(ping_count) == 3:
|
||||
retrieval_released.set()
|
||||
|
||||
await client_internal_user.app(scope, receive, observe_send)
|
||||
|
||||
async def slow_aquery(**kwargs):
|
||||
await asyncio.sleep(0.3)
|
||||
await asyncio.wait_for(retrieval_released.wait(), timeout=10)
|
||||
return await litellm_module.acompletion(
|
||||
model="gpt-4o-mini",
|
||||
messages=[{"role": "user", "content": "What is the codename?"}],
|
||||
|
|
@ -2252,7 +2266,7 @@ def test_rag_query_stream_pings_while_retrieval_is_still_running(client_internal
|
|||
"litellm.proxy.proxy_server.prisma_client", None
|
||||
), # test-quality-ok: proxy module global, no injection seam
|
||||
):
|
||||
response = client_internal_user.post(
|
||||
response = TestClient(observe_pings).post(
|
||||
"/v1/rag/query",
|
||||
json={
|
||||
"model": "gpt-4o-mini",
|
||||
|
|
|
|||
|
|
@ -14513,6 +14513,109 @@ class TestHeuristicFirstConfig:
|
|||
assert ComplexityRouterConfig(tiers=dict(HEURISTIC_FIRST_TIERS)).uses_llm_classifier is False
|
||||
|
||||
|
||||
@pytest.mark.parametrize("limit", (0, -1, 1.5, True))
|
||||
def test_heuristic_context_limit_rejects_invalid_values(limit: object) -> None:
|
||||
with pytest.raises(ValidationError, match="heuristic_first_max_context_tokens"):
|
||||
_heuristic_first_router(MagicMock(), heuristic_first_max_context_tokens=limit)
|
||||
|
||||
|
||||
def test_heuristic_context_limit_requires_heuristic_first() -> None:
|
||||
with pytest.raises(ValidationError, match="heuristic_first_max_context_tokens"):
|
||||
ComplexityRouterConfig(tiers=dict(HEURISTIC_FIRST_TIERS), heuristic_first_max_context_tokens=8000)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"history_content",
|
||||
(
|
||||
"trace " * 6000,
|
||||
[{"type": "text", "text": "trace " * 6000}],
|
||||
[{"type": "tool_result", "tool_use_id": "call_1", "content": "trace " * 6000}],
|
||||
[{"type": "tool_result", "tool_use_id": "call_1", "content": [{"type": "text", "text": "trace " * 6000}]}],
|
||||
),
|
||||
ids=("plain", "text-parts", "tool-result", "nested-tool-result"),
|
||||
)
|
||||
async def test_short_followup_in_long_conversation_reaches_classifier(
|
||||
mock_router_instance: MagicMock, history_content: object
|
||||
) -> None:
|
||||
mock_router_instance.acompletion = AsyncMock(return_value=_llm_response('{"tier": "COMPLEX"}'))
|
||||
router: Final = _heuristic_first_router(mock_router_instance, heuristic_first_max_context_tokens=8000)
|
||||
messages: Final[list[dict[str, object]]] = [
|
||||
{"role": "user", "content": "debug this failure"},
|
||||
{"role": "user", "content": history_content},
|
||||
{"role": "user", "content": "why did that fail?"},
|
||||
]
|
||||
result: Final = await router.async_pre_routing_hook(
|
||||
model="test-complexity-router", messages=messages, request_kwargs={}
|
||||
)
|
||||
assert result is not None
|
||||
assert result.model == HEURISTIC_FIRST_TIERS["COMPLEX"]
|
||||
assert result.routing_decision["cause"] == "llm_classifier"
|
||||
mock_router_instance.acompletion.assert_awaited_once()
|
||||
classifier_messages: Final = mock_router_instance.acompletion.call_args.kwargs["messages"]
|
||||
assert "Conversation so far: ~9008 tokens" in str(classifier_messages)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"limit, history_chars, cause",
|
||||
(
|
||||
(None, 40000, "heuristic_first_short_circuit"),
|
||||
(10, 35, "heuristic_first_short_circuit"),
|
||||
(10, 36, "heuristic_first_short_circuit"),
|
||||
(10, 40, "llm_classifier"),
|
||||
),
|
||||
)
|
||||
async def test_heuristic_context_limit_boundary_and_disabled(
|
||||
mock_router_instance: MagicMock, limit: int | None, history_chars: int, cause: str
|
||||
) -> None:
|
||||
mock_router_instance.acompletion = AsyncMock(return_value=_llm_response('{"tier": "COMPLEX"}'))
|
||||
router: Final = _heuristic_first_router(mock_router_instance, heuristic_first_max_context_tokens=limit)
|
||||
messages: Final = [{"role": "system", "content": "x" * history_chars}, {"role": "user", "content": "hi!!"}]
|
||||
outcome: Final = await router.aclassify("hi!!", messages=messages)
|
||||
assert outcome.cause == cause
|
||||
assert mock_router_instance.acompletion.await_count == int(cause == "llm_classifier")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_long_context_preserves_tool_pin_then_classifies_next_human_ask(mock_router_instance: MagicMock) -> None:
|
||||
mock_router_instance.health.return_value.session_pins = memory_pins()
|
||||
mock_router_instance.acompletion = AsyncMock(return_value=_llm_response('{"tier": "COMPLEX"}'))
|
||||
router: Final = _heuristic_first_router(
|
||||
mock_router_instance,
|
||||
heuristic_first_max_context_tokens=8000,
|
||||
classification_mode="user_turn",
|
||||
session_affinity=False,
|
||||
)
|
||||
first: Final[list[dict[str, object]]] = [{"role": "user", "content": "hi"}]
|
||||
continuation: Final[list[dict[str, object]]] = [
|
||||
*first,
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": None,
|
||||
"tool_calls": [{"id": "call_1", "type": "function", "function": {"name": "read", "arguments": "{}"}}],
|
||||
},
|
||||
{"role": "tool", "tool_call_id": "call_1", "content": "trace " * 6000},
|
||||
]
|
||||
for messages, cause, model in (
|
||||
(first, "heuristic_first_short_circuit", HEURISTIC_FIRST_TIERS["SIMPLE"]),
|
||||
(continuation, "user_turn_continuation", HEURISTIC_FIRST_TIERS["SIMPLE"]),
|
||||
(
|
||||
[*continuation, {"role": "user", "content": "why did that fail?"}],
|
||||
"llm_classifier",
|
||||
HEURISTIC_FIRST_TIERS["COMPLEX"],
|
||||
),
|
||||
):
|
||||
result: Final = await router.async_pre_routing_hook(
|
||||
model="test-complexity-router",
|
||||
messages=messages,
|
||||
request_kwargs={"metadata": {"session_id": "context-test"}},
|
||||
)
|
||||
assert result is not None
|
||||
assert (result.model, result.routing_decision["cause"]) == (model, cause)
|
||||
mock_router_instance.acompletion.assert_awaited_once()
|
||||
|
||||
|
||||
class TestHeuristicFirst:
|
||||
"""Behavior of the heuristic-first chain: when the classifier call is skipped, and when it is not."""
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue