mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-07 08:26:10 +00:00
feat(ui): add stalled task escalation controls to the auto-router form
Adds an "Advanced: Stalled Task Escalation" section to the complexity router config: a toggle plus the repeat threshold and the window of recent tool calls to examine. Both knobs are seeded on enable and cleared on disable, so an off router sends none of the three keys, which is what the backend requires next to session pinning and a custom tier set. The toggle locks out with an explanation when "How often to classify" is set to once-per-session or new-user-message, since both replay a held routing decision instead of classifying and a stall would never reach the classifier. The keys join the custom-tier restriction registry, which both strips them from a custom-tier save and marks the section restricted. ResponseFormatControls moves into its own file to keep ComplexityRouterConfig.tsx under the 800-line lint ceiling, matching the one-file-per-control layout its siblings already use.
This commit is contained in:
parent
7c6638e5c3
commit
01b55daee9
11 changed files with 395 additions and 19 deletions
|
|
@ -33,6 +33,8 @@ import { ModelGroup } from "@/components/llm_calls/fetch_models";
|
|||
import AdaptiveRoutingConfig from "./AdaptiveRoutingConfig";
|
||||
import ClassificationMethodConfig from "./ClassificationMethodConfig";
|
||||
import ContextWindowEscalationConfig from "./ContextWindowEscalationConfig";
|
||||
import ResponseFormatControls from "./ResponseFormatControls";
|
||||
import StallEscalationConfig from "./StallEscalationConfig";
|
||||
import { Restricted, restrictedBy } from "./TierRestrictions";
|
||||
import { type TierSetAction, applyTierSetAction, setFallbackTier } from "./tier_set_actions";
|
||||
import {
|
||||
|
|
@ -418,6 +420,14 @@ export interface ComplexityRouterConfigValue {
|
|||
deployment_affinity?: boolean;
|
||||
/** Plan-mode floor as a tier ROW ID, unset meaning off. The wire carries the row's name. */
|
||||
plan_mode_min_tier?: string;
|
||||
/**
|
||||
* Mid-task stall escalation. Undefined means off, which keeps all three keys out of the payload:
|
||||
* the backend rejects them alongside session pinning, user-turn classification and a custom tier
|
||||
* set, so an off router must stay silent about them rather than send an explicit false.
|
||||
*/
|
||||
stall_escalation_enabled?: boolean;
|
||||
stall_escalation_window?: number;
|
||||
stall_escalation_repeat_threshold?: number;
|
||||
adaptive?: boolean;
|
||||
adaptive_weights?: AdaptiveRouterWeights;
|
||||
tier_distance_penalty?: number;
|
||||
|
|
@ -571,25 +581,6 @@ const PlanModeOverrideControls: React.FC<{
|
|||
</>
|
||||
);
|
||||
|
||||
const ResponseFormatControls: React.FC<{
|
||||
value: ComplexityRouterConfigValue;
|
||||
onChange: (value: ComplexityRouterConfigValue) => void;
|
||||
}> = ({ value, onChange }) => (
|
||||
<>
|
||||
<div className="flex items-center gap-2 mb-2">
|
||||
<Switch
|
||||
checked={value.return_raw_model_name ?? false}
|
||||
onCheckedChange={(returnRawModelName) => onChange({ ...value, return_raw_model_name: returnRawModelName })}
|
||||
aria-label="Return raw model name"
|
||||
/>
|
||||
<strong className="font-semibold">Return raw model name</strong>
|
||||
</div>
|
||||
<span className="block text-xs text-muted-foreground">
|
||||
Return the resolved underlying model name in responses instead of the autorouter alias.
|
||||
</span>
|
||||
</>
|
||||
);
|
||||
|
||||
const ComplexityRouterConfig: React.FC<ComplexityRouterConfigProps> = ({
|
||||
modelInfo,
|
||||
value,
|
||||
|
|
@ -855,6 +846,15 @@ const ComplexityRouterConfig: React.FC<ComplexityRouterConfigProps> = ({
|
|||
label: <strong className="text-foreground font-semibold">Advanced: Context Window Escalation</strong>,
|
||||
children: <ContextWindowEscalationConfig value={value} onChange={onChange} />,
|
||||
},
|
||||
{
|
||||
key: "stall-escalation",
|
||||
label: <strong className="text-foreground font-semibold">Advanced: Stalled Task Escalation</strong>,
|
||||
children: (
|
||||
<Restricted by={restrictedBy(value, "stallEscalation")}>
|
||||
<StallEscalationConfig value={value} onChange={onChange} />
|
||||
</Restricted>
|
||||
),
|
||||
},
|
||||
{
|
||||
key: "response",
|
||||
label: <strong className="text-foreground font-semibold">Advanced: Response Format</strong>,
|
||||
|
|
|
|||
|
|
@ -0,0 +1,24 @@
|
|||
import { Switch } from "@/components/ui/switch";
|
||||
import React from "react";
|
||||
import type { ComplexityRouterConfigValue } from "./ComplexityRouterConfig";
|
||||
|
||||
const ResponseFormatControls: React.FC<{
|
||||
value: ComplexityRouterConfigValue;
|
||||
onChange: (value: ComplexityRouterConfigValue) => void;
|
||||
}> = ({ value, onChange }) => (
|
||||
<>
|
||||
<div className="flex items-center gap-2 mb-2">
|
||||
<Switch
|
||||
checked={value.return_raw_model_name ?? false}
|
||||
onCheckedChange={(returnRawModelName) => onChange({ ...value, return_raw_model_name: returnRawModelName })}
|
||||
aria-label="Return raw model name"
|
||||
/>
|
||||
<strong className="font-semibold">Return raw model name</strong>
|
||||
</div>
|
||||
<span className="block text-xs text-muted-foreground">
|
||||
Return the resolved underlying model name in responses instead of the autorouter alias.
|
||||
</span>
|
||||
</>
|
||||
);
|
||||
|
||||
export default ResponseFormatControls;
|
||||
|
|
@ -0,0 +1,108 @@
|
|||
import { fireEvent, renderWithProviders, screen } from "../../../tests/test-utils";
|
||||
import { vi } from "vitest";
|
||||
import type { ComplexityRouterConfigValue } from "./ComplexityRouterConfig";
|
||||
import StallEscalationConfig, { stallEscalationBlockedReason } from "./StallEscalationConfig";
|
||||
|
||||
const tiers = { SIMPLE: "gpt-4o-mini", MEDIUM: "gpt-4o", COMPLEX: "claude-sonnet-4", REASONING: "o1-preview" };
|
||||
|
||||
const baseValue: ComplexityRouterConfigValue = {
|
||||
tiers,
|
||||
classifier_type: "heuristic",
|
||||
};
|
||||
|
||||
const renderConfig = (value: Partial<ComplexityRouterConfigValue> = {}) => {
|
||||
const onChange = vi.fn();
|
||||
renderWithProviders(<StallEscalationConfig value={{ ...baseValue, ...value }} onChange={onChange} />);
|
||||
return onChange;
|
||||
};
|
||||
|
||||
const toggle = () => screen.getByRole("switch", { name: "Escalate a stalled task to a stronger model" });
|
||||
|
||||
describe("stallEscalationBlockedReason", () => {
|
||||
it("blocks on session pinning, which replays a model instead of classifying", () => {
|
||||
expect(stallEscalationBlockedReason({ ...baseValue, session_affinity: true })).toContain("Classification Method");
|
||||
});
|
||||
|
||||
it("blocks on user-turn classification, which skips the agent-loop turns a stall shows up in", () => {
|
||||
expect(stallEscalationBlockedReason({ ...baseValue, classification_mode: "user_turn" })).toContain("every request");
|
||||
});
|
||||
|
||||
it("allows the default every-request router", () => {
|
||||
expect(stallEscalationBlockedReason(baseValue)).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe("StallEscalationConfig", () => {
|
||||
it("hides the knobs until the feature is turned on", () => {
|
||||
renderConfig();
|
||||
expect(toggle()).not.toBeChecked();
|
||||
expect(screen.queryByLabelText("Repeats before escalating")).not.toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("turning it on seeds both knobs so the saved config is explicit rather than half-set", () => {
|
||||
const onChange = renderConfig();
|
||||
fireEvent.click(toggle());
|
||||
expect(onChange).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
stall_escalation_enabled: true,
|
||||
stall_escalation_window: 6,
|
||||
stall_escalation_repeat_threshold: 3,
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("turning it off clears all three keys, since the backend rejects them next to session pinning", () => {
|
||||
const onChange = renderConfig({
|
||||
stall_escalation_enabled: true,
|
||||
stall_escalation_window: 6,
|
||||
stall_escalation_repeat_threshold: 3,
|
||||
});
|
||||
fireEvent.click(toggle());
|
||||
expect(onChange).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
stall_escalation_enabled: undefined,
|
||||
stall_escalation_window: undefined,
|
||||
stall_escalation_repeat_threshold: undefined,
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("raises the window to match a larger threshold, which could otherwise never be reached", () => {
|
||||
const onChange = renderConfig({
|
||||
stall_escalation_enabled: true,
|
||||
stall_escalation_window: 4,
|
||||
stall_escalation_repeat_threshold: 3,
|
||||
});
|
||||
fireEvent.change(screen.getByLabelText("Repeats before escalating"), { target: { value: "9" } });
|
||||
expect(onChange).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ stall_escalation_repeat_threshold: 9, stall_escalation_window: 9 }),
|
||||
);
|
||||
});
|
||||
|
||||
it("holds the window at the threshold when someone types a smaller one", () => {
|
||||
const onChange = renderConfig({
|
||||
stall_escalation_enabled: true,
|
||||
stall_escalation_window: 6,
|
||||
stall_escalation_repeat_threshold: 3,
|
||||
});
|
||||
fireEvent.change(screen.getByLabelText("Recent calls examined"), { target: { value: "1" } });
|
||||
expect(onChange).toHaveBeenCalledWith(expect.objectContaining({ stall_escalation_window: 3 }));
|
||||
});
|
||||
|
||||
it("floors the threshold at 2, below which a single ordinary retry would escalate", () => {
|
||||
const onChange = renderConfig({ stall_escalation_enabled: true });
|
||||
fireEvent.change(screen.getByLabelText("Repeats before escalating"), { target: { value: "1" } });
|
||||
expect(onChange).toHaveBeenCalledWith(expect.objectContaining({ stall_escalation_repeat_threshold: 2 }));
|
||||
});
|
||||
|
||||
it("disables the toggle and says why when session pinning is on", () => {
|
||||
renderConfig({ session_affinity: true });
|
||||
expect(toggle()).toHaveAttribute("aria-disabled", "true");
|
||||
expect(screen.getByText(/How often to classify/)).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("hides the knobs when a blocker is switched on under an already-enabled router", () => {
|
||||
renderConfig({ stall_escalation_enabled: true, session_affinity: true });
|
||||
expect(screen.queryByLabelText("Repeats before escalating")).not.toBeInTheDocument();
|
||||
});
|
||||
});
|
||||
|
|
@ -0,0 +1,116 @@
|
|||
import { Input } from "@/components/ui/input";
|
||||
import { Switch } from "@/components/ui/switch";
|
||||
import React from "react";
|
||||
import { type ComplexityRouterConfigValue, classificationFrequency } from "./ComplexityRouterConfig";
|
||||
|
||||
export const DEFAULT_STALL_ESCALATION_WINDOW = 6;
|
||||
export const DEFAULT_STALL_ESCALATION_REPEAT_THRESHOLD = 3;
|
||||
|
||||
/**
|
||||
* Why the toggle is unavailable, or null when it can be turned on. Both blockers replay a held
|
||||
* routing decision instead of classifying most turns, so detection would never see the tool
|
||||
* calls it reads.
|
||||
*/
|
||||
export const stallEscalationBlockedReason = (value: ComplexityRouterConfigValue): string | null => {
|
||||
const frequency = classificationFrequency(value);
|
||||
if (frequency === "session")
|
||||
return 'Set "How often to classify" to every request under Advanced: Classification Method to use this. Scoring once per session replays that model instead of classifying, so a stall never reaches the classifier.';
|
||||
if (frequency === "user_turn")
|
||||
return 'Set "How often to classify" to every request under Advanced: Classification Method to use this. Scoring only new user messages skips the tool-call turns a stall shows up in.';
|
||||
return null;
|
||||
};
|
||||
|
||||
const clampedInt = (raw: string, min: number, fallback: number): number => {
|
||||
const parsed = Number(raw);
|
||||
if (!Number.isFinite(parsed)) return fallback;
|
||||
return Math.max(min, Math.trunc(parsed));
|
||||
};
|
||||
|
||||
const StallEscalationConfig: React.FC<{
|
||||
value: ComplexityRouterConfigValue;
|
||||
onChange: (value: ComplexityRouterConfigValue) => void;
|
||||
}> = ({ value, onChange }) => {
|
||||
const enabled = value.stall_escalation_enabled ?? false;
|
||||
const blockedReason = stallEscalationBlockedReason(value);
|
||||
const window = value.stall_escalation_window ?? DEFAULT_STALL_ESCALATION_WINDOW;
|
||||
const threshold = value.stall_escalation_repeat_threshold ?? DEFAULT_STALL_ESCALATION_REPEAT_THRESHOLD;
|
||||
// A threshold above the window can never be reached, and the backend rejects the pair, so the
|
||||
// window rises with the threshold rather than letting the form save something inert.
|
||||
const commitThreshold = (raw: string) => {
|
||||
const nextThreshold = clampedInt(raw, 2, DEFAULT_STALL_ESCALATION_REPEAT_THRESHOLD);
|
||||
onChange({
|
||||
...value,
|
||||
stall_escalation_repeat_threshold: nextThreshold,
|
||||
stall_escalation_window: Math.max(window, nextThreshold),
|
||||
});
|
||||
};
|
||||
const commitWindow = (raw: string) => {
|
||||
const nextWindow = clampedInt(raw, 1, DEFAULT_STALL_ESCALATION_WINDOW);
|
||||
onChange({
|
||||
...value,
|
||||
stall_escalation_window: Math.max(nextWindow, threshold),
|
||||
});
|
||||
};
|
||||
const toggle = (next: boolean) => {
|
||||
const enabledValue: ComplexityRouterConfigValue = {
|
||||
...value,
|
||||
stall_escalation_enabled: next || undefined,
|
||||
stall_escalation_window: next ? window : undefined,
|
||||
stall_escalation_repeat_threshold: next ? threshold : undefined,
|
||||
};
|
||||
onChange(enabledValue);
|
||||
};
|
||||
return (
|
||||
<>
|
||||
<div className="flex items-center gap-2 mb-2">
|
||||
<Switch
|
||||
checked={enabled}
|
||||
disabled={blockedReason !== null}
|
||||
onCheckedChange={toggle}
|
||||
aria-label="Escalate a stalled task to a stronger model"
|
||||
/>
|
||||
<strong className="font-semibold">Escalate a stalled task to a stronger model</strong>
|
||||
</div>
|
||||
<span className="block text-xs mb-3 text-muted-foreground">
|
||||
When the model keeps repeating the same tool call, or the same call keeps erroring, bump the request one tier
|
||||
higher for as long as it looks stuck. The automatic counterpart to an escalation keyword: nobody has to notice
|
||||
the loop and ask. Off means a stuck task keeps the model it was classified onto.
|
||||
{blockedReason !== null && ` ${blockedReason}`}
|
||||
</span>
|
||||
{enabled && blockedReason === null && (
|
||||
<div className="flex flex-wrap gap-4">
|
||||
<div style={{ maxWidth: 240 }}>
|
||||
<label className="block text-sm font-medium mb-1" htmlFor="stall-escalation-repeat-threshold">
|
||||
Repeats before escalating
|
||||
</label>
|
||||
<Input
|
||||
id="stall-escalation-repeat-threshold"
|
||||
inputMode="numeric"
|
||||
value={threshold}
|
||||
onChange={(event) => commitThreshold(event.target.value)}
|
||||
/>
|
||||
<span className="block text-xs mt-1 text-muted-foreground">
|
||||
How many identical or failing calls count as stuck. At least 2; lower reacts sooner and misfires more.
|
||||
</span>
|
||||
</div>
|
||||
<div style={{ maxWidth: 240 }}>
|
||||
<label className="block text-sm font-medium mb-1" htmlFor="stall-escalation-window">
|
||||
Recent calls examined
|
||||
</label>
|
||||
<Input
|
||||
id="stall-escalation-window"
|
||||
inputMode="numeric"
|
||||
value={window}
|
||||
onChange={(event) => commitWindow(event.target.value)}
|
||||
/>
|
||||
<span className="block text-xs mt-1 text-muted-foreground">
|
||||
How far back to look, in tool calls. Never below the repeat count, since that could never be reached.
|
||||
</span>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
</>
|
||||
);
|
||||
};
|
||||
|
||||
export default StallEscalationConfig;
|
||||
|
|
@ -394,6 +394,9 @@ const AddAutoRouterTab: React.FC<AddAutoRouterTabProps> = ({
|
|||
embeddingModel,
|
||||
matchThreshold,
|
||||
escalationKeywords,
|
||||
stallEscalationEnabled: complexityRouterConfig.stall_escalation_enabled,
|
||||
stallEscalationWindow: complexityRouterConfig.stall_escalation_window,
|
||||
stallEscalationRepeatThreshold: complexityRouterConfig.stall_escalation_repeat_threshold,
|
||||
adaptive: complexityRouterConfig.adaptive ?? false,
|
||||
adaptiveWeights: complexityRouterConfig.adaptive_weights ?? DEFAULT_ADAPTIVE_WEIGHTS,
|
||||
tierDistancePenalty: complexityRouterConfig.tier_distance_penalty ?? DEFAULT_TIER_DISTANCE_PENALTY,
|
||||
|
|
|
|||
|
|
@ -1020,6 +1020,9 @@ describe("buildComplexityRouterConfig with an edited tier set", () => {
|
|||
heuristicFirstMaxTier: "SIMPLE",
|
||||
hybridBoundaryMargin: 0.03,
|
||||
customTechnicalKeywords: ["kubernetes"],
|
||||
stallEscalationEnabled: true,
|
||||
stallEscalationWindow: 6,
|
||||
stallEscalationRepeatThreshold: 3,
|
||||
};
|
||||
const emittingType = key === "heuristic_first_max_tier" ? "heuristic_first" : "llm";
|
||||
const typeForKey = key === "hybrid_boundary_margin" ? "hybrid" : emittingType;
|
||||
|
|
@ -1114,6 +1117,34 @@ describe("hydrateCustomTierSet", () => {
|
|||
});
|
||||
});
|
||||
|
||||
describe("buildComplexityRouterConfig stall escalation", () => {
|
||||
it("omits all three keys when the toggle is off, since the backend rejects them next to session pinning", () => {
|
||||
const config = buildComplexityRouterConfig({ ...baseParams, stallEscalationEnabled: false });
|
||||
expect(config).not.toHaveProperty("stall_escalation_enabled");
|
||||
expect(config).not.toHaveProperty("stall_escalation_window");
|
||||
expect(config).not.toHaveProperty("stall_escalation_repeat_threshold");
|
||||
});
|
||||
|
||||
it("emits the toggle and both knobs when it is on", () => {
|
||||
const config = buildComplexityRouterConfig({
|
||||
...baseParams,
|
||||
stallEscalationEnabled: true,
|
||||
stallEscalationWindow: 8,
|
||||
stallEscalationRepeatThreshold: 4,
|
||||
});
|
||||
expect(config.stall_escalation_enabled).toBe(true);
|
||||
expect(config.stall_escalation_window).toBe(8);
|
||||
expect(config.stall_escalation_repeat_threshold).toBe(4);
|
||||
});
|
||||
|
||||
it("emits the toggle alone when neither knob was touched, so both track the backend defaults", () => {
|
||||
const config = buildComplexityRouterConfig({ ...baseParams, stallEscalationEnabled: true });
|
||||
expect(config.stall_escalation_enabled).toBe(true);
|
||||
expect(config).not.toHaveProperty("stall_escalation_window");
|
||||
expect(config).not.toHaveProperty("stall_escalation_repeat_threshold");
|
||||
});
|
||||
});
|
||||
|
||||
describe("dryRunRejection", () => {
|
||||
it("blocks the save on a rejection whose message is missing, which the write would return as a raw 400", () => {
|
||||
expect(dryRunRejection({ valid: false })).toBe("The proxy rejected this auto-router configuration");
|
||||
|
|
|
|||
|
|
@ -126,6 +126,9 @@ export interface BuildComplexityRouterConfigParams {
|
|||
embeddingModel: string | undefined;
|
||||
matchThreshold: number;
|
||||
escalationKeywords: string[];
|
||||
stallEscalationEnabled?: boolean;
|
||||
stallEscalationWindow?: number;
|
||||
stallEscalationRepeatThreshold?: number;
|
||||
adaptive: boolean;
|
||||
adaptiveWeights: AdaptiveRouterWeights;
|
||||
tierDistancePenalty: number;
|
||||
|
|
@ -186,6 +189,9 @@ export interface ComplexityRouterConfigPayload {
|
|||
embedding_model?: string;
|
||||
match_threshold?: number;
|
||||
escalation_keywords?: string[];
|
||||
stall_escalation_enabled?: boolean;
|
||||
stall_escalation_window?: number;
|
||||
stall_escalation_repeat_threshold?: number;
|
||||
adaptive?: boolean;
|
||||
adaptive_weights?: AdaptiveRouterWeights;
|
||||
tier_distance_penalty?: number;
|
||||
|
|
@ -446,6 +452,9 @@ export const buildComplexityRouterConfig = ({
|
|||
embeddingModel,
|
||||
matchThreshold,
|
||||
escalationKeywords,
|
||||
stallEscalationEnabled,
|
||||
stallEscalationWindow,
|
||||
stallEscalationRepeatThreshold,
|
||||
adaptive,
|
||||
adaptiveWeights,
|
||||
tierDistancePenalty,
|
||||
|
|
@ -507,6 +516,15 @@ export const buildComplexityRouterConfig = ({
|
|||
...(customTechnicalKeywords.length > 0 && { custom_technical_keywords: customTechnicalKeywords }),
|
||||
...(cleanedKeywordTierRules.length > 0 && { keyword_tier_rules: cleanedKeywordTierRules }),
|
||||
escalation_keywords: cleanedEscalationKeywords,
|
||||
// Only written when on: the backend rejects it alongside session_affinity, user_turn mode and
|
||||
// a custom tier set, so an off router must not carry the key into any of those saves.
|
||||
...(stallEscalationEnabled && {
|
||||
stall_escalation_enabled: true,
|
||||
...(stallEscalationWindow !== undefined && { stall_escalation_window: stallEscalationWindow }),
|
||||
...(stallEscalationRepeatThreshold !== undefined && {
|
||||
stall_escalation_repeat_threshold: stallEscalationRepeatThreshold,
|
||||
}),
|
||||
}),
|
||||
...(semanticMatchingEnabled && {
|
||||
semantic_keyword_matching: true,
|
||||
embedding_model: embeddingModel,
|
||||
|
|
|
|||
|
|
@ -113,6 +113,10 @@ export const CUSTOM_TIER_RESTRICTIONS = {
|
|||
omit: ["escalation_keywords"],
|
||||
reason: "Escalation bumps a request along the built-in tier ladder, which your tier set replaces",
|
||||
},
|
||||
stallEscalation: {
|
||||
omit: ["stall_escalation_enabled", "stall_escalation_window", "stall_escalation_repeat_threshold"],
|
||||
reason: "Stall escalation bumps a request along the built-in tier ladder, which your tier set replaces",
|
||||
},
|
||||
adaptive: {
|
||||
omit: ["adaptive", "adaptive_weights", "tier_distance_penalty", "adaptive_eligible"],
|
||||
reason: "Adaptive routing scores models along the built-in tier ladder, which your tier set replaces",
|
||||
|
|
|
|||
|
|
@ -593,16 +593,54 @@ describe("managed keys survive an untouched open-and-save", () => {
|
|||
"hybrid_boundary_margin",
|
||||
]);
|
||||
|
||||
// The stall keys are rejected beside the session pinning and user-turn classification this
|
||||
// fixture sets, so they get their own round trip below rather than widening this one.
|
||||
const KEYS_ANOTHER_CLASSIFICATION_FREQUENCY_OWNS = new Set([
|
||||
"stall_escalation_enabled",
|
||||
"stall_escalation_window",
|
||||
"stall_escalation_repeat_threshold",
|
||||
]);
|
||||
|
||||
it("carries every managed key a built-in router can hold through hydrate then save", () => {
|
||||
const hydrated = hydrateComplexityRouterConfig(STORED_ALL_MANAGED, undefined);
|
||||
const saved = buildUpdatedComplexityRouterConfig(STORED_ALL_MANAGED, hydrated);
|
||||
|
||||
const dropped = [...MANAGED_COMPLEXITY_ROUTER_KEYS]
|
||||
.filter((key) => !KEYS_ANOTHER_CLASSIFIER_TYPE_OWNS.has(key))
|
||||
.filter((key) => !KEYS_ANOTHER_CLASSIFICATION_FREQUENCY_OWNS.has(key))
|
||||
.filter((key) => saved[key] === undefined);
|
||||
expect(dropped).toEqual([]);
|
||||
});
|
||||
|
||||
it("carries the stall-escalation keys through their own round trip", () => {
|
||||
const stored: Record<string, unknown> = {
|
||||
...STORED_ALL_MANAGED,
|
||||
session_affinity: false,
|
||||
classification_mode: "every_request",
|
||||
stall_escalation_enabled: true,
|
||||
stall_escalation_window: 8,
|
||||
stall_escalation_repeat_threshold: 4,
|
||||
};
|
||||
const hydrated = hydrateComplexityRouterConfig(stored, undefined);
|
||||
const saved = buildUpdatedComplexityRouterConfig(stored, hydrated);
|
||||
|
||||
expect(saved.stall_escalation_enabled).toBe(true);
|
||||
expect(saved.stall_escalation_window).toBe(8);
|
||||
expect(saved.stall_escalation_repeat_threshold).toBe(4);
|
||||
});
|
||||
|
||||
it("leaves the stall keys out of a saved config that never had them on", () => {
|
||||
const stored: Record<string, unknown> = {
|
||||
...STORED_ALL_MANAGED,
|
||||
session_affinity: false,
|
||||
classification_mode: "every_request",
|
||||
};
|
||||
const hydrated = hydrateComplexityRouterConfig(stored, undefined);
|
||||
const saved = buildUpdatedComplexityRouterConfig(stored, hydrated);
|
||||
|
||||
expect(saved).not.toHaveProperty("stall_escalation_enabled");
|
||||
});
|
||||
|
||||
it("drops a stored local-scorer threshold when the operator converts the router to custom tiers", () => {
|
||||
const hydrated = hydrateComplexityRouterConfig(STORED_ALL_MANAGED, undefined);
|
||||
const converted = {
|
||||
|
|
|
|||
|
|
@ -115,6 +115,9 @@ export interface StoredComplexityRouterConfig {
|
|||
return_raw_model_name?: boolean;
|
||||
enable_context_window_escalation?: unknown;
|
||||
context_window_escalation_buffer?: unknown;
|
||||
stall_escalation_enabled?: unknown;
|
||||
stall_escalation_window?: unknown;
|
||||
stall_escalation_repeat_threshold?: unknown;
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -208,6 +211,13 @@ export const hydrateComplexityRouterConfig = (
|
|||
typeof parsedConfig.context_window_escalation_buffer === "number"
|
||||
? parsedConfig.context_window_escalation_buffer
|
||||
: undefined,
|
||||
stall_escalation_enabled: parsedConfig.stall_escalation_enabled === true || undefined,
|
||||
stall_escalation_window:
|
||||
typeof parsedConfig.stall_escalation_window === "number" ? parsedConfig.stall_escalation_window : undefined,
|
||||
stall_escalation_repeat_threshold:
|
||||
typeof parsedConfig.stall_escalation_repeat_threshold === "number"
|
||||
? parsedConfig.stall_escalation_repeat_threshold
|
||||
: undefined,
|
||||
};
|
||||
};
|
||||
|
||||
|
|
@ -245,6 +255,9 @@ export const MANAGED_COMPLEXITY_ROUTER_KEYS = new Set([
|
|||
"reasoning_override_min_score",
|
||||
"enable_context_window_escalation",
|
||||
"context_window_escalation_buffer",
|
||||
"stall_escalation_enabled",
|
||||
"stall_escalation_window",
|
||||
"stall_escalation_repeat_threshold",
|
||||
]);
|
||||
|
||||
// Managed only when the caller passes the corresponding state. A caller that does not render
|
||||
|
|
@ -351,6 +364,9 @@ export const buildUpdatedComplexityRouterConfig = (
|
|||
tierModelParams: value.tier_model_params,
|
||||
enableContextWindowEscalation: value.enable_context_window_escalation,
|
||||
contextWindowEscalationBuffer: value.context_window_escalation_buffer,
|
||||
stallEscalationEnabled: value.stall_escalation_enabled,
|
||||
stallEscalationWindow: value.stall_escalation_window,
|
||||
stallEscalationRepeatThreshold: value.stall_escalation_repeat_threshold,
|
||||
};
|
||||
const built = buildComplexityRouterConfig(builderParams);
|
||||
|
||||
|
|
|
|||
18
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
18
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
|
|
@ -34896,6 +34896,24 @@ export interface components {
|
|||
* @description Keywords indicating simple/basic queries
|
||||
*/
|
||||
simple_keywords?: string[] | null;
|
||||
/**
|
||||
* Stall Escalation Enabled
|
||||
* @description Escalate mid-task to the next-higher configured tier when the assistant's own recent tool calls look stuck: stall_escalation_repeat_threshold or more of the last stall_escalation_window tool calls are identical repeats (same tool, same arguments) or came back as errors. One tier at most, on the same ladder escalation_keywords bumps along, and never above the highest configured tier. Detection re-runs on every classified turn from the tool calls visible in that request, so it needs no state and nothing survives past the task: once the recent tool calls stop looking stuck, the next classified turn routes normally again. Mutually exclusive with session_affinity and classification_mode='user_turn', which both replay a held routing decision instead of classifying most turns, so this would never see the tool calls to look at. Off by default.
|
||||
* @default false
|
||||
*/
|
||||
stall_escalation_enabled: boolean;
|
||||
/**
|
||||
* Stall Escalation Repeat Threshold
|
||||
* @description How many of the last stall_escalation_window tool calls must be identical repeats, or error results, before the task counts as stalled. Must not exceed stall_escalation_window, or the condition could never be reached.
|
||||
* @default 3
|
||||
*/
|
||||
stall_escalation_repeat_threshold: number;
|
||||
/**
|
||||
* Stall Escalation Window
|
||||
* @description How many of the assistant's most recent tool calls stall detection looks at, oldest ones dropped as new calls happen. Counted across the whole visible conversation rather than reset at the newest human ask, so evidence from before a plain follow-up message like 'try again' is still visible on the turn after it.
|
||||
* @default 6
|
||||
*/
|
||||
stall_escalation_window: number;
|
||||
/**
|
||||
* Technical Keywords
|
||||
* @description Keywords indicating technical content
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue