scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-06 02:48:13 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 103

Merge 0356e9a506 into b781d157d7

Browse source
This commit is contained in:
tin-berri 2026-09-30 10:31:40 -04:00 • committed by GitHub
commit 2dc931954f
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
16 changed files with 441 additions and 13 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

88
ui/litellm-dashboard/src/components/add_model/CacheAwareRoutingConfig.test.tsx Normal file
View file

@ -0,0 +1,88 @@
import { useState } from "react";
import { describe, expect, it } from "vitest";
import { fireEvent, render, screen } from "@testing-library/react";
import CacheAwareRoutingConfig from "./CacheAwareRoutingConfig";
import type { ComplexityRouterConfigValue } from "./ComplexityRouterConfig";
const initial: ComplexityRouterConfigValue = {
classifier_type: "heuristic",
tiers: { SIMPLE: ["small"], MEDIUM: [], COMPLEX: ["large"], REASONING: [] },
};
const Harness = ({ value = initial }: { value?: ComplexityRouterConfigValue }) => {
const [config, setConfig] = useState(value);
return <CacheAwareRoutingConfig value={config} onChange={setConfig} />;
};
describe("CacheAwareRoutingConfig", () => {
it("starts off and reveals optional cost settings only after opting in", () => {
render(<Harness />);
expect(screen.getByRole("switch", { name: "Cache-aware routing" })).not.toBeChecked();
expect(screen.queryByLabelText("Expected output tokens")).not.toBeInTheDocument();
fireEvent.click(screen.getByRole("switch", { name: "Cache-aware routing" }));
expect(screen.getByRole("switch", { name: "Cache-aware routing" })).toBeChecked();
expect(screen.getByLabelText("Expected output tokens")).toHaveValue(null);
expect(screen.getByLabelText("Prediction timeout (ms)")).toHaveValue(null);
});
it("accepts zero output, clamps invalid bounds, and lets blank fields return to defaults", () => {
render(<Harness value={{ ...initial, cache_aware_routing: true }} />);
const output = screen.getByLabelText("Expected output tokens");
const timeout = screen.getByLabelText("Prediction timeout (ms)");
fireEvent.change(output, { target: { value: "-1" } });
fireEvent.change(timeout, { target: { value: "0" } });
expect(output).toHaveValue(0);
expect(timeout).toHaveValue(1);
fireEvent.change(output, { target: { value: "512.5" } });
fireEvent.change(timeout, { target: { value: "750.5" } });
expect(output).toHaveValue(512);
expect(timeout).toHaveValue(750);
fireEvent.change(output, { target: { value: "" } });
fireEvent.change(timeout, { target: { value: "" } });
expect(output).toHaveValue(null);
expect(timeout).toHaveValue(null);
});
it("keeps the cost settings when temporarily turning routing off", () => {
render(
<Harness
value={{
...initial,
cache_aware_routing: true,
cache_aware_routing_output_tokens: 0,
cache_aware_routing_timeout_ms: 500,
}}
/>,
);
fireEvent.click(screen.getByRole("switch", { name: "Cache-aware routing" }));
expect(screen.queryByLabelText("Expected output tokens")).not.toBeInTheDocument();
fireEvent.click(screen.getByRole("switch", { name: "Cache-aware routing" }));
expect(screen.getByLabelText("Expected output tokens")).toHaveValue(0);
expect(screen.getByLabelText("Prediction timeout (ms)")).toHaveValue(500);
});
it.each<{ patch: Partial<ComplexityRouterConfigValue>; reason: string }>([
{ patch: { adaptive: true }, reason: "Turn off Adaptive Routing" },
{ patch: { session_affinity: true }, reason: 'Set "How often to classify"' },
{ patch: { classification_mode: "user_turn" }, reason: 'Set "How often to classify"' },
{ patch: { tiers: { ...initial.tiers, SIMPLE: ["small", "other"] } }, reason: "Choose one model per tier" },
{
patch: { tier_model_params: { SIMPLE: { small: { reasoning_effort: "high" } } } },
reason: "Remove per-model parameter overrides",
},
{ patch: { custom_tier_set: { tiers: [], fallback_tier_id: "custom" } }, reason: "Use the built-in tiers" },
])(
"explains an incompatible setting and still allows an existing opt-in to be disabled: $reason",
({ patch, reason }) => {
const view = render(<Harness value={{ ...initial, ...patch }} />);
expect(screen.getByRole("switch", { name: "Cache-aware routing" })).toHaveAttribute("aria-disabled", "true");
fireEvent.click(screen.getByRole("switch", { name: "Cache-aware routing" }));
expect(screen.getByRole("switch", { name: "Cache-aware routing" })).not.toBeChecked();
expect(screen.getByRole("status")).toHaveTextContent(reason);
view.unmount();
render(<Harness value={{ ...initial, ...patch, cache_aware_routing: true }} />);
fireEvent.click(screen.getByRole("switch", { name: "Cache-aware routing" }));
expect(screen.getByRole("switch", { name: "Cache-aware routing" })).not.toBeChecked();
},
);
});

115
ui/litellm-dashboard/src/components/add_model/CacheAwareRoutingConfig.tsx Normal file
View file

@ -0,0 +1,115 @@
import { Input } from "@/components/ui/input";
import { Switch } from "@/components/ui/switch";
import { classificationFrequency, type ComplexityRouterConfigValue } from "./ComplexityRouterConfig";
const blockedReason = (value: ComplexityRouterConfigValue): string | null => {
if (value.custom_tier_set) return "Use the built-in tiers to enable cache-aware routing.";
if (value.adaptive) return "Turn off Adaptive Routing to enable cache-aware routing.";
if (classificationFrequency(value) !== "every_request") {
return 'Set "How often to classify" to every request to enable cache-aware routing.';
}
if (Object.values(value.tiers).some((models) => models.length > 1)) {
return "Choose one model per tier to enable cache-aware routing.";
}
if (
Object.values(value.tier_model_params ?? {}).some((models) =>
Object.values(models).some((params) => Object.keys(params).length > 0),
)
) {
return "Remove per-model parameter overrides to enable cache-aware routing.";
}
return null;
};
const optionalInteger = (raw: string, minimum: number): number | undefined => {
if (raw.trim() === "") return undefined;
const parsed = Number(raw);
return Number.isFinite(parsed) ? Math.max(minimum, Math.trunc(parsed)) : undefined;
};
const CacheAwareRoutingConfig = ({
value,
onChange,
}: {
value: ComplexityRouterConfigValue;
onChange: (value: ComplexityRouterConfigValue) => void;
}) => {
const enabled = value.cache_aware_routing ?? false;
const reason = blockedReason(value);
return (
<div className="space-y-3">
<div className="flex items-center gap-2">
<Switch
checked={enabled}
disabled={!enabled && reason !== null}
onCheckedChange={(next) => onChange({ ...value, cache_aware_routing: next })}
aria-label="Cache-aware routing"
/>
<span className="text-sm font-medium">Consider prompt-cache savings</span>
</div>
<p className="text-xs text-muted-foreground">
Disabled by default. Reuse a model with a warm prompt cache when its estimated total cost is lower and it meets
the selected tier or higher. Supports native Anthropic Messages with explicit prompt caching; unsupported
requests keep their usual route.
</p>
{reason && (
<p className="text-xs text-muted-foreground" role="status">
{enabled && "Cache-aware routing is currently skipped. "}
{reason}
</p>
)}
{enabled && (
<div className="grid gap-4 sm:grid-cols-2">
<div>
<label className="block text-sm font-medium mb-1" htmlFor="cache-aware-output-tokens">
Expected output tokens
</label>
<Input
id="cache-aware-output-tokens"
type="number"
min={0}
step={1}
placeholder="1024"
value={value.cache_aware_routing_output_tokens ?? ""}
onChange={(event) =>
onChange({
...value,
cache_aware_routing_output_tokens: optionalInteger(event.target.value, 0),
})
}
aria-describedby="cache-aware-output-help"
/>
<p id="cache-aware-output-help" className="mt-1 text-xs text-muted-foreground">
Used to estimate cost, not to limit the response. Leave blank to use the default of 1024.
</p>
</div>
<div>
<label className="block text-sm font-medium mb-1" htmlFor="cache-aware-timeout">
Prediction timeout (ms)
</label>
<Input
id="cache-aware-timeout"
type="number"
min={1}
step={1}
placeholder="2000"
value={value.cache_aware_routing_timeout_ms ?? ""}
onChange={(event) =>
onChange({
...value,
cache_aware_routing_timeout_ms: optionalInteger(event.target.value, 1),
})
}
aria-describedby="cache-aware-timeout-help"
/>
<p id="cache-aware-timeout-help" className="mt-1 text-xs text-muted-foreground">
Keep the original route if the comparison takes too long. Leave blank to use the default of 2000 ms.
</p>
</div>
</div>
)}
</div>
);
};
export default CacheAwareRoutingConfig;

8
ui/litellm-dashboard/src/components/add_model/ComplexityRouterAdvancedSections.tsx
View file

@ -5,6 +5,7 @@ import { Separator } from "@/components/ui/separator";
import type { ModelGroup } from "@/components/llm_calls/fetch_models";
import type { ComplexityRouterConfigValue } from "./ComplexityRouterConfig";
import AdaptiveRoutingConfig from "./AdaptiveRoutingConfig";
import CacheAwareRoutingConfig from "./CacheAwareRoutingConfig";
import ClassificationMethodConfig from "./ClassificationMethodConfig";
import ForecastClassifierConfig from "./ForecastClassifierConfig";
import ContextWindowEscalationConfig from "./ContextWindowEscalationConfig";
@ -141,6 +142,11 @@ const ComplexityRouterAdvancedSections: React.FC<ComplexityRouterAdvancedSection
label: <strong className="text-foreground font-semibold">Affinity</strong>,
children: <AffinityControls value={value} onChange={onChange} />,
},
{
key: "cache-aware",
label: <strong className="text-foreground font-semibold">Cache-aware routing</strong>,
children: <CacheAwareRoutingConfig value={value} onChange={onChange} />,
},
{
key: "modality",
label: <strong className="text-foreground font-semibold">Modality Routing</strong>,
@ -253,7 +259,7 @@ const ComplexityRouterAdvancedSections: React.FC<ComplexityRouterAdvancedSection
"keyword-semantic",
],
},
{ label: "Sessions and efficiency", keys: ["affinity", "adaptive", "compression"] },
{ label: "Sessions and efficiency", keys: ["affinity", "adaptive", "cache-aware", "compression"] },
{ label: "Compatibility", keys: ["response"] },
];
const [openGroups, setOpenGroups] = React.useState<string[]>(() =>

3
ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx
View file

@ -405,6 +405,9 @@ export interface ComplexityRouterConfigValue {
*/
enable_context_window_escalation?: boolean;
context_window_escalation_buffer?: number;
cache_aware_routing?: boolean;
cache_aware_routing_output_tokens?: number;
cache_aware_routing_timeout_ms?: number;
/**
* Heuristic scorer knobs. Undefined means the operator never touched them, which keeps the key out of the
* payload so the router tracks the backend defaults rather than freezing today's numbers.

3
ui/litellm-dashboard/src/components/add_model/JevConnectionTest.integration.test.tsx
View file

@ -1,4 +1,5 @@
import { afterEach, describe, expect, it, vi } from "vitest";
import { normalizeTierModels } from "./complexity_router_tiers";
import { fireEvent, renderWithProviders, screen, waitFor } from "../../../tests/test-utils";
import AutoRouterConnectionTest from "./auto_router_connection_test";
import AutoRouterRoutingTest from "./AutoRouterRoutingTest";
@ -53,7 +54,7 @@ const request = buildSavedJevConnectionTestRequest(
"saved-id",
);
const targets = buildAutoRouterTestTargets({
tiers: Object.entries(config.tiers),
tiers: Object.entries(config.tiers).map(([tier, models]) => [tier, normalizeTierModels(models)]),
semanticMatchingEnabled: false,
embeddingModel: undefined,
});

43
ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.integration.test.tsx
View file

@ -1,3 +1,4 @@
import { normalizeTierModels } from "./complexity_router_tiers";
import {
openAutoRouterAdvanced,
selectAutoRouterOption,
@ -971,6 +972,32 @@ describe("AddAutoRouterTab", () => {
);
});
it.each([false, true])("creates a router with cache routing only after an explicit opt-in: %s", async (enabled) => {
const user = userEvent.setup();
mockFetchAvailableModels.mockResolvedValue(ALL_FAMILY_MODELS);
renderWithProviders(<Harness />);
const setup = await screen.findByRole("button", { name: "Choose models for me" });
await waitFor(() => expect(setup).toBeEnabled());
await user.click(setup);
fireEvent.change(screen.getByLabelText("Auto Router Name"), { target: { value: "cache-aware-router" } });
openAutoRouterAdvanced("Cache-aware routing");
const toggle = screen.getByRole("switch", { name: "Cache-aware routing" });
expect(toggle).not.toBeChecked();
if (enabled) {
await user.click(toggle);
fireEvent.change(screen.getByLabelText("Expected output tokens"), { target: { value: "512" } });
fireEvent.change(screen.getByLabelText("Prediction timeout (ms)"), { target: { value: "750" } });
}
await waitFor(() => expect(screen.getByRole("button", { name: "Add Auto Router" })).toBeEnabled());
await user.click(screen.getByRole("button", { name: "Add Auto Router" }));
await waitFor(() => expect(handleAddAutoRouterSubmit).toHaveBeenCalledOnce());
const config = vi.mocked(handleAddAutoRouterSubmit).mock.calls.at(-1)?.[0].complexity_router_config;
expect(config?.cache_aware_routing ?? false).toBe(enabled);
expect(config?.cache_aware_routing_output_tokens).toBe(enabled ? 512 : undefined);
expect(config?.cache_aware_routing_timeout_ms).toBe(enabled ? 750 : undefined);
expect(Object.values(config?.tiers ?? {}).every((models) => typeof models === "string")).toBe(enabled);
});
it("starts context-window escalation disabled and carries an explicit opt-in to the create payload", async () => {
const user = userEvent.setup();
vi.mocked(getMissingTiersError).mockReturnValue(null);
@ -1711,10 +1738,10 @@ describe("AddAutoRouterTab", () => {
expect(vi.mocked(handleAddAutoRouterSubmit).mock.calls.at(-1)?.[0]).toMatchObject({
complexity_router_config: {
tiers: {
SIMPLE: ANTHROPIC_TIERS.SIMPLE.map(nativeGroupFor),
MEDIUM: ANTHROPIC_TIERS.MEDIUM.map(nativeGroupFor),
COMPLEX: ANTHROPIC_TIERS.COMPLEX.map(nativeGroupFor),
REASONING: ANTHROPIC_TIERS.REASONING.map(nativeGroupFor),
SIMPLE: normalizeTierModels(ANTHROPIC_TIERS.SIMPLE).map(nativeGroupFor),
MEDIUM: normalizeTierModels(ANTHROPIC_TIERS.MEDIUM).map(nativeGroupFor),
COMPLEX: normalizeTierModels(ANTHROPIC_TIERS.COMPLEX).map(nativeGroupFor),
REASONING: normalizeTierModels(ANTHROPIC_TIERS.REASONING).map(nativeGroupFor),
},
},
});
@ -1806,10 +1833,10 @@ describe("AddAutoRouterTab", () => {
expect(vi.mocked(handleAddAutoRouterSubmit).mock.calls.at(-1)?.[0]).toMatchObject({
complexity_router_config: {
tiers: {
SIMPLE: ANTHROPIC_TIERS.SIMPLE.map(expandedGroupFor),
MEDIUM: ANTHROPIC_TIERS.MEDIUM.map(expandedGroupFor),
COMPLEX: ANTHROPIC_TIERS.COMPLEX.map(expandedGroupFor),
REASONING: ANTHROPIC_TIERS.REASONING.map(expandedGroupFor),
SIMPLE: normalizeTierModels(ANTHROPIC_TIERS.SIMPLE).map(expandedGroupFor),
MEDIUM: normalizeTierModels(ANTHROPIC_TIERS.MEDIUM).map(expandedGroupFor),
COMPLEX: normalizeTierModels(ANTHROPIC_TIERS.COMPLEX).map(expandedGroupFor),
REASONING: normalizeTierModels(ANTHROPIC_TIERS.REASONING).map(expandedGroupFor),
},
},
});

35
ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.test.ts
View file

@ -57,6 +57,41 @@ const baseParams: BuildComplexityRouterConfigParams = {
};
describe("buildComplexityRouterConfig", () => {
it.each([undefined, false, true])(
"keeps cache routing opt-in and uses eligible single-model tiers only when enabled: %s",
(enabled) => {
const config = buildComplexityRouterConfig({ ...baseParams, cacheAwareRouting: enabled });
expect(config.cache_aware_routing).toBe(enabled);
expect(Object.hasOwn(config, "cache_aware_routing")).toBe(enabled !== undefined);
expect(config.tiers).toEqual(
enabled ? Object.fromEntries(Object.entries(tiers).map(([tier, models]) => [tier, models[0]])) : tiers,
);
expect(config).not.toHaveProperty("cache_aware_routing_output_tokens");
expect(config).not.toHaveProperty("cache_aware_routing_timeout_ms");
expect(config).not.toHaveProperty("enable_context_window_escalation");
expect(config).not.toHaveProperty("max_tokens_from_tier_model");
},
);
it("keeps zero-output estimates and drops empty tiers without flattening real model pools", () => {
const params = {
...baseParams,
cacheAwareRouting: true,
tiers: { ...tiers, MEDIUM: [], COMPLEX: ["first", "second"] },
cacheAwareRoutingOutputTokens: 0,
cacheAwareRoutingTimeoutMs: 750,
};
const config = buildComplexityRouterConfig(params);
const expected = {
cache_aware_routing: true,
cache_aware_routing_output_tokens: 0,
cache_aware_routing_timeout_ms: 750,
tiers: { SIMPLE: tiers.SIMPLE[0], COMPLEX: ["first", "second"], REASONING: tiers.REASONING[0] },
};
expect(config).toMatchObject(expected);
expect(config.tiers).not.toHaveProperty("MEDIUM");
});
it("accepts built-in JEV defaults without an LLM classifier model", () => {
expect(getClassifierModelError({ classifier_type: "jev" })).toBeNull();
});

32
ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.ts
View file

@ -182,6 +182,9 @@ export interface StoredComplexityRouterConfig {
return_raw_model_name?: boolean;
enable_context_window_escalation?: unknown;
context_window_escalation_buffer?: unknown;
cache_aware_routing?: unknown;
cache_aware_routing_output_tokens?: unknown;
cache_aware_routing_timeout_ms?: unknown;
stall_escalation_enabled?: unknown;
stall_escalation_window?: unknown;
stall_escalation_repeat_threshold?: unknown;
@ -246,6 +249,9 @@ export interface BuildComplexityRouterConfigParams {
tierModelParams?: TierModelParamsByTier;
enableContextWindowEscalation?: boolean;
contextWindowEscalationBuffer?: number;
cacheAwareRouting?: boolean;
cacheAwareRoutingOutputTokens?: number;
cacheAwareRoutingTimeoutMs?: number;
sessionAffinityTtlSeconds?: number;
codeKeywords?: string[];
reasoningKeywords?: string[];
@ -276,7 +282,7 @@ export interface TierDefinitionPayload {
}
export interface ComplexityRouterConfigPayload {
tiers: ComplexityTiers | Record<string, string[]>;
tiers: Record<string, string | string[]>;
enable_non_reasoning_tier?: boolean;
tier_definitions?: TierDefinitionPayload[];
fallback_tier?: string;
@ -325,6 +331,9 @@ export interface ComplexityRouterConfigPayload {
reasoning_override_min_score?: number;
enable_context_window_escalation?: boolean;
context_window_escalation_buffer?: number;
cache_aware_routing?: boolean;
cache_aware_routing_output_tokens?: number;
cache_aware_routing_timeout_ms?: number;
tier_model_configs?: Record<string, { model_name: string; litellm_params: TierModelParams }[]>;
code_keywords?: string[];
reasoning_keywords?: string[];
@ -693,6 +702,9 @@ export const buildComplexityRouterConfig = ({
tierModelParams,
enableContextWindowEscalation,
contextWindowEscalationBuffer,
cacheAwareRouting,
cacheAwareRoutingOutputTokens,
cacheAwareRoutingTimeoutMs,
sessionAffinityTtlSeconds,
codeKeywords,
reasoningKeywords,
@ -761,8 +773,17 @@ export const buildComplexityRouterConfig = ({
classifierPluginTimeoutMs > 0;
const supportsOpeningPrompt = !customTierSet && !forecast && usesLlmClassifier(effectiveType);
const populatedTiers =
forecast || cacheAwareRouting
? Object.fromEntries(Object.entries(tiers).filter(([, models]) => models.length > 0))
: tiers;
const payload: ComplexityRouterConfigPayload = {
tiers: forecast ? Object.fromEntries(Object.entries(tiers).filter(([, models]) => models.length > 0)) : tiers,
tiers:
cacheAwareRouting && !customTierSet
? Object.fromEntries(
Object.entries(populatedTiers).map(([tier, models]) => [tier, models.length === 1 ? models[0] : models]),
)
: populatedTiers,
// The backend rejects the flag beside a custom tier set.
...(!customTierSet && enableNonReasoningTier && { enable_non_reasoning_tier: true }),
...(serializedTierModelConfigs && { tier_model_configs: serializedTierModelConfigs }),
@ -816,6 +837,13 @@ export const buildComplexityRouterConfig = ({
adaptive_eligible: adaptiveEligible,
}),
...(returnRawModelName && { return_raw_model_name: true }),
...(cacheAwareRouting !== undefined && { cache_aware_routing: cacheAwareRouting }),
...(cacheAwareRoutingOutputTokens !== undefined && {
cache_aware_routing_output_tokens: cacheAwareRoutingOutputTokens,
}),
...(cacheAwareRoutingTimeoutMs !== undefined && {
cache_aware_routing_timeout_ms: cacheAwareRoutingTimeoutMs,
}),
...((forecast || enableContextWindowEscalation !== undefined) && {
enable_context_window_escalation: enableContextWindowEscalation ?? false,
}),

3
ui/litellm-dashboard/src/components/add_model/complexity_router_builder_params.ts
View file

@ -58,6 +58,9 @@ export const builderParamsFromValue = (
tierModelParams: value.tier_model_params,
enableContextWindowEscalation: value.enable_context_window_escalation,
contextWindowEscalationBuffer: value.context_window_escalation_buffer,
cacheAwareRouting: value.cache_aware_routing,
cacheAwareRoutingOutputTokens: value.cache_aware_routing_output_tokens,
cacheAwareRoutingTimeoutMs: value.cache_aware_routing_timeout_ms,
stallEscalationEnabled: value.stall_escalation_enabled,
stallEscalationWindow: value.stall_escalation_window,
stallEscalationRepeatThreshold: value.stall_escalation_repeat_threshold,

50
ui/litellm-dashboard/src/components/edit_auto_router/build_updated_complexity_router_config.test.ts
View file

@ -48,6 +48,53 @@ const hydratedState: KeywordMatchingState = {
};
describe("buildUpdatedComplexityRouterConfig keyword matching", () => {
it.each([undefined, false, true])("preserves cache settings through edit and save: %s", (enabled) => {
const stored = {
...STORED,
classifier_type: "heuristic" as const,
cache_aware_routing: enabled,
cache_aware_routing_output_tokens: 0,
cache_aware_routing_timeout_ms: 750,
};
const hydrated = hydrateComplexityRouterConfig(stored, undefined);
expect(hydrated.cache_aware_routing).toBe(enabled);
const saved = buildUpdatedComplexityRouterConfig(stored, hydrated);
expect(saved.cache_aware_routing).toBe(enabled);
expect(Object.hasOwn(saved, "cache_aware_routing")).toBe(enabled !== undefined);
expect(saved).toMatchObject({
cache_aware_routing_output_tokens: 0,
cache_aware_routing_timeout_ms: 750,
some_future_backend_key: STORED.some_future_backend_key,
});
});
it("disables cache routing and removes cleared overrides without changing context or output limits", () => {
const stored = {
...STORED,
classifier_type: "heuristic" as const,
cache_aware_routing: true,
cache_aware_routing_output_tokens: 512,
cache_aware_routing_timeout_ms: 750,
enable_context_window_escalation: false,
max_tokens_from_tier_model: false,
};
const hydrated = hydrateComplexityRouterConfig(stored, undefined);
const edited = {
...hydrated,
cache_aware_routing: false,
cache_aware_routing_output_tokens: undefined,
cache_aware_routing_timeout_ms: undefined,
};
const saved = buildUpdatedComplexityRouterConfig(stored, edited);
expect(saved).toMatchObject({
cache_aware_routing: false,
enable_context_window_escalation: false,
max_tokens_from_tier_model: false,
});
expect(saved).not.toHaveProperty("cache_aware_routing_output_tokens");
expect(saved).not.toHaveProperty("cache_aware_routing_timeout_ms");
});
it.each([false, true])("omits masked JEV credentials from dashboard saves, edited: %s", (edited) => {
const stored = {
classifier_type: "jev" as const,
@ -863,6 +910,9 @@ describe("managed keys survive an untouched open-and-save", () => {
reasoning_override_min_score: 0.3,
enable_context_window_escalation: false,
context_window_escalation_buffer: 0.9,
cache_aware_routing: false,
cache_aware_routing_output_tokens: 512,
cache_aware_routing_timeout_ms: 750,
code_keywords: ["async", "await"],
reasoning_keywords: ["prove"],
technical_keywords: ["api"],

33
ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.integration.test.tsx
View file

@ -84,6 +84,39 @@ describe("EditAutoRouterModal keyword matching", () => {
modelPatchUpdateCall.mockClear();
});
it("reopens saved cache settings and persists an explicit opt-out and cleared estimates", async () => {
const user = userEvent.setup();
renderModal({
modelData: {
...MODEL_DATA,
litellm_params: {
...MODEL_DATA.litellm_params,
complexity_router_config: {
...STORED_CONFIG,
cache_aware_routing: true,
cache_aware_routing_output_tokens: 512,
cache_aware_routing_timeout_ms: 750,
},
},
},
});
await screen.findByRole("textbox", { name: "Auto Router Name" });
openAutoRouterAdvanced("Cache-aware routing");
const toggle = screen.getByRole("switch", { name: "Cache-aware routing" });
expect(toggle).toBeChecked();
expect(screen.getByLabelText("Expected output tokens")).toHaveValue(512);
expect(screen.getByLabelText("Prediction timeout (ms)")).toHaveValue(750);
fireEvent.change(screen.getByLabelText("Expected output tokens"), { target: { value: "" } });
fireEvent.change(screen.getByLabelText("Prediction timeout (ms)"), { target: { value: "" } });
await user.click(toggle);
await waitFor(() => expect(screen.getByRole("button", { name: /save changes/i })).toBeEnabled());
await user.click(screen.getByRole("button", { name: /save changes/i }));
await waitFor(() => expect(modelPatchUpdateCall).toHaveBeenCalledOnce());
expect(savedConfig().cache_aware_routing).toBe(false);
expect(savedConfig()).not.toHaveProperty("cache_aware_routing_output_tokens");
expect(savedConfig()).not.toHaveProperty("cache_aware_routing_timeout_ms");
});
it("saves a member's changed routing config without resending administrator settings", async () => {
const user = userEvent.setup();
renderModal({

3
ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.tsx
View file

@ -128,6 +128,9 @@ export const MANAGED_COMPLEXITY_ROUTER_KEYS = new Set([
"reasoning_override_min_score",
"enable_context_window_escalation",
"context_window_escalation_buffer",
"cache_aware_routing",
"cache_aware_routing_output_tokens",
"cache_aware_routing_timeout_ms",
"stall_escalation_enabled",
"stall_escalation_window",
"stall_escalation_repeat_threshold",

10
ui/litellm-dashboard/src/components/edit_auto_router/hydrate_complexity_router_config.ts
View file

@ -153,6 +153,16 @@ export const hydrateComplexityRouterConfig = (
typeof parsedConfig.context_window_escalation_buffer === "number"
? parsedConfig.context_window_escalation_buffer
: undefined,
cache_aware_routing:
typeof parsedConfig.cache_aware_routing === "boolean" ? parsedConfig.cache_aware_routing : undefined,
cache_aware_routing_output_tokens:
typeof parsedConfig.cache_aware_routing_output_tokens === "number"
? parsedConfig.cache_aware_routing_output_tokens
: undefined,
cache_aware_routing_timeout_ms:
typeof parsedConfig.cache_aware_routing_timeout_ms === "number"
? parsedConfig.cache_aware_routing_timeout_ms
: undefined,
stall_escalation_enabled: parsedConfig.stall_escalation_enabled === true || undefined,
stall_escalation_window:
typeof parsedConfig.stall_escalation_window === "number" ? parsedConfig.stall_escalation_window : undefined,

21
ui/litellm-dashboard/src/lib/autorouter_presets.test.ts
View file

@ -942,6 +942,27 @@ describe("autorouter_presets", () => {
expect(prefill.complexityRouterConfig.context_window_escalation_buffer).toBe(0.9);
});
it.each([undefined, false, true])(
"preserves cache-routing settings and scalar tier models from a preset: %s",
(enabled) => {
const config = {
...getPresetByKey("anthropic_family")!.complexity_router_config,
tiers: { SIMPLE: "small", MEDIUM: "large", COMPLEX: "large", REASONING: "large" },
cache_aware_routing: enabled,
cache_aware_routing_output_tokens: 0,
cache_aware_routing_timeout_ms: 750,
};
const prefill = buildPresetPrefill(config, groupsOnly(["small", "large"]));
const expected = {
tiers: { SIMPLE: ["small"], MEDIUM: ["large"], COMPLEX: ["large"], REASONING: ["large"] },
cache_aware_routing: enabled,
cache_aware_routing_output_tokens: 0,
cache_aware_routing_timeout_ms: 750,
};
expect(prefill.complexityRouterConfig).toMatchObject(expected);
},
);
it("carries a preset's classification_mode and defaults it when the preset omits one", () => {
const tiers = { SIMPLE: ["gpt-5-nano"], MEDIUM: [], COMPLEX: [], REASONING: [] };
const base = {

6
ui/litellm-dashboard/src/lib/autorouter_presets.ts
View file

@ -24,6 +24,7 @@ import {
TierModelParams,
TierModelParamsByTier,
hydrateTierModelParams,
normalizeTierModels,
} from "@/components/add_model/complexity_router_tiers";
import { DEFAULT_ESCALATION_KEYWORDS } from "@/components/add_model/EscalationKeywords";
import { DEFAULT_MATCH_THRESHOLD } from "@/components/add_model/SemanticKeywordMatching";
@ -303,7 +304,7 @@ export const buildPresetPrefill = (
availability: ModelAvailability,
): PresetPrefill => {
const resolve = (model: string): string => resolveAvailableModel(model, availability) ?? model;
const resolveTier = (models: string[]): string[] => models.map(resolve);
const resolveTier = (models: string | string[]): string[] => normalizeTierModels(models).map(resolve);
// Params key on the model name the preset spells while every tier entry is rewritten to the
// caller's registered spelling, so the keys have to be rewritten the same way. Otherwise
// serializeTierModelConfigs drops them for naming a model the tier no longer holds.
@ -363,6 +364,9 @@ export const buildPresetPrefill = (
reasoning_override_min_score: hydrateReasoningOverrideMinScore(config.reasoning_override_min_score),
enable_context_window_escalation: config.enable_context_window_escalation,
context_window_escalation_buffer: config.context_window_escalation_buffer,
cache_aware_routing: config.cache_aware_routing,
cache_aware_routing_output_tokens: config.cache_aware_routing_output_tokens,
cache_aware_routing_timeout_ms: config.cache_aware_routing_timeout_ms,
},
customTechnicalKeywords: config.custom_technical_keywords ?? [],
keywordTierRules: hydrateKeywordTierRules(config.keyword_tier_rules ?? []),

1
ui/litellm-dashboard/tests/autoRouterSetup.ts
View file

@ -7,6 +7,7 @@ const groups: Record<string, string> = {
"Ignore Custom Tags": "Classifier tuning",
Affinity: "Sessions and efficiency",
"Adaptive Routing": "Sessions and efficiency",
"Cache-aware routing": "Sessions and efficiency",
Compression: "Sessions and efficiency",
"Response Format": "Compatibility",
};