From 0356e9a506fe588b62273c06166d5d8529af6f57 Mon Sep 17 00:00:00 2001 From: Tin Chi Lo Date: Sat, 26 Sep 2026 18:33:33 -0700 Subject: [PATCH] feat(ui): configure cache-aware auto routing --- .../CacheAwareRoutingConfig.test.tsx | 88 ++++++++++++++ .../add_model/CacheAwareRoutingConfig.tsx | 115 ++++++++++++++++++ .../ComplexityRouterAdvancedSections.tsx | 8 +- .../add_model/ComplexityRouterConfig.tsx | 3 + .../JevConnectionTest.integration.test.tsx | 3 +- .../add_auto_router_tab.integration.test.tsx | 43 +++++-- .../build_complexity_router_config.test.ts | 35 ++++++ .../build_complexity_router_config.ts | 32 ++++- .../complexity_router_builder_params.ts | 3 + ...d_updated_complexity_router_config.test.ts | 50 ++++++++ ...dit_auto_router_modal.integration.test.tsx | 33 +++++ .../edit_auto_router_modal.tsx | 3 + .../hydrate_complexity_router_config.ts | 10 ++ .../src/lib/autorouter_presets.test.ts | 21 ++++ .../src/lib/autorouter_presets.ts | 6 +- ui/litellm-dashboard/tests/autoRouterSetup.ts | 1 + 16 files changed, 441 insertions(+), 13 deletions(-) create mode 100644 ui/litellm-dashboard/src/components/add_model/CacheAwareRoutingConfig.test.tsx create mode 100644 ui/litellm-dashboard/src/components/add_model/CacheAwareRoutingConfig.tsx diff --git a/ui/litellm-dashboard/src/components/add_model/CacheAwareRoutingConfig.test.tsx b/ui/litellm-dashboard/src/components/add_model/CacheAwareRoutingConfig.test.tsx new file mode 100644 index 00000000000..843b4238a48 --- /dev/null +++ b/ui/litellm-dashboard/src/components/add_model/CacheAwareRoutingConfig.test.tsx @@ -0,0 +1,88 @@ +import { useState } from "react"; +import { describe, expect, it } from "vitest"; +import { fireEvent, render, screen } from "@testing-library/react"; +import CacheAwareRoutingConfig from "./CacheAwareRoutingConfig"; +import type { ComplexityRouterConfigValue } from "./ComplexityRouterConfig"; + +const initial: ComplexityRouterConfigValue = { + classifier_type: "heuristic", + tiers: { SIMPLE: ["small"], MEDIUM: [], COMPLEX: ["large"], REASONING: [] }, +}; + +const Harness = ({ value = initial }: { value?: ComplexityRouterConfigValue }) => { + const [config, setConfig] = useState(value); + return ; +}; + +describe("CacheAwareRoutingConfig", () => { + it("starts off and reveals optional cost settings only after opting in", () => { + render(); + expect(screen.getByRole("switch", { name: "Cache-aware routing" })).not.toBeChecked(); + expect(screen.queryByLabelText("Expected output tokens")).not.toBeInTheDocument(); + fireEvent.click(screen.getByRole("switch", { name: "Cache-aware routing" })); + expect(screen.getByRole("switch", { name: "Cache-aware routing" })).toBeChecked(); + expect(screen.getByLabelText("Expected output tokens")).toHaveValue(null); + expect(screen.getByLabelText("Prediction timeout (ms)")).toHaveValue(null); + }); + + it("accepts zero output, clamps invalid bounds, and lets blank fields return to defaults", () => { + render(); + const output = screen.getByLabelText("Expected output tokens"); + const timeout = screen.getByLabelText("Prediction timeout (ms)"); + fireEvent.change(output, { target: { value: "-1" } }); + fireEvent.change(timeout, { target: { value: "0" } }); + expect(output).toHaveValue(0); + expect(timeout).toHaveValue(1); + fireEvent.change(output, { target: { value: "512.5" } }); + fireEvent.change(timeout, { target: { value: "750.5" } }); + expect(output).toHaveValue(512); + expect(timeout).toHaveValue(750); + fireEvent.change(output, { target: { value: "" } }); + fireEvent.change(timeout, { target: { value: "" } }); + expect(output).toHaveValue(null); + expect(timeout).toHaveValue(null); + }); + + it("keeps the cost settings when temporarily turning routing off", () => { + render( + , + ); + fireEvent.click(screen.getByRole("switch", { name: "Cache-aware routing" })); + expect(screen.queryByLabelText("Expected output tokens")).not.toBeInTheDocument(); + fireEvent.click(screen.getByRole("switch", { name: "Cache-aware routing" })); + expect(screen.getByLabelText("Expected output tokens")).toHaveValue(0); + expect(screen.getByLabelText("Prediction timeout (ms)")).toHaveValue(500); + }); + + it.each<{ patch: Partial; reason: string }>([ + { patch: { adaptive: true }, reason: "Turn off Adaptive Routing" }, + { patch: { session_affinity: true }, reason: 'Set "How often to classify"' }, + { patch: { classification_mode: "user_turn" }, reason: 'Set "How often to classify"' }, + { patch: { tiers: { ...initial.tiers, SIMPLE: ["small", "other"] } }, reason: "Choose one model per tier" }, + { + patch: { tier_model_params: { SIMPLE: { small: { reasoning_effort: "high" } } } }, + reason: "Remove per-model parameter overrides", + }, + { patch: { custom_tier_set: { tiers: [], fallback_tier_id: "custom" } }, reason: "Use the built-in tiers" }, + ])( + "explains an incompatible setting and still allows an existing opt-in to be disabled: $reason", + ({ patch, reason }) => { + const view = render(); + expect(screen.getByRole("switch", { name: "Cache-aware routing" })).toHaveAttribute("aria-disabled", "true"); + fireEvent.click(screen.getByRole("switch", { name: "Cache-aware routing" })); + expect(screen.getByRole("switch", { name: "Cache-aware routing" })).not.toBeChecked(); + expect(screen.getByRole("status")).toHaveTextContent(reason); + view.unmount(); + render(); + fireEvent.click(screen.getByRole("switch", { name: "Cache-aware routing" })); + expect(screen.getByRole("switch", { name: "Cache-aware routing" })).not.toBeChecked(); + }, + ); +}); diff --git a/ui/litellm-dashboard/src/components/add_model/CacheAwareRoutingConfig.tsx b/ui/litellm-dashboard/src/components/add_model/CacheAwareRoutingConfig.tsx new file mode 100644 index 00000000000..f71d5f2e17b --- /dev/null +++ b/ui/litellm-dashboard/src/components/add_model/CacheAwareRoutingConfig.tsx @@ -0,0 +1,115 @@ +import { Input } from "@/components/ui/input"; +import { Switch } from "@/components/ui/switch"; +import { classificationFrequency, type ComplexityRouterConfigValue } from "./ComplexityRouterConfig"; + +const blockedReason = (value: ComplexityRouterConfigValue): string | null => { + if (value.custom_tier_set) return "Use the built-in tiers to enable cache-aware routing."; + if (value.adaptive) return "Turn off Adaptive Routing to enable cache-aware routing."; + if (classificationFrequency(value) !== "every_request") { + return 'Set "How often to classify" to every request to enable cache-aware routing.'; + } + if (Object.values(value.tiers).some((models) => models.length > 1)) { + return "Choose one model per tier to enable cache-aware routing."; + } + if ( + Object.values(value.tier_model_params ?? {}).some((models) => + Object.values(models).some((params) => Object.keys(params).length > 0), + ) + ) { + return "Remove per-model parameter overrides to enable cache-aware routing."; + } + return null; +}; + +const optionalInteger = (raw: string, minimum: number): number | undefined => { + if (raw.trim() === "") return undefined; + const parsed = Number(raw); + return Number.isFinite(parsed) ? Math.max(minimum, Math.trunc(parsed)) : undefined; +}; + +const CacheAwareRoutingConfig = ({ + value, + onChange, +}: { + value: ComplexityRouterConfigValue; + onChange: (value: ComplexityRouterConfigValue) => void; +}) => { + const enabled = value.cache_aware_routing ?? false; + const reason = blockedReason(value); + return ( +
+
+ onChange({ ...value, cache_aware_routing: next })} + aria-label="Cache-aware routing" + /> + Consider prompt-cache savings +
+

+ Disabled by default. Reuse a model with a warm prompt cache when its estimated total cost is lower and it meets + the selected tier or higher. Supports native Anthropic Messages with explicit prompt caching; unsupported + requests keep their usual route. +

+ {reason && ( +

+ {enabled && "Cache-aware routing is currently skipped. "} + {reason} +

+ )} + {enabled && ( +
+
+ + + onChange({ + ...value, + cache_aware_routing_output_tokens: optionalInteger(event.target.value, 0), + }) + } + aria-describedby="cache-aware-output-help" + /> +

+ Used to estimate cost, not to limit the response. Leave blank to use the default of 1024. +

+
+
+ + + onChange({ + ...value, + cache_aware_routing_timeout_ms: optionalInteger(event.target.value, 1), + }) + } + aria-describedby="cache-aware-timeout-help" + /> +

+ Keep the original route if the comparison takes too long. Leave blank to use the default of 2000 ms. +

+
+
+ )} +
+ ); +}; + +export default CacheAwareRoutingConfig; diff --git a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterAdvancedSections.tsx b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterAdvancedSections.tsx index a4e833152a9..98960897bb5 100644 --- a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterAdvancedSections.tsx +++ b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterAdvancedSections.tsx @@ -5,6 +5,7 @@ import { Separator } from "@/components/ui/separator"; import type { ModelGroup } from "@/components/llm_calls/fetch_models"; import type { ComplexityRouterConfigValue } from "./ComplexityRouterConfig"; import AdaptiveRoutingConfig from "./AdaptiveRoutingConfig"; +import CacheAwareRoutingConfig from "./CacheAwareRoutingConfig"; import ClassificationMethodConfig from "./ClassificationMethodConfig"; import ForecastClassifierConfig from "./ForecastClassifierConfig"; import ContextWindowEscalationConfig from "./ContextWindowEscalationConfig"; @@ -141,6 +142,11 @@ const ComplexityRouterAdvancedSections: React.FCAffinity, children: , }, + { + key: "cache-aware", + label: Cache-aware routing, + children: , + }, { key: "modality", label: Modality Routing, @@ -253,7 +259,7 @@ const ComplexityRouterAdvancedSections: React.FC(() => diff --git a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx index 32f9ebf97ad..12ca8701596 100644 --- a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx +++ b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx @@ -405,6 +405,9 @@ export interface ComplexityRouterConfigValue { */ enable_context_window_escalation?: boolean; context_window_escalation_buffer?: number; + cache_aware_routing?: boolean; + cache_aware_routing_output_tokens?: number; + cache_aware_routing_timeout_ms?: number; /** * Heuristic scorer knobs. Undefined means the operator never touched them, which keeps the key out of the * payload so the router tracks the backend defaults rather than freezing today's numbers. diff --git a/ui/litellm-dashboard/src/components/add_model/JevConnectionTest.integration.test.tsx b/ui/litellm-dashboard/src/components/add_model/JevConnectionTest.integration.test.tsx index ff147e20d1a..2f9d99e68e8 100644 --- a/ui/litellm-dashboard/src/components/add_model/JevConnectionTest.integration.test.tsx +++ b/ui/litellm-dashboard/src/components/add_model/JevConnectionTest.integration.test.tsx @@ -1,4 +1,5 @@ import { afterEach, describe, expect, it, vi } from "vitest"; +import { normalizeTierModels } from "./complexity_router_tiers"; import { fireEvent, renderWithProviders, screen, waitFor } from "../../../tests/test-utils"; import AutoRouterConnectionTest from "./auto_router_connection_test"; import AutoRouterRoutingTest from "./AutoRouterRoutingTest"; @@ -53,7 +54,7 @@ const request = buildSavedJevConnectionTestRequest( "saved-id", ); const targets = buildAutoRouterTestTargets({ - tiers: Object.entries(config.tiers), + tiers: Object.entries(config.tiers).map(([tier, models]) => [tier, normalizeTierModels(models)]), semanticMatchingEnabled: false, embeddingModel: undefined, }); diff --git a/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.integration.test.tsx b/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.integration.test.tsx index 0337d35d698..cd82135cd6b 100644 --- a/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.integration.test.tsx +++ b/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.integration.test.tsx @@ -1,3 +1,4 @@ +import { normalizeTierModels } from "./complexity_router_tiers"; import { openAutoRouterAdvanced, selectAutoRouterOption, @@ -971,6 +972,32 @@ describe("AddAutoRouterTab", () => { ); }); + it.each([false, true])("creates a router with cache routing only after an explicit opt-in: %s", async (enabled) => { + const user = userEvent.setup(); + mockFetchAvailableModels.mockResolvedValue(ALL_FAMILY_MODELS); + renderWithProviders(); + const setup = await screen.findByRole("button", { name: "Choose models for me" }); + await waitFor(() => expect(setup).toBeEnabled()); + await user.click(setup); + fireEvent.change(screen.getByLabelText("Auto Router Name"), { target: { value: "cache-aware-router" } }); + openAutoRouterAdvanced("Cache-aware routing"); + const toggle = screen.getByRole("switch", { name: "Cache-aware routing" }); + expect(toggle).not.toBeChecked(); + if (enabled) { + await user.click(toggle); + fireEvent.change(screen.getByLabelText("Expected output tokens"), { target: { value: "512" } }); + fireEvent.change(screen.getByLabelText("Prediction timeout (ms)"), { target: { value: "750" } }); + } + await waitFor(() => expect(screen.getByRole("button", { name: "Add Auto Router" })).toBeEnabled()); + await user.click(screen.getByRole("button", { name: "Add Auto Router" })); + await waitFor(() => expect(handleAddAutoRouterSubmit).toHaveBeenCalledOnce()); + const config = vi.mocked(handleAddAutoRouterSubmit).mock.calls.at(-1)?.[0].complexity_router_config; + expect(config?.cache_aware_routing ?? false).toBe(enabled); + expect(config?.cache_aware_routing_output_tokens).toBe(enabled ? 512 : undefined); + expect(config?.cache_aware_routing_timeout_ms).toBe(enabled ? 750 : undefined); + expect(Object.values(config?.tiers ?? {}).every((models) => typeof models === "string")).toBe(enabled); + }); + it("starts context-window escalation disabled and carries an explicit opt-in to the create payload", async () => { const user = userEvent.setup(); vi.mocked(getMissingTiersError).mockReturnValue(null); @@ -1711,10 +1738,10 @@ describe("AddAutoRouterTab", () => { expect(vi.mocked(handleAddAutoRouterSubmit).mock.calls.at(-1)?.[0]).toMatchObject({ complexity_router_config: { tiers: { - SIMPLE: ANTHROPIC_TIERS.SIMPLE.map(nativeGroupFor), - MEDIUM: ANTHROPIC_TIERS.MEDIUM.map(nativeGroupFor), - COMPLEX: ANTHROPIC_TIERS.COMPLEX.map(nativeGroupFor), - REASONING: ANTHROPIC_TIERS.REASONING.map(nativeGroupFor), + SIMPLE: normalizeTierModels(ANTHROPIC_TIERS.SIMPLE).map(nativeGroupFor), + MEDIUM: normalizeTierModels(ANTHROPIC_TIERS.MEDIUM).map(nativeGroupFor), + COMPLEX: normalizeTierModels(ANTHROPIC_TIERS.COMPLEX).map(nativeGroupFor), + REASONING: normalizeTierModels(ANTHROPIC_TIERS.REASONING).map(nativeGroupFor), }, }, }); @@ -1806,10 +1833,10 @@ describe("AddAutoRouterTab", () => { expect(vi.mocked(handleAddAutoRouterSubmit).mock.calls.at(-1)?.[0]).toMatchObject({ complexity_router_config: { tiers: { - SIMPLE: ANTHROPIC_TIERS.SIMPLE.map(expandedGroupFor), - MEDIUM: ANTHROPIC_TIERS.MEDIUM.map(expandedGroupFor), - COMPLEX: ANTHROPIC_TIERS.COMPLEX.map(expandedGroupFor), - REASONING: ANTHROPIC_TIERS.REASONING.map(expandedGroupFor), + SIMPLE: normalizeTierModels(ANTHROPIC_TIERS.SIMPLE).map(expandedGroupFor), + MEDIUM: normalizeTierModels(ANTHROPIC_TIERS.MEDIUM).map(expandedGroupFor), + COMPLEX: normalizeTierModels(ANTHROPIC_TIERS.COMPLEX).map(expandedGroupFor), + REASONING: normalizeTierModels(ANTHROPIC_TIERS.REASONING).map(expandedGroupFor), }, }, }); diff --git a/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.test.ts b/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.test.ts index 6d458d61797..b2aad778897 100644 --- a/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.test.ts +++ b/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.test.ts @@ -57,6 +57,41 @@ const baseParams: BuildComplexityRouterConfigParams = { }; describe("buildComplexityRouterConfig", () => { + it.each([undefined, false, true])( + "keeps cache routing opt-in and uses eligible single-model tiers only when enabled: %s", + (enabled) => { + const config = buildComplexityRouterConfig({ ...baseParams, cacheAwareRouting: enabled }); + expect(config.cache_aware_routing).toBe(enabled); + expect(Object.hasOwn(config, "cache_aware_routing")).toBe(enabled !== undefined); + expect(config.tiers).toEqual( + enabled ? Object.fromEntries(Object.entries(tiers).map(([tier, models]) => [tier, models[0]])) : tiers, + ); + expect(config).not.toHaveProperty("cache_aware_routing_output_tokens"); + expect(config).not.toHaveProperty("cache_aware_routing_timeout_ms"); + expect(config).not.toHaveProperty("enable_context_window_escalation"); + expect(config).not.toHaveProperty("max_tokens_from_tier_model"); + }, + ); + + it("keeps zero-output estimates and drops empty tiers without flattening real model pools", () => { + const params = { + ...baseParams, + cacheAwareRouting: true, + tiers: { ...tiers, MEDIUM: [], COMPLEX: ["first", "second"] }, + cacheAwareRoutingOutputTokens: 0, + cacheAwareRoutingTimeoutMs: 750, + }; + const config = buildComplexityRouterConfig(params); + const expected = { + cache_aware_routing: true, + cache_aware_routing_output_tokens: 0, + cache_aware_routing_timeout_ms: 750, + tiers: { SIMPLE: tiers.SIMPLE[0], COMPLEX: ["first", "second"], REASONING: tiers.REASONING[0] }, + }; + expect(config).toMatchObject(expected); + expect(config.tiers).not.toHaveProperty("MEDIUM"); + }); + it("accepts built-in JEV defaults without an LLM classifier model", () => { expect(getClassifierModelError({ classifier_type: "jev" })).toBeNull(); }); diff --git a/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.ts b/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.ts index 13df40464f0..0e72dc9580e 100644 --- a/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.ts +++ b/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.ts @@ -182,6 +182,9 @@ export interface StoredComplexityRouterConfig { return_raw_model_name?: boolean; enable_context_window_escalation?: unknown; context_window_escalation_buffer?: unknown; + cache_aware_routing?: unknown; + cache_aware_routing_output_tokens?: unknown; + cache_aware_routing_timeout_ms?: unknown; stall_escalation_enabled?: unknown; stall_escalation_window?: unknown; stall_escalation_repeat_threshold?: unknown; @@ -246,6 +249,9 @@ export interface BuildComplexityRouterConfigParams { tierModelParams?: TierModelParamsByTier; enableContextWindowEscalation?: boolean; contextWindowEscalationBuffer?: number; + cacheAwareRouting?: boolean; + cacheAwareRoutingOutputTokens?: number; + cacheAwareRoutingTimeoutMs?: number; sessionAffinityTtlSeconds?: number; codeKeywords?: string[]; reasoningKeywords?: string[]; @@ -276,7 +282,7 @@ export interface TierDefinitionPayload { } export interface ComplexityRouterConfigPayload { - tiers: ComplexityTiers | Record; + tiers: Record; enable_non_reasoning_tier?: boolean; tier_definitions?: TierDefinitionPayload[]; fallback_tier?: string; @@ -325,6 +331,9 @@ export interface ComplexityRouterConfigPayload { reasoning_override_min_score?: number; enable_context_window_escalation?: boolean; context_window_escalation_buffer?: number; + cache_aware_routing?: boolean; + cache_aware_routing_output_tokens?: number; + cache_aware_routing_timeout_ms?: number; tier_model_configs?: Record; code_keywords?: string[]; reasoning_keywords?: string[]; @@ -693,6 +702,9 @@ export const buildComplexityRouterConfig = ({ tierModelParams, enableContextWindowEscalation, contextWindowEscalationBuffer, + cacheAwareRouting, + cacheAwareRoutingOutputTokens, + cacheAwareRoutingTimeoutMs, sessionAffinityTtlSeconds, codeKeywords, reasoningKeywords, @@ -761,8 +773,17 @@ export const buildComplexityRouterConfig = ({ classifierPluginTimeoutMs > 0; const supportsOpeningPrompt = !customTierSet && !forecast && usesLlmClassifier(effectiveType); + const populatedTiers = + forecast || cacheAwareRouting + ? Object.fromEntries(Object.entries(tiers).filter(([, models]) => models.length > 0)) + : tiers; const payload: ComplexityRouterConfigPayload = { - tiers: forecast ? Object.fromEntries(Object.entries(tiers).filter(([, models]) => models.length > 0)) : tiers, + tiers: + cacheAwareRouting && !customTierSet + ? Object.fromEntries( + Object.entries(populatedTiers).map(([tier, models]) => [tier, models.length === 1 ? models[0] : models]), + ) + : populatedTiers, // The backend rejects the flag beside a custom tier set. ...(!customTierSet && enableNonReasoningTier && { enable_non_reasoning_tier: true }), ...(serializedTierModelConfigs && { tier_model_configs: serializedTierModelConfigs }), @@ -816,6 +837,13 @@ export const buildComplexityRouterConfig = ({ adaptive_eligible: adaptiveEligible, }), ...(returnRawModelName && { return_raw_model_name: true }), + ...(cacheAwareRouting !== undefined && { cache_aware_routing: cacheAwareRouting }), + ...(cacheAwareRoutingOutputTokens !== undefined && { + cache_aware_routing_output_tokens: cacheAwareRoutingOutputTokens, + }), + ...(cacheAwareRoutingTimeoutMs !== undefined && { + cache_aware_routing_timeout_ms: cacheAwareRoutingTimeoutMs, + }), ...((forecast || enableContextWindowEscalation !== undefined) && { enable_context_window_escalation: enableContextWindowEscalation ?? false, }), diff --git a/ui/litellm-dashboard/src/components/add_model/complexity_router_builder_params.ts b/ui/litellm-dashboard/src/components/add_model/complexity_router_builder_params.ts index 124a85ce9a3..2cfccebc119 100644 --- a/ui/litellm-dashboard/src/components/add_model/complexity_router_builder_params.ts +++ b/ui/litellm-dashboard/src/components/add_model/complexity_router_builder_params.ts @@ -58,6 +58,9 @@ export const builderParamsFromValue = ( tierModelParams: value.tier_model_params, enableContextWindowEscalation: value.enable_context_window_escalation, contextWindowEscalationBuffer: value.context_window_escalation_buffer, + cacheAwareRouting: value.cache_aware_routing, + cacheAwareRoutingOutputTokens: value.cache_aware_routing_output_tokens, + cacheAwareRoutingTimeoutMs: value.cache_aware_routing_timeout_ms, stallEscalationEnabled: value.stall_escalation_enabled, stallEscalationWindow: value.stall_escalation_window, stallEscalationRepeatThreshold: value.stall_escalation_repeat_threshold, diff --git a/ui/litellm-dashboard/src/components/edit_auto_router/build_updated_complexity_router_config.test.ts b/ui/litellm-dashboard/src/components/edit_auto_router/build_updated_complexity_router_config.test.ts index 4b8e298191d..072bd89b9bb 100644 --- a/ui/litellm-dashboard/src/components/edit_auto_router/build_updated_complexity_router_config.test.ts +++ b/ui/litellm-dashboard/src/components/edit_auto_router/build_updated_complexity_router_config.test.ts @@ -48,6 +48,53 @@ const hydratedState: KeywordMatchingState = { }; describe("buildUpdatedComplexityRouterConfig keyword matching", () => { + it.each([undefined, false, true])("preserves cache settings through edit and save: %s", (enabled) => { + const stored = { + ...STORED, + classifier_type: "heuristic" as const, + cache_aware_routing: enabled, + cache_aware_routing_output_tokens: 0, + cache_aware_routing_timeout_ms: 750, + }; + const hydrated = hydrateComplexityRouterConfig(stored, undefined); + expect(hydrated.cache_aware_routing).toBe(enabled); + const saved = buildUpdatedComplexityRouterConfig(stored, hydrated); + expect(saved.cache_aware_routing).toBe(enabled); + expect(Object.hasOwn(saved, "cache_aware_routing")).toBe(enabled !== undefined); + expect(saved).toMatchObject({ + cache_aware_routing_output_tokens: 0, + cache_aware_routing_timeout_ms: 750, + some_future_backend_key: STORED.some_future_backend_key, + }); + }); + + it("disables cache routing and removes cleared overrides without changing context or output limits", () => { + const stored = { + ...STORED, + classifier_type: "heuristic" as const, + cache_aware_routing: true, + cache_aware_routing_output_tokens: 512, + cache_aware_routing_timeout_ms: 750, + enable_context_window_escalation: false, + max_tokens_from_tier_model: false, + }; + const hydrated = hydrateComplexityRouterConfig(stored, undefined); + const edited = { + ...hydrated, + cache_aware_routing: false, + cache_aware_routing_output_tokens: undefined, + cache_aware_routing_timeout_ms: undefined, + }; + const saved = buildUpdatedComplexityRouterConfig(stored, edited); + expect(saved).toMatchObject({ + cache_aware_routing: false, + enable_context_window_escalation: false, + max_tokens_from_tier_model: false, + }); + expect(saved).not.toHaveProperty("cache_aware_routing_output_tokens"); + expect(saved).not.toHaveProperty("cache_aware_routing_timeout_ms"); + }); + it.each([false, true])("omits masked JEV credentials from dashboard saves, edited: %s", (edited) => { const stored = { classifier_type: "jev" as const, @@ -863,6 +910,9 @@ describe("managed keys survive an untouched open-and-save", () => { reasoning_override_min_score: 0.3, enable_context_window_escalation: false, context_window_escalation_buffer: 0.9, + cache_aware_routing: false, + cache_aware_routing_output_tokens: 512, + cache_aware_routing_timeout_ms: 750, code_keywords: ["async", "await"], reasoning_keywords: ["prove"], technical_keywords: ["api"], diff --git a/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.integration.test.tsx b/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.integration.test.tsx index 33d35b9677c..07b03e82bd6 100644 --- a/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.integration.test.tsx +++ b/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.integration.test.tsx @@ -84,6 +84,39 @@ describe("EditAutoRouterModal keyword matching", () => { modelPatchUpdateCall.mockClear(); }); + it("reopens saved cache settings and persists an explicit opt-out and cleared estimates", async () => { + const user = userEvent.setup(); + renderModal({ + modelData: { + ...MODEL_DATA, + litellm_params: { + ...MODEL_DATA.litellm_params, + complexity_router_config: { + ...STORED_CONFIG, + cache_aware_routing: true, + cache_aware_routing_output_tokens: 512, + cache_aware_routing_timeout_ms: 750, + }, + }, + }, + }); + await screen.findByRole("textbox", { name: "Auto Router Name" }); + openAutoRouterAdvanced("Cache-aware routing"); + const toggle = screen.getByRole("switch", { name: "Cache-aware routing" }); + expect(toggle).toBeChecked(); + expect(screen.getByLabelText("Expected output tokens")).toHaveValue(512); + expect(screen.getByLabelText("Prediction timeout (ms)")).toHaveValue(750); + fireEvent.change(screen.getByLabelText("Expected output tokens"), { target: { value: "" } }); + fireEvent.change(screen.getByLabelText("Prediction timeout (ms)"), { target: { value: "" } }); + await user.click(toggle); + await waitFor(() => expect(screen.getByRole("button", { name: /save changes/i })).toBeEnabled()); + await user.click(screen.getByRole("button", { name: /save changes/i })); + await waitFor(() => expect(modelPatchUpdateCall).toHaveBeenCalledOnce()); + expect(savedConfig().cache_aware_routing).toBe(false); + expect(savedConfig()).not.toHaveProperty("cache_aware_routing_output_tokens"); + expect(savedConfig()).not.toHaveProperty("cache_aware_routing_timeout_ms"); + }); + it("saves a member's changed routing config without resending administrator settings", async () => { const user = userEvent.setup(); renderModal({ diff --git a/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.tsx b/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.tsx index 2fd4fba2b31..3e6cef5d9f9 100644 --- a/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.tsx +++ b/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.tsx @@ -128,6 +128,9 @@ export const MANAGED_COMPLEXITY_ROUTER_KEYS = new Set([ "reasoning_override_min_score", "enable_context_window_escalation", "context_window_escalation_buffer", + "cache_aware_routing", + "cache_aware_routing_output_tokens", + "cache_aware_routing_timeout_ms", "stall_escalation_enabled", "stall_escalation_window", "stall_escalation_repeat_threshold", diff --git a/ui/litellm-dashboard/src/components/edit_auto_router/hydrate_complexity_router_config.ts b/ui/litellm-dashboard/src/components/edit_auto_router/hydrate_complexity_router_config.ts index 6dbd2b19b52..02310e1b8ac 100644 --- a/ui/litellm-dashboard/src/components/edit_auto_router/hydrate_complexity_router_config.ts +++ b/ui/litellm-dashboard/src/components/edit_auto_router/hydrate_complexity_router_config.ts @@ -153,6 +153,16 @@ export const hydrateComplexityRouterConfig = ( typeof parsedConfig.context_window_escalation_buffer === "number" ? parsedConfig.context_window_escalation_buffer : undefined, + cache_aware_routing: + typeof parsedConfig.cache_aware_routing === "boolean" ? parsedConfig.cache_aware_routing : undefined, + cache_aware_routing_output_tokens: + typeof parsedConfig.cache_aware_routing_output_tokens === "number" + ? parsedConfig.cache_aware_routing_output_tokens + : undefined, + cache_aware_routing_timeout_ms: + typeof parsedConfig.cache_aware_routing_timeout_ms === "number" + ? parsedConfig.cache_aware_routing_timeout_ms + : undefined, stall_escalation_enabled: parsedConfig.stall_escalation_enabled === true || undefined, stall_escalation_window: typeof parsedConfig.stall_escalation_window === "number" ? parsedConfig.stall_escalation_window : undefined, diff --git a/ui/litellm-dashboard/src/lib/autorouter_presets.test.ts b/ui/litellm-dashboard/src/lib/autorouter_presets.test.ts index c67befe8fa5..62b9eea55e8 100644 --- a/ui/litellm-dashboard/src/lib/autorouter_presets.test.ts +++ b/ui/litellm-dashboard/src/lib/autorouter_presets.test.ts @@ -942,6 +942,27 @@ describe("autorouter_presets", () => { expect(prefill.complexityRouterConfig.context_window_escalation_buffer).toBe(0.9); }); + it.each([undefined, false, true])( + "preserves cache-routing settings and scalar tier models from a preset: %s", + (enabled) => { + const config = { + ...getPresetByKey("anthropic_family")!.complexity_router_config, + tiers: { SIMPLE: "small", MEDIUM: "large", COMPLEX: "large", REASONING: "large" }, + cache_aware_routing: enabled, + cache_aware_routing_output_tokens: 0, + cache_aware_routing_timeout_ms: 750, + }; + const prefill = buildPresetPrefill(config, groupsOnly(["small", "large"])); + const expected = { + tiers: { SIMPLE: ["small"], MEDIUM: ["large"], COMPLEX: ["large"], REASONING: ["large"] }, + cache_aware_routing: enabled, + cache_aware_routing_output_tokens: 0, + cache_aware_routing_timeout_ms: 750, + }; + expect(prefill.complexityRouterConfig).toMatchObject(expected); + }, + ); + it("carries a preset's classification_mode and defaults it when the preset omits one", () => { const tiers = { SIMPLE: ["gpt-5-nano"], MEDIUM: [], COMPLEX: [], REASONING: [] }; const base = { diff --git a/ui/litellm-dashboard/src/lib/autorouter_presets.ts b/ui/litellm-dashboard/src/lib/autorouter_presets.ts index bbbf151e49b..f5e805c2a0e 100644 --- a/ui/litellm-dashboard/src/lib/autorouter_presets.ts +++ b/ui/litellm-dashboard/src/lib/autorouter_presets.ts @@ -24,6 +24,7 @@ import { TierModelParams, TierModelParamsByTier, hydrateTierModelParams, + normalizeTierModels, } from "@/components/add_model/complexity_router_tiers"; import { DEFAULT_ESCALATION_KEYWORDS } from "@/components/add_model/EscalationKeywords"; import { DEFAULT_MATCH_THRESHOLD } from "@/components/add_model/SemanticKeywordMatching"; @@ -303,7 +304,7 @@ export const buildPresetPrefill = ( availability: ModelAvailability, ): PresetPrefill => { const resolve = (model: string): string => resolveAvailableModel(model, availability) ?? model; - const resolveTier = (models: string[]): string[] => models.map(resolve); + const resolveTier = (models: string | string[]): string[] => normalizeTierModels(models).map(resolve); // Params key on the model name the preset spells while every tier entry is rewritten to the // caller's registered spelling, so the keys have to be rewritten the same way. Otherwise // serializeTierModelConfigs drops them for naming a model the tier no longer holds. @@ -363,6 +364,9 @@ export const buildPresetPrefill = ( reasoning_override_min_score: hydrateReasoningOverrideMinScore(config.reasoning_override_min_score), enable_context_window_escalation: config.enable_context_window_escalation, context_window_escalation_buffer: config.context_window_escalation_buffer, + cache_aware_routing: config.cache_aware_routing, + cache_aware_routing_output_tokens: config.cache_aware_routing_output_tokens, + cache_aware_routing_timeout_ms: config.cache_aware_routing_timeout_ms, }, customTechnicalKeywords: config.custom_technical_keywords ?? [], keywordTierRules: hydrateKeywordTierRules(config.keyword_tier_rules ?? []), diff --git a/ui/litellm-dashboard/tests/autoRouterSetup.ts b/ui/litellm-dashboard/tests/autoRouterSetup.ts index 46286718699..f574af31c5b 100644 --- a/ui/litellm-dashboard/tests/autoRouterSetup.ts +++ b/ui/litellm-dashboard/tests/autoRouterSetup.ts @@ -7,6 +7,7 @@ const groups: Record = { "Ignore Custom Tags": "Classifier tuning", Affinity: "Sessions and efficiency", "Adaptive Routing": "Sessions and efficiency", + "Cache-aware routing": "Sessions and efficiency", Compression: "Sessions and efficiency", "Response Format": "Compatibility", };