diff --git a/ui/litellm-dashboard/src/autorouter_presets.json b/ui/litellm-dashboard/src/autorouter_presets.json index da35d6171dc..b977bd484ac 100644 --- a/ui/litellm-dashboard/src/autorouter_presets.json +++ b/ui/litellm-dashboard/src/autorouter_presets.json @@ -16,6 +16,7 @@ "escalation_keywords": ["LITELLM ESCALATE"], "classification_mode": "every_request", "session_affinity": false, + "modality_routing": false, "deployment_affinity": true } }, @@ -33,6 +34,7 @@ "escalation_keywords": ["LITELLM ESCALATE"], "classification_mode": "every_request", "session_affinity": false, + "modality_routing": false, "deployment_affinity": true } }, @@ -60,6 +62,7 @@ "escalation_keywords": ["LITELLM ESCALATE"], "classification_mode": "every_request", "session_affinity": false, + "modality_routing": false, "deployment_affinity": true } }, @@ -77,6 +80,7 @@ "escalation_keywords": ["LITELLM ESCALATE"], "classification_mode": "every_request", "session_affinity": false, + "modality_routing": false, "deployment_affinity": true } } diff --git a/ui/litellm-dashboard/src/components/add_model/AffinityControls.tsx b/ui/litellm-dashboard/src/components/add_model/AffinityControls.tsx new file mode 100644 index 00000000000..4d9e2122739 --- /dev/null +++ b/ui/litellm-dashboard/src/components/add_model/AffinityControls.tsx @@ -0,0 +1,26 @@ +import React from "react"; + +import { Switch } from "@/components/ui/switch"; + +import type { ComplexityRouterConfigValue } from "./ComplexityRouterConfig"; +import { DEFAULT_DEPLOYMENT_AFFINITY } from "./ComplexityRouterConfig"; + +export const AffinityControls: React.FC<{ + value: ComplexityRouterConfigValue; + onChange: (value: ComplexityRouterConfigValue) => void; +}> = ({ value, onChange }) => ( + <> +
+ onChange({ ...value, deployment_affinity: deploymentAffinity })} + aria-label="Pin a session to one deployment per model group" + /> + Pin a session to one deployment per model group +
+ + Keeps a session on the same deployment within a group, so provider prompt caches stay warm. Turn off to + load-balance every turn. + + +); diff --git a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.test.tsx b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.test.tsx index bbee03135c7..751f8870561 100644 --- a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.test.tsx +++ b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.test.tsx @@ -836,6 +836,27 @@ describe("ComplexityRouterConfig tier labels", () => { }); }); +describe("ComplexityRouterConfig modality panel", () => { + it("defaults the image-routing switch off and writes modality_routing through onChange", () => { + const onChange = vi.fn(); + renderWithProviders(); + fireEvent.click(screen.getByText("Advanced: Modality Routing")); + + const toggle = screen.getByRole("switch", { name: "Route image requests to vision-capable models" }); + expect(toggle).not.toBeChecked(); + fireEvent.click(toggle); + + expect(onChange).toHaveBeenCalledWith({ ...defaultValue, modality_routing: true }); + }); + + it("renders a stored modality_routing=true as on", () => { + renderWithProviders(); + fireEvent.click(screen.getByText("Advanced: Modality Routing")); + + expect(screen.getByRole("switch", { name: "Route image requests to vision-capable models" })).toBeChecked(); + }); +}); + describe("ComplexityRouterConfig affinity panel", () => { it("holds the deployment switch at its backend default, session pinning having moved to the frequency choice", () => { renderWithProviders(); diff --git a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx index 5fde87fd638..6062705aa82 100644 --- a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx +++ b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx @@ -4,6 +4,9 @@ import { SearchSelect } from "@/components/shared/SearchSelect"; import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select"; import { ChevronRight, Info, Plus, Trash2, X } from "lucide-react"; import { Switch } from "@/components/ui/switch"; + +import { AffinityControls } from "./AffinityControls"; +import { ModalityRoutingControls } from "./ModalityRoutingControls"; import { Card, CardContent } from "@/components/ui/card"; import { Collapsible, CollapsibleContent, CollapsibleTrigger } from "@/components/ui/collapsible"; import { InputGroup, InputGroupAddon, InputGroupButton, InputGroupInput } from "@/components/ui/input-group"; @@ -396,6 +399,7 @@ export interface ComplexityRouterConfigValue { heuristic_first_max_tier?: string; classification_mode?: ClassificationMode; session_affinity?: boolean; + modality_routing?: boolean; deployment_affinity?: boolean; /** Plan-mode floor as a tier ROW ID, unset meaning off. The wire carries the row's name. */ plan_mode_min_tier?: string; @@ -511,26 +515,6 @@ export const DEFAULT_HEURISTIC_FIRST_MAX_TIER = "SIMPLE"; */ export const HEURISTIC_FIRST_MAX_TIER_KEYS = TIER_KEYS.slice(0, -1); -const AffinityControls: React.FC<{ - value: ComplexityRouterConfigValue; - onChange: (value: ComplexityRouterConfigValue) => void; -}> = ({ value, onChange }) => ( - <> -
- onChange({ ...value, deployment_affinity: deploymentAffinity })} - aria-label="Pin a session to one deployment per model group" - /> - Pin a session to one deployment per model group -
- - Keeps a session on the same deployment within a group, so provider prompt caches stay warm. Turn off to - load-balance every turn. - - -); - const PlanModeOverrideControls: React.FC<{ value: ComplexityRouterConfigValue; onChange: (value: ComplexityRouterConfigValue) => void; @@ -841,6 +825,11 @@ const ComplexityRouterConfig: React.FC = ({ label: Advanced: Affinity, children: , }, + { + key: "modality", + label: Advanced: Modality Routing, + children: , + }, { key: "plan-mode", label: Advanced: Plan-Mode Override, diff --git a/ui/litellm-dashboard/src/components/add_model/ModalityRoutingControls.tsx b/ui/litellm-dashboard/src/components/add_model/ModalityRoutingControls.tsx new file mode 100644 index 00000000000..dd697b35239 --- /dev/null +++ b/ui/litellm-dashboard/src/components/add_model/ModalityRoutingControls.tsx @@ -0,0 +1,26 @@ +import React from "react"; + +import { Switch } from "@/components/ui/switch"; + +import type { ComplexityRouterConfigValue } from "./ComplexityRouterConfig"; + +export const ModalityRoutingControls: React.FC<{ + value: ComplexityRouterConfigValue; + onChange: (value: ComplexityRouterConfigValue) => void; +}> = ({ value, onChange }) => ( + <> +
+ onChange({ ...value, modality_routing: modalityRouting })} + aria-label="Route image requests to vision-capable models" + /> + Route image requests to vision-capable models +
+ + Replaces a routed model that cannot take image input with the nearest higher tier that can, then the default + model, instead of failing with a provider 400. Only models explicitly declared supports_vision false are replaced, + and a kept session pin still wins. + + +); diff --git a/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx b/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx index c60ce5e4959..318adcce369 100644 --- a/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx +++ b/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx @@ -351,6 +351,7 @@ const AddAutoRouterTab: React.FC = ({ classifierContextIncludeAssistantTurns: complexityRouterConfig.classifier_context_include_assistant_turns, classifierFallback: complexityRouterConfig.classifier_fallback, sessionAffinity: complexityRouterConfig.session_affinity ?? DEFAULT_SESSION_AFFINITY, + modalityRouting: complexityRouterConfig.modality_routing ?? false, deploymentAffinity: complexityRouterConfig.deployment_affinity ?? DEFAULT_DEPLOYMENT_AFFINITY, customTechnicalKeywords, keywordTierRules, diff --git a/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.test.ts b/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.test.ts index 40addf086b6..af87cc6c8fb 100644 --- a/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.test.ts +++ b/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.test.ts @@ -55,6 +55,7 @@ describe("buildComplexityRouterConfig", () => { classification_mode: "every_request", session_affinity: false, deployment_affinity: true, + modality_routing: false, escalation_keywords: ["LITELLM ESCALATE"], }; expect(config).toEqual(expected); @@ -248,6 +249,12 @@ describe("buildComplexityRouterConfig", () => { expect(config.return_raw_model_name).toBeUndefined(); }); + it("writes modality_routing explicitly both ways, so the stored config never relies on the backend default", () => { + expect(buildComplexityRouterConfig({ ...baseParams, modalityRouting: true }).modality_routing).toBe(true); + expect(buildComplexityRouterConfig(baseParams).modality_routing).toBe(false); + expect(buildComplexityRouterConfig({ ...baseParams, modalityRouting: false }).modality_routing).toBe(false); + }); + it("writes session_affinity=true so turning the toggle on overrides the backend's off-by-default", () => { const config = buildComplexityRouterConfig({ ...baseParams, sessionAffinity: true }); expect(config.session_affinity).toBe(true); diff --git a/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.ts b/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.ts index 6a087649c00..af34dc92c0f 100644 --- a/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.ts +++ b/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.ts @@ -109,6 +109,7 @@ export interface BuildComplexityRouterConfigParams { heuristicFirstMaxTier: string | undefined; classificationMode: ClassificationMode | undefined; sessionAffinity: boolean; + modalityRouting?: boolean; deploymentAffinity: boolean; customTechnicalKeywords: string[]; keywordTierRules: KeywordTierRule[]; @@ -165,6 +166,7 @@ export interface ComplexityRouterConfigPayload { classification_mode: ClassificationMode; session_affinity: boolean; deployment_affinity: boolean; + modality_routing: boolean; custom_technical_keywords?: string[]; keyword_tier_rules?: { keywords: string[]; tier: KeywordTierRule["tier"] }[]; semantic_keyword_matching?: boolean; @@ -399,6 +401,7 @@ export const buildComplexityRouterConfig = ({ heuristicFirstMaxTier, classificationMode, sessionAffinity, + modalityRouting, deploymentAffinity, customTechnicalKeywords, keywordTierRules, @@ -460,6 +463,7 @@ export const buildComplexityRouterConfig = ({ classification_mode: classificationMode ?? DEFAULT_CLASSIFICATION_MODE, session_affinity: sessionAffinity, deployment_affinity: deploymentAffinity, + modality_routing: modalityRouting ?? false, ...(customTechnicalKeywords.length > 0 && { custom_technical_keywords: customTechnicalKeywords }), ...(cleanedKeywordTierRules.length > 0 && { keyword_tier_rules: cleanedKeywordTierRules }), escalation_keywords: cleanedEscalationKeywords, diff --git a/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.test.ts b/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.test.ts index d199af51b41..481ba2b6b00 100644 --- a/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.test.ts +++ b/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.test.ts @@ -1,4 +1,4 @@ -import { buildUpdatedComplexityRouterConfig } from "./edit_auto_router_modal"; +import { buildUpdatedComplexityRouterConfig, hydrateComplexityRouterConfig } from "./edit_auto_router_modal"; const storedConfigValue = { tiers: { @@ -50,6 +50,7 @@ const expectedClassifiedTierConfig = { classification_mode: "every_request", session_affinity: false, deployment_affinity: true, + modality_routing: false, adaptive: true, adaptive_weights: { quality: 0.4, cost: 0.6 }, adaptive_eligible: "classified_tier", @@ -72,6 +73,7 @@ const expectedAdaptiveDisabledConfig = { classification_mode: "every_request", session_affinity: false, deployment_affinity: true, + modality_routing: false, }; describe("buildUpdatedComplexityRouterConfig", () => { @@ -87,6 +89,26 @@ describe("buildUpdatedComplexityRouterConfig", () => { expect(updatedConfig).toEqual(expectedAdaptiveDisabledConfig); }); + it("hydrates a stored modality_routing into form state and defaults absent to off", () => { + expect(hydrateComplexityRouterConfig({ ...storedConfig, modality_routing: true }, null).modality_routing).toBe( + true, + ); + expect(hydrateComplexityRouterConfig(storedConfig, null).modality_routing).toBe(false); + }); + + it("round-trips modality_routing explicitly in both directions", () => { + const enabled = buildUpdatedComplexityRouterConfig(storedConfig, { + ...classifiedTierValue, + modality_routing: true, + }); + expect(enabled.modality_routing).toBe(true); + const disabled = buildUpdatedComplexityRouterConfig( + { ...storedConfig, modality_routing: true }, + { ...classifiedTierValue, modality_routing: false }, + ); + expect(disabled.modality_routing).toBe(false); + }); + it("includes return_raw_model_name only when enabled", () => { const updatedConfig = buildUpdatedComplexityRouterConfig(storedConfig, { ...classifiedTierValue, diff --git a/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.tsx b/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.tsx index 2aecf0b3483..e18582f77a0 100644 --- a/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.tsx +++ b/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.tsx @@ -102,6 +102,7 @@ export interface StoredComplexityRouterConfig { dimension_weights?: unknown; reasoning_override_min_score?: unknown; session_affinity?: unknown; + modality_routing?: unknown; deployment_affinity?: unknown; adaptive?: boolean; adaptive_weights?: AdaptiveRouterWeights; @@ -176,6 +177,7 @@ export const hydrateComplexityRouterConfig = ( reasoning_override_min_score: hydrateReasoningOverrideMinScore(parsedConfig.reasoning_override_min_score), session_affinity: typeof parsedConfig.session_affinity === "boolean" ? parsedConfig.session_affinity : DEFAULT_SESSION_AFFINITY, + modality_routing: typeof parsedConfig.modality_routing === "boolean" ? parsedConfig.modality_routing : false, deployment_affinity: typeof parsedConfig.deployment_affinity === "boolean" ? parsedConfig.deployment_affinity @@ -214,6 +216,7 @@ export const MANAGED_COMPLEXITY_ROUTER_KEYS = new Set([ "heuristic_first_max_tier", "classification_mode", "session_affinity", + "modality_routing", "deployment_affinity", "adaptive", "adaptive_weights", @@ -309,6 +312,7 @@ export const buildUpdatedComplexityRouterConfig = ( classifierContextIncludeAssistantTurns: value.classifier_context_include_assistant_turns, classifierFallback: value.classifier_fallback, sessionAffinity: value.session_affinity ?? DEFAULT_SESSION_AFFINITY, + modalityRouting: value.modality_routing ?? false, deploymentAffinity: value.deployment_affinity ?? DEFAULT_DEPLOYMENT_AFFINITY, customTechnicalKeywords: customTechnicalKeywords ?? [], keywordTierRules: keywordMatching?.keywordTierRules ?? [], diff --git a/ui/litellm-dashboard/src/lib/autorouter_presets.test.ts b/ui/litellm-dashboard/src/lib/autorouter_presets.test.ts index 5632d6d947f..2a8306473b8 100644 --- a/ui/litellm-dashboard/src/lib/autorouter_presets.test.ts +++ b/ui/litellm-dashboard/src/lib/autorouter_presets.test.ts @@ -129,6 +129,18 @@ describe("autorouter_presets", () => { } }); + it("carries a preset's modality_routing into the prefilled form state", () => { + const preset = getPresetByKey("anthropic_family")!; + const withFlag = { ...preset.complexity_router_config, modality_routing: true }; + const prefill = buildPresetPrefill(withFlag, groupsOnly(getRequiredModelsInPreset(preset))); + expect(prefill.complexityRouterConfig.modality_routing).toBe(true); + const withoutFlag = buildPresetPrefill( + preset.complexity_router_config, + groupsOnly(getRequiredModelsInPreset(preset)), + ); + expect(withoutFlag.complexityRouterConfig.modality_routing).toBe(false); + }); + it("prefills the anthropic preset's effort through to tier_model_params", () => { const preset = getPresetByKey("anthropic_family")!; const prefill = buildPresetPrefill(preset.complexity_router_config, groupsOnly(getRequiredModelsInPreset(preset))); diff --git a/ui/litellm-dashboard/src/lib/autorouter_presets.ts b/ui/litellm-dashboard/src/lib/autorouter_presets.ts index 4c1f08b62e9..721bd6f2b2a 100644 --- a/ui/litellm-dashboard/src/lib/autorouter_presets.ts +++ b/ui/litellm-dashboard/src/lib/autorouter_presets.ts @@ -292,6 +292,7 @@ export const buildPresetPrefill = ( classification_mode: config.classification_mode ?? DEFAULT_CLASSIFICATION_MODE, session_affinity: config.session_affinity ?? DEFAULT_SESSION_AFFINITY, deployment_affinity: config.deployment_affinity ?? DEFAULT_DEPLOYMENT_AFFINITY, + modality_routing: config.modality_routing ?? false, adaptive: config.adaptive, adaptive_weights: config.adaptive_weights, tier_distance_penalty: config.tier_distance_penalty,