diff --git a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma index f18556ac329..55ff02b9954 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma +++ b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma @@ -1097,6 +1097,26 @@ model LiteLLM_AccessGroupTable { updated_at DateTime @default(now()) @updatedAt updated_by String? } +// Routing Groups table for storing named routing pipelines +model LiteLLM_RoutingGroupTable { + routing_group_id String @id @default(uuid()) + routing_group_name String @unique + description String? + routing_strategy String + deployments Json + fallback_config Json? @default("{}") + retry_config Json? @default("{}") + cooldown_config Json? @default("{}") + settings Json? @default("{}") + assigned_team_ids String[] @default([]) + assigned_key_ids String[] @default([]) + is_active Boolean @default(true) + created_at DateTime @default(now()) + created_by String + updated_at DateTime @default(now()) @updatedAt + updated_by String +} + // Claude Code Plugin Marketplace table model LiteLLM_ClaudeCodePluginTable { id String @id @default(uuid()) diff --git a/ui/litellm-dashboard/src/components/routing_groups/RoutingFlowVisualization.tsx b/ui/litellm-dashboard/src/components/routing_groups/RoutingFlowVisualization.tsx index b9dfd84631a..b2ac5add32e 100644 --- a/ui/litellm-dashboard/src/components/routing_groups/RoutingFlowVisualization.tsx +++ b/ui/litellm-dashboard/src/components/routing_groups/RoutingFlowVisualization.tsx @@ -15,7 +15,7 @@ interface RoutingFlowVisualizationProps { deployments: Deployment[]; } -const DEPLOY_COLORS = ["#10b981", "#f59e0b", "#3b82f6", "#ef4444", "#8b5cf6", "#ec4899", "#06b6d4", "#84cc16"]; +const DEPLOY_COLORS = ["#3b82f6", "#6366f1", "#0ea5e9"]; const VerticalLine: React.FC<{ height?: number }> = ({ height = 28 }) => (
@@ -27,63 +27,22 @@ const StatusDot: React.FC = () => (
); -const KillTag: React.FC = () => ( - - kill - -); - const FailConnector: React.FC = () => (
- - + - - FAIL → TRY NEXT - + FAIL → TRY NEXT
); -const ProviderIcon: React.FC<{ color: string; name: string }> = ({ color, name }) => ( -
- {name.charAt(0).toUpperCase()} -
-); - const TriggerCard: React.FC<{ subtitle: string }> = ({ subtitle }) => ( -
-
- TRIGGER -
+
+
TRIGGER
@@ -97,19 +56,8 @@ const TriggerCard: React.FC<{ subtitle: string }> = ({ subtitle }) => ( ); const OutputCard: React.FC<{ subtitle: string }> = ({ subtitle }) => ( -
-
- OUTPUT -
+
+
OUTPUT
@@ -124,36 +72,20 @@ const OutputCard: React.FC<{ subtitle: string }> = ({ subtitle }) => (
); -const ModelCard: React.FC<{ - label: string; - name: string; - subName: string; - color: string; -}> = ({ label, name, subName, color }) => ( -
+const ModelNodeCard: React.FC<{ label: string; name: string; subName: string }> = ({ label, name, subName }) => ( +
-
- {label} -
+
{label}
- +
+ {name.charAt(0).toUpperCase()} +
{name}
{subName}
-
); @@ -165,14 +97,9 @@ const FailoverFlow: React.FC<{ deployments: Deployment[] }> = ({ deployments }) const label = i === 0 ? "PRIMARY" : `FALLBACK-${i}`; const shortName = dep.model_name.split("/").pop() || dep.model_name; return ( - + {i === 0 ? : } - + ); })} @@ -189,123 +116,139 @@ const WeightedFlow: React.FC<{ deployments: Deployment[] }> = ({ deployments }) - {/* Load balancer split node */} -
+
- - LOAD BALANCER - + LOAD BALANCER
- {/* Weight bar */} -
+ +
{deployments.map((dep, i) => { const pct = totalWeight > 0 ? ((dep.weight ?? 1) / totalWeight) * 100 : 0; - return ( -
- ); + return
; })}
-
- {/* Fan-out lines from center */} -
- {/* Vertical stem */} -
- {/* Horizontal bar */} -
- {/* Branch lines + cards */} -
+
{deployments.map((dep, i) => { const pct = totalWeight > 0 ? Math.round(((dep.weight ?? 1) / totalWeight) * 100) : 0; const shortName = dep.model_name.split("/").pop() || dep.model_name; + const color = DEPLOY_COLORS[i % DEPLOY_COLORS.length]; return ( -
- -
-
-
- {pct}% +
+
+
+ {shortName.charAt(0).toUpperCase()}
- -
- {shortName} -
-
- {dep.model_name} +
+
{shortName}
+
{dep.model_name}
- +
+ {pct}% +
+
); })}
- {/* Fan-in horizontal bar */} -
-
+
); }; +const STRATEGY_META: Record = { + "simple-shuffle": { label: "SHUFFLE ROUTER", subtitle: "→ random selection", output: "random deployment selected", color: "#6b7280" }, + "latency-based-routing": { label: "LATENCY ROUTER", subtitle: "→ latency-based", output: "fastest model wins", color: "#6366f1" }, + "usage-based-routing-v2": { label: "USAGE ROUTER", subtitle: "→ usage-based", output: "least busy model selected", color: "#6366f1" }, + "cost-based-routing": { label: "COST ROUTER", subtitle: "→ cost-based", output: "cheapest deployment selected", color: "#059669" }, + "least-busy": { label: "LEAST BUSY ROUTER", subtitle: "→ least busy", output: "deployment with fewest requests", color: "#6366f1" }, +}; + +const COMPLEXITY_TIER_LABELS = [ + { tier: "SIMPLE", label: "Simple", threshold: "< 0.15" }, + { tier: "MEDIUM", label: "Medium", threshold: "0.15 – 0.35" }, + { tier: "COMPLEX", label: "Complex", threshold: "0.35 – 0.60" }, + { tier: "REASONING", label: "Reasoning", threshold: "> 0.60" }, +]; + +const ComplexityFlow: React.FC<{ deployments: Deployment[] }> = ({ deployments }) => ( +
+ + + +
+
+ + + + AUTO ROUTER +
+
Rule-based scoring · no API calls · <1ms
+ +
+ {COMPLEXITY_TIER_LABELS.map((t, i) => { + const dep = deployments[i]; + const shortName = dep ? (dep.model_name.split("/").pop() || dep.model_name) : null; + return ( +
+
+
+ {t.label} + score {t.threshold} +
+
+ {shortName ? ( +
+
+ {shortName.charAt(0).toUpperCase()} +
+ {shortName} + +
+ ) : ( +
No model assigned
+ )} +
+ ); + })} +
+
+ + + +
+); + const SmartRoutingFlow: React.FC<{ deployments: Deployment[]; strategy: string }> = ({ deployments, strategy }) => { - const isLatency = strategy === "latency-based-routing"; + const meta = STRATEGY_META[strategy] ?? STRATEGY_META["simple-shuffle"]; return (
- + - -
-
- {isLatency ? "LATENCY ROUTER" : "USAGE ROUTER"} +
+
+ {meta.label}
{deployments.map((dep, i) => { const shortName = dep.model_name.split("/").pop() || dep.model_name; return ( -
+
- +
+ {shortName.charAt(0).toUpperCase()} +
{shortName}
{dep.model_name}
@@ -317,9 +260,8 @@ const SmartRoutingFlow: React.FC<{ deployments: Deployment[]; strategy: string } })}
- - +
); }; @@ -337,13 +279,18 @@ const EmptyFlow: React.FC = () => ( ); export default function RoutingFlowVisualization({ strategy, deployments }: RoutingFlowVisualizationProps) { + if (strategy === "complexity-router") return ; if (deployments.length === 0) return ; switch (strategy) { case "priority-failover": return ; case "weighted": return ; + case "simple-shuffle": case "latency-based-routing": - case "usage-based-routing-v2": return ; - default: return ; + case "usage-based-routing-v2": + case "cost-based-routing": + case "least-busy": + return ; + default: return ; } } diff --git a/ui/litellm-dashboard/src/components/routing_groups/RoutingGroupBuilder.tsx b/ui/litellm-dashboard/src/components/routing_groups/RoutingGroupBuilder.tsx index 94f9a5a473c..94087bd36de 100644 --- a/ui/litellm-dashboard/src/components/routing_groups/RoutingGroupBuilder.tsx +++ b/ui/litellm-dashboard/src/components/routing_groups/RoutingGroupBuilder.tsx @@ -1,17 +1,21 @@ "use client"; import React, { useEffect, useMemo, useRef, useState } from "react"; -import { Button, InputNumber } from "antd"; +import { Button, InputNumber, message } from "antd"; import { ArrowLeftOutlined, CodeOutlined, DeleteOutlined, + DollarOutlined, + ForkOutlined, HolderOutlined, OrderedListOutlined, PlusOutlined, PercentageOutlined, + SwapOutlined, ThunderboltOutlined, DashboardOutlined, + NodeIndexOutlined, } from "@ant-design/icons"; import { routingGroupCreateCall, routingGroupUpdateCall } from "@/components/networking"; import ModelSelector from "@/components/common_components/ModelSelector"; @@ -34,6 +38,12 @@ interface RoutingGroupBuilderProps { } const STRATEGIES = [ + { + value: "simple-shuffle", + label: "Simple Shuffle", + desc: "Random selection across deployments", + icon: , + }, { value: "priority-failover", label: "Ordered Fallback", @@ -58,6 +68,31 @@ const STRATEGIES = [ desc: "Route to fastest responding backend", icon: , }, + { + value: "cost-based-routing", + label: "Cost-Based", + desc: "Route to cheapest deployment", + icon: , + }, + { + value: "least-busy", + label: "Least Busy", + desc: "Route to deployment with fewest active requests", + icon: , + }, + { + value: "complexity-router", + label: "Auto Router", + desc: "Route by request complexity, no API calls, <1ms", + icon: , + }, +]; + +const COMPLEXITY_TIERS: { key: string; label: string; desc: string; examples: string }[] = [ + { key: "SIMPLE", label: "Simple", desc: "Basic questions, greetings, simple facts", examples: '"Hello!", "What is Python?"' }, + { key: "MEDIUM", label: "Medium", desc: "Standard queries requiring explanation", examples: '"Explain REST APIs", "Debug this error"' }, + { key: "COMPLEX", label: "Complex", desc: "Technical, multi-part deep requests", examples: '"Design a microservices arch"' }, + { key: "REASONING", label: "Reasoning", desc: "Chain-of-thought, analysis", examples: '"Think step by step..."' }, ]; const INITIAL_SLOT_COUNT = 3; @@ -71,11 +106,12 @@ export default function RoutingGroupBuilder({ const { accessToken: authToken } = useAuthorized(); const effectiveToken = accessToken ?? authToken ?? ""; - const [strategy, setStrategy] = useState("priority-failover"); + const [strategy, setStrategy] = useState("simple-shuffle"); const [deployments, setDeployments] = useState([]); const [emptySlots, setEmptySlots] = useState(INITIAL_SLOT_COUNT); const [logicalName, setLogicalName] = useState(""); const [submitting, setSubmitting] = useState(false); + const [tierModels, setTierModels] = useState>({ SIMPLE: "", MEDIUM: "", COMPLEX: "", REASONING: "" }); const dragItem = useRef(null); const dragOverItem = useRef(null); @@ -105,27 +141,46 @@ export default function RoutingGroupBuilder({ [deployments] ); + const isComplexityRouter = strategy === "complexity-router"; + + const handleTierModelChange = (tierKey: string, model: string) => { + const updated = { ...tierModels, [tierKey]: model }; + setTierModels(updated); + const deps = COMPLEXITY_TIERS + .map((t) => updated[t.key]) + .filter(Boolean) + .map((name) => ({ model_id: name, model_name: name, provider: "-" })); + setDeployments(deps); + }; + const handleStrategyChange = (value: string) => { setStrategy(value); - setDeployments((prev) => - prev.map((d, i) => ({ + setDeployments((prev) => { + const equal = prev.length > 0 ? Math.max(1, Math.round(100 / prev.length)) : 1; + return prev.map((d, i) => ({ ...d, - weight: value === "weighted" ? (d.weight ?? 1) : undefined, + weight: value === "weighted" ? equal : undefined, priority: value === "priority-failover" ? i + 1 : undefined, - })) - ); + })); + }); }; const handleSlotSelected = (slotIndex: number, modelGroup: string) => { if (!modelGroup || deployments.some((d) => d.model_name === modelGroup)) return; + const count = deployments.length + 1; + const equal = Math.max(1, Math.round(100 / count)); const newDep: Deployment = { model_id: modelGroup, model_name: modelGroup, provider: "-", - weight: strategy === "weighted" ? 1 : undefined, - priority: strategy === "priority-failover" ? deployments.length + 1 : undefined, + weight: strategy === "weighted" ? equal : undefined, + priority: strategy === "priority-failover" ? count : undefined, }; - setDeployments((prev) => [...prev, newDep]); + if (strategy === "weighted") { + setDeployments((prev) => [...prev.map((d) => ({ ...d, weight: equal })), newDep]); + } else { + setDeployments((prev) => [...prev, newDep]); + } setEmptySlots((prev) => Math.max(0, prev - 1)); }; @@ -151,10 +206,16 @@ export default function RoutingGroupBuilder({ }); }; - const updateWeight = (index: number, weight: number | null) => { - setDeployments((prev) => - prev.map((d, i) => (i === index ? { ...d, weight: weight ?? 1 } : d)) - ); + const updatePercent = (index: number, newPct: number | null) => { + const pct = Math.max(1, Math.min(99, newPct ?? 1)); + setDeployments((prev) => { + const otherTotal = prev.reduce((s, d, i) => i === index ? s : s + (d.weight ?? 1), 0); + if (otherTotal === 0) return prev; + const scale = (100 - pct) / otherTotal; + return prev.map((d, i) => + i === index ? { ...d, weight: pct } : { ...d, weight: Math.max(1, Math.round((d.weight ?? 1) * scale)) } + ); + }); }; const handleDragStart = (index: number) => { dragItem.current = index; }; @@ -180,7 +241,19 @@ export default function RoutingGroupBuilder({ }; const handleSubmit = async () => { - if (!accessToken || !logicalName.trim() || deployments.length === 0) return; + const token = accessToken ?? effectiveToken; + if (!token) { + message.error("No access token available"); + return; + } + if (!logicalName.trim()) { + message.error("Please enter a logical model name"); + return; + } + if (deployments.length === 0) { + message.error("Please add at least one deployment"); + return; + } try { setSubmitting(true); const payload: Record = { @@ -189,13 +262,16 @@ export default function RoutingGroupBuilder({ deployments, }; if (isEditing && editTarget?.routing_group_id) { - await routingGroupUpdateCall(accessToken, editTarget.routing_group_id as string, payload); + await routingGroupUpdateCall(token, editTarget.routing_group_id as string, payload); + message.success("Routing group updated"); } else { - await routingGroupCreateCall(accessToken, payload); + await routingGroupCreateCall(token, payload); + message.success("Routing group created"); } onSuccess(); } catch (err) { console.error("Failed to save routing group:", err); + message.error("Failed to save: " + (err instanceof Error ? err.message : String(err))); } finally { setSubmitting(false); } @@ -207,7 +283,7 @@ export default function RoutingGroupBuilder({
{/* ── LEFT: Config Panel ── */} -
+
{s.label}
{s.desc}
- {isActive && ( - - Active - - )} ); })}
- {/* 2 · Deployments */} + {/* 2 · Deployments / Tier Config */}
-
-
2 · Deployments
- -
- - {/* Pipeline breadcrumb chain */} - {deployments.length > 0 && ( -
- {deployments.map((dep, i) => { - const shortName = dep.model_name.split("/").pop() || dep.model_name; - return ( - - {i > 0 && →} - - {i + 1} - {shortName} - - - ); - })} -
- )} - - {/* Weight distribution bar */} - {strategy === "weighted" && deployments.length > 0 && ( -
-
- {deployments.map((dep, i) => { - const pct = totalWeight > 0 ? ((dep.weight ?? 1) / totalWeight) * 100 : 0; - return
; - })} + {isComplexityRouter ? ( + <> +
2 · Complexity Tiers
+
Assign a model to each complexity tier. Simple queries go to cheaper models, complex ones to more capable models.
+
+ {COMPLEXITY_TIERS.map((t) => ( +
+
+ {t.label} + {t.examples} +
+
{t.desc}
+ handleTierModelChange(t.key, model)} + /> +
+ ))}
-
- {deployments.map((dep, i) => { + + ) : ( + <> +
+
2 · Deployments
+ +
+ + {deployments.length > 0 && ( +
+ {deployments.map((dep, i) => { + const shortName = dep.model_name.split("/").pop() || dep.model_name; + return ( + + {i > 0 && →} + + {i + 1} + {shortName} + + + ); + })} +
+ )} + + {strategy === "weighted" && deployments.length > 0 && ( +
+
+ {deployments.map((dep, i) => { + const pct = totalWeight > 0 ? ((dep.weight ?? 1) / totalWeight) * 100 : 0; + return
; + })} +
+
+ {deployments.map((dep, i) => { + const pct = totalWeight > 0 ? Math.round(((dep.weight ?? 1) / totalWeight) * 100) : 0; + const shortName = dep.model_name.split("/").pop() || dep.model_name; + return ( +
+
+ {shortName}: {pct}% +
+ ); + })} +
+
+ )} + +
+ {deployments.map((dep, idx) => { const pct = totalWeight > 0 ? Math.round(((dep.weight ?? 1) / totalWeight) * 100) : 0; - const shortName = dep.model_name.split("/").pop() || dep.model_name; return ( -
-
- {shortName}: {pct}% +
handleDragStart(idx)} + onDragEnter={() => handleDragEnter(idx)} + onDragEnd={handleDragEnd} + onDragOver={(e) => e.preventDefault()} + className="rounded-lg border border-gray-200 bg-white px-3 py-2.5 cursor-grab active:cursor-grabbing hover:border-gray-300 transition-colors" + > +
+ + {idx + 1} +
e.stopPropagation()}> + handleDeploymentChange(idx, model)} + /> +
+ +
+ {strategy === "weighted" && ( +
e.stopPropagation()}> + updatePercent(idx, val)} + addonAfter="%" + style={{ width: 80 }} + /> + traffic +
+ )}
); })} + + {Array.from({ length: emptySlots }).map((_, slotIdx) => ( +
+
+ {deployments.length + slotIdx + 1} +
+ handleSlotSelected(slotIdx, model)} + /> +
+
+
+ ))}
-
+ )} - -
- {deployments.map((dep, idx) => { - const pct = totalWeight > 0 ? Math.round(((dep.weight ?? 1) / totalWeight) * 100) : 0; - return ( -
handleDragStart(idx)} - onDragEnter={() => handleDragEnter(idx)} - onDragEnd={handleDragEnd} - onDragOver={(e) => e.preventDefault()} - className="rounded-lg border border-gray-200 bg-white px-3 py-2.5 cursor-grab active:cursor-grabbing hover:border-gray-300 transition-colors" - > -
- - {idx + 1} -
e.stopPropagation()}> - handleDeploymentChange(idx, model)} - /> -
- -
- {strategy === "weighted" && ( -
e.stopPropagation()}> - Weight - updateWeight(idx, val)} - addonAfter="%" - style={{ width: 90 }} - /> - = {pct}% traffic -
- )} -
- ); - })} - - {Array.from({ length: emptySlots }).map((_, slotIdx) => ( -
-
- {deployments.length + slotIdx + 1} -
- handleSlotSelected(slotIdx, model)} - /> -
-
-
- ))} -
{/* 3 · Logical Model Name */} @@ -455,28 +550,6 @@ export default function RoutingGroupBuilder({
- {/* Stats footer */} -
-
- {[ - { label: "REQUESTS", value: "0" }, - { label: "SUCCESS", value: "100%", color: "#22c55e" }, - { label: "AVG LATENCY", value: "—" }, - { label: "FALLBACKS", value: "0" }, - ].map((stat, i) => ( -
0 ? "1px solid #e5e7eb" : "none", paddingLeft: i > 0 ? 16 : 0 }}> -
{stat.label}
-
{stat.value}
-
- ))} -
-
- - - - Request Log -
-