diff --git a/PR_DESCRIPTION.md b/PR_DESCRIPTION.md new file mode 100644 index 00000000000..c75f2f5ec36 --- /dev/null +++ b/PR_DESCRIPTION.md @@ -0,0 +1,135 @@ +# feat(router): Add complexity-based auto routing strategy + +## Summary + +This PR adds a new routing strategy called `complexity_router` that classifies requests by complexity using rule-based scoring and routes them to appropriate models - **with zero API calls and sub-millisecond latency**. + +Unlike the existing `auto_router` which uses embedding-based semantic matching, this approach: +- **Zero external API calls** - all scoring is local +- **Sub-millisecond latency** - typically <1ms per classification (vs 100-500ms for embedding API) +- **Predictable behavior** - deterministic rule-based scoring +- **No training required** - works out of the box, no utterance examples needed + +Inspired by [ClawRouter](https://github.com/BlockRunAI/ClawRouter). + +## How It Works + +The router scores each request across 7 weighted dimensions: + +| Dimension | Description | Weight | +|-----------|-------------|--------| +| `tokenCount` | Short prompts = simple, long = complex | 0.15 | +| `codePresence` | Code keywords (function, class, async, etc.) | 0.20 | +| `reasoningMarkers` | "step by step", "think through", etc. | 0.25 | +| `technicalTerms` | Domain complexity indicators | 0.15 | +| `simpleIndicators` | "what is", "define" (negative weight) | 0.15 | +| `multiStepPatterns` | "first...then", numbered steps | 0.05 | +| `questionComplexity` | Multiple question marks | 0.05 | + +The weighted sum maps to tiers: +- **SIMPLE** (< 0.25): Basic questions, greetings → cheap/fast models +- **MEDIUM** (0.25 - 0.50): Standard queries → balanced models +- **COMPLEX** (0.50 - 0.75): Technical, multi-part requests → capable models +- **REASONING** (> 0.75): Chain-of-thought, analysis → reasoning models + +### Special: Reasoning Override +If 2+ reasoning markers are detected in the user message, the request automatically routes to REASONING tier regardless of score. + +## Usage + +```yaml +model_list: + - model_name: smart-router + litellm_params: + model: auto_router/complexity_router + complexity_router_config: + tiers: + SIMPLE: gpt-4o-mini + MEDIUM: gpt-4o + COMPLEX: claude-sonnet-4 + REASONING: o1-preview +``` + +Then use like any other model: +```python +response = litellm.completion( + model="smart-router", + messages=[{"role": "user", "content": "What is 2+2?"}] +) +# Routes to SIMPLE tier (gpt-4o-mini) +``` + +## Full Configuration Options + +```yaml +complexity_router_config: + tiers: + SIMPLE: gpt-4o-mini + MEDIUM: gpt-4o + COMPLEX: claude-sonnet-4 + REASONING: o1-preview + + # Optional: override tier boundaries (normalized scores) + tier_boundaries: + simple_medium: 0.25 + medium_complex: 0.50 + complex_reasoning: 0.75 + + # Optional: override token count thresholds + token_thresholds: + simple: 50 # Below this = "short" + complex: 500 # Above this = "long" + + # Optional: override dimension weights + dimension_weights: + tokenCount: 0.15 + codePresence: 0.20 + reasoningMarkers: 0.25 + technicalTerms: 0.15 + simpleIndicators: 0.15 + multiStepPatterns: 0.05 + questionComplexity: 0.05 + + # Optional: fallback model + default_model: gpt-4o +``` + +## Files Changed + +### New Files +- `litellm/router_strategy/complexity_router/complexity_router.py` - Main router class +- `litellm/router_strategy/complexity_router/config.py` - Configuration and defaults +- `litellm/router_strategy/complexity_router/__init__.py` - Package exports +- `litellm/router_strategy/complexity_router/README.md` - Documentation +- `tests/test_litellm/router_strategy/test_complexity_router.py` - Test suite (37 tests) + +### Modified Files +- `litellm/router.py` - Integration with pre_routing_hook +- `litellm/types/router.py` - New config params + +## Testing + +```bash +pytest tests/test_litellm/router_strategy/test_complexity_router.py -v +# 37 tests pass +``` + +## Use Cases + +1. **Cost optimization**: Route simple queries ("What is X?") to cheap models, complex queries to capable models +2. **Latency optimization**: Simple greetings get fast responses, complex analysis gets thorough responses +3. **Resource management**: Expensive reasoning models only used when actually needed + +## Comparison with auto_router + +| Feature | complexity_router | auto_router | +|---------|-------------------|-------------| +| Classification | Rule-based scoring | Semantic embedding | +| Latency | <1ms | ~100-500ms (embedding API) | +| API Calls | None | Requires embedding model | +| Training | None | Requires utterance examples | +| Best For | Cost optimization | Intent routing | + +--- + +cc @ishaan-jaff for review diff --git a/litellm/router_strategy/complexity_router/config.py b/litellm/router_strategy/complexity_router/config.py index 112893cabe7..07a2589cba0 100644 --- a/litellm/router_strategy/complexity_router/config.py +++ b/litellm/router_strategy/complexity_router/config.py @@ -80,11 +80,11 @@ DEFAULT_CREATIVE_KEYWORDS: List[str] = [ # ─── Default Dimension Weights ─── DEFAULT_DIMENSION_WEIGHTS: Dict[str, float] = { - "tokenCount": 0.15, - "codePresence": 0.20, - "reasoningMarkers": 0.25, - "technicalTerms": 0.15, - "simpleIndicators": 0.15, + "tokenCount": 0.10, # Reduced - length is less important than content + "codePresence": 0.25, # Increased - code requests need capable models + "reasoningMarkers": 0.25, # High - explicit reasoning requests + "technicalTerms": 0.20, # Increased - technical content matters + "simpleIndicators": 0.10, # Reduced - don't over-penalize simple patterns "multiStepPatterns": 0.05, "questionComplexity": 0.05, } @@ -102,30 +102,83 @@ DEFAULT_TIER_BOUNDARIES: Dict[str, float] = { # ─── Default Token Thresholds ─── DEFAULT_TOKEN_THRESHOLDS: Dict[str, int] = { - "simple": 50, # Requests under 50 tokens are likely simple - "complex": 500, # Requests over 500 tokens are likely complex + "simple": 15, # Only very short prompts (<15 tokens) are penalized + "complex": 400, # Long prompts (>400 tokens) get complexity boost } # ─── Default Tier to Model Mapping ─── +# Standard defaults - best cost/performance for most users DEFAULT_TIER_MODELS: Dict[str, str] = { "SIMPLE": "gpt-4o-mini", "MEDIUM": "gpt-4o", "COMPLEX": "claude-sonnet-4-20250514", - "REASONING": "claude-sonnet-4-20250514", # or o1/o3 when available + "REASONING": "claude-sonnet-4-20250514", +} + +# Enterprise presets - for teams using specific cloud providers +ENTERPRISE_TIER_PRESETS: Dict[str, Dict[str, str]] = { + # AWS Bedrock - for enterprises on AWS + "bedrock": { + "SIMPLE": "bedrock/anthropic.claude-3-haiku-20240307-v1:0", + "MEDIUM": "bedrock/anthropic.claude-3-5-sonnet-20241022-v2:0", + "COMPLEX": "bedrock/anthropic.claude-3-5-sonnet-20241022-v2:0", + "REASONING": "bedrock/anthropic.claude-3-5-sonnet-20241022-v2:0", + }, + # Google Vertex AI - for enterprises on GCP + "vertex": { + "SIMPLE": "vertex_ai/gemini-2.0-flash", + "MEDIUM": "vertex_ai/gemini-2.0-flash", + "COMPLEX": "vertex_ai/gemini-2.5-pro", + "REASONING": "vertex_ai/gemini-2.5-pro", + }, + # Azure OpenAI - for enterprises on Azure + "azure": { + "SIMPLE": "azure/gpt-4o-mini", + "MEDIUM": "azure/gpt-4o", + "COMPLEX": "azure/gpt-4o", + "REASONING": "azure/o1", + }, + # Direct API (OpenAI + Anthropic) - best quality, recommended for startups + "standard": { + "SIMPLE": "gpt-4o-mini", + "MEDIUM": "gpt-4o", + "COMPLEX": "claude-sonnet-4-20250514", + "REASONING": "claude-sonnet-4-20250514", + }, + # Cost-optimized - maximum savings + "cost_optimized": { + "SIMPLE": "gemini/gemini-2.0-flash", + "MEDIUM": "gpt-4o-mini", + "COMPLEX": "claude-3-5-sonnet-20241022", + "REASONING": "claude-sonnet-4-20250514", + }, } class ComplexityRouterConfig(BaseModel): """Configuration for the ComplexityRouter.""" + # Preset name (bedrock, vertex, azure, standard, cost_optimized) + # If set, overrides 'tiers' with the preset values + preset: Optional[str] = Field( + default=None, + description="Preset name: bedrock, vertex, azure, standard, cost_optimized", + ) + # Tier to model mapping tiers: Dict[str, str] = Field( default_factory=lambda: DEFAULT_TIER_MODELS.copy(), description="Mapping of complexity tiers to model names", ) + def model_post_init(self, __context) -> None: + """Apply preset if specified.""" + if self.preset and self.preset in ENTERPRISE_TIER_PRESETS: + # Override tiers with preset values + self.tiers = ENTERPRISE_TIER_PRESETS[self.preset].copy() + # Tier boundaries (normalized scores) tier_boundaries: Dict[str, float] = Field( default_factory=lambda: DEFAULT_TIER_BOUNDARIES.copy(), diff --git a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx new file mode 100644 index 00000000000..8a7bba95853 --- /dev/null +++ b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx @@ -0,0 +1,119 @@ +import React from "react"; +import { Card, Select as AntdSelect, Typography, Tooltip } from "antd"; +import { InfoCircleOutlined } from "@ant-design/icons"; +import { ModelGroup } from "../playground/llm_calls/fetch_models"; + +const { Text } = Typography; + +interface ComplexityTiers { + SIMPLE: string; + MEDIUM: string; + COMPLEX: string; + REASONING: string; +} + +interface ComplexityRouterConfigProps { + modelInfo: ModelGroup[]; + value?: ComplexityTiers; + onChange?: (tiers: ComplexityTiers) => void; +} + +const tierDescriptions = { + SIMPLE: "Quick questions, greetings, simple lookups (e.g., \"What is the capital of France?\")", + MEDIUM: "Moderate complexity, explanations, summaries", + COMPLEX: "Code generation, technical analysis, detailed research", + REASONING: "Multi-step reasoning, complex problem solving, chain-of-thought tasks", +}; + +const tierLabels = { + SIMPLE: "Simple Tasks", + MEDIUM: "Medium Tasks", + COMPLEX: "Complex Tasks", + REASONING: "Reasoning Tasks", +}; + +const ComplexityRouterConfig: React.FC = ({ + modelInfo, + value, + onChange +}) => { + const tiers: ComplexityTiers = value || { + SIMPLE: "", + MEDIUM: "", + COMPLEX: "", + REASONING: "", + }; + + const handleTierChange = (tier: keyof ComplexityTiers, model: string) => { + const updatedTiers = { ...tiers, [tier]: model }; + onChange?.(updatedTiers); + }; + + // Prepare model options for dropdowns + const modelOptions = Array.from( + new Set(modelInfo.map((model) => model.model_group)) + ).map((model_group) => ({ + value: model_group, + label: model_group, + })); + + return ( +
+
+ + Configure which model handles each complexity tier. Requests are automatically classified and routed — no training data needed. + +
+ + +
+ {(["SIMPLE", "MEDIUM", "COMPLEX", "REASONING"] as const).map((tier) => ( +
+
+ {tierLabels[tier]} + + + +
+ handleTierChange(tier, value)} + placeholder={`Select model for ${tierLabels[tier].toLowerCase()}`} + showSearch + style={{ width: "100%" }} + options={modelOptions} + allowClear + /> + + {tierDescriptions[tier]} + +
+ ))} +
+ + + {/* Recommendations */} + +
+ +
+ + Recommendations + + + • Simple: Use fast, cheap models (e.g., GPT-4o-mini, Gemini Flash) +
+ • Medium: Balanced models (e.g., GPT-4o, Claude Sonnet) +
+ • Complex: Capable models (e.g., Claude Sonnet, GPT-4o) +
+ • Reasoning: Best reasoning models (e.g., Claude Opus, o1-preview) + +
+
+ +
+ ); +}; + +export default ComplexityRouterConfig; diff --git a/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx b/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx index fb68d6f59e3..e52c67539db 100644 --- a/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx +++ b/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx @@ -1,5 +1,5 @@ import React, { useEffect, useState } from "react"; -import { Card, Form, Button, Tooltip, Typography, Select as AntdSelect, Modal } from "antd"; +import { Card, Form, Button, Tooltip, Typography, Select as AntdSelect, Modal, Radio, Badge, Space } from "antd"; import type { FormInstance } from "antd"; import { Text, TextInput } from "@tremor/react"; import { modelAvailableCall } from "../networking"; @@ -8,7 +8,9 @@ import { all_admin_roles } from "@/utils/roles"; import { handleAddAutoRouterSubmit } from "./handle_add_auto_router_submit"; import { fetchAvailableModels, ModelGroup } from "../playground/llm_calls/fetch_models"; import RouterConfigBuilder from "./RouterConfigBuilder"; +import ComplexityRouterConfig from "./ComplexityRouterConfig"; import NotificationManager from "../molecules/notifications_manager"; +import { ThunderboltOutlined, BranchesOutlined } from "@ant-design/icons"; interface AddAutoRouterTabProps { form: FormInstance; @@ -17,6 +19,15 @@ interface AddAutoRouterTabProps { userRole: string; } +type RouterType = "complexity" | "semantic"; + +interface ComplexityTiers { + SIMPLE: string; + MEDIUM: string; + COMPLEX: string; + REASONING: string; +} + const { Title, Link } = Typography; const AddAutoRouterTab: React.FC = ({ form, handleOk, accessToken, userRole }) => { @@ -29,7 +40,20 @@ const AddAutoRouterTab: React.FC = ({ form, handleOk, acc const [modelInfo, setModelInfo] = useState([]); const [showCustomDefaultModel, setShowCustomDefaultModel] = useState(false); const [showCustomEmbeddingModel, setShowCustomEmbeddingModel] = useState(false); + + // Router type state - default to complexity router + const [routerType, setRouterType] = useState("complexity"); + + // Semantic router config (existing) const [routerConfig, setRouterConfig] = useState(null); + + // Complexity router config (new) + const [complexityTiers, setComplexityTiers] = useState({ + SIMPLE: "", + MEDIUM: "", + COMPLEX: "", + REASONING: "", + }); useEffect(() => { const fetchModelAccessGroups = async () => { @@ -64,7 +88,8 @@ const AddAutoRouterTab: React.FC = ({ form, handleOk, acc // Auto router specific form submit handler const handleAutoRouterSubmit = () => { console.log("Auto router submit triggered!"); - console.log("Router config:", routerConfig); + console.log("Router type:", routerType); + const currentFormValues = form.getFieldsValue(); console.log("Form values:", currentFormValues); @@ -74,79 +99,165 @@ const AddAutoRouterTab: React.FC = ({ form, handleOk, acc return; } - if (!currentFormValues.auto_router_default_model) { - NotificationManager.fromBackend("Please select a Default Model"); - return; - } + // Validation differs based on router type + if (routerType === "complexity") { + // Complexity Router validation + const filledTiers = Object.values(complexityTiers).filter(Boolean); + if (filledTiers.length === 0) { + NotificationManager.fromBackend("Please select at least one model for a complexity tier"); + return; + } - // Set auto router specific form values that are required by the regular model form - form.setFieldsValue({ - custom_llm_provider: "auto_router", - model: currentFormValues.auto_router_name, - // api_key is not needed for auto router, but form expects it - api_key: "not_required_for_auto_router", - }); - - // Custom validation for router config - if (!routerConfig || !routerConfig.routes || routerConfig.routes.length === 0) { - NotificationManager.fromBackend("Please configure at least one route for the auto router"); - return; - } - - // Check if all routes have required fields - const invalidRoutes = routerConfig.routes.filter( - (route: any) => !route.name || !route.description || route.utterances.length === 0, - ); - - if (invalidRoutes.length > 0) { - NotificationManager.fromBackend( - "Please ensure all routes have a target model, description, and at least one utterance", - ); - return; - } - - form - .validateFields() - .then((values) => { - console.log("Form validation passed, submitting with values:", values); - // Add the router config to form values - const submitValues = { - ...values, - auto_router_config: routerConfig, - }; - console.log("Final submit values:", submitValues); - handleAddAutoRouterSubmit(submitValues, accessToken, form, handleOk); - }) - .catch((error) => { - console.error("Validation failed:", error); - - // Extract specific field errors - const fieldErrors = error.errorFields || []; - if (fieldErrors.length > 0) { - const missingFields = fieldErrors.map((field: any) => { - const fieldName = field.name[0]; - const friendlyNames: { [key: string]: string } = { - auto_router_name: "Auto Router Name", - auto_router_default_model: "Default Model", - auto_router_embedding_model: "Embedding Model", - }; - return friendlyNames[fieldName] || fieldName; - }); - NotificationManager.fromBackend(`Please fill in the following required fields: ${missingFields.join(", ")}`); - } else { - NotificationManager.fromBackend("Please fill in all required fields"); - } + // For complexity router, use the first non-empty tier as default + const defaultModel = complexityTiers.MEDIUM || complexityTiers.SIMPLE || complexityTiers.COMPLEX || complexityTiers.REASONING; + + // Set form values for complexity router + form.setFieldsValue({ + custom_llm_provider: "auto_router", + model: currentFormValues.auto_router_name, + api_key: "not_required_for_auto_router", + auto_router_default_model: defaultModel, }); + + form + .validateFields(["auto_router_name"]) + .then((values) => { + console.log("Complexity router validation passed"); + + // Build the complexity router config + const submitValues = { + ...values, + auto_router_name: currentFormValues.auto_router_name, + auto_router_default_model: defaultModel, + // Use special model prefix for complexity router + model_type: "complexity_router", + complexity_router_config: { + tiers: complexityTiers, + }, + model_access_group: currentFormValues.model_access_group, + }; + + console.log("Final submit values:", submitValues); + handleAddAutoRouterSubmit(submitValues, accessToken, form, handleOk); + }) + .catch((error) => { + console.error("Validation failed:", error); + NotificationManager.fromBackend("Please fill in all required fields"); + }); + + } else { + // Semantic Router validation (existing logic) + if (!currentFormValues.auto_router_default_model) { + NotificationManager.fromBackend("Please select a Default Model"); + return; + } + + form.setFieldsValue({ + custom_llm_provider: "auto_router", + model: currentFormValues.auto_router_name, + api_key: "not_required_for_auto_router", + }); + + // Custom validation for router config + if (!routerConfig || !routerConfig.routes || routerConfig.routes.length === 0) { + NotificationManager.fromBackend("Please configure at least one route for the auto router"); + return; + } + + // Check if all routes have required fields + const invalidRoutes = routerConfig.routes.filter( + (route: any) => !route.name || !route.description || route.utterances.length === 0, + ); + + if (invalidRoutes.length > 0) { + NotificationManager.fromBackend( + "Please ensure all routes have a target model, description, and at least one utterance", + ); + return; + } + + form + .validateFields() + .then((values) => { + console.log("Form validation passed, submitting with values:", values); + const submitValues = { + ...values, + auto_router_config: routerConfig, + model_type: "semantic_router", + }; + console.log("Final submit values:", submitValues); + handleAddAutoRouterSubmit(submitValues, accessToken, form, handleOk); + }) + .catch((error) => { + console.error("Validation failed:", error); + const fieldErrors = error.errorFields || []; + if (fieldErrors.length > 0) { + const missingFields = fieldErrors.map((field: any) => { + const fieldName = field.name[0]; + const friendlyNames: { [key: string]: string } = { + auto_router_name: "Auto Router Name", + auto_router_default_model: "Default Model", + auto_router_embedding_model: "Embedding Model", + }; + return friendlyNames[fieldName] || fieldName; + }); + NotificationManager.fromBackend(`Please fill in the following required fields: ${missingFields.join(", ")}`); + } else { + NotificationManager.fromBackend("Please fill in all required fields"); + } + }); + } }; return ( <> Add Auto Router - Create an auto router with intelligent routing logic that automatically selects the best model based on user - input patterns and semantic matching. + Create an auto router that automatically selects the best model based on request complexity or semantic matching. + +
+ Router Type + setRouterType(e.target.value)} + className="w-full" + > + + +
+ + Complexity Router + +
+
+ Automatically routes based on request complexity. No training data needed — just pick 4 models and go. +
+ ✓ Zero API calls · ✓ <1ms latency · ✓ No cost +
+ + +
+ + Semantic Router +
+
+ Routes based on semantic similarity to example utterances. Requires embedding model and training examples. +
+ + + +
+ +
= ({ form, handleOk, acc labelCol={{ span: 10 }} labelAlign="left" > - + - {/* Router Configuration Builder */} -
- { - setRouterConfig(config); - form.setFieldValue("auto_router_config", config); - }} - /> -
+ {/* Conditional rendering based on router type */} + {routerType === "complexity" ? ( + /* Complexity Router Configuration */ +
+ { + setComplexityTiers(tiers); + }} + /> +
+ ) : ( + /* Semantic Router Configuration (existing) */ + <> + {/* Router Configuration Builder */} +
+ { + setRouterConfig(config); + form.setFieldValue("auto_router_config", config); + }} + /> +
- {/* Auto Router Default Model */} - - { - setShowCustomDefaultModel(value === "custom"); - }} - options={[ - ...Array.from(new Set(modelInfo.map((option) => option.model_group))).map((model_group) => ({ - value: model_group, - label: model_group, - })), - { value: "custom", label: "Enter custom model name" }, - ]} - style={{ width: "100%" }} - showSearch={true} - /> - + {/* Auto Router Default Model */} + + { + setShowCustomDefaultModel(value === "custom"); + }} + options={[ + ...Array.from(new Set(modelInfo.map((option) => option.model_group))).map((model_group) => ({ + value: model_group, + label: model_group, + })), + { value: "custom", label: "Enter custom model name" }, + ]} + style={{ width: "100%" }} + showSearch={true} + /> + + + {/* Auto Router Embedding Model */} + + { + setShowCustomEmbeddingModel(value === "custom"); + form.setFieldValue("auto_router_embedding_model", value); + }} + options={[ + ...Array.from(new Set(modelInfo.map((option) => option.model_group))).map((model_group) => ({ + value: model_group, + label: model_group, + })), + { value: "custom", label: "Enter custom model name" }, + ]} + style={{ width: "100%" }} + showSearch={true} + allowClear + /> + + + )} - {/* Auto Router Embedding Model */} - - { - setShowCustomEmbeddingModel(value === "custom"); - form.setFieldValue("auto_router_embedding_model", value); - }} - options={[ - ...Array.from(new Set(modelInfo.map((option) => option.model_group))).map((model_group) => ({ - value: model_group, - label: model_group, - })), - { value: "custom", label: "Enter custom model name" }, - ]} - style={{ width: "100%" }} - showSearch={true} - allowClear - /> -
Additional Settings @@ -268,13 +397,12 @@ const AddAutoRouterTab: React.FC = ({ form, handleOk, acc