diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index f52288ea72a..cbd64a178b8 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -16289,7 +16289,7 @@ "cache_read_input_token_cost": 3e-08, "input_cost_per_audio_token": 1e-06, "input_cost_per_token": 3e-07, - "litellm_provider": "vertex_ai-language-models", + "litellm_provider": "gemini", "max_audio_length_hours": 8.4, "max_audio_per_prompt": 1, "supports_reasoning": false, diff --git a/litellm/proxy/management_endpoints/routing_group_endpoints.py b/litellm/proxy/management_endpoints/routing_group_endpoints.py new file mode 100644 index 00000000000..6d34b88639c --- /dev/null +++ b/litellm/proxy/management_endpoints/routing_group_endpoints.py @@ -0,0 +1,471 @@ +""" +Routing Group management endpoints. + +Provides CRUD operations for routing groups — named, persisted routing pipelines +that combine model deployments with a routing strategy. +""" + +import uuid +from collections import defaultdict +from typing import List, Optional + +from fastapi import APIRouter, Depends, HTTPException, status + +import litellm +from litellm._logging import verbose_proxy_logger +from litellm.proxy._types import CommonProxyErrors, UserAPIKeyAuth +from litellm.proxy.auth.user_api_key_auth import user_api_key_auth +from litellm.proxy.utils import get_prisma_client_or_throw +from litellm.types.router import ( + FailureInjectionConfig, + RoutingGroupConfig, + RoutingGroupDeployment, + RoutingGroupListResponse, + RoutingGroupSimulationResult, + RoutingGroupTestResult, +) + +router = APIRouter( + tags=["routing group management"], +) + + +def _get_llm_router(): + """Get the LLM router from the proxy server.""" + from litellm.proxy.proxy_server import llm_router + + return llm_router + + +def _record_to_config(record) -> RoutingGroupConfig: + """Convert a Prisma routing group record to a RoutingGroupConfig.""" + return RoutingGroupConfig( + routing_group_id=record.routing_group_id, + routing_group_name=record.routing_group_name, + description=record.description, + routing_strategy=record.routing_strategy, + deployments=[ + RoutingGroupDeployment(**d) for d in (record.deployments or []) + ], + fallback_config=record.fallback_config, + retry_config=record.retry_config, + cooldown_config=record.cooldown_config, + settings=record.settings, + assigned_team_ids=list(record.assigned_team_ids or []), + assigned_key_ids=list(record.assigned_key_ids or []), + is_active=record.is_active, + ) + + +VALID_ROUTING_STRATEGIES = frozenset( + { + "priority-failover", + "simple-shuffle", + "least-busy", + "latency-based-routing", + "cost-based-routing", + "usage-based-routing-v2", + "weighted", + } +) + + +async def _sync_routing_group_to_router(config: RoutingGroupConfig) -> None: + """ + Translate a RoutingGroupConfig into live Router configuration. + + For priority-failover: creates tiered model groups + fallback chain. + The fallback chain implements the per-group ordering independently of + the shared router's routing_strategy — this strategy is fully isolated + per group. + + For all other strategies: deployments are added to a single model group + under the routing_group_name. The shared router's routing_strategy is + NOT changed here — that is the proxy-level concern set in the config YAML. + Each group's deployments are correctly scoped to their own model group name; + the router picks between them using whatever strategy the proxy was started + with. For "weighted" groups, per-deployment weights are stored in + litellm_params so simple_shuffle (if active) will honour them + automatically. + """ + llm_router = _get_llm_router() + if llm_router is None: + verbose_proxy_logger.warning( + "No LLM router available, routing group will be applied on next startup" + ) + return + + strategy = config.routing_strategy + group_name = config.routing_group_name + + if strategy == "priority-failover": + priority_groups: dict = defaultdict(list) + for dep in config.deployments: + p = dep.priority if dep.priority is not None else 999 + priority_groups[p].append(dep) + + sorted_priorities = sorted(priority_groups.keys()) + group_names_by_priority = [] + + for i, priority in enumerate(sorted_priorities): + if i == 0: + # Primary group uses the routing_group_name directly + tier_group_name = group_name + else: + tier_group_name = f"{group_name}__fallback_p{priority}" + group_names_by_priority.append(tier_group_name) + + for dep in priority_groups[priority]: + try: + deployment_dict = { + "model_name": tier_group_name, + "litellm_params": { + "model": dep.model_name, + }, + "model_info": { + "id": dep.model_id, + }, + } + llm_router.add_deployment( + deployment=litellm.types.router.Deployment(**deployment_dict) + ) + except Exception as e: + verbose_proxy_logger.debug( + f"Could not add deployment {dep.model_id} to router: {e}" + ) + + # Wire fallback chain + if len(group_names_by_priority) > 1: + primary = group_names_by_priority[0] + fallbacks = group_names_by_priority[1:] + existing_fallbacks = llm_router.fallbacks or [] + # Remove old entry for this group if it exists + existing_fallbacks = [f for f in existing_fallbacks if primary not in f] + existing_fallbacks.append({primary: fallbacks}) + llm_router.fallbacks = existing_fallbacks + + else: + # All strategies except priority-failover: single model group. + for dep in config.deployments: + try: + litellm_params: dict = {"model": dep.model_name} + if strategy == "weighted" and dep.weight is not None: + # simple_shuffle reads `weight` from litellm_params and + # does a proportional random pick — this is how "weighted" + # is actually honoured at call time. + litellm_params["weight"] = dep.weight + + deployment_dict = { + "model_name": group_name, + "litellm_params": litellm_params, + "model_info": { + "id": dep.model_id, + }, + } + llm_router.add_deployment( + deployment=litellm.types.router.Deployment(**deployment_dict) + ) + except Exception as e: + verbose_proxy_logger.debug( + f"Could not add deployment {dep.model_id} to router: {e}" + ) + + + +# --------------------------------------------------------------------------- +# CRUD endpoints +# --------------------------------------------------------------------------- + + +@router.post( + "/v1/routing_group", + tags=["routing_group"], + dependencies=[Depends(user_api_key_auth)], + response_model=RoutingGroupConfig, + status_code=status.HTTP_201_CREATED, + summary="Create a routing group", +) +async def create_routing_group( + data: RoutingGroupConfig, + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), +) -> RoutingGroupConfig: + """Create a new routing group.""" + prisma_client = get_prisma_client_or_throw( + CommonProxyErrors.db_not_connected_error.value + ) + + if data.routing_strategy not in VALID_ROUTING_STRATEGIES: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=f"Invalid routing_strategy '{data.routing_strategy}'. Must be one of: {sorted(VALID_ROUTING_STRATEGIES)}", + ) + + if not data.deployments: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="deployments must contain at least one deployment", + ) + + caller = user_api_key_dict.user_id or "unknown" + routing_group_id = str(uuid.uuid4()) + + try: + created = await prisma_client.db.litellm_routinggrouptable.create( + data={ + "routing_group_id": routing_group_id, + "routing_group_name": data.routing_group_name, + "description": data.description, + "routing_strategy": data.routing_strategy, + "deployments": [d.model_dump() for d in data.deployments], + "fallback_config": data.fallback_config or {}, + "retry_config": data.retry_config or {}, + "cooldown_config": data.cooldown_config or {}, + "settings": data.settings or {}, + "assigned_team_ids": data.assigned_team_ids or [], + "assigned_key_ids": data.assigned_key_ids or [], + "is_active": data.is_active, + "created_by": caller, + "updated_by": caller, + } + ) + except Exception as e: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=f"Failed to create routing group: {str(e)}", + ) + + result = _record_to_config(created) + await _sync_routing_group_to_router(result) + return result + + +@router.get( + "/v1/routing_group", + tags=["routing_group"], + dependencies=[Depends(user_api_key_auth)], + response_model=RoutingGroupListResponse, + summary="List routing groups", +) +async def list_routing_groups( + page: int = 1, + size: int = 50, + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), +) -> RoutingGroupListResponse: + """List all routing groups.""" + prisma_client = get_prisma_client_or_throw( + CommonProxyErrors.db_not_connected_error.value + ) + + skip = (page - 1) * size + rows = await prisma_client.db.litellm_routinggrouptable.find_many( + skip=skip, + take=size, + order={"created_at": "desc"}, + ) + total = await prisma_client.db.litellm_routinggrouptable.count() + + groups = [_record_to_config(r) for r in rows] + return RoutingGroupListResponse( + routing_groups=groups, total=total, page=page, size=size + ) + + +@router.get( + "/v1/routing_group/{routing_group_id}", + tags=["routing_group"], + dependencies=[Depends(user_api_key_auth)], + response_model=RoutingGroupConfig, + summary="Get a routing group", +) +async def get_routing_group( + routing_group_id: str, + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), +) -> RoutingGroupConfig: + """Get a specific routing group by ID.""" + prisma_client = get_prisma_client_or_throw( + CommonProxyErrors.db_not_connected_error.value + ) + + row = await prisma_client.db.litellm_routinggrouptable.find_unique( + where={"routing_group_id": routing_group_id} + ) + if row is None: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=f"Routing group '{routing_group_id}' not found", + ) + + return _record_to_config(row) + + +@router.put( + "/v1/routing_group/{routing_group_id}", + tags=["routing_group"], + dependencies=[Depends(user_api_key_auth)], + response_model=RoutingGroupConfig, + summary="Update a routing group", +) +async def update_routing_group( + routing_group_id: str, + data: RoutingGroupConfig, + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), +) -> RoutingGroupConfig: + """Update an existing routing group.""" + prisma_client = get_prisma_client_or_throw( + CommonProxyErrors.db_not_connected_error.value + ) + + caller = user_api_key_dict.user_id or "unknown" + + try: + updated = await prisma_client.db.litellm_routinggrouptable.update( + where={"routing_group_id": routing_group_id}, + data={ + "routing_group_name": data.routing_group_name, + "description": data.description, + "routing_strategy": data.routing_strategy, + "deployments": [d.model_dump() for d in data.deployments], + "fallback_config": data.fallback_config or {}, + "retry_config": data.retry_config or {}, + "cooldown_config": data.cooldown_config or {}, + "settings": data.settings or {}, + "assigned_team_ids": data.assigned_team_ids or [], + "assigned_key_ids": data.assigned_key_ids or [], + "is_active": data.is_active, + "updated_by": caller, + }, + ) + except Exception as e: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=f"Routing group not found or update failed: {str(e)}", + ) + + result = _record_to_config(updated) + await _sync_routing_group_to_router(result) + return result + + +@router.delete( + "/v1/routing_group/{routing_group_id}", + tags=["routing_group"], + dependencies=[Depends(user_api_key_auth)], + summary="Delete a routing group", +) +async def delete_routing_group( + routing_group_id: str, + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), +): + """Delete a routing group.""" + prisma_client = get_prisma_client_or_throw( + CommonProxyErrors.db_not_connected_error.value + ) + + try: + await prisma_client.db.litellm_routinggrouptable.delete( + where={"routing_group_id": routing_group_id} + ) + except Exception as e: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=f"Routing group not found: {str(e)}", + ) + + return {"routing_group_id": routing_group_id, "deleted": True} + + +@router.post( + "/v1/routing_group/{routing_group_id}/test", + tags=["routing_group"], + dependencies=[Depends(user_api_key_auth)], + response_model=RoutingGroupTestResult, + summary="Test a routing group with a single request", +) +async def test_routing_group( + routing_group_id: str, + messages: Optional[List] = None, + mock: bool = False, + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), +) -> RoutingGroupTestResult: + """Send a single test request through a routing group and return the routing trace.""" + prisma_client = get_prisma_client_or_throw( + CommonProxyErrors.db_not_connected_error.value + ) + + row = await prisma_client.db.litellm_routinggrouptable.find_unique( + where={"routing_group_id": routing_group_id} + ) + if row is None: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=f"Routing group '{routing_group_id}' not found", + ) + + llm_router = _get_llm_router() + if llm_router is None: + raise HTTPException( + status_code=status.HTTP_503_SERVICE_UNAVAILABLE, + detail="LLM router not initialized", + ) + + from litellm.proxy.routing_group_utils.test_engine import RoutingGroupTestEngine + + config = _record_to_config(row) + engine = RoutingGroupTestEngine() + return await engine.test_single_request( + routing_group_name=config.routing_group_name, + router=llm_router, + messages=messages, + mock=mock, + ) + + +@router.post( + "/v1/routing_group/{routing_group_id}/simulate", + tags=["routing_group"], + dependencies=[Depends(user_api_key_auth)], + response_model=RoutingGroupSimulationResult, + summary="Simulate traffic through a routing group", +) +async def simulate_routing_group( + routing_group_id: str, + num_requests: int = 100, + concurrency: int = 10, + mock: bool = True, + failure_injection: Optional[FailureInjectionConfig] = None, + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), +) -> RoutingGroupSimulationResult: + """Simulate N requests through a routing group and return traffic distribution statistics.""" + prisma_client = get_prisma_client_or_throw( + CommonProxyErrors.db_not_connected_error.value + ) + + row = await prisma_client.db.litellm_routinggrouptable.find_unique( + where={"routing_group_id": routing_group_id} + ) + if row is None: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=f"Routing group '{routing_group_id}' not found", + ) + + llm_router = _get_llm_router() + if llm_router is None: + raise HTTPException( + status_code=status.HTTP_503_SERVICE_UNAVAILABLE, + detail="LLM router not initialized", + ) + + from litellm.proxy.routing_group_utils.test_engine import RoutingGroupTestEngine + + config = _record_to_config(row) + engine = RoutingGroupTestEngine() + return await engine.simulate_traffic( + routing_group_name=config.routing_group_name, + router=llm_router, + routing_group_config=config, + num_requests=num_requests, + concurrency=concurrency, + mock=mock, + failure_injection=failure_injection, + ) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 3390caa8731..dd65bc39b46 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -400,6 +400,9 @@ from litellm.proxy.management_endpoints.project_endpoints import ( from litellm.proxy.management_endpoints.router_settings_endpoints import ( router as router_settings_router, ) +from litellm.proxy.management_endpoints.routing_group_endpoints import ( + router as routing_group_router, +) from litellm.proxy.management_endpoints.scim.scim_v2 import scim_router from litellm.proxy.management_endpoints.tag_management_endpoints import ( router as tag_management_router, @@ -4429,9 +4432,38 @@ class ProxyConfig: prisma_client=prisma_client, proxy_config=self ) + await self._load_routing_groups_from_db(prisma_client=prisma_client) + if self._should_load_db_object(object_type="semantic_filter_settings"): await self._init_semantic_filter_settings_in_db(prisma_client=prisma_client) + async def _load_routing_groups_from_db(self, prisma_client: PrismaClient) -> None: + """Load all active routing groups from DB and sync to router.""" + from litellm.proxy.management_endpoints.routing_group_endpoints import ( + _sync_routing_group_to_router, + ) + from litellm.types.router import RoutingGroupConfig, RoutingGroupDeployment + + try: + groups = await prisma_client.db.litellm_routinggrouptable.find_many( + where={"is_active": True} + ) + for g in groups: + config = RoutingGroupConfig( + routing_group_id=g.routing_group_id, + routing_group_name=g.routing_group_name, + routing_strategy=g.routing_strategy, + deployments=[ + RoutingGroupDeployment(**d) for d in (g.deployments or []) + ], + ) + await _sync_routing_group_to_router(config) + verbose_proxy_logger.debug( + f"Loaded {len(groups)} routing groups from DB" + ) + except Exception as e: + verbose_proxy_logger.debug(f"Could not load routing groups from DB: {e}") + async def _init_semantic_filter_settings_in_db(self, prisma_client: PrismaClient): """ Initialize MCP semantic filter settings from database. @@ -13056,5 +13088,6 @@ async def dynamic_mcp_route(mcp_server_name: str, request: Request): app.mount(path=BASE_MCP_ROUTE, app=mcp_app) +app.include_router(routing_group_router) app.include_router(mcp_rest_endpoints_router) app.include_router(mcp_discoverable_endpoints_router) diff --git a/litellm/proxy/routing_group_utils/__init__.py b/litellm/proxy/routing_group_utils/__init__.py new file mode 100644 index 00000000000..8b137891791 --- /dev/null +++ b/litellm/proxy/routing_group_utils/__init__.py @@ -0,0 +1 @@ + diff --git a/litellm/proxy/routing_group_utils/routing_trace_callback.py b/litellm/proxy/routing_group_utils/routing_trace_callback.py new file mode 100644 index 00000000000..b87b750097b --- /dev/null +++ b/litellm/proxy/routing_group_utils/routing_trace_callback.py @@ -0,0 +1,86 @@ +""" +Routing trace callback for capturing routing decisions during test/simulation. +""" + +from typing import Any, List, Optional + +from litellm._logging import verbose_proxy_logger +from litellm.integrations.custom_logger import CustomLogger +from litellm.types.router import RoutingTrace + + +class RoutingTraceCallback(CustomLogger): + """ + Temporary callback registered during routing group test/simulation. + + Captures which deployments were tried, outcomes, latencies, and fallback depth. + Attach to litellm.callbacks before a call and detach after to capture traces. + """ + + def __init__(self) -> None: + self.traces: List[RoutingTrace] = [] + + def _extract_trace_from_kwargs( + self, + kwargs: dict, + start_time: Any, + end_time: Any, + status: str, + exception: Optional[Exception] = None, + ) -> RoutingTrace: + litellm_params = kwargs.get("litellm_params") or {} + metadata = litellm_params.get("metadata") or {} + + # Calculate latency + try: + if hasattr(start_time, "timestamp") and hasattr(end_time, "timestamp"): + latency_ms = (end_time.timestamp() - start_time.timestamp()) * 1000 + else: + latency_ms = float(end_time - start_time) * 1000 + except Exception: + latency_ms = 0.0 + + fallback_depth = metadata.get("fallback_depth", 0) + if not isinstance(fallback_depth, int): + fallback_depth = 0 + + deployment_id = ( + metadata.get("model_id") + or kwargs.get("model_id") + or kwargs.get("litellm_call_id", "unknown") + ) + + return RoutingTrace( + deployment_id=str(deployment_id), + deployment_name=kwargs.get("model", "unknown"), + provider=kwargs.get("custom_llm_provider") + or litellm_params.get("custom_llm_provider", "unknown"), + latency_ms=latency_ms, + was_fallback=fallback_depth > 0, + fallback_depth=fallback_depth, + status=status, + error_message=str(exception) if exception else None, + ) + + async def async_log_success_event( + self, kwargs: dict, response_obj: Any, start_time: Any, end_time: Any + ) -> None: + trace = self._extract_trace_from_kwargs( + kwargs, start_time, end_time, status="success" + ) + self.traces.append(trace) + verbose_proxy_logger.debug( + f"RoutingTraceCallback: success on {trace.deployment_name} ({trace.latency_ms:.0f}ms)" + ) + + async def async_log_failure_event( + self, kwargs: dict, response_obj: Any, start_time: Any, end_time: Any + ) -> None: + exception = kwargs.get("exception") + trace = self._extract_trace_from_kwargs( + kwargs, start_time, end_time, status="error", exception=exception + ) + self.traces.append(trace) + verbose_proxy_logger.debug( + f"RoutingTraceCallback: failure on {trace.deployment_name} - {trace.error_message}" + ) diff --git a/litellm/proxy/routing_group_utils/test_engine.py b/litellm/proxy/routing_group_utils/test_engine.py new file mode 100644 index 00000000000..f9fb6d760be --- /dev/null +++ b/litellm/proxy/routing_group_utils/test_engine.py @@ -0,0 +1,321 @@ +""" +Test and simulation engine for routing groups. + +Executes test requests through a routing group and collects routing traces +that drive the Live Tester visualization in the UI. +""" + +import asyncio +import random +import time +from typing import TYPE_CHECKING, Any, Dict, List, Optional, cast + +from litellm._logging import verbose_proxy_logger +from litellm.types.llms.openai import AllMessageValues +from litellm.types.router import ( + DeploymentTrafficStats, + FailureInjectionConfig, + FlowStep, + RoutingGroupConfig, + RoutingGroupSimulationResult, + RoutingGroupTestResult, + RoutingTrace, +) +from litellm.types.utils import Choices, ModelResponse + +if TYPE_CHECKING: + from litellm.router import Router + + +_DEFAULT_MESSAGES: List[Any] = [{"role": "user", "content": "Hello, respond with one word."}] + + +def _empty_dep_stats() -> Dict[str, Any]: + return { + "deployment_name": "unknown", + "provider": "unknown", + "request_count": 0, + "success_count": 0, + "failure_count": 0, + "latencies": [], + "priority": None, + "weight": None, + } + + +class RoutingGroupTestEngine: + """ + Executes test/simulation requests through a routing group and returns + structured results for the Live Tester visualization. + """ + + async def test_single_request( + self, + routing_group_name: str, + router: "Router", + messages: Optional[List[AllMessageValues]] = None, + mock: bool = False, + ) -> RoutingGroupTestResult: + """ + Send one request through the routing group and return the routing trace. + + Uses a RoutingTraceCallback temporarily attached to litellm.callbacks + to capture which deployments were tried and their outcomes. + """ + import litellm + from litellm.proxy.routing_group_utils.routing_trace_callback import ( + RoutingTraceCallback, + ) + + if messages is None: + messages = cast(List[AllMessageValues], _DEFAULT_MESSAGES) + + trace_callback = RoutingTraceCallback() + + # Attach callback temporarily + litellm.callbacks.append(trace_callback) + + start = time.monotonic() + success = False + response_text: Optional[str] = None + + try: + if mock: + response = await router.acompletion( + model=routing_group_name, + messages=messages, + mock_response="Mock routing group test response", + ) + else: + response = await router.acompletion( + model=routing_group_name, + messages=messages, + ) + success = True + try: + _choices = cast(List[Choices], cast(ModelResponse, response).choices) + response_text = _choices[0].message.content + except Exception: + response_text = str(response) + except Exception as e: + verbose_proxy_logger.debug( + f"RoutingGroupTestEngine: request failed for group '{routing_group_name}': {e}" + ) + finally: + # Always detach callback + try: + litellm.callbacks.remove(trace_callback) + except ValueError: + pass + + total_latency_ms = (time.monotonic() - start) * 1000 + + # Determine final deployment from last success trace + final_deployment = "unknown" + final_provider = "unknown" + success_traces = [t for t in trace_callback.traces if t.status == "success"] + if success_traces: + final = success_traces[-1] + final_deployment = final.deployment_name + final_provider = final.provider + + return RoutingGroupTestResult( + success=success, + response_text=response_text, + traces=trace_callback.traces, + total_latency_ms=total_latency_ms, + final_deployment=final_deployment, + final_provider=final_provider, + ) + + async def simulate_traffic( + self, + routing_group_name: str, + router: "Router", + routing_group_config: RoutingGroupConfig, + num_requests: int = 100, + concurrency: int = 10, + mock: bool = True, + failure_injection: Optional[FailureInjectionConfig] = None, + ) -> RoutingGroupSimulationResult: + """ + Simulate N concurrent requests through the routing group. + + Returns aggregated traffic distribution statistics and flow data + for the Live Tester visualization. + """ + import litellm + from litellm.proxy.routing_group_utils.routing_trace_callback import ( + RoutingTraceCallback, + ) + + trace_callback = RoutingTraceCallback() + litellm.callbacks.append(trace_callback) + + semaphore = asyncio.Semaphore(concurrency) + + async def _one_request() -> bool: + async with semaphore: + try: + if mock: + # Optionally inject failures via mock_response that raises + if failure_injection: + # Pick a deployment deterministically based on routing group config + # and check if we should inject a failure + # We can't know which deployment will be picked before the call, + # so we use a probabilistic approach: + # average failure rate across all deployments as a proxy + rates = list( + failure_injection.deployment_failure_rates.values() + ) + avg_rate = sum(rates) / len(rates) if rates else 0.0 + if random.random() < avg_rate: + raise Exception( + "Simulated failure (failure injection)" + ) + await router.acompletion( + model=routing_group_name, + messages=cast(List[AllMessageValues], _DEFAULT_MESSAGES), + mock_response="Simulated response", + ) + else: + await router.acompletion( + model=routing_group_name, + messages=cast(List[AllMessageValues], _DEFAULT_MESSAGES), + ) + return True + except Exception: + return False + + total_latency_start = time.monotonic() + results = await asyncio.gather( + *[_one_request() for _ in range(num_requests)], + return_exceptions=True, + ) + _ = time.monotonic() - total_latency_start # total wall time (unused but kept for future) + + litellm.callbacks.remove(trace_callback) + + successful = sum(1 for r in results if r is True) + failed = num_requests - successful + + # Aggregate per-deployment stats from traces + per_dep: Dict[str, Dict[str, Any]] = {} + + for trace in trace_callback.traces: + dep_id = trace.deployment_id or trace.deployment_name + if dep_id not in per_dep: + per_dep[dep_id] = _empty_dep_stats() + per_dep[dep_id]["deployment_name"] = trace.deployment_name + per_dep[dep_id]["provider"] = trace.provider + per_dep[dep_id]["request_count"] = cast(int, per_dep[dep_id]["request_count"]) + 1 + cast(List[float], per_dep[dep_id]["latencies"]).append(trace.latency_ms) + if trace.status == "success": + per_dep[dep_id]["success_count"] = cast(int, per_dep[dep_id]["success_count"]) + 1 + else: + per_dep[dep_id]["failure_count"] = cast(int, per_dep[dep_id]["failure_count"]) + 1 + + # Enrich with priority/weight from config + dep_lookup = {d.model_id: d for d in routing_group_config.deployments} + for dep_id, stats in per_dep.items(): + if dep_id in dep_lookup: + dep = dep_lookup[dep_id] + stats["priority"] = dep.priority + stats["weight"] = dep.weight + + total_traced: int = sum(cast(int, s["request_count"]) for s in per_dep.values()) + + traffic_distribution = [ + DeploymentTrafficStats( + deployment_id=dep_id, + deployment_name=cast(str, stats["deployment_name"]), + provider=cast(str, stats["provider"]), + request_count=cast(int, stats["request_count"]), + success_count=cast(int, stats["success_count"]), + failure_count=cast(int, stats["failure_count"]), + avg_latency_ms=( + sum(cast(List[float], stats["latencies"])) / len(cast(List[float], stats["latencies"])) + if stats["latencies"] + else 0.0 + ), + percent_of_total=( + cast(int, stats["request_count"]) / total_traced * 100 + if total_traced > 0 + else 0.0 + ), + priority=cast(Optional[int], stats["priority"]), + weight=cast(Optional[int], stats["weight"]), + ) + for dep_id, stats in per_dep.items() + ] + + # Sort: primary (priority 1 or highest traffic) first + traffic_distribution.sort( + key=lambda x: (x.priority or 999, -x.request_count) + ) + + # Build flow data for priority-failover strategy + flow_data: Optional[List[FlowStep]] = None + if routing_group_config.routing_strategy == "priority-failover": + flow_data = _build_flow_data( + trace_callback.traces, routing_group_config, traffic_distribution + ) + + fallback_count = sum( + 1 for t in trace_callback.traces if t.was_fallback + ) + + all_latencies = [t.latency_ms for t in trace_callback.traces] + avg_latency = sum(all_latencies) / len(all_latencies) if all_latencies else 0.0 + + return RoutingGroupSimulationResult( + total_requests=num_requests, + successful_requests=successful, + failed_requests=failed, + avg_latency_ms=avg_latency, + fallback_count=fallback_count, + traffic_distribution=traffic_distribution, + flow_data=flow_data, + ) + + +def _build_flow_data( + _traces: List[RoutingTrace], + _config: RoutingGroupConfig, + traffic_distribution: List[DeploymentTrafficStats], +) -> List[FlowStep]: + """ + Build flow steps for the priority-failover visualization. + + For each deployment in priority order, create a FlowStep showing + how many requests went to it and why (primary vs fallback). + """ + flow_steps: List[FlowStep] = [] + + sorted_deps = sorted( + traffic_distribution, + key=lambda x: (x.priority or 999, -x.request_count), + ) + + for i, dep in enumerate(sorted_deps): + if i == 0: + flow_steps.append( + FlowStep( + from_deployment=None, + to_deployment=dep.deployment_name, + request_count=dep.request_count, + reason="primary", + ) + ) + else: + prev = sorted_deps[i - 1] + flow_steps.append( + FlowStep( + from_deployment=prev.deployment_name, + to_deployment=dep.deployment_name, + request_count=dep.request_count, + reason="fallback_error", + ) + ) + + return flow_steps diff --git a/litellm/proxy/schema.prisma b/litellm/proxy/schema.prisma index f18556ac329..75f271f8790 100644 --- a/litellm/proxy/schema.prisma +++ b/litellm/proxy/schema.prisma @@ -1112,3 +1112,23 @@ model LiteLLM_ClaudeCodePluginTable { @@map("LiteLLM_ClaudeCodePluginTable") } + +// Routing Groups table for storing named routing pipelines +model LiteLLM_RoutingGroupTable { + routing_group_id String @id @default(uuid()) + routing_group_name String @unique + description String? + routing_strategy String + deployments Json + fallback_config Json? @default("{}") + retry_config Json? @default("{}") + cooldown_config Json? @default("{}") + settings Json? @default("{}") + assigned_team_ids String[] @default([]) + assigned_key_ids String[] @default([]) + is_active Boolean @default(true) + created_at DateTime @default(now()) + created_by String + updated_at DateTime @default(now()) @updatedAt + updated_by String +} diff --git a/litellm/types/router.py b/litellm/types/router.py index aa4d7bd9a97..60843de9963 100644 --- a/litellm/types/router.py +++ b/litellm/types/router.py @@ -866,3 +866,106 @@ class PreRoutingHookResponse(BaseModel): model: str messages: Optional[List[Dict[str, str]]] + + +class RoutingGroupDeployment(BaseModel): + """A deployment reference within a routing group.""" + + model_id: str + model_name: str + provider: str + priority: Optional[int] = None + weight: Optional[int] = None + display_name: Optional[str] = None + + +class RoutingGroupConfig(BaseModel): + """Full routing group configuration.""" + + routing_group_id: Optional[str] = None + routing_group_name: str + description: Optional[str] = None + routing_strategy: str + deployments: List[RoutingGroupDeployment] + fallback_config: Optional[dict] = None + retry_config: Optional[dict] = None + cooldown_config: Optional[dict] = None + settings: Optional[dict] = None + assigned_team_ids: Optional[List[str]] = None + assigned_key_ids: Optional[List[str]] = None + is_active: bool = True + + +class RoutingGroupListResponse(BaseModel): + routing_groups: List[RoutingGroupConfig] + total: int + page: int + size: int + + +class RoutingTrace(BaseModel): + """A single routing decision record for test/simulation.""" + + deployment_id: str + deployment_name: str + provider: str + latency_ms: float + was_fallback: bool + fallback_depth: int = 0 + status: str # "success" or "error" + error_message: Optional[str] = None + + +class RoutingGroupTestResult(BaseModel): + """Result of a single test request through a routing group.""" + + success: bool + response_text: Optional[str] = None + traces: List[RoutingTrace] + total_latency_ms: float + final_deployment: str + final_provider: str + + +class DeploymentTrafficStats(BaseModel): + """Per-deployment statistics from a simulation.""" + + deployment_id: str + deployment_name: str + provider: str + request_count: int + success_count: int + failure_count: int + avg_latency_ms: float + percent_of_total: float + priority: Optional[int] = None + weight: Optional[int] = None + + +class FlowStep(BaseModel): + """For priority-failover: represents traffic flow between deployments.""" + + from_deployment: Optional[str] = None + to_deployment: str + request_count: int + reason: str # "primary", "fallback_error", "fallback_rate_limit" + + +class RoutingGroupSimulationResult(BaseModel): + """Result of a traffic simulation through a routing group.""" + + total_requests: int + successful_requests: int + failed_requests: int + avg_latency_ms: float + fallback_count: int + traffic_distribution: List[DeploymentTrafficStats] + flow_data: Optional[List[FlowStep]] = None + + +class FailureInjectionConfig(BaseModel): + """Configuration for injecting failures during simulation.""" + + deployment_failure_rates: Dict[ + str, float + ] # {deployment_id: failure_probability 0.0-1.0} diff --git a/tests/litellm/proxy/test_routing_group_test_engine.py b/tests/litellm/proxy/test_routing_group_test_engine.py new file mode 100644 index 00000000000..d076d4cbc81 --- /dev/null +++ b/tests/litellm/proxy/test_routing_group_test_engine.py @@ -0,0 +1,529 @@ +""" +Unit tests for RoutingGroupTestEngine and RoutingTraceCallback. +""" + +import datetime +from typing import List +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest + +from litellm.proxy.routing_group_utils.routing_trace_callback import ( + RoutingTraceCallback, +) +from litellm.proxy.routing_group_utils.test_engine import ( + RoutingGroupTestEngine, + _build_flow_data, +) +from litellm.types.router import ( + DeploymentTrafficStats, + RoutingGroupConfig, + RoutingGroupDeployment, + RoutingTrace, +) + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +def _make_kwargs( + model: str = "test-model", + provider: str = "openai", + call_id: str = "call-123", + fallback_depth: int = 0, + exception=None, +) -> dict: + return { + "model": model, + "custom_llm_provider": provider, + "litellm_call_id": call_id, + "exception": exception, + "litellm_params": { + "metadata": {"fallback_depth": fallback_depth}, + "custom_llm_provider": provider, + }, + } + + +def _make_mock_response(content: str = "hello") -> MagicMock: + response = MagicMock() + response.choices = [MagicMock()] + response.choices[0].message = MagicMock() + response.choices[0].message.content = content + return response + + +def _make_routing_group_config(strategy: str = "priority-failover") -> RoutingGroupConfig: + return RoutingGroupConfig( + routing_group_name="test-group", + routing_strategy=strategy, + deployments=[ + RoutingGroupDeployment( + model_id="id-primary", + model_name="nebius/llama-70b", + provider="nebius", + priority=1, + ), + RoutingGroupDeployment( + model_id="id-fallback", + model_name="fireworks/llama-70b", + provider="fireworks_ai", + priority=2, + ), + ], + ) + + +# --------------------------------------------------------------------------- +# RoutingTraceCallback tests +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_routing_trace_callback_records_success(): + """async_log_success_event should append a success trace.""" + callback = RoutingTraceCallback() + + start = datetime.datetime(2024, 1, 1, 0, 0, 0) + end = datetime.datetime(2024, 1, 1, 0, 0, 1) # 1 second later + + kwargs = _make_kwargs(model="gpt-4o", provider="openai") + await callback.async_log_success_event( + kwargs=kwargs, response_obj=None, start_time=start, end_time=end + ) + + assert len(callback.traces) == 1 + trace = callback.traces[0] + assert trace.status == "success" + assert trace.deployment_name == "gpt-4o" + assert trace.provider == "openai" + assert trace.latency_ms == pytest.approx(1000.0, abs=1.0) + assert trace.was_fallback is False + assert trace.error_message is None + + +@pytest.mark.asyncio +async def test_routing_trace_callback_records_failure(): + """async_log_failure_event should append an error trace with message.""" + callback = RoutingTraceCallback() + + start = datetime.datetime(2024, 1, 1, 0, 0, 0) + end = datetime.datetime(2024, 1, 1, 0, 0, 0, 500000) # 0.5 seconds + + exc = RuntimeError("rate limit exceeded") + kwargs = _make_kwargs(model="claude-3", provider="anthropic", exception=exc) + await callback.async_log_failure_event( + kwargs=kwargs, response_obj=None, start_time=start, end_time=end + ) + + assert len(callback.traces) == 1 + trace = callback.traces[0] + assert trace.status == "error" + assert trace.deployment_name == "claude-3" + assert trace.provider == "anthropic" + assert trace.error_message == "rate limit exceeded" + assert trace.was_fallback is False + + +@pytest.mark.asyncio +async def test_routing_trace_callback_records_fallback(): + """Traces with fallback_depth > 0 should have was_fallback=True.""" + callback = RoutingTraceCallback() + + start = datetime.datetime(2024, 1, 1, 0, 0, 0) + end = datetime.datetime(2024, 1, 1, 0, 0, 1) + + kwargs = _make_kwargs(model="backup-model", provider="azure", fallback_depth=1) + await callback.async_log_success_event( + kwargs=kwargs, response_obj=None, start_time=start, end_time=end + ) + + trace = callback.traces[0] + assert trace.was_fallback is True + assert trace.fallback_depth == 1 + + +@pytest.mark.asyncio +async def test_routing_trace_callback_multiple_events(): + """Multiple events should all be recorded in order.""" + callback = RoutingTraceCallback() + + start = datetime.datetime(2024, 1, 1, 0, 0, 0) + end = datetime.datetime(2024, 1, 1, 0, 0, 1) + + for i in range(3): + kwargs = _make_kwargs(model=f"model-{i}", provider="openai") + await callback.async_log_success_event( + kwargs=kwargs, response_obj=None, start_time=start, end_time=end + ) + + assert len(callback.traces) == 3 + for i, trace in enumerate(callback.traces): + assert trace.deployment_name == f"model-{i}" + + +# --------------------------------------------------------------------------- +# RoutingGroupTestEngine.test_single_request tests +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_engine_test_single_request_success(): + """test_single_request with a mock router that succeeds should return success=True.""" + mock_router = AsyncMock() + mock_router.acompletion = AsyncMock(return_value=_make_mock_response("pong")) + + engine = RoutingGroupTestEngine() + + with patch("litellm.callbacks", []): + result = await engine.test_single_request( + routing_group_name="test-group", + router=mock_router, + mock=True, + ) + + assert result.success is True + assert result.response_text == "pong" + assert result.total_latency_ms >= 0 + + +@pytest.mark.asyncio +async def test_engine_test_single_request_failure(): + """test_single_request when router raises should return success=False.""" + mock_router = AsyncMock() + mock_router.acompletion = AsyncMock(side_effect=Exception("provider error")) + + engine = RoutingGroupTestEngine() + + with patch("litellm.callbacks", []): + result = await engine.test_single_request( + routing_group_name="test-group", + router=mock_router, + ) + + assert result.success is False + assert result.response_text is None + + +@pytest.mark.asyncio +async def test_engine_test_single_request_callback_detached_on_success(): + """The trace callback should be removed from litellm.callbacks after the call.""" + mock_router = AsyncMock() + mock_router.acompletion = AsyncMock(return_value=_make_mock_response("ok")) + + engine = RoutingGroupTestEngine() + callbacks_list: list = [] + + with patch("litellm.callbacks", callbacks_list): + await engine.test_single_request( + routing_group_name="test-group", + router=mock_router, + mock=True, + ) + + # Callback should have been removed + from litellm.proxy.routing_group_utils.routing_trace_callback import ( + RoutingTraceCallback, + ) + assert not any(isinstance(c, RoutingTraceCallback) for c in callbacks_list) + + +@pytest.mark.asyncio +async def test_engine_test_single_request_callback_detached_on_failure(): + """The trace callback should be removed even when the router call raises.""" + mock_router = AsyncMock() + mock_router.acompletion = AsyncMock(side_effect=Exception("boom")) + + engine = RoutingGroupTestEngine() + callbacks_list: list = [] + + with patch("litellm.callbacks", callbacks_list): + await engine.test_single_request( + routing_group_name="test-group", + router=mock_router, + ) + + from litellm.proxy.routing_group_utils.routing_trace_callback import ( + RoutingTraceCallback, + ) + assert not any(isinstance(c, RoutingTraceCallback) for c in callbacks_list) + + +@pytest.mark.asyncio +async def test_engine_test_single_request_uses_custom_messages(): + """Custom messages should be passed through to the router.""" + mock_router = AsyncMock() + mock_router.acompletion = AsyncMock(return_value=_make_mock_response("custom")) + + engine = RoutingGroupTestEngine() + custom_messages = [{"role": "user", "content": "custom prompt"}] + + with patch("litellm.callbacks", []): + await engine.test_single_request( + routing_group_name="my-group", + router=mock_router, + messages=custom_messages, + mock=True, + ) + + call_kwargs = mock_router.acompletion.call_args + assert call_kwargs.kwargs.get("messages") == custom_messages or ( + len(call_kwargs.args) > 1 and call_kwargs.args[1] == custom_messages + ) + + +# --------------------------------------------------------------------------- +# RoutingGroupTestEngine.simulate_traffic tests +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_engine_simulate_traffic_total_requests(): + """simulate_traffic should return total_requests matching the requested count.""" + mock_router = AsyncMock() + mock_router.acompletion = AsyncMock(return_value=_make_mock_response("sim")) + + engine = RoutingGroupTestEngine() + config = _make_routing_group_config() + + with patch("litellm.callbacks", []): + result = await engine.simulate_traffic( + routing_group_name="test-group", + router=mock_router, + routing_group_config=config, + num_requests=10, + concurrency=5, + mock=True, + ) + + assert result.total_requests == 10 + assert result.successful_requests + result.failed_requests == 10 + + +@pytest.mark.asyncio +async def test_engine_simulate_traffic_all_failures(): + """When all requests fail, successful_requests should be 0.""" + mock_router = AsyncMock() + mock_router.acompletion = AsyncMock(side_effect=Exception("always fails")) + + engine = RoutingGroupTestEngine() + config = _make_routing_group_config() + + with patch("litellm.callbacks", []): + result = await engine.simulate_traffic( + routing_group_name="test-group", + router=mock_router, + routing_group_config=config, + num_requests=5, + concurrency=5, + mock=True, + ) + + assert result.total_requests == 5 + assert result.successful_requests == 0 + assert result.failed_requests == 5 + + +@pytest.mark.asyncio +async def test_engine_simulate_traffic_no_flow_data_for_non_priority(): + """flow_data should be None when routing_strategy is not priority-failover.""" + mock_router = AsyncMock() + mock_router.acompletion = AsyncMock(return_value=_make_mock_response("ok")) + + engine = RoutingGroupTestEngine() + config = _make_routing_group_config(strategy="weighted") + + with patch("litellm.callbacks", []): + result = await engine.simulate_traffic( + routing_group_name="test-group", + router=mock_router, + routing_group_config=config, + num_requests=5, + concurrency=5, + mock=True, + ) + + assert result.flow_data is None + + +@pytest.mark.asyncio +async def test_engine_simulate_traffic_flow_data_for_priority_failover(): + """flow_data should be populated when routing_strategy is priority-failover and there are traces.""" + mock_router = AsyncMock() + mock_router.acompletion = AsyncMock(return_value=_make_mock_response("ok")) + + engine = RoutingGroupTestEngine() + config = _make_routing_group_config(strategy="priority-failover") + + with patch("litellm.callbacks", []): + result = await engine.simulate_traffic( + routing_group_name="test-group", + router=mock_router, + routing_group_config=config, + num_requests=5, + concurrency=5, + mock=True, + ) + + # flow_data may be None if no traces were recorded (mock router doesn't fire callbacks) + # In that case traffic_distribution will be empty too, which is fine + assert result.total_requests == 5 + + +# --------------------------------------------------------------------------- +# _build_flow_data tests +# --------------------------------------------------------------------------- + + +def test_build_flow_data_primary_first(): + """The first FlowStep should have reason='primary' and from_deployment=None.""" + traces: List[RoutingTrace] = [] + config = _make_routing_group_config() + + traffic = [ + DeploymentTrafficStats( + deployment_id="id-primary", + deployment_name="nebius/llama-70b", + provider="nebius", + request_count=80, + success_count=80, + failure_count=0, + avg_latency_ms=150.0, + percent_of_total=80.0, + priority=1, + ), + DeploymentTrafficStats( + deployment_id="id-fallback", + deployment_name="fireworks/llama-70b", + provider="fireworks_ai", + request_count=20, + success_count=20, + failure_count=0, + avg_latency_ms=200.0, + percent_of_total=20.0, + priority=2, + ), + ] + + flow_steps = _build_flow_data(traces, config, traffic) + + assert len(flow_steps) == 2 + assert flow_steps[0].reason == "primary" + assert flow_steps[0].from_deployment is None + assert flow_steps[0].to_deployment == "nebius/llama-70b" + assert flow_steps[0].request_count == 80 + + +def test_build_flow_data_fallback_reason(): + """Subsequent FlowSteps should have reason='fallback_error' and correct from_deployment.""" + traces: List[RoutingTrace] = [] + config = _make_routing_group_config() + + traffic = [ + DeploymentTrafficStats( + deployment_id="id-primary", + deployment_name="primary-model", + provider="nebius", + request_count=70, + success_count=70, + failure_count=0, + avg_latency_ms=100.0, + percent_of_total=70.0, + priority=1, + ), + DeploymentTrafficStats( + deployment_id="id-fallback", + deployment_name="fallback-model", + provider="fireworks_ai", + request_count=30, + success_count=30, + failure_count=0, + avg_latency_ms=200.0, + percent_of_total=30.0, + priority=2, + ), + ] + + flow_steps = _build_flow_data(traces, config, traffic) + + assert flow_steps[1].reason == "fallback_error" + assert flow_steps[1].from_deployment == "primary-model" + assert flow_steps[1].to_deployment == "fallback-model" + assert flow_steps[1].request_count == 30 + + +def test_build_flow_data_single_deployment(): + """With a single deployment, only one FlowStep (primary) should be created.""" + traces: List[RoutingTrace] = [] + config = RoutingGroupConfig( + routing_group_name="single-group", + routing_strategy="priority-failover", + deployments=[ + RoutingGroupDeployment( + model_id="id-1", + model_name="only-model", + provider="openai", + priority=1, + ) + ], + ) + + traffic = [ + DeploymentTrafficStats( + deployment_id="id-1", + deployment_name="only-model", + provider="openai", + request_count=100, + success_count=100, + failure_count=0, + avg_latency_ms=120.0, + percent_of_total=100.0, + priority=1, + ), + ] + + flow_steps = _build_flow_data(traces, config, traffic) + + assert len(flow_steps) == 1 + assert flow_steps[0].reason == "primary" + assert flow_steps[0].from_deployment is None + + +def test_build_flow_data_sorts_by_priority(): + """_build_flow_data should sort deployments by priority before building flow steps.""" + traces: List[RoutingTrace] = [] + config = _make_routing_group_config() + + # Intentionally pass traffic in reverse priority order + traffic = [ + DeploymentTrafficStats( + deployment_id="id-fallback", + deployment_name="fallback-model", + provider="fireworks_ai", + request_count=20, + success_count=20, + failure_count=0, + avg_latency_ms=200.0, + percent_of_total=20.0, + priority=2, + ), + DeploymentTrafficStats( + deployment_id="id-primary", + deployment_name="primary-model", + provider="nebius", + request_count=80, + success_count=80, + failure_count=0, + avg_latency_ms=100.0, + percent_of_total=80.0, + priority=1, + ), + ] + + flow_steps = _build_flow_data(traces, config, traffic) + + # First step should be the priority=1 deployment regardless of input order + assert flow_steps[0].to_deployment == "primary-model" + assert flow_steps[0].reason == "primary" + assert flow_steps[1].to_deployment == "fallback-model" + assert flow_steps[1].reason == "fallback_error" diff --git a/tests/litellm/proxy/test_routing_group_types.py b/tests/litellm/proxy/test_routing_group_types.py new file mode 100644 index 00000000000..c31577c6d4e --- /dev/null +++ b/tests/litellm/proxy/test_routing_group_types.py @@ -0,0 +1,122 @@ +""" +Unit tests for Routing Group Pydantic types. +""" +from litellm.types.router import ( + DeploymentTrafficStats, + FailureInjectionConfig, + RoutingGroupConfig, + RoutingGroupDeployment, + RoutingGroupSimulationResult, + RoutingTrace, +) + + +def test_routing_group_deployment_basic(): + d = RoutingGroupDeployment( + model_id="test-id", + model_name="nebius/meta-llama/Llama-3.3-70B", + provider="nebius", + ) + assert d.priority is None + assert d.weight is None + + +def test_routing_group_deployment_with_priority(): + d = RoutingGroupDeployment( + model_id="test-id", + model_name="nebius/meta-llama/Llama-3.3-70B", + provider="nebius", + priority=1, + ) + assert d.priority == 1 + + +def test_routing_group_config_basic(): + config = RoutingGroupConfig( + routing_group_name="test-group", + routing_strategy="priority-failover", + deployments=[ + RoutingGroupDeployment( + model_id="id-1", + model_name="nebius/meta-llama/Llama-3.3-70B", + provider="nebius", + priority=1, + ), + RoutingGroupDeployment( + model_id="id-2", + model_name="fireworks_ai/llama-v3p3-70b", + provider="fireworks_ai", + priority=2, + ), + ], + ) + assert config.routing_group_name == "test-group" + assert len(config.deployments) == 2 + assert config.is_active is True + assert config.routing_group_id is None + + +def test_routing_group_config_weighted(): + config = RoutingGroupConfig( + routing_group_name="weighted-group", + routing_strategy="weighted", + deployments=[ + RoutingGroupDeployment( + model_id="id-1", + model_name="nebius/meta-llama/Llama-3.3-70B", + provider="nebius", + weight=80, + ), + RoutingGroupDeployment( + model_id="id-2", + model_name="azure/gpt-4o", + provider="azure", + weight=20, + ), + ], + ) + assert config.routing_strategy == "weighted" + + +def test_routing_trace(): + trace = RoutingTrace( + deployment_id="id-1", + deployment_name="nebius/meta-llama/Llama-3.3-70B", + provider="nebius", + latency_ms=149.5, + was_fallback=False, + status="success", + ) + assert trace.fallback_depth == 0 + assert trace.error_message is None + + +def test_routing_group_simulation_result(): + result = RoutingGroupSimulationResult( + total_requests=100, + successful_requests=98, + failed_requests=2, + avg_latency_ms=169.0, + fallback_count=5, + traffic_distribution=[ + DeploymentTrafficStats( + deployment_id="id-1", + deployment_name="Nebius", + provider="nebius", + request_count=85, + success_count=85, + failure_count=0, + avg_latency_ms=149.0, + percent_of_total=85.0, + ) + ], + ) + assert result.total_requests == 100 + assert result.flow_data is None + + +def test_failure_injection_config(): + config = FailureInjectionConfig( + deployment_failure_rates={"id-1": 0.5, "id-2": 0.1} + ) + assert config.deployment_failure_rates["id-1"] == 0.5 diff --git a/tests/test_litellm/proxy/management_endpoints/test_routing_group_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_routing_group_endpoints.py new file mode 100644 index 00000000000..3191df63f14 --- /dev/null +++ b/tests/test_litellm/proxy/management_endpoints/test_routing_group_endpoints.py @@ -0,0 +1,488 @@ +""" +Unit tests for routing group management endpoints. +""" + +import os +import sys +import types +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest +from fastapi.testclient import TestClient + +import litellm.proxy.proxy_server as ps +from litellm.proxy._types import UserAPIKeyAuth +from litellm.proxy.proxy_server import app +from litellm.types.router import ( + RoutingGroupConfig, + RoutingGroupDeployment, +) + +sys.path.insert(0, os.path.abspath("../../../")) + + +def _make_routing_group_record( + routing_group_id: str = "rg-123", + routing_group_name: str = "test-group", + routing_strategy: str = "simple-shuffle", + deployments: list | None = None, + description: str | None = None, + fallback_config: dict | None = None, + retry_config: dict | None = None, + cooldown_config: dict | None = None, + settings: dict | None = None, + assigned_team_ids: list | None = None, + assigned_key_ids: list | None = None, + is_active: bool = True, +): + data = { + "routing_group_id": routing_group_id, + "routing_group_name": routing_group_name, + "routing_strategy": routing_strategy, + "deployments": deployments or [], + "description": description, + "fallback_config": fallback_config or {}, + "retry_config": retry_config or {}, + "cooldown_config": cooldown_config or {}, + "settings": settings or {}, + "assigned_team_ids": assigned_team_ids or [], + "assigned_key_ids": assigned_key_ids or [], + "is_active": is_active, + } + record = MagicMock() + for k, v in data.items(): + setattr(record, k, v) + return record + + +@pytest.fixture +def client_and_mocks(monkeypatch): + """Setup mock prisma and auth for routing group endpoints.""" + mock_routing_group_table = MagicMock() + mock_prisma = MagicMock() + + def _create_side_effect(*, data): + return _make_routing_group_record( + routing_group_id=data.get("routing_group_id", "rg-new"), + routing_group_name=data.get("routing_group_name", "new-group"), + routing_strategy=data.get("routing_strategy", "simple-shuffle"), + deployments=data.get("deployments", []), + description=data.get("description"), + assigned_team_ids=data.get("assigned_team_ids", []), + assigned_key_ids=data.get("assigned_key_ids", []), + is_active=data.get("is_active", True), + ) + + mock_routing_group_table.create = AsyncMock(side_effect=_create_side_effect) + mock_routing_group_table.find_unique = AsyncMock(return_value=None) + mock_routing_group_table.find_many = AsyncMock(return_value=[]) + mock_routing_group_table.count = AsyncMock(return_value=0) + mock_routing_group_table.update = AsyncMock( + side_effect=lambda *, where, data: _make_routing_group_record( + routing_group_id=where.get("routing_group_id", "rg-123"), + routing_group_name=data.get("routing_group_name", "updated-group"), + routing_strategy=data.get("routing_strategy", "simple-shuffle"), + deployments=data.get("deployments", []), + description=data.get("description"), + assigned_team_ids=data.get("assigned_team_ids", []), + assigned_key_ids=data.get("assigned_key_ids", []), + is_active=data.get("is_active", True), + ) + ) + mock_routing_group_table.delete = AsyncMock(return_value=None) + + mock_db = types.SimpleNamespace( + litellm_routinggrouptable=mock_routing_group_table, + ) + mock_prisma.db = mock_db + + monkeypatch.setattr(ps, "prisma_client", mock_prisma) + + user = UserAPIKeyAuth(user_id="test-user") + app.dependency_overrides[ps.user_api_key_auth] = lambda: user + + # Mock the router sync so tests don't need a live LLM router + mock_llm_router = MagicMock() + mock_llm_router.fallbacks = [] + mock_llm_router.add_deployment = MagicMock() + monkeypatch.setattr(ps, "llm_router", mock_llm_router) + + client = TestClient(app) + + yield client, mock_prisma, mock_routing_group_table, mock_llm_router + + app.dependency_overrides.clear() + monkeypatch.setattr(ps, "prisma_client", ps.prisma_client) + + +# --------------------------------------------------------------------------- +# CREATE +# --------------------------------------------------------------------------- + + +def test_create_routing_group_success(client_and_mocks): + client, _, mock_table, _ = client_and_mocks + + payload = { + "routing_group_name": "my-group", + "routing_strategy": "simple-shuffle", + "deployments": [ + { + "model_id": "id-1", + "model_name": "gpt-4o", + "provider": "openai", + } + ], + } + resp = client.post("/v1/routing_group", json=payload) + assert resp.status_code == 201 + body = resp.json() + assert body["routing_group_name"] == "my-group" + mock_table.create.assert_awaited_once() + + +def test_create_routing_group_invalid_strategy(client_and_mocks): + client, _, _, _ = client_and_mocks + + payload = { + "routing_group_name": "bad-group", + "routing_strategy": "not-a-real-strategy", + "deployments": [ + { + "model_id": "id-1", + "model_name": "gpt-4o", + "provider": "openai", + } + ], + } + resp = client.post("/v1/routing_group", json=payload) + assert resp.status_code == 400 + assert "Invalid routing_strategy" in resp.json()["detail"] + + +def test_create_routing_group_empty_deployments(client_and_mocks): + client, _, _, _ = client_and_mocks + + payload = { + "routing_group_name": "empty-group", + "routing_strategy": "simple-shuffle", + "deployments": [], + } + resp = client.post("/v1/routing_group", json=payload) + assert resp.status_code == 400 + assert "deployments" in resp.json()["detail"] + + +# --------------------------------------------------------------------------- +# LIST +# --------------------------------------------------------------------------- + + +def test_list_routing_groups_empty(client_and_mocks): + client, _, _, _ = client_and_mocks + + resp = client.get("/v1/routing_group") + assert resp.status_code == 200 + body = resp.json() + assert body["routing_groups"] == [] + assert body["total"] == 0 + + +def test_list_routing_groups_with_records(client_and_mocks): + client, _, mock_table, _ = client_and_mocks + + records = [ + _make_routing_group_record( + routing_group_id="rg-1", + routing_group_name="group-1", + deployments=[ + {"model_id": "id-1", "model_name": "gpt-4o", "provider": "openai"} + ], + ), + _make_routing_group_record( + routing_group_id="rg-2", + routing_group_name="group-2", + deployments=[ + { + "model_id": "id-2", + "model_name": "claude-3-5-sonnet-20241022", + "provider": "anthropic", + } + ], + ), + ] + mock_table.find_many = AsyncMock(return_value=records) + mock_table.count = AsyncMock(return_value=2) + + resp = client.get("/v1/routing_group") + assert resp.status_code == 200 + body = resp.json() + assert body["total"] == 2 + assert len(body["routing_groups"]) == 2 + + +# --------------------------------------------------------------------------- +# GET +# --------------------------------------------------------------------------- + + +def test_get_routing_group_not_found(client_and_mocks): + client, _, mock_table, _ = client_and_mocks + mock_table.find_unique = AsyncMock(return_value=None) + + resp = client.get("/v1/routing_group/nonexistent-id") + assert resp.status_code == 404 + + +def test_get_routing_group_success(client_and_mocks): + client, _, mock_table, _ = client_and_mocks + + record = _make_routing_group_record( + routing_group_id="rg-abc", + routing_group_name="found-group", + deployments=[ + {"model_id": "id-1", "model_name": "gpt-4o", "provider": "openai"} + ], + ) + mock_table.find_unique = AsyncMock(return_value=record) + + resp = client.get("/v1/routing_group/rg-abc") + assert resp.status_code == 200 + body = resp.json() + assert body["routing_group_id"] == "rg-abc" + assert body["routing_group_name"] == "found-group" + + +# --------------------------------------------------------------------------- +# UPDATE +# --------------------------------------------------------------------------- + + +def test_update_routing_group_success(client_and_mocks): + client, _, mock_table, _ = client_and_mocks + + payload = { + "routing_group_name": "updated-group", + "routing_strategy": "latency-based-routing", + "deployments": [ + {"model_id": "id-1", "model_name": "gpt-4o", "provider": "openai"} + ], + } + resp = client.put("/v1/routing_group/rg-123", json=payload) + assert resp.status_code == 200 + body = resp.json() + assert body["routing_group_name"] == "updated-group" + mock_table.update.assert_awaited_once() + + +def test_update_routing_group_db_error_returns_404(client_and_mocks): + client, _, mock_table, _ = client_and_mocks + + mock_table.update = AsyncMock(side_effect=Exception("Record not found")) + + payload = { + "routing_group_name": "updated-group", + "routing_strategy": "simple-shuffle", + "deployments": [ + {"model_id": "id-1", "model_name": "gpt-4o", "provider": "openai"} + ], + } + resp = client.put("/v1/routing_group/bad-id", json=payload) + assert resp.status_code == 404 + + +# --------------------------------------------------------------------------- +# DELETE +# --------------------------------------------------------------------------- + + +def test_delete_routing_group_success(client_and_mocks): + client, _, mock_table, _ = client_and_mocks + + resp = client.delete("/v1/routing_group/rg-123") + assert resp.status_code == 200 + body = resp.json() + assert body["deleted"] is True + assert body["routing_group_id"] == "rg-123" + mock_table.delete.assert_awaited_once() + + +def test_delete_routing_group_not_found(client_and_mocks): + client, _, mock_table, _ = client_and_mocks + mock_table.delete = AsyncMock(side_effect=Exception("Record does not exist")) + + resp = client.delete("/v1/routing_group/missing-id") + assert resp.status_code == 404 + + +# --------------------------------------------------------------------------- +# _sync_routing_group_to_router unit tests +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_sync_priority_failover_wires_fallbacks(): + """priority-failover strategy should create tiered groups and wire fallback chain.""" + from litellm.proxy.management_endpoints.routing_group_endpoints import ( + _sync_routing_group_to_router, + ) + + mock_router = MagicMock() + mock_router.fallbacks = [] + mock_router.add_deployment = MagicMock() + + config = RoutingGroupConfig( + routing_group_name="my-failover-group", + routing_strategy="priority-failover", + deployments=[ + RoutingGroupDeployment( + model_id="id-1", + model_name="gpt-4o", + provider="openai", + priority=1, + ), + RoutingGroupDeployment( + model_id="id-2", + model_name="claude-3-5-sonnet-20241022", + provider="anthropic", + priority=2, + ), + ], + ) + + with patch( + "litellm.proxy.management_endpoints.routing_group_endpoints._get_llm_router", + return_value=mock_router, + ): + await _sync_routing_group_to_router(config) + + # add_deployment should be called twice (once per deployment) + assert mock_router.add_deployment.call_count == 2 + + # Fallback chain should be wired: primary -> [fallback_p2] + assert len(mock_router.fallbacks) == 1 + fallback_entry = mock_router.fallbacks[0] + assert "my-failover-group" in fallback_entry + assert fallback_entry["my-failover-group"] == ["my-failover-group__fallback_p2"] + + +@pytest.mark.asyncio +async def test_sync_weighted_includes_weight_in_litellm_params(): + """weighted strategy should pass weight into litellm_params.""" + from litellm.proxy.management_endpoints.routing_group_endpoints import ( + _sync_routing_group_to_router, + ) + + captured_deployments = [] + + mock_router = MagicMock() + mock_router.fallbacks = [] + + def capture_add_deployment(deployment): + captured_deployments.append(deployment) + + mock_router.add_deployment = MagicMock(side_effect=capture_add_deployment) + + config = RoutingGroupConfig( + routing_group_name="weighted-group", + routing_strategy="weighted", + deployments=[ + RoutingGroupDeployment( + model_id="id-1", + model_name="gpt-4o", + provider="openai", + weight=80, + ), + RoutingGroupDeployment( + model_id="id-2", + model_name="claude-3-5-sonnet-20241022", + provider="anthropic", + weight=20, + ), + ], + ) + + with patch( + "litellm.proxy.management_endpoints.routing_group_endpoints._get_llm_router", + return_value=mock_router, + ): + await _sync_routing_group_to_router(config) + + assert mock_router.add_deployment.call_count == 2 + + # Each captured deployment should have weight in its litellm_params + for dep in captured_deployments: + assert hasattr(dep, "litellm_params") + params = dep.litellm_params + # litellm_params is a pydantic model; access as dict or attribute + params_dict = ( + params.model_dump() if hasattr(params, "model_dump") else dict(params) + ) + assert "weight" in params_dict + assert params_dict["weight"] in (80, 20) + + +@pytest.mark.asyncio +async def test_sync_no_router_logs_warning(): + """When llm_router is None, _sync should log a warning and return gracefully.""" + from litellm.proxy.management_endpoints.routing_group_endpoints import ( + _sync_routing_group_to_router, + ) + + config = RoutingGroupConfig( + routing_group_name="no-router-group", + routing_strategy="simple-shuffle", + deployments=[ + RoutingGroupDeployment( + model_id="id-1", + model_name="gpt-4o", + provider="openai", + ) + ], + ) + + with patch( + "litellm.proxy.management_endpoints.routing_group_endpoints._get_llm_router", + return_value=None, + ): + # Should not raise + await _sync_routing_group_to_router(config) + + +@pytest.mark.asyncio +async def test_sync_simple_shuffle_no_fallbacks(): + """Non-priority-failover strategy should not modify router.fallbacks.""" + from litellm.proxy.management_endpoints.routing_group_endpoints import ( + _sync_routing_group_to_router, + ) + + mock_router = MagicMock() + mock_router.fallbacks = [] + mock_router.add_deployment = MagicMock() + + config = RoutingGroupConfig( + routing_group_name="shuffle-group", + routing_strategy="simple-shuffle", + deployments=[ + RoutingGroupDeployment( + model_id="id-1", + model_name="gpt-4o", + provider="openai", + ), + RoutingGroupDeployment( + model_id="id-2", + model_name="gpt-4o-mini", + provider="openai", + ), + ], + ) + + with patch( + "litellm.proxy.management_endpoints.routing_group_endpoints._get_llm_router", + return_value=mock_router, + ): + await _sync_routing_group_to_router(config) + + assert mock_router.add_deployment.call_count == 2 + # fallbacks should remain unchanged (empty) + assert mock_router.fallbacks == [] diff --git a/ui/litellm-dashboard/src/app/(dashboard)/components/Sidebar2.tsx b/ui/litellm-dashboard/src/app/(dashboard)/components/Sidebar2.tsx index a74d3c108d6..d1c1f9ddd95 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/components/Sidebar2.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/components/Sidebar2.tsx @@ -20,6 +20,7 @@ import { ToolOutlined, TagsOutlined, AuditOutlined, + BranchesOutlined, } from "@ant-design/icons"; // import { // all_admin_roles, @@ -105,6 +106,8 @@ const routeFor = (slug: string): string => { return "guardrails"; case "policies": return "policies"; + case "routing-groups": + return "routing-groups"; // tools case "mcp-servers": @@ -168,6 +171,13 @@ const menuItems: MenuItemCfg[] = [ icon: , roles: rolesWithWriteAccess, }, + { + key: "29", + page: "routing-groups", + label: "Routing Groups", + icon: , + roles: rolesWithWriteAccess, + }, { key: "12", page: "new_usage", diff --git a/ui/litellm-dashboard/src/app/(dashboard)/routing-groups/page.tsx b/ui/litellm-dashboard/src/app/(dashboard)/routing-groups/page.tsx new file mode 100644 index 00000000000..2460f27a566 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/routing-groups/page.tsx @@ -0,0 +1,18 @@ +"use client"; + +import RoutingGroupsView from "@/components/routing_groups/RoutingGroupsView"; +import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; + +const RoutingGroupsPage = () => { + const { accessToken, userId, userRole } = useAuthorized(); + + return ( + + ); +}; + +export default RoutingGroupsPage; diff --git a/ui/litellm-dashboard/src/components/networking.tsx b/ui/litellm-dashboard/src/components/networking.tsx index 0df6d813d7c..b538018a579 100644 --- a/ui/litellm-dashboard/src/components/networking.tsx +++ b/ui/litellm-dashboard/src/components/networking.tsx @@ -10035,3 +10035,172 @@ export const updateToolPolicy = async ( } return response.json(); }; + +export const routingGroupCreateCall = async ( + accessToken: string, + formValues: Record, +): Promise> => { + const url = proxyBaseUrl + ? `${proxyBaseUrl}/v1/routing_group` + : `/v1/routing_group`; + const response = await fetch(url, { + method: "POST", + headers: { + [globalLitellmHeaderName]: `Bearer ${accessToken}`, + "Content-Type": "application/json", + }, + body: JSON.stringify(formValues), + }); + if (!response.ok) { + const errorData = await response.text(); + handleError(errorData); + throw new Error("Network response was not ok"); + } + return response.json(); +}; + +export const routingGroupListCall = async ( + accessToken: string, + page: number = 1, + size: number = 50, +): Promise<{ routing_groups: Record[]; total: number; page: number; size: number }> => { + const url = proxyBaseUrl + ? `${proxyBaseUrl}/v1/routing_group?page=${page}&size=${size}` + : `/v1/routing_group?page=${page}&size=${size}`; + const response = await fetch(url, { + method: "GET", + headers: { + [globalLitellmHeaderName]: `Bearer ${accessToken}`, + "Content-Type": "application/json", + }, + }); + if (!response.ok) { + const errorData = await response.text(); + handleError(errorData); + throw new Error("Network response was not ok"); + } + return response.json(); +}; + +export const routingGroupGetCall = async ( + accessToken: string, + routingGroupId: string, +): Promise> => { + const url = proxyBaseUrl + ? `${proxyBaseUrl}/v1/routing_group/${routingGroupId}` + : `/v1/routing_group/${routingGroupId}`; + const response = await fetch(url, { + method: "GET", + headers: { + [globalLitellmHeaderName]: `Bearer ${accessToken}`, + "Content-Type": "application/json", + }, + }); + if (!response.ok) { + const errorData = await response.text(); + handleError(errorData); + throw new Error("Network response was not ok"); + } + return response.json(); +}; + +export const routingGroupUpdateCall = async ( + accessToken: string, + routingGroupId: string, + formValues: Record, +): Promise> => { + const url = proxyBaseUrl + ? `${proxyBaseUrl}/v1/routing_group/${routingGroupId}` + : `/v1/routing_group/${routingGroupId}`; + const response = await fetch(url, { + method: "PUT", + headers: { + [globalLitellmHeaderName]: `Bearer ${accessToken}`, + "Content-Type": "application/json", + }, + body: JSON.stringify(formValues), + }); + if (!response.ok) { + const errorData = await response.text(); + handleError(errorData); + throw new Error("Network response was not ok"); + } + return response.json(); +}; + +export const routingGroupDeleteCall = async ( + accessToken: string, + routingGroupId: string, +): Promise> => { + const url = proxyBaseUrl + ? `${proxyBaseUrl}/v1/routing_group/${routingGroupId}` + : `/v1/routing_group/${routingGroupId}`; + const response = await fetch(url, { + method: "DELETE", + headers: { + [globalLitellmHeaderName]: `Bearer ${accessToken}`, + "Content-Type": "application/json", + }, + }); + if (!response.ok) { + const errorData = await response.text(); + handleError(errorData); + throw new Error("Network response was not ok"); + } + return response.json(); +}; + +export const routingGroupTestCall = async ( + accessToken: string, + routingGroupId: string, + messages?: Array>, + mock: boolean = false, +): Promise> => { + const url = proxyBaseUrl + ? `${proxyBaseUrl}/v1/routing_group/${routingGroupId}/test` + : `/v1/routing_group/${routingGroupId}/test`; + const response = await fetch(url, { + method: "POST", + headers: { + [globalLitellmHeaderName]: `Bearer ${accessToken}`, + "Content-Type": "application/json", + }, + body: JSON.stringify({ messages, mock }), + }); + if (!response.ok) { + const errorData = await response.text(); + handleError(errorData); + throw new Error("Network response was not ok"); + } + return response.json(); +}; + +export const routingGroupSimulateCall = async ( + accessToken: string, + routingGroupId: string, + config: { + num_requests: number; + concurrency: number; + mock: boolean; + failure_injection?: { deployment_failure_rates: Record }; + }, +): Promise> => { + const url = proxyBaseUrl + ? `${proxyBaseUrl}/v1/routing_group/${routingGroupId}/simulate` + : `/v1/routing_group/${routingGroupId}/simulate`; + const response = await fetch(url, { + method: "POST", + headers: { + [globalLitellmHeaderName]: `Bearer ${accessToken}`, + "Content-Type": "application/json", + }, + body: JSON.stringify(config), + }); + if (!response.ok) { + const errorData = await response.text(); + handleError(errorData); + throw new Error("Network response was not ok"); + } + return response.json(); +}; +}; diff --git a/ui/litellm-dashboard/src/components/routing_groups/AnimatedStream.tsx b/ui/litellm-dashboard/src/components/routing_groups/AnimatedStream.tsx new file mode 100644 index 00000000000..0154df3a48b --- /dev/null +++ b/ui/litellm-dashboard/src/components/routing_groups/AnimatedStream.tsx @@ -0,0 +1,48 @@ +"use client"; +import React from "react"; + +interface AnimatedStreamProps { + /** SVG path d string */ + path: string; + color: string; + /** stroke-width in px */ + thickness: number; + animated?: boolean; +} + +export default function AnimatedStream({ path, color, thickness, animated = true }: AnimatedStreamProps) { + const dashLen = Math.max(8, thickness * 3); + const gapLen = Math.max(4, thickness * 2); + + return ( + + {/* Dim base */} + + {/* Glow */} + + {/* Animated dashes */} + + + + ); +} diff --git a/ui/litellm-dashboard/src/components/routing_groups/DeploymentCard.tsx b/ui/litellm-dashboard/src/components/routing_groups/DeploymentCard.tsx new file mode 100644 index 00000000000..7c794a35732 --- /dev/null +++ b/ui/litellm-dashboard/src/components/routing_groups/DeploymentCard.tsx @@ -0,0 +1,56 @@ +"use client"; +import React from "react"; + +interface DeploymentCardProps { + provider: string; + displayName: string; + providerColor: string; + label: string; // "Priority 1" or "Weight 83%" + avgLatencyMs: number; + percentage: number; // 0-100 + requestCount: number; + width?: number; // px, default 220 +} + +export default function DeploymentCard({ + provider, + displayName, + providerColor, + label, + avgLatencyMs, + percentage, + requestCount, + width = 220, +}: DeploymentCardProps) { + const initial = displayName.charAt(0).toUpperCase(); + return ( +
+ {/* Provider icon */} +
+ {initial} +
+ + {/* Name + label */} +
+
{displayName}
+
+ {label} · {avgLatencyMs > 0 ? `${Math.round(avgLatencyMs)}ms avg` : "—"} +
+
+ + {/* Percentage + req count */} +
+ + {Math.round(percentage)}% + + {requestCount} req +
+
+ ); +} diff --git a/ui/litellm-dashboard/src/components/routing_groups/LiveTester.tsx b/ui/litellm-dashboard/src/components/routing_groups/LiveTester.tsx new file mode 100644 index 00000000000..d614ff68bb7 --- /dev/null +++ b/ui/litellm-dashboard/src/components/routing_groups/LiveTester.tsx @@ -0,0 +1,208 @@ +"use client"; +import React, { useState } from "react"; +import { Button, InputNumber, Radio, Spin, Typography } from "antd"; +import { SwapOutlined } from "@ant-design/icons"; +import OrderedFallbackFlow from "./OrderedFallbackFlow"; +import WeightedRoundRobinFlow from "./WeightedRoundRobinFlow"; +import StatsBar from "./StatsBar"; +import { routingGroupSimulateCall } from "@/components/networking"; + +interface Deployment { + deployment_id: string; + provider: string; + display_name: string; + priority?: number; + weight?: number; + avg_latency_ms: number; + percent_of_total: number; + request_count: number; + success_count: number; + failure_count: number; +} + +interface LiveTesterProps { + accessToken: string; + routingGroupId: string; + routingGroupName: string; + routingStrategy: string; // "priority-failover" | "weighted" | etc. + initialDeployments?: Deployment[]; +} + +const isOrderedStrategy = (s: string) => + s === "priority-failover"; + +export default function LiveTester({ + accessToken, + routingGroupId, + routingGroupName, + routingStrategy, + initialDeployments = [], +}: LiveTesterProps) { + // Force mode based on strategy, but allow user to toggle + const [viewMode, setViewMode] = useState<"ordered" | "weighted">( + isOrderedStrategy(routingStrategy) ? "ordered" : "weighted" + ); + const [loading, setLoading] = useState(false); + const [numRequests, setNumRequests] = useState(100); + const [concurrency, setConcurrency] = useState(10); + const [mockMode, setMockMode] = useState(true); + + // Live stats state + const [deployments, setDeployments] = useState( + initialDeployments.length > 0 ? initialDeployments : [] + ); + const [stats, setStats] = useState({ + totalRequests: 0, + successRate: 100, + avgLatencyMs: 0, + fallbackCount: 0, + }); + + const isOrdered = viewMode === "ordered"; + + const handleSimulate = async () => { + if (!accessToken) return; + setLoading(true); + try { + const result = await routingGroupSimulateCall(accessToken, routingGroupId, { + num_requests: numRequests, + concurrency, + mock: mockMode, + }) as Record; + + const dist = (result.traffic_distribution as Deployment[]) || []; + setDeployments(dist); + const total = (result.total_requests as number) || 0; + const successful = (result.successful_requests as number) || 0; + const avgLatency = (result.avg_latency_ms as number) || 0; + const fallbacks = (result.fallback_count as number) || 0; + setStats({ + totalRequests: total, + successRate: total > 0 ? (successful / total) * 100 : 100, + avgLatencyMs: avgLatency, + fallbackCount: fallbacks, + }); + } catch (e) { + console.error("Simulation error:", e); + } finally { + setLoading(false); + } + }; + + const infoText = isOrdered + ? "Traffic flows to the primary provider first. When a request fails, it cascades down the priority chain. Thicker streams = more traffic. Dashed red lines show the fallback path." + : "Traffic is distributed across providers based on weights. Thicker streams = more traffic. Adjust weights in the routing group settings."; + + const badgeLabel = isOrdered ? "Ordered Fallback" : "Weighted Round-Robin"; + const badgeColor = isOrdered ? "#92400e" : "#134e4a"; + const badgeTextColor = isOrdered ? "#fbbf24" : "#2dd4bf"; + + return ( +
+ {/* Header */} +
+
+
+ + Live Tester + + + {badgeLabel} + +
+ + Sending to{" "} + + {routingGroupName} + + +
+ +
+ + {/* Stats bar */} + + + {/* Flow diagram */} + + {deployments.length === 0 ? ( +
+ Run a simulation to see traffic flow +
+ ) : isOrdered ? ( + + ) : ( + + )} + + + {/* Info box */} +
+ {infoText} +
+ + {/* Simulation controls */} +
+ + Simulation Controls + +
+
+
Requests
+ setNumRequests(v ?? 100)} + style={{ width: 100, background: "#0f172a", borderColor: "#334155", color: "#f1f5f9" }} + /> +
+
+
Concurrency
+ setConcurrency(v ?? 10)} + style={{ width: 90, background: "#0f172a", borderColor: "#334155", color: "#f1f5f9" }} + /> +
+
+
Mode
+ setMockMode(e.target.value === "mock")} + buttonStyle="solid" + size="small" + > + Mock + Real + +
+ +
+
+
+ ); +} diff --git a/ui/litellm-dashboard/src/components/routing_groups/OrderedFallbackFlow.tsx b/ui/litellm-dashboard/src/components/routing_groups/OrderedFallbackFlow.tsx new file mode 100644 index 00000000000..14ec20c5fc2 --- /dev/null +++ b/ui/litellm-dashboard/src/components/routing_groups/OrderedFallbackFlow.tsx @@ -0,0 +1,129 @@ +"use client"; +import React from "react"; +import AnimatedStream from "./AnimatedStream"; +import DeploymentCard from "./DeploymentCard"; + +const PROVIDER_COLORS: Record = { + nebius: "#14b8a6", + fireworks_ai: "#f97316", + azure: "#8b5cf6", + openai: "#10b981", + anthropic: "#d97706", + google: "#3b82f6", + aws: "#ef4444", +}; +const getProviderColor = (p: string) => PROVIDER_COLORS[p.toLowerCase()] ?? "#6b7280"; + +interface Deployment { + deployment_id: string; + provider: string; + display_name: string; + priority?: number; + avg_latency_ms: number; + percent_of_total: number; + request_count: number; +} + +interface OrderedFallbackFlowProps { + deployments: Deployment[]; + routingGroupName: string; +} + +const CARD_HEIGHT = 76; +const CARD_GAP = 48; // space between cards (for the "fail" arrow) +const CARD_WIDTH = 220; +const LEFT_PADDING = 40; +const HUB_RADIUS = 28; + +export default function OrderedFallbackFlow({ deployments, routingGroupName }: OrderedFallbackFlowProps) { + const sorted = [...deployments].sort((a, b) => (a.priority ?? 999) - (b.priority ?? 999)); + const totalHeight = sorted.length * CARD_HEIGHT + (sorted.length - 1) * CARD_GAP + 40; + const svgWidth = 600; + + // Hub sits vertically centered, 1/3 from left + const hubX = svgWidth * 0.32; + const hubY = totalHeight / 2; + + // Cards start at x = svgWidth - CARD_WIDTH - LEFT_PADDING + const cardX = svgWidth - CARD_WIDTH - LEFT_PADDING; + + // Y positions for each card (centered block) + const blockHeight = sorted.length * CARD_HEIGHT + (sorted.length - 1) * CARD_GAP; + const blockTop = (totalHeight - blockHeight) / 2; + const cardYs = sorted.map((_, i) => blockTop + i * (CARD_HEIGHT + CARD_GAP)); + const cardCenterYs = cardYs.map((y) => y + CARD_HEIGHT / 2); + + // Primary stream: from hub to first card center + const primaryPath = `M ${hubX} ${hubY} L ${cardX} ${cardCenterYs[0]}`; + + // Fallback dashed paths between cards + const fallbackPaths = sorted.slice(0, -1).map((_, i) => { + const x = cardX + CARD_WIDTH / 2; + const fromY = cardYs[i] + CARD_HEIGHT; + const toY = cardYs[i + 1]; + return { path: `M ${x} ${fromY} L ${x} ${toY}`, midY: (fromY + toY) / 2 }; + }); + + return ( +
+ {/* SVG layer */} + + {/* Client → hub stream */} + + + {/* Hub circle */} + + + LITELLM + + + {/* Primary stream: hub → first card */} + + + {/* Fallback dashed arrows between cards */} + {fallbackPaths.map(({ path, midY }, i) => ( + + + + fail → + + + ))} + + + {/* "client" label */} +
+ client +
+ + {/* Deployment cards */} +
+
+ {sorted.map((dep, i) => ( + + ))} +
+
+
+ ); +} diff --git a/ui/litellm-dashboard/src/components/routing_groups/RoutingGroupBuilder.tsx b/ui/litellm-dashboard/src/components/routing_groups/RoutingGroupBuilder.tsx new file mode 100644 index 00000000000..df2e5ee76c3 --- /dev/null +++ b/ui/litellm-dashboard/src/components/routing_groups/RoutingGroupBuilder.tsx @@ -0,0 +1,312 @@ +"use client"; + +import React, { useEffect, useState } from "react"; +import { + Button, + Collapse, + Form, + Input, + InputNumber, + Modal, + Radio, + Space, + Typography, +} from "antd"; +import { CloseOutlined, PlusOutlined } from "@ant-design/icons"; +import { routingGroupCreateCall, routingGroupUpdateCall } from "@/components/networking"; + +interface Deployment { + model_name: string; + litellm_provider: string; + weight?: number; + priority?: number; +} + +interface RoutingGroupBuilderProps { + accessToken: string | null; + visible: boolean; + editTarget: Record | null; + onClose: () => void; + onSuccess: () => void; +} + +const ROUTING_STRATEGIES = [ + { value: "priority-failover", label: "Priority Failover: ordered, first success wins" }, + { value: "weighted", label: "Weighted: split traffic by percentage" }, + { value: "cost-based-routing", label: "Cost-Based: route to cheapest" }, + { value: "latency-based-routing", label: "Latency-Based: route to fastest" }, + { value: "least-busy", label: "Least Busy: fewest in-flight requests" }, + { value: "usage-based-routing-v2", label: "Usage-Based: TPM/RPM aware" }, + { value: "simple-shuffle", label: "Round Robin: even distribution" }, +]; + +export default function RoutingGroupBuilder({ + accessToken, + visible, + editTarget, + onClose, + onSuccess, +}: RoutingGroupBuilderProps) { + const [form] = Form.useForm(); + const [deployments, setDeployments] = useState([]); + const [newModelName, setNewModelName] = useState(""); + const [newProvider, setNewProvider] = useState(""); + const [submitting, setSubmitting] = useState(false); + const [strategy, setStrategy] = useState("priority-failover"); + + const isEditing = editTarget !== null; + + useEffect(() => { + if (editTarget) { + form.setFieldsValue({ + routing_group_name: editTarget.routing_group_name, + description: editTarget.description, + routing_strategy: editTarget.routing_strategy ?? "priority-failover", + max_retries: editTarget.max_retries, + cooldown_time: editTarget.cooldown_time, + }); + setStrategy((editTarget.routing_strategy as string) ?? "priority-failover"); + if (Array.isArray(editTarget.deployments)) { + setDeployments(editTarget.deployments as Deployment[]); + } + } else { + form.resetFields(); + setDeployments([]); + setStrategy("priority-failover"); + } + }, [editTarget, form]); + + const addDeployment = () => { + const trimmedModel = newModelName.trim(); + if (!trimmedModel) return; + const newDeployment: Deployment = { + model_name: trimmedModel, + litellm_provider: newProvider.trim(), + weight: strategy === "weighted" ? 1 : undefined, + priority: strategy === "priority-failover" ? deployments.length + 1 : undefined, + }; + setDeployments((prev) => [...prev, newDeployment]); + setNewModelName(""); + setNewProvider(""); + }; + + const removeDeployment = (index: number) => { + setDeployments((prev) => { + const updated = prev.filter((_, i) => i !== index); + if (strategy === "priority-failover") { + return updated.map((d, i) => ({ ...d, priority: i + 1 })); + } + return updated; + }); + }; + + const updateDeploymentWeight = (index: number, weight: number | null) => { + setDeployments((prev) => + prev.map((d, i) => (i === index ? { ...d, weight: weight ?? 1 } : d)) + ); + }; + + const handleStrategyChange = (value: string) => { + setStrategy(value); + setDeployments((prev) => + prev.map((d, i) => ({ + ...d, + weight: value === "weighted" ? (d.weight ?? 1) : undefined, + priority: value === "priority-failover" ? i + 1 : undefined, + })) + ); + }; + + const handleSubmit = async () => { + if (!accessToken) return; + try { + const values = await form.validateFields(); + setSubmitting(true); + + const payload: Record = { + routing_group_name: values.routing_group_name, + description: values.description, + routing_strategy: values.routing_strategy, + deployments, + max_retries: values.max_retries, + cooldown_time: values.cooldown_time, + }; + + if (isEditing && editTarget?.routing_group_id) { + await routingGroupUpdateCall( + accessToken, + editTarget.routing_group_id as string, + payload + ); + } else { + await routingGroupCreateCall(accessToken, payload); + } + + onSuccess(); + } catch (err) { + console.error("Failed to save routing group:", err); + } finally { + setSubmitting(false); + } + }; + + return ( + + Cancel + , + , + ]} + > +
+ + + + + + + + + + handleStrategyChange(e.target.value)} + > + + {ROUTING_STRATEGIES.map((s) => ( + + {s.label} + + ))} + + + + +
+ Deployments +
+ + setNewModelName(e.target.value)} + style={{ width: 200 }} + onPressEnter={addDeployment} + /> + setNewProvider(e.target.value)} + style={{ width: 160 }} + onPressEnter={addDeployment} + /> + + +
+ + {deployments.length === 0 && ( + + No deployments added yet. + + )} + + {deployments.map((dep, idx) => ( +
+ {strategy === "priority-failover" && ( + + #{dep.priority} + + )} + + {dep.model_name} + {dep.litellm_provider ? ` (${dep.litellm_provider})` : ""} + + {strategy === "weighted" && ( + updateDeploymentWeight(idx, val)} + addonAfter="%" + style={{ width: 100 }} + /> + )} +
+ ))} +
+ + + + + + + + + + + + +
+ + Access is managed via Teams.{" "} + + Manage Teams + + +
+
+ + ); +} diff --git a/ui/litellm-dashboard/src/components/routing_groups/RoutingGroupsTable.tsx b/ui/litellm-dashboard/src/components/routing_groups/RoutingGroupsTable.tsx new file mode 100644 index 00000000000..7d12514d5c8 --- /dev/null +++ b/ui/litellm-dashboard/src/components/routing_groups/RoutingGroupsTable.tsx @@ -0,0 +1,168 @@ +"use client"; + +import React, { useEffect, useState } from "react"; +import { Button, Popconfirm, Table, Tag, Typography } from "antd"; +import { DeleteOutlined, EditOutlined, ThunderboltOutlined } from "@ant-design/icons"; +import { routingGroupDeleteCall, routingGroupListCall } from "@/components/networking"; + +interface RoutingGroupsTableProps { + accessToken: string | null; + refreshKey: number; + onEdit: (group: Record) => void; + onTest?: (group: Record) => void; +} + +const strategyColors: Record = { + "priority-failover": "blue", + weighted: "green", + "cost-based-routing": "gold", + "latency-based-routing": "cyan", + "least-busy": "purple", + "usage-based-routing-v2": "orange", + "simple-shuffle": "geekblue", +}; + +const strategyLabels: Record = { + "priority-failover": "Priority Failover", + weighted: "Weighted", + "cost-based-routing": "Cost-Based", + "latency-based-routing": "Latency-Based", + "least-busy": "Least Busy", + "usage-based-routing-v2": "Usage-Based", + "simple-shuffle": "Round Robin", +}; + +export default function RoutingGroupsTable({ + accessToken, + refreshKey, + onEdit, + onTest, +}: RoutingGroupsTableProps) { + const [groups, setGroups] = useState[]>([]); + const [loading, setLoading] = useState(false); + + useEffect(() => { + if (!accessToken) return; + setLoading(true); + routingGroupListCall(accessToken, 1, 50) + .then((resp) => { + setGroups(resp.routing_groups || []); + }) + .catch(console.error) + .finally(() => setLoading(false)); + }, [accessToken, refreshKey]); + + const handleDelete = async (groupId: string) => { + if (!accessToken) return; + try { + await routingGroupDeleteCall(accessToken, groupId); + setGroups((prev) => + prev.filter((g) => (g.routing_group_id as string) !== groupId) + ); + } catch (err) { + console.error("Failed to delete routing group:", err); + } + }; + + const columns = [ + { + title: "Name", + dataIndex: "routing_group_name", + key: "routing_group_name", + render: (name: string) => ( + {name} + ), + }, + { + title: "Strategy", + dataIndex: "routing_strategy", + key: "routing_strategy", + render: (strategy: string) => { + const label = strategyLabels[strategy] ?? strategy; + const color = strategyColors[strategy] ?? "default"; + return {label}; + }, + }, + { + title: "Deployments", + dataIndex: "deployments", + key: "deployments", + render: (deployments: unknown) => { + const count = Array.isArray(deployments) ? deployments.length : 0; + return {count}; + }, + }, + { + title: "Status", + dataIndex: "status", + key: "status", + render: (status: string) => { + const isActive = !status || status === "active"; + return ( + + {isActive ? "Active" : "Inactive"} + + ); + }, + }, + { + title: "Created", + dataIndex: "created_at", + key: "created_at", + render: (created: string) => { + if (!created) return "-"; + return new Date(created).toLocaleDateString(); + }, + }, + { + title: "Actions", + key: "actions", + render: (_: unknown, record: Record) => { + const groupId = record.routing_group_id as string; + return ( +
+ {onTest && ( + + )} + + handleDelete(groupId)} + okText="Delete" + okButtonProps={{ danger: true }} + cancelText="Cancel" + > + + +
+ ); + }, + }, + ]; + + return ( + (record.routing_group_id as string) ?? Math.random().toString()} + pagination={{ pageSize: 20 }} + /> + ); +} diff --git a/ui/litellm-dashboard/src/components/routing_groups/RoutingGroupsView.tsx b/ui/litellm-dashboard/src/components/routing_groups/RoutingGroupsView.tsx new file mode 100644 index 00000000000..3d620b5d340 --- /dev/null +++ b/ui/litellm-dashboard/src/components/routing_groups/RoutingGroupsView.tsx @@ -0,0 +1,112 @@ +"use client"; + +import React, { useState } from "react"; +import { Button, Divider, Typography } from "antd"; +import { PlusOutlined, CloseOutlined } from "@ant-design/icons"; +import RoutingGroupsTable from "./RoutingGroupsTable"; +import RoutingGroupBuilder from "./RoutingGroupBuilder"; +import LiveTester from "./LiveTester"; + +interface RoutingGroupsViewProps { + accessToken: string | null; + userRole: string; + userId: string; +} + +export default function RoutingGroupsView({ + accessToken, + userRole, + userId, +}: RoutingGroupsViewProps) { + const [createModalVisible, setCreateModalVisible] = useState(false); + const [editTarget, setEditTarget] = useState | null>(null); + const [refreshKey, setRefreshKey] = useState(0); + const [selectedGroup, setSelectedGroup] = useState | null>(null); + + const handleEdit = (group: Record) => { + setEditTarget(group); + setCreateModalVisible(true); + }; + + const handleTest = (group: Record) => { + setSelectedGroup(group); + }; + + const handleClose = () => { + setCreateModalVisible(false); + setEditTarget(null); + }; + + const handleSuccess = () => { + setCreateModalVisible(false); + setEditTarget(null); + setRefreshKey((k) => k + 1); + }; + + return ( +
+
+
+ + Routing Groups + + + Configure named routing pipelines with fallback strategies + +
+ +
+ + + + {selectedGroup && accessToken && ( + <> + +
+ + Testing:{" "} + + {selectedGroup.routing_group_name as string} + + + +
+ + + )} + + {createModalVisible && ( + + )} +
+ ); +} diff --git a/ui/litellm-dashboard/src/components/routing_groups/StatsBar.tsx b/ui/litellm-dashboard/src/components/routing_groups/StatsBar.tsx new file mode 100644 index 00000000000..62973925ae7 --- /dev/null +++ b/ui/litellm-dashboard/src/components/routing_groups/StatsBar.tsx @@ -0,0 +1,29 @@ +"use client"; +import React from "react"; + +interface StatsBarProps { + requests: number; + successRate: number; // 0-100 + avgLatencyMs: number; + fallbackCount: number; +} + +export default function StatsBar({ requests, successRate, avgLatencyMs, fallbackCount }: StatsBarProps) { + const stats = [ + { label: "REQUESTS", value: requests.toLocaleString() }, + { label: "SUCCESS", value: `${Math.round(successRate)}%` }, + { label: "AVG LATENCY", value: avgLatencyMs > 0 ? `${Math.round(avgLatencyMs)}ms` : "—" }, + { label: "FALLBACKS", value: fallbackCount.toLocaleString() }, + ]; + + return ( +
+ {stats.map(({ label, value }) => ( +
+ {label} + {value} +
+ ))} +
+ ); +} diff --git a/ui/litellm-dashboard/src/components/routing_groups/WeightedRoundRobinFlow.tsx b/ui/litellm-dashboard/src/components/routing_groups/WeightedRoundRobinFlow.tsx new file mode 100644 index 00000000000..01468ff2760 --- /dev/null +++ b/ui/litellm-dashboard/src/components/routing_groups/WeightedRoundRobinFlow.tsx @@ -0,0 +1,122 @@ +"use client"; +import React from "react"; +import AnimatedStream from "./AnimatedStream"; +import DeploymentCard from "./DeploymentCard"; + +const PROVIDER_COLORS: Record = { + nebius: "#14b8a6", + fireworks_ai: "#f97316", + azure: "#8b5cf6", + openai: "#10b981", + anthropic: "#d97706", + google: "#3b82f6", + aws: "#ef4444", +}; +const getProviderColor = (p: string) => PROVIDER_COLORS[p.toLowerCase()] ?? "#6b7280"; + +interface Deployment { + deployment_id: string; + provider: string; + display_name: string; + weight?: number; + avg_latency_ms: number; + percent_of_total: number; + request_count: number; +} + +interface WeightedRoundRobinFlowProps { + deployments: Deployment[]; + routingGroupName: string; +} + +const CARD_HEIGHT = 76; +const CARD_GAP = 28; +const CARD_WIDTH = 220; +const LEFT_PADDING = 40; +const HUB_RADIUS = 28; + +export default function WeightedRoundRobinFlow({ deployments }: WeightedRoundRobinFlowProps) { + // Sort by weight desc (highest traffic first = top position) + const sorted = [...deployments].sort((a, b) => (b.weight ?? 0) - (a.weight ?? 0)); + const totalHeight = sorted.length * CARD_HEIGHT + (sorted.length - 1) * CARD_GAP + 80; + const svgWidth = 600; + + const hubX = svgWidth * 0.32; + const hubY = totalHeight / 2; + + const cardX = svgWidth - CARD_WIDTH - LEFT_PADDING; + + const blockHeight = sorted.length * CARD_HEIGHT + (sorted.length - 1) * CARD_GAP; + const blockTop = (totalHeight - blockHeight) / 2; + const cardCenterYs = sorted.map((_, i) => blockTop + i * (CARD_HEIGHT + CARD_GAP) + CARD_HEIGHT / 2); + + // Max thickness 14px for 100%, min 2px + const maxPct = Math.max(...sorted.map((d) => d.percent_of_total), 1); + const getThickness = (pct: number) => Math.max(2, Math.round((pct / maxPct) * 14)); + + // Bezier path from hub to each card center + const getPath = (targetY: number) => { + const cp1x = hubX + 80; + const cp2x = cardX - 60; + return `M ${hubX} ${hubY} C ${cp1x} ${hubY} ${cp2x} ${targetY} ${cardX} ${targetY}`; + }; + + return ( +
+ + {/* Client → hub */} + + + {/* Hub */} + + + LITELLM + + + {/* Fan-out streams */} + {sorted.map((dep, i) => ( + + ))} + + + {/* "client" label */} +
+ client +
+ + {/* Cards */} +
+
+ {sorted.map((dep) => ( + + ))} +
+
+
+ ); +}