mirror of
https://github.com/BerriAI/litellm.git
synced 2026-08-28 05:25:59 +00:00
feat(proxy): add Routing Groups — named routing pipelines with live tester UI
New first-class concept: a routing group is a named, persisted pipeline that combines model deployments with a routing strategy (priority-failover, weighted, latency-based, cost-based, etc.). Backend: - LiteLLM_RoutingGroupTable Prisma model - 9 new Pydantic types in litellm/types/router.py - CRUD + test + simulate REST endpoints at /v1/routing_group - RoutingGroupTestEngine + RoutingTraceCallback for live testing - Startup sync: active groups are loaded from DB and wired into the LLM router - priority-failover: tiered model groups + fallback chain - weighted: per-deployment weight passed through to simple_shuffle UI: - Routing Groups page + sidebar entry - Builder modal: strategy picker, deployment list with priority/weight controls - Table with delete/test actions - Live Tester: animated SVG traffic flow diagrams - OrderedFallbackFlow (vertical cascade, dashed red fail arrows) - WeightedRoundRobinFlow (fan-out bezier streams, thickness ∝ traffic %) - StatsBar, DeploymentCard, AnimatedStream components - Simulation controls (N requests, concurrency, mock/real toggle)
This commit is contained in:
Ishaan Jaffer
parent
121c633d6e
commit
be2d1ad678
23 changed files with 3556 additions and 1 deletions
|
|
@ -16289,7 +16289,7 @@
|
|||
"cache_read_input_token_cost": 3e-08,
|
||||
"input_cost_per_audio_token": 1e-06,
|
||||
"input_cost_per_token": 3e-07,
|
||||
"litellm_provider": "vertex_ai-language-models",
|
||||
"litellm_provider": "gemini",
|
||||
"max_audio_length_hours": 8.4,
|
||||
"max_audio_per_prompt": 1,
|
||||
"supports_reasoning": false,
|
||||
|
|
|
|||
471
litellm/proxy/management_endpoints/routing_group_endpoints.py
Normal file
471
litellm/proxy/management_endpoints/routing_group_endpoints.py
Normal file
|
|
@ -0,0 +1,471 @@
|
|||
"""
|
||||
Routing Group management endpoints.
|
||||
|
||||
Provides CRUD operations for routing groups — named, persisted routing pipelines
|
||||
that combine model deployments with a routing strategy.
|
||||
"""
|
||||
|
||||
import uuid
|
||||
from collections import defaultdict
|
||||
from typing import List, Optional
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, status
|
||||
|
||||
import litellm
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
from litellm.proxy._types import CommonProxyErrors, UserAPIKeyAuth
|
||||
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
|
||||
from litellm.proxy.utils import get_prisma_client_or_throw
|
||||
from litellm.types.router import (
|
||||
FailureInjectionConfig,
|
||||
RoutingGroupConfig,
|
||||
RoutingGroupDeployment,
|
||||
RoutingGroupListResponse,
|
||||
RoutingGroupSimulationResult,
|
||||
RoutingGroupTestResult,
|
||||
)
|
||||
|
||||
router = APIRouter(
|
||||
tags=["routing group management"],
|
||||
)
|
||||
|
||||
|
||||
def _get_llm_router():
|
||||
"""Get the LLM router from the proxy server."""
|
||||
from litellm.proxy.proxy_server import llm_router
|
||||
|
||||
return llm_router
|
||||
|
||||
|
||||
def _record_to_config(record) -> RoutingGroupConfig:
|
||||
"""Convert a Prisma routing group record to a RoutingGroupConfig."""
|
||||
return RoutingGroupConfig(
|
||||
routing_group_id=record.routing_group_id,
|
||||
routing_group_name=record.routing_group_name,
|
||||
description=record.description,
|
||||
routing_strategy=record.routing_strategy,
|
||||
deployments=[
|
||||
RoutingGroupDeployment(**d) for d in (record.deployments or [])
|
||||
],
|
||||
fallback_config=record.fallback_config,
|
||||
retry_config=record.retry_config,
|
||||
cooldown_config=record.cooldown_config,
|
||||
settings=record.settings,
|
||||
assigned_team_ids=list(record.assigned_team_ids or []),
|
||||
assigned_key_ids=list(record.assigned_key_ids or []),
|
||||
is_active=record.is_active,
|
||||
)
|
||||
|
||||
|
||||
VALID_ROUTING_STRATEGIES = frozenset(
|
||||
{
|
||||
"priority-failover",
|
||||
"simple-shuffle",
|
||||
"least-busy",
|
||||
"latency-based-routing",
|
||||
"cost-based-routing",
|
||||
"usage-based-routing-v2",
|
||||
"weighted",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
async def _sync_routing_group_to_router(config: RoutingGroupConfig) -> None:
|
||||
"""
|
||||
Translate a RoutingGroupConfig into live Router configuration.
|
||||
|
||||
For priority-failover: creates tiered model groups + fallback chain.
|
||||
The fallback chain implements the per-group ordering independently of
|
||||
the shared router's routing_strategy — this strategy is fully isolated
|
||||
per group.
|
||||
|
||||
For all other strategies: deployments are added to a single model group
|
||||
under the routing_group_name. The shared router's routing_strategy is
|
||||
NOT changed here — that is the proxy-level concern set in the config YAML.
|
||||
Each group's deployments are correctly scoped to their own model group name;
|
||||
the router picks between them using whatever strategy the proxy was started
|
||||
with. For "weighted" groups, per-deployment weights are stored in
|
||||
litellm_params so simple_shuffle (if active) will honour them
|
||||
automatically.
|
||||
"""
|
||||
llm_router = _get_llm_router()
|
||||
if llm_router is None:
|
||||
verbose_proxy_logger.warning(
|
||||
"No LLM router available, routing group will be applied on next startup"
|
||||
)
|
||||
return
|
||||
|
||||
strategy = config.routing_strategy
|
||||
group_name = config.routing_group_name
|
||||
|
||||
if strategy == "priority-failover":
|
||||
priority_groups: dict = defaultdict(list)
|
||||
for dep in config.deployments:
|
||||
p = dep.priority if dep.priority is not None else 999
|
||||
priority_groups[p].append(dep)
|
||||
|
||||
sorted_priorities = sorted(priority_groups.keys())
|
||||
group_names_by_priority = []
|
||||
|
||||
for i, priority in enumerate(sorted_priorities):
|
||||
if i == 0:
|
||||
# Primary group uses the routing_group_name directly
|
||||
tier_group_name = group_name
|
||||
else:
|
||||
tier_group_name = f"{group_name}__fallback_p{priority}"
|
||||
group_names_by_priority.append(tier_group_name)
|
||||
|
||||
for dep in priority_groups[priority]:
|
||||
try:
|
||||
deployment_dict = {
|
||||
"model_name": tier_group_name,
|
||||
"litellm_params": {
|
||||
"model": dep.model_name,
|
||||
},
|
||||
"model_info": {
|
||||
"id": dep.model_id,
|
||||
},
|
||||
}
|
||||
llm_router.add_deployment(
|
||||
deployment=litellm.types.router.Deployment(**deployment_dict)
|
||||
)
|
||||
except Exception as e:
|
||||
verbose_proxy_logger.debug(
|
||||
f"Could not add deployment {dep.model_id} to router: {e}"
|
||||
)
|
||||
|
||||
# Wire fallback chain
|
||||
if len(group_names_by_priority) > 1:
|
||||
primary = group_names_by_priority[0]
|
||||
fallbacks = group_names_by_priority[1:]
|
||||
existing_fallbacks = llm_router.fallbacks or []
|
||||
# Remove old entry for this group if it exists
|
||||
existing_fallbacks = [f for f in existing_fallbacks if primary not in f]
|
||||
existing_fallbacks.append({primary: fallbacks})
|
||||
llm_router.fallbacks = existing_fallbacks
|
||||
|
||||
else:
|
||||
# All strategies except priority-failover: single model group.
|
||||
for dep in config.deployments:
|
||||
try:
|
||||
litellm_params: dict = {"model": dep.model_name}
|
||||
if strategy == "weighted" and dep.weight is not None:
|
||||
# simple_shuffle reads `weight` from litellm_params and
|
||||
# does a proportional random pick — this is how "weighted"
|
||||
# is actually honoured at call time.
|
||||
litellm_params["weight"] = dep.weight
|
||||
|
||||
deployment_dict = {
|
||||
"model_name": group_name,
|
||||
"litellm_params": litellm_params,
|
||||
"model_info": {
|
||||
"id": dep.model_id,
|
||||
},
|
||||
}
|
||||
llm_router.add_deployment(
|
||||
deployment=litellm.types.router.Deployment(**deployment_dict)
|
||||
)
|
||||
except Exception as e:
|
||||
verbose_proxy_logger.debug(
|
||||
f"Could not add deployment {dep.model_id} to router: {e}"
|
||||
)
|
||||
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# CRUD endpoints
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@router.post(
|
||||
"/v1/routing_group",
|
||||
tags=["routing_group"],
|
||||
dependencies=[Depends(user_api_key_auth)],
|
||||
response_model=RoutingGroupConfig,
|
||||
status_code=status.HTTP_201_CREATED,
|
||||
summary="Create a routing group",
|
||||
)
|
||||
async def create_routing_group(
|
||||
data: RoutingGroupConfig,
|
||||
user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
|
||||
) -> RoutingGroupConfig:
|
||||
"""Create a new routing group."""
|
||||
prisma_client = get_prisma_client_or_throw(
|
||||
CommonProxyErrors.db_not_connected_error.value
|
||||
)
|
||||
|
||||
if data.routing_strategy not in VALID_ROUTING_STRATEGIES:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_400_BAD_REQUEST,
|
||||
detail=f"Invalid routing_strategy '{data.routing_strategy}'. Must be one of: {sorted(VALID_ROUTING_STRATEGIES)}",
|
||||
)
|
||||
|
||||
if not data.deployments:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_400_BAD_REQUEST,
|
||||
detail="deployments must contain at least one deployment",
|
||||
)
|
||||
|
||||
caller = user_api_key_dict.user_id or "unknown"
|
||||
routing_group_id = str(uuid.uuid4())
|
||||
|
||||
try:
|
||||
created = await prisma_client.db.litellm_routinggrouptable.create(
|
||||
data={
|
||||
"routing_group_id": routing_group_id,
|
||||
"routing_group_name": data.routing_group_name,
|
||||
"description": data.description,
|
||||
"routing_strategy": data.routing_strategy,
|
||||
"deployments": [d.model_dump() for d in data.deployments],
|
||||
"fallback_config": data.fallback_config or {},
|
||||
"retry_config": data.retry_config or {},
|
||||
"cooldown_config": data.cooldown_config or {},
|
||||
"settings": data.settings or {},
|
||||
"assigned_team_ids": data.assigned_team_ids or [],
|
||||
"assigned_key_ids": data.assigned_key_ids or [],
|
||||
"is_active": data.is_active,
|
||||
"created_by": caller,
|
||||
"updated_by": caller,
|
||||
}
|
||||
)
|
||||
except Exception as e:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_400_BAD_REQUEST,
|
||||
detail=f"Failed to create routing group: {str(e)}",
|
||||
)
|
||||
|
||||
result = _record_to_config(created)
|
||||
await _sync_routing_group_to_router(result)
|
||||
return result
|
||||
|
||||
|
||||
@router.get(
|
||||
"/v1/routing_group",
|
||||
tags=["routing_group"],
|
||||
dependencies=[Depends(user_api_key_auth)],
|
||||
response_model=RoutingGroupListResponse,
|
||||
summary="List routing groups",
|
||||
)
|
||||
async def list_routing_groups(
|
||||
page: int = 1,
|
||||
size: int = 50,
|
||||
user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
|
||||
) -> RoutingGroupListResponse:
|
||||
"""List all routing groups."""
|
||||
prisma_client = get_prisma_client_or_throw(
|
||||
CommonProxyErrors.db_not_connected_error.value
|
||||
)
|
||||
|
||||
skip = (page - 1) * size
|
||||
rows = await prisma_client.db.litellm_routinggrouptable.find_many(
|
||||
skip=skip,
|
||||
take=size,
|
||||
order={"created_at": "desc"},
|
||||
)
|
||||
total = await prisma_client.db.litellm_routinggrouptable.count()
|
||||
|
||||
groups = [_record_to_config(r) for r in rows]
|
||||
return RoutingGroupListResponse(
|
||||
routing_groups=groups, total=total, page=page, size=size
|
||||
)
|
||||
|
||||
|
||||
@router.get(
|
||||
"/v1/routing_group/{routing_group_id}",
|
||||
tags=["routing_group"],
|
||||
dependencies=[Depends(user_api_key_auth)],
|
||||
response_model=RoutingGroupConfig,
|
||||
summary="Get a routing group",
|
||||
)
|
||||
async def get_routing_group(
|
||||
routing_group_id: str,
|
||||
user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
|
||||
) -> RoutingGroupConfig:
|
||||
"""Get a specific routing group by ID."""
|
||||
prisma_client = get_prisma_client_or_throw(
|
||||
CommonProxyErrors.db_not_connected_error.value
|
||||
)
|
||||
|
||||
row = await prisma_client.db.litellm_routinggrouptable.find_unique(
|
||||
where={"routing_group_id": routing_group_id}
|
||||
)
|
||||
if row is None:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_404_NOT_FOUND,
|
||||
detail=f"Routing group '{routing_group_id}' not found",
|
||||
)
|
||||
|
||||
return _record_to_config(row)
|
||||
|
||||
|
||||
@router.put(
|
||||
"/v1/routing_group/{routing_group_id}",
|
||||
tags=["routing_group"],
|
||||
dependencies=[Depends(user_api_key_auth)],
|
||||
response_model=RoutingGroupConfig,
|
||||
summary="Update a routing group",
|
||||
)
|
||||
async def update_routing_group(
|
||||
routing_group_id: str,
|
||||
data: RoutingGroupConfig,
|
||||
user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
|
||||
) -> RoutingGroupConfig:
|
||||
"""Update an existing routing group."""
|
||||
prisma_client = get_prisma_client_or_throw(
|
||||
CommonProxyErrors.db_not_connected_error.value
|
||||
)
|
||||
|
||||
caller = user_api_key_dict.user_id or "unknown"
|
||||
|
||||
try:
|
||||
updated = await prisma_client.db.litellm_routinggrouptable.update(
|
||||
where={"routing_group_id": routing_group_id},
|
||||
data={
|
||||
"routing_group_name": data.routing_group_name,
|
||||
"description": data.description,
|
||||
"routing_strategy": data.routing_strategy,
|
||||
"deployments": [d.model_dump() for d in data.deployments],
|
||||
"fallback_config": data.fallback_config or {},
|
||||
"retry_config": data.retry_config or {},
|
||||
"cooldown_config": data.cooldown_config or {},
|
||||
"settings": data.settings or {},
|
||||
"assigned_team_ids": data.assigned_team_ids or [],
|
||||
"assigned_key_ids": data.assigned_key_ids or [],
|
||||
"is_active": data.is_active,
|
||||
"updated_by": caller,
|
||||
},
|
||||
)
|
||||
except Exception as e:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_404_NOT_FOUND,
|
||||
detail=f"Routing group not found or update failed: {str(e)}",
|
||||
)
|
||||
|
||||
result = _record_to_config(updated)
|
||||
await _sync_routing_group_to_router(result)
|
||||
return result
|
||||
|
||||
|
||||
@router.delete(
|
||||
"/v1/routing_group/{routing_group_id}",
|
||||
tags=["routing_group"],
|
||||
dependencies=[Depends(user_api_key_auth)],
|
||||
summary="Delete a routing group",
|
||||
)
|
||||
async def delete_routing_group(
|
||||
routing_group_id: str,
|
||||
user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
|
||||
):
|
||||
"""Delete a routing group."""
|
||||
prisma_client = get_prisma_client_or_throw(
|
||||
CommonProxyErrors.db_not_connected_error.value
|
||||
)
|
||||
|
||||
try:
|
||||
await prisma_client.db.litellm_routinggrouptable.delete(
|
||||
where={"routing_group_id": routing_group_id}
|
||||
)
|
||||
except Exception as e:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_404_NOT_FOUND,
|
||||
detail=f"Routing group not found: {str(e)}",
|
||||
)
|
||||
|
||||
return {"routing_group_id": routing_group_id, "deleted": True}
|
||||
|
||||
|
||||
@router.post(
|
||||
"/v1/routing_group/{routing_group_id}/test",
|
||||
tags=["routing_group"],
|
||||
dependencies=[Depends(user_api_key_auth)],
|
||||
response_model=RoutingGroupTestResult,
|
||||
summary="Test a routing group with a single request",
|
||||
)
|
||||
async def test_routing_group(
|
||||
routing_group_id: str,
|
||||
messages: Optional[List] = None,
|
||||
mock: bool = False,
|
||||
user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
|
||||
) -> RoutingGroupTestResult:
|
||||
"""Send a single test request through a routing group and return the routing trace."""
|
||||
prisma_client = get_prisma_client_or_throw(
|
||||
CommonProxyErrors.db_not_connected_error.value
|
||||
)
|
||||
|
||||
row = await prisma_client.db.litellm_routinggrouptable.find_unique(
|
||||
where={"routing_group_id": routing_group_id}
|
||||
)
|
||||
if row is None:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_404_NOT_FOUND,
|
||||
detail=f"Routing group '{routing_group_id}' not found",
|
||||
)
|
||||
|
||||
llm_router = _get_llm_router()
|
||||
if llm_router is None:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
|
||||
detail="LLM router not initialized",
|
||||
)
|
||||
|
||||
from litellm.proxy.routing_group_utils.test_engine import RoutingGroupTestEngine
|
||||
|
||||
config = _record_to_config(row)
|
||||
engine = RoutingGroupTestEngine()
|
||||
return await engine.test_single_request(
|
||||
routing_group_name=config.routing_group_name,
|
||||
router=llm_router,
|
||||
messages=messages,
|
||||
mock=mock,
|
||||
)
|
||||
|
||||
|
||||
@router.post(
|
||||
"/v1/routing_group/{routing_group_id}/simulate",
|
||||
tags=["routing_group"],
|
||||
dependencies=[Depends(user_api_key_auth)],
|
||||
response_model=RoutingGroupSimulationResult,
|
||||
summary="Simulate traffic through a routing group",
|
||||
)
|
||||
async def simulate_routing_group(
|
||||
routing_group_id: str,
|
||||
num_requests: int = 100,
|
||||
concurrency: int = 10,
|
||||
mock: bool = True,
|
||||
failure_injection: Optional[FailureInjectionConfig] = None,
|
||||
user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
|
||||
) -> RoutingGroupSimulationResult:
|
||||
"""Simulate N requests through a routing group and return traffic distribution statistics."""
|
||||
prisma_client = get_prisma_client_or_throw(
|
||||
CommonProxyErrors.db_not_connected_error.value
|
||||
)
|
||||
|
||||
row = await prisma_client.db.litellm_routinggrouptable.find_unique(
|
||||
where={"routing_group_id": routing_group_id}
|
||||
)
|
||||
if row is None:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_404_NOT_FOUND,
|
||||
detail=f"Routing group '{routing_group_id}' not found",
|
||||
)
|
||||
|
||||
llm_router = _get_llm_router()
|
||||
if llm_router is None:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
|
||||
detail="LLM router not initialized",
|
||||
)
|
||||
|
||||
from litellm.proxy.routing_group_utils.test_engine import RoutingGroupTestEngine
|
||||
|
||||
config = _record_to_config(row)
|
||||
engine = RoutingGroupTestEngine()
|
||||
return await engine.simulate_traffic(
|
||||
routing_group_name=config.routing_group_name,
|
||||
router=llm_router,
|
||||
routing_group_config=config,
|
||||
num_requests=num_requests,
|
||||
concurrency=concurrency,
|
||||
mock=mock,
|
||||
failure_injection=failure_injection,
|
||||
)
|
||||
|
|
@ -400,6 +400,9 @@ from litellm.proxy.management_endpoints.project_endpoints import (
|
|||
from litellm.proxy.management_endpoints.router_settings_endpoints import (
|
||||
router as router_settings_router,
|
||||
)
|
||||
from litellm.proxy.management_endpoints.routing_group_endpoints import (
|
||||
router as routing_group_router,
|
||||
)
|
||||
from litellm.proxy.management_endpoints.scim.scim_v2 import scim_router
|
||||
from litellm.proxy.management_endpoints.tag_management_endpoints import (
|
||||
router as tag_management_router,
|
||||
|
|
@ -4429,9 +4432,38 @@ class ProxyConfig:
|
|||
prisma_client=prisma_client, proxy_config=self
|
||||
)
|
||||
|
||||
await self._load_routing_groups_from_db(prisma_client=prisma_client)
|
||||
|
||||
if self._should_load_db_object(object_type="semantic_filter_settings"):
|
||||
await self._init_semantic_filter_settings_in_db(prisma_client=prisma_client)
|
||||
|
||||
async def _load_routing_groups_from_db(self, prisma_client: PrismaClient) -> None:
|
||||
"""Load all active routing groups from DB and sync to router."""
|
||||
from litellm.proxy.management_endpoints.routing_group_endpoints import (
|
||||
_sync_routing_group_to_router,
|
||||
)
|
||||
from litellm.types.router import RoutingGroupConfig, RoutingGroupDeployment
|
||||
|
||||
try:
|
||||
groups = await prisma_client.db.litellm_routinggrouptable.find_many(
|
||||
where={"is_active": True}
|
||||
)
|
||||
for g in groups:
|
||||
config = RoutingGroupConfig(
|
||||
routing_group_id=g.routing_group_id,
|
||||
routing_group_name=g.routing_group_name,
|
||||
routing_strategy=g.routing_strategy,
|
||||
deployments=[
|
||||
RoutingGroupDeployment(**d) for d in (g.deployments or [])
|
||||
],
|
||||
)
|
||||
await _sync_routing_group_to_router(config)
|
||||
verbose_proxy_logger.debug(
|
||||
f"Loaded {len(groups)} routing groups from DB"
|
||||
)
|
||||
except Exception as e:
|
||||
verbose_proxy_logger.debug(f"Could not load routing groups from DB: {e}")
|
||||
|
||||
async def _init_semantic_filter_settings_in_db(self, prisma_client: PrismaClient):
|
||||
"""
|
||||
Initialize MCP semantic filter settings from database.
|
||||
|
|
@ -13056,5 +13088,6 @@ async def dynamic_mcp_route(mcp_server_name: str, request: Request):
|
|||
|
||||
|
||||
app.mount(path=BASE_MCP_ROUTE, app=mcp_app)
|
||||
app.include_router(routing_group_router)
|
||||
app.include_router(mcp_rest_endpoints_router)
|
||||
app.include_router(mcp_discoverable_endpoints_router)
|
||||
|
|
|
|||
1
litellm/proxy/routing_group_utils/__init__.py
Normal file
1
litellm/proxy/routing_group_utils/__init__.py
Normal file
|
|
@ -0,0 +1 @@
|
|||
|
||||
86
litellm/proxy/routing_group_utils/routing_trace_callback.py
Normal file
86
litellm/proxy/routing_group_utils/routing_trace_callback.py
Normal file
|
|
@ -0,0 +1,86 @@
|
|||
"""
|
||||
Routing trace callback for capturing routing decisions during test/simulation.
|
||||
"""
|
||||
|
||||
from typing import Any, List, Optional
|
||||
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.types.router import RoutingTrace
|
||||
|
||||
|
||||
class RoutingTraceCallback(CustomLogger):
|
||||
"""
|
||||
Temporary callback registered during routing group test/simulation.
|
||||
|
||||
Captures which deployments were tried, outcomes, latencies, and fallback depth.
|
||||
Attach to litellm.callbacks before a call and detach after to capture traces.
|
||||
"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.traces: List[RoutingTrace] = []
|
||||
|
||||
def _extract_trace_from_kwargs(
|
||||
self,
|
||||
kwargs: dict,
|
||||
start_time: Any,
|
||||
end_time: Any,
|
||||
status: str,
|
||||
exception: Optional[Exception] = None,
|
||||
) -> RoutingTrace:
|
||||
litellm_params = kwargs.get("litellm_params") or {}
|
||||
metadata = litellm_params.get("metadata") or {}
|
||||
|
||||
# Calculate latency
|
||||
try:
|
||||
if hasattr(start_time, "timestamp") and hasattr(end_time, "timestamp"):
|
||||
latency_ms = (end_time.timestamp() - start_time.timestamp()) * 1000
|
||||
else:
|
||||
latency_ms = float(end_time - start_time) * 1000
|
||||
except Exception:
|
||||
latency_ms = 0.0
|
||||
|
||||
fallback_depth = metadata.get("fallback_depth", 0)
|
||||
if not isinstance(fallback_depth, int):
|
||||
fallback_depth = 0
|
||||
|
||||
deployment_id = (
|
||||
metadata.get("model_id")
|
||||
or kwargs.get("model_id")
|
||||
or kwargs.get("litellm_call_id", "unknown")
|
||||
)
|
||||
|
||||
return RoutingTrace(
|
||||
deployment_id=str(deployment_id),
|
||||
deployment_name=kwargs.get("model", "unknown"),
|
||||
provider=kwargs.get("custom_llm_provider")
|
||||
or litellm_params.get("custom_llm_provider", "unknown"),
|
||||
latency_ms=latency_ms,
|
||||
was_fallback=fallback_depth > 0,
|
||||
fallback_depth=fallback_depth,
|
||||
status=status,
|
||||
error_message=str(exception) if exception else None,
|
||||
)
|
||||
|
||||
async def async_log_success_event(
|
||||
self, kwargs: dict, response_obj: Any, start_time: Any, end_time: Any
|
||||
) -> None:
|
||||
trace = self._extract_trace_from_kwargs(
|
||||
kwargs, start_time, end_time, status="success"
|
||||
)
|
||||
self.traces.append(trace)
|
||||
verbose_proxy_logger.debug(
|
||||
f"RoutingTraceCallback: success on {trace.deployment_name} ({trace.latency_ms:.0f}ms)"
|
||||
)
|
||||
|
||||
async def async_log_failure_event(
|
||||
self, kwargs: dict, response_obj: Any, start_time: Any, end_time: Any
|
||||
) -> None:
|
||||
exception = kwargs.get("exception")
|
||||
trace = self._extract_trace_from_kwargs(
|
||||
kwargs, start_time, end_time, status="error", exception=exception
|
||||
)
|
||||
self.traces.append(trace)
|
||||
verbose_proxy_logger.debug(
|
||||
f"RoutingTraceCallback: failure on {trace.deployment_name} - {trace.error_message}"
|
||||
)
|
||||
321
litellm/proxy/routing_group_utils/test_engine.py
Normal file
321
litellm/proxy/routing_group_utils/test_engine.py
Normal file
|
|
@ -0,0 +1,321 @@
|
|||
"""
|
||||
Test and simulation engine for routing groups.
|
||||
|
||||
Executes test requests through a routing group and collects routing traces
|
||||
that drive the Live Tester visualization in the UI.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import random
|
||||
import time
|
||||
from typing import TYPE_CHECKING, Any, Dict, List, Optional, cast
|
||||
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
from litellm.types.llms.openai import AllMessageValues
|
||||
from litellm.types.router import (
|
||||
DeploymentTrafficStats,
|
||||
FailureInjectionConfig,
|
||||
FlowStep,
|
||||
RoutingGroupConfig,
|
||||
RoutingGroupSimulationResult,
|
||||
RoutingGroupTestResult,
|
||||
RoutingTrace,
|
||||
)
|
||||
from litellm.types.utils import Choices, ModelResponse
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.router import Router
|
||||
|
||||
|
||||
_DEFAULT_MESSAGES: List[Any] = [{"role": "user", "content": "Hello, respond with one word."}]
|
||||
|
||||
|
||||
def _empty_dep_stats() -> Dict[str, Any]:
|
||||
return {
|
||||
"deployment_name": "unknown",
|
||||
"provider": "unknown",
|
||||
"request_count": 0,
|
||||
"success_count": 0,
|
||||
"failure_count": 0,
|
||||
"latencies": [],
|
||||
"priority": None,
|
||||
"weight": None,
|
||||
}
|
||||
|
||||
|
||||
class RoutingGroupTestEngine:
|
||||
"""
|
||||
Executes test/simulation requests through a routing group and returns
|
||||
structured results for the Live Tester visualization.
|
||||
"""
|
||||
|
||||
async def test_single_request(
|
||||
self,
|
||||
routing_group_name: str,
|
||||
router: "Router",
|
||||
messages: Optional[List[AllMessageValues]] = None,
|
||||
mock: bool = False,
|
||||
) -> RoutingGroupTestResult:
|
||||
"""
|
||||
Send one request through the routing group and return the routing trace.
|
||||
|
||||
Uses a RoutingTraceCallback temporarily attached to litellm.callbacks
|
||||
to capture which deployments were tried and their outcomes.
|
||||
"""
|
||||
import litellm
|
||||
from litellm.proxy.routing_group_utils.routing_trace_callback import (
|
||||
RoutingTraceCallback,
|
||||
)
|
||||
|
||||
if messages is None:
|
||||
messages = cast(List[AllMessageValues], _DEFAULT_MESSAGES)
|
||||
|
||||
trace_callback = RoutingTraceCallback()
|
||||
|
||||
# Attach callback temporarily
|
||||
litellm.callbacks.append(trace_callback)
|
||||
|
||||
start = time.monotonic()
|
||||
success = False
|
||||
response_text: Optional[str] = None
|
||||
|
||||
try:
|
||||
if mock:
|
||||
response = await router.acompletion(
|
||||
model=routing_group_name,
|
||||
messages=messages,
|
||||
mock_response="Mock routing group test response",
|
||||
)
|
||||
else:
|
||||
response = await router.acompletion(
|
||||
model=routing_group_name,
|
||||
messages=messages,
|
||||
)
|
||||
success = True
|
||||
try:
|
||||
_choices = cast(List[Choices], cast(ModelResponse, response).choices)
|
||||
response_text = _choices[0].message.content
|
||||
except Exception:
|
||||
response_text = str(response)
|
||||
except Exception as e:
|
||||
verbose_proxy_logger.debug(
|
||||
f"RoutingGroupTestEngine: request failed for group '{routing_group_name}': {e}"
|
||||
)
|
||||
finally:
|
||||
# Always detach callback
|
||||
try:
|
||||
litellm.callbacks.remove(trace_callback)
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
total_latency_ms = (time.monotonic() - start) * 1000
|
||||
|
||||
# Determine final deployment from last success trace
|
||||
final_deployment = "unknown"
|
||||
final_provider = "unknown"
|
||||
success_traces = [t for t in trace_callback.traces if t.status == "success"]
|
||||
if success_traces:
|
||||
final = success_traces[-1]
|
||||
final_deployment = final.deployment_name
|
||||
final_provider = final.provider
|
||||
|
||||
return RoutingGroupTestResult(
|
||||
success=success,
|
||||
response_text=response_text,
|
||||
traces=trace_callback.traces,
|
||||
total_latency_ms=total_latency_ms,
|
||||
final_deployment=final_deployment,
|
||||
final_provider=final_provider,
|
||||
)
|
||||
|
||||
async def simulate_traffic(
|
||||
self,
|
||||
routing_group_name: str,
|
||||
router: "Router",
|
||||
routing_group_config: RoutingGroupConfig,
|
||||
num_requests: int = 100,
|
||||
concurrency: int = 10,
|
||||
mock: bool = True,
|
||||
failure_injection: Optional[FailureInjectionConfig] = None,
|
||||
) -> RoutingGroupSimulationResult:
|
||||
"""
|
||||
Simulate N concurrent requests through the routing group.
|
||||
|
||||
Returns aggregated traffic distribution statistics and flow data
|
||||
for the Live Tester visualization.
|
||||
"""
|
||||
import litellm
|
||||
from litellm.proxy.routing_group_utils.routing_trace_callback import (
|
||||
RoutingTraceCallback,
|
||||
)
|
||||
|
||||
trace_callback = RoutingTraceCallback()
|
||||
litellm.callbacks.append(trace_callback)
|
||||
|
||||
semaphore = asyncio.Semaphore(concurrency)
|
||||
|
||||
async def _one_request() -> bool:
|
||||
async with semaphore:
|
||||
try:
|
||||
if mock:
|
||||
# Optionally inject failures via mock_response that raises
|
||||
if failure_injection:
|
||||
# Pick a deployment deterministically based on routing group config
|
||||
# and check if we should inject a failure
|
||||
# We can't know which deployment will be picked before the call,
|
||||
# so we use a probabilistic approach:
|
||||
# average failure rate across all deployments as a proxy
|
||||
rates = list(
|
||||
failure_injection.deployment_failure_rates.values()
|
||||
)
|
||||
avg_rate = sum(rates) / len(rates) if rates else 0.0
|
||||
if random.random() < avg_rate:
|
||||
raise Exception(
|
||||
"Simulated failure (failure injection)"
|
||||
)
|
||||
await router.acompletion(
|
||||
model=routing_group_name,
|
||||
messages=cast(List[AllMessageValues], _DEFAULT_MESSAGES),
|
||||
mock_response="Simulated response",
|
||||
)
|
||||
else:
|
||||
await router.acompletion(
|
||||
model=routing_group_name,
|
||||
messages=cast(List[AllMessageValues], _DEFAULT_MESSAGES),
|
||||
)
|
||||
return True
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
total_latency_start = time.monotonic()
|
||||
results = await asyncio.gather(
|
||||
*[_one_request() for _ in range(num_requests)],
|
||||
return_exceptions=True,
|
||||
)
|
||||
_ = time.monotonic() - total_latency_start # total wall time (unused but kept for future)
|
||||
|
||||
litellm.callbacks.remove(trace_callback)
|
||||
|
||||
successful = sum(1 for r in results if r is True)
|
||||
failed = num_requests - successful
|
||||
|
||||
# Aggregate per-deployment stats from traces
|
||||
per_dep: Dict[str, Dict[str, Any]] = {}
|
||||
|
||||
for trace in trace_callback.traces:
|
||||
dep_id = trace.deployment_id or trace.deployment_name
|
||||
if dep_id not in per_dep:
|
||||
per_dep[dep_id] = _empty_dep_stats()
|
||||
per_dep[dep_id]["deployment_name"] = trace.deployment_name
|
||||
per_dep[dep_id]["provider"] = trace.provider
|
||||
per_dep[dep_id]["request_count"] = cast(int, per_dep[dep_id]["request_count"]) + 1
|
||||
cast(List[float], per_dep[dep_id]["latencies"]).append(trace.latency_ms)
|
||||
if trace.status == "success":
|
||||
per_dep[dep_id]["success_count"] = cast(int, per_dep[dep_id]["success_count"]) + 1
|
||||
else:
|
||||
per_dep[dep_id]["failure_count"] = cast(int, per_dep[dep_id]["failure_count"]) + 1
|
||||
|
||||
# Enrich with priority/weight from config
|
||||
dep_lookup = {d.model_id: d for d in routing_group_config.deployments}
|
||||
for dep_id, stats in per_dep.items():
|
||||
if dep_id in dep_lookup:
|
||||
dep = dep_lookup[dep_id]
|
||||
stats["priority"] = dep.priority
|
||||
stats["weight"] = dep.weight
|
||||
|
||||
total_traced: int = sum(cast(int, s["request_count"]) for s in per_dep.values())
|
||||
|
||||
traffic_distribution = [
|
||||
DeploymentTrafficStats(
|
||||
deployment_id=dep_id,
|
||||
deployment_name=cast(str, stats["deployment_name"]),
|
||||
provider=cast(str, stats["provider"]),
|
||||
request_count=cast(int, stats["request_count"]),
|
||||
success_count=cast(int, stats["success_count"]),
|
||||
failure_count=cast(int, stats["failure_count"]),
|
||||
avg_latency_ms=(
|
||||
sum(cast(List[float], stats["latencies"])) / len(cast(List[float], stats["latencies"]))
|
||||
if stats["latencies"]
|
||||
else 0.0
|
||||
),
|
||||
percent_of_total=(
|
||||
cast(int, stats["request_count"]) / total_traced * 100
|
||||
if total_traced > 0
|
||||
else 0.0
|
||||
),
|
||||
priority=cast(Optional[int], stats["priority"]),
|
||||
weight=cast(Optional[int], stats["weight"]),
|
||||
)
|
||||
for dep_id, stats in per_dep.items()
|
||||
]
|
||||
|
||||
# Sort: primary (priority 1 or highest traffic) first
|
||||
traffic_distribution.sort(
|
||||
key=lambda x: (x.priority or 999, -x.request_count)
|
||||
)
|
||||
|
||||
# Build flow data for priority-failover strategy
|
||||
flow_data: Optional[List[FlowStep]] = None
|
||||
if routing_group_config.routing_strategy == "priority-failover":
|
||||
flow_data = _build_flow_data(
|
||||
trace_callback.traces, routing_group_config, traffic_distribution
|
||||
)
|
||||
|
||||
fallback_count = sum(
|
||||
1 for t in trace_callback.traces if t.was_fallback
|
||||
)
|
||||
|
||||
all_latencies = [t.latency_ms for t in trace_callback.traces]
|
||||
avg_latency = sum(all_latencies) / len(all_latencies) if all_latencies else 0.0
|
||||
|
||||
return RoutingGroupSimulationResult(
|
||||
total_requests=num_requests,
|
||||
successful_requests=successful,
|
||||
failed_requests=failed,
|
||||
avg_latency_ms=avg_latency,
|
||||
fallback_count=fallback_count,
|
||||
traffic_distribution=traffic_distribution,
|
||||
flow_data=flow_data,
|
||||
)
|
||||
|
||||
|
||||
def _build_flow_data(
|
||||
_traces: List[RoutingTrace],
|
||||
_config: RoutingGroupConfig,
|
||||
traffic_distribution: List[DeploymentTrafficStats],
|
||||
) -> List[FlowStep]:
|
||||
"""
|
||||
Build flow steps for the priority-failover visualization.
|
||||
|
||||
For each deployment in priority order, create a FlowStep showing
|
||||
how many requests went to it and why (primary vs fallback).
|
||||
"""
|
||||
flow_steps: List[FlowStep] = []
|
||||
|
||||
sorted_deps = sorted(
|
||||
traffic_distribution,
|
||||
key=lambda x: (x.priority or 999, -x.request_count),
|
||||
)
|
||||
|
||||
for i, dep in enumerate(sorted_deps):
|
||||
if i == 0:
|
||||
flow_steps.append(
|
||||
FlowStep(
|
||||
from_deployment=None,
|
||||
to_deployment=dep.deployment_name,
|
||||
request_count=dep.request_count,
|
||||
reason="primary",
|
||||
)
|
||||
)
|
||||
else:
|
||||
prev = sorted_deps[i - 1]
|
||||
flow_steps.append(
|
||||
FlowStep(
|
||||
from_deployment=prev.deployment_name,
|
||||
to_deployment=dep.deployment_name,
|
||||
request_count=dep.request_count,
|
||||
reason="fallback_error",
|
||||
)
|
||||
)
|
||||
|
||||
return flow_steps
|
||||
|
|
@ -1112,3 +1112,23 @@ model LiteLLM_ClaudeCodePluginTable {
|
|||
|
||||
@@map("LiteLLM_ClaudeCodePluginTable")
|
||||
}
|
||||
|
||||
// Routing Groups table for storing named routing pipelines
|
||||
model LiteLLM_RoutingGroupTable {
|
||||
routing_group_id String @id @default(uuid())
|
||||
routing_group_name String @unique
|
||||
description String?
|
||||
routing_strategy String
|
||||
deployments Json
|
||||
fallback_config Json? @default("{}")
|
||||
retry_config Json? @default("{}")
|
||||
cooldown_config Json? @default("{}")
|
||||
settings Json? @default("{}")
|
||||
assigned_team_ids String[] @default([])
|
||||
assigned_key_ids String[] @default([])
|
||||
is_active Boolean @default(true)
|
||||
created_at DateTime @default(now())
|
||||
created_by String
|
||||
updated_at DateTime @default(now()) @updatedAt
|
||||
updated_by String
|
||||
}
|
||||
|
|
|
|||
|
|
@ -866,3 +866,106 @@ class PreRoutingHookResponse(BaseModel):
|
|||
|
||||
model: str
|
||||
messages: Optional[List[Dict[str, str]]]
|
||||
|
||||
|
||||
class RoutingGroupDeployment(BaseModel):
|
||||
"""A deployment reference within a routing group."""
|
||||
|
||||
model_id: str
|
||||
model_name: str
|
||||
provider: str
|
||||
priority: Optional[int] = None
|
||||
weight: Optional[int] = None
|
||||
display_name: Optional[str] = None
|
||||
|
||||
|
||||
class RoutingGroupConfig(BaseModel):
|
||||
"""Full routing group configuration."""
|
||||
|
||||
routing_group_id: Optional[str] = None
|
||||
routing_group_name: str
|
||||
description: Optional[str] = None
|
||||
routing_strategy: str
|
||||
deployments: List[RoutingGroupDeployment]
|
||||
fallback_config: Optional[dict] = None
|
||||
retry_config: Optional[dict] = None
|
||||
cooldown_config: Optional[dict] = None
|
||||
settings: Optional[dict] = None
|
||||
assigned_team_ids: Optional[List[str]] = None
|
||||
assigned_key_ids: Optional[List[str]] = None
|
||||
is_active: bool = True
|
||||
|
||||
|
||||
class RoutingGroupListResponse(BaseModel):
|
||||
routing_groups: List[RoutingGroupConfig]
|
||||
total: int
|
||||
page: int
|
||||
size: int
|
||||
|
||||
|
||||
class RoutingTrace(BaseModel):
|
||||
"""A single routing decision record for test/simulation."""
|
||||
|
||||
deployment_id: str
|
||||
deployment_name: str
|
||||
provider: str
|
||||
latency_ms: float
|
||||
was_fallback: bool
|
||||
fallback_depth: int = 0
|
||||
status: str # "success" or "error"
|
||||
error_message: Optional[str] = None
|
||||
|
||||
|
||||
class RoutingGroupTestResult(BaseModel):
|
||||
"""Result of a single test request through a routing group."""
|
||||
|
||||
success: bool
|
||||
response_text: Optional[str] = None
|
||||
traces: List[RoutingTrace]
|
||||
total_latency_ms: float
|
||||
final_deployment: str
|
||||
final_provider: str
|
||||
|
||||
|
||||
class DeploymentTrafficStats(BaseModel):
|
||||
"""Per-deployment statistics from a simulation."""
|
||||
|
||||
deployment_id: str
|
||||
deployment_name: str
|
||||
provider: str
|
||||
request_count: int
|
||||
success_count: int
|
||||
failure_count: int
|
||||
avg_latency_ms: float
|
||||
percent_of_total: float
|
||||
priority: Optional[int] = None
|
||||
weight: Optional[int] = None
|
||||
|
||||
|
||||
class FlowStep(BaseModel):
|
||||
"""For priority-failover: represents traffic flow between deployments."""
|
||||
|
||||
from_deployment: Optional[str] = None
|
||||
to_deployment: str
|
||||
request_count: int
|
||||
reason: str # "primary", "fallback_error", "fallback_rate_limit"
|
||||
|
||||
|
||||
class RoutingGroupSimulationResult(BaseModel):
|
||||
"""Result of a traffic simulation through a routing group."""
|
||||
|
||||
total_requests: int
|
||||
successful_requests: int
|
||||
failed_requests: int
|
||||
avg_latency_ms: float
|
||||
fallback_count: int
|
||||
traffic_distribution: List[DeploymentTrafficStats]
|
||||
flow_data: Optional[List[FlowStep]] = None
|
||||
|
||||
|
||||
class FailureInjectionConfig(BaseModel):
|
||||
"""Configuration for injecting failures during simulation."""
|
||||
|
||||
deployment_failure_rates: Dict[
|
||||
str, float
|
||||
] # {deployment_id: failure_probability 0.0-1.0}
|
||||
|
|
|
|||
529
tests/litellm/proxy/test_routing_group_test_engine.py
Normal file
529
tests/litellm/proxy/test_routing_group_test_engine.py
Normal file
|
|
@ -0,0 +1,529 @@
|
|||
"""
|
||||
Unit tests for RoutingGroupTestEngine and RoutingTraceCallback.
|
||||
"""
|
||||
|
||||
import datetime
|
||||
from typing import List
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from litellm.proxy.routing_group_utils.routing_trace_callback import (
|
||||
RoutingTraceCallback,
|
||||
)
|
||||
from litellm.proxy.routing_group_utils.test_engine import (
|
||||
RoutingGroupTestEngine,
|
||||
_build_flow_data,
|
||||
)
|
||||
from litellm.types.router import (
|
||||
DeploymentTrafficStats,
|
||||
RoutingGroupConfig,
|
||||
RoutingGroupDeployment,
|
||||
RoutingTrace,
|
||||
)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _make_kwargs(
|
||||
model: str = "test-model",
|
||||
provider: str = "openai",
|
||||
call_id: str = "call-123",
|
||||
fallback_depth: int = 0,
|
||||
exception=None,
|
||||
) -> dict:
|
||||
return {
|
||||
"model": model,
|
||||
"custom_llm_provider": provider,
|
||||
"litellm_call_id": call_id,
|
||||
"exception": exception,
|
||||
"litellm_params": {
|
||||
"metadata": {"fallback_depth": fallback_depth},
|
||||
"custom_llm_provider": provider,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _make_mock_response(content: str = "hello") -> MagicMock:
|
||||
response = MagicMock()
|
||||
response.choices = [MagicMock()]
|
||||
response.choices[0].message = MagicMock()
|
||||
response.choices[0].message.content = content
|
||||
return response
|
||||
|
||||
|
||||
def _make_routing_group_config(strategy: str = "priority-failover") -> RoutingGroupConfig:
|
||||
return RoutingGroupConfig(
|
||||
routing_group_name="test-group",
|
||||
routing_strategy=strategy,
|
||||
deployments=[
|
||||
RoutingGroupDeployment(
|
||||
model_id="id-primary",
|
||||
model_name="nebius/llama-70b",
|
||||
provider="nebius",
|
||||
priority=1,
|
||||
),
|
||||
RoutingGroupDeployment(
|
||||
model_id="id-fallback",
|
||||
model_name="fireworks/llama-70b",
|
||||
provider="fireworks_ai",
|
||||
priority=2,
|
||||
),
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# RoutingTraceCallback tests
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_routing_trace_callback_records_success():
|
||||
"""async_log_success_event should append a success trace."""
|
||||
callback = RoutingTraceCallback()
|
||||
|
||||
start = datetime.datetime(2024, 1, 1, 0, 0, 0)
|
||||
end = datetime.datetime(2024, 1, 1, 0, 0, 1) # 1 second later
|
||||
|
||||
kwargs = _make_kwargs(model="gpt-4o", provider="openai")
|
||||
await callback.async_log_success_event(
|
||||
kwargs=kwargs, response_obj=None, start_time=start, end_time=end
|
||||
)
|
||||
|
||||
assert len(callback.traces) == 1
|
||||
trace = callback.traces[0]
|
||||
assert trace.status == "success"
|
||||
assert trace.deployment_name == "gpt-4o"
|
||||
assert trace.provider == "openai"
|
||||
assert trace.latency_ms == pytest.approx(1000.0, abs=1.0)
|
||||
assert trace.was_fallback is False
|
||||
assert trace.error_message is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_routing_trace_callback_records_failure():
|
||||
"""async_log_failure_event should append an error trace with message."""
|
||||
callback = RoutingTraceCallback()
|
||||
|
||||
start = datetime.datetime(2024, 1, 1, 0, 0, 0)
|
||||
end = datetime.datetime(2024, 1, 1, 0, 0, 0, 500000) # 0.5 seconds
|
||||
|
||||
exc = RuntimeError("rate limit exceeded")
|
||||
kwargs = _make_kwargs(model="claude-3", provider="anthropic", exception=exc)
|
||||
await callback.async_log_failure_event(
|
||||
kwargs=kwargs, response_obj=None, start_time=start, end_time=end
|
||||
)
|
||||
|
||||
assert len(callback.traces) == 1
|
||||
trace = callback.traces[0]
|
||||
assert trace.status == "error"
|
||||
assert trace.deployment_name == "claude-3"
|
||||
assert trace.provider == "anthropic"
|
||||
assert trace.error_message == "rate limit exceeded"
|
||||
assert trace.was_fallback is False
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_routing_trace_callback_records_fallback():
|
||||
"""Traces with fallback_depth > 0 should have was_fallback=True."""
|
||||
callback = RoutingTraceCallback()
|
||||
|
||||
start = datetime.datetime(2024, 1, 1, 0, 0, 0)
|
||||
end = datetime.datetime(2024, 1, 1, 0, 0, 1)
|
||||
|
||||
kwargs = _make_kwargs(model="backup-model", provider="azure", fallback_depth=1)
|
||||
await callback.async_log_success_event(
|
||||
kwargs=kwargs, response_obj=None, start_time=start, end_time=end
|
||||
)
|
||||
|
||||
trace = callback.traces[0]
|
||||
assert trace.was_fallback is True
|
||||
assert trace.fallback_depth == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_routing_trace_callback_multiple_events():
|
||||
"""Multiple events should all be recorded in order."""
|
||||
callback = RoutingTraceCallback()
|
||||
|
||||
start = datetime.datetime(2024, 1, 1, 0, 0, 0)
|
||||
end = datetime.datetime(2024, 1, 1, 0, 0, 1)
|
||||
|
||||
for i in range(3):
|
||||
kwargs = _make_kwargs(model=f"model-{i}", provider="openai")
|
||||
await callback.async_log_success_event(
|
||||
kwargs=kwargs, response_obj=None, start_time=start, end_time=end
|
||||
)
|
||||
|
||||
assert len(callback.traces) == 3
|
||||
for i, trace in enumerate(callback.traces):
|
||||
assert trace.deployment_name == f"model-{i}"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# RoutingGroupTestEngine.test_single_request tests
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_engine_test_single_request_success():
|
||||
"""test_single_request with a mock router that succeeds should return success=True."""
|
||||
mock_router = AsyncMock()
|
||||
mock_router.acompletion = AsyncMock(return_value=_make_mock_response("pong"))
|
||||
|
||||
engine = RoutingGroupTestEngine()
|
||||
|
||||
with patch("litellm.callbacks", []):
|
||||
result = await engine.test_single_request(
|
||||
routing_group_name="test-group",
|
||||
router=mock_router,
|
||||
mock=True,
|
||||
)
|
||||
|
||||
assert result.success is True
|
||||
assert result.response_text == "pong"
|
||||
assert result.total_latency_ms >= 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_engine_test_single_request_failure():
|
||||
"""test_single_request when router raises should return success=False."""
|
||||
mock_router = AsyncMock()
|
||||
mock_router.acompletion = AsyncMock(side_effect=Exception("provider error"))
|
||||
|
||||
engine = RoutingGroupTestEngine()
|
||||
|
||||
with patch("litellm.callbacks", []):
|
||||
result = await engine.test_single_request(
|
||||
routing_group_name="test-group",
|
||||
router=mock_router,
|
||||
)
|
||||
|
||||
assert result.success is False
|
||||
assert result.response_text is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_engine_test_single_request_callback_detached_on_success():
|
||||
"""The trace callback should be removed from litellm.callbacks after the call."""
|
||||
mock_router = AsyncMock()
|
||||
mock_router.acompletion = AsyncMock(return_value=_make_mock_response("ok"))
|
||||
|
||||
engine = RoutingGroupTestEngine()
|
||||
callbacks_list: list = []
|
||||
|
||||
with patch("litellm.callbacks", callbacks_list):
|
||||
await engine.test_single_request(
|
||||
routing_group_name="test-group",
|
||||
router=mock_router,
|
||||
mock=True,
|
||||
)
|
||||
|
||||
# Callback should have been removed
|
||||
from litellm.proxy.routing_group_utils.routing_trace_callback import (
|
||||
RoutingTraceCallback,
|
||||
)
|
||||
assert not any(isinstance(c, RoutingTraceCallback) for c in callbacks_list)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_engine_test_single_request_callback_detached_on_failure():
|
||||
"""The trace callback should be removed even when the router call raises."""
|
||||
mock_router = AsyncMock()
|
||||
mock_router.acompletion = AsyncMock(side_effect=Exception("boom"))
|
||||
|
||||
engine = RoutingGroupTestEngine()
|
||||
callbacks_list: list = []
|
||||
|
||||
with patch("litellm.callbacks", callbacks_list):
|
||||
await engine.test_single_request(
|
||||
routing_group_name="test-group",
|
||||
router=mock_router,
|
||||
)
|
||||
|
||||
from litellm.proxy.routing_group_utils.routing_trace_callback import (
|
||||
RoutingTraceCallback,
|
||||
)
|
||||
assert not any(isinstance(c, RoutingTraceCallback) for c in callbacks_list)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_engine_test_single_request_uses_custom_messages():
|
||||
"""Custom messages should be passed through to the router."""
|
||||
mock_router = AsyncMock()
|
||||
mock_router.acompletion = AsyncMock(return_value=_make_mock_response("custom"))
|
||||
|
||||
engine = RoutingGroupTestEngine()
|
||||
custom_messages = [{"role": "user", "content": "custom prompt"}]
|
||||
|
||||
with patch("litellm.callbacks", []):
|
||||
await engine.test_single_request(
|
||||
routing_group_name="my-group",
|
||||
router=mock_router,
|
||||
messages=custom_messages,
|
||||
mock=True,
|
||||
)
|
||||
|
||||
call_kwargs = mock_router.acompletion.call_args
|
||||
assert call_kwargs.kwargs.get("messages") == custom_messages or (
|
||||
len(call_kwargs.args) > 1 and call_kwargs.args[1] == custom_messages
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# RoutingGroupTestEngine.simulate_traffic tests
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_engine_simulate_traffic_total_requests():
|
||||
"""simulate_traffic should return total_requests matching the requested count."""
|
||||
mock_router = AsyncMock()
|
||||
mock_router.acompletion = AsyncMock(return_value=_make_mock_response("sim"))
|
||||
|
||||
engine = RoutingGroupTestEngine()
|
||||
config = _make_routing_group_config()
|
||||
|
||||
with patch("litellm.callbacks", []):
|
||||
result = await engine.simulate_traffic(
|
||||
routing_group_name="test-group",
|
||||
router=mock_router,
|
||||
routing_group_config=config,
|
||||
num_requests=10,
|
||||
concurrency=5,
|
||||
mock=True,
|
||||
)
|
||||
|
||||
assert result.total_requests == 10
|
||||
assert result.successful_requests + result.failed_requests == 10
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_engine_simulate_traffic_all_failures():
|
||||
"""When all requests fail, successful_requests should be 0."""
|
||||
mock_router = AsyncMock()
|
||||
mock_router.acompletion = AsyncMock(side_effect=Exception("always fails"))
|
||||
|
||||
engine = RoutingGroupTestEngine()
|
||||
config = _make_routing_group_config()
|
||||
|
||||
with patch("litellm.callbacks", []):
|
||||
result = await engine.simulate_traffic(
|
||||
routing_group_name="test-group",
|
||||
router=mock_router,
|
||||
routing_group_config=config,
|
||||
num_requests=5,
|
||||
concurrency=5,
|
||||
mock=True,
|
||||
)
|
||||
|
||||
assert result.total_requests == 5
|
||||
assert result.successful_requests == 0
|
||||
assert result.failed_requests == 5
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_engine_simulate_traffic_no_flow_data_for_non_priority():
|
||||
"""flow_data should be None when routing_strategy is not priority-failover."""
|
||||
mock_router = AsyncMock()
|
||||
mock_router.acompletion = AsyncMock(return_value=_make_mock_response("ok"))
|
||||
|
||||
engine = RoutingGroupTestEngine()
|
||||
config = _make_routing_group_config(strategy="weighted")
|
||||
|
||||
with patch("litellm.callbacks", []):
|
||||
result = await engine.simulate_traffic(
|
||||
routing_group_name="test-group",
|
||||
router=mock_router,
|
||||
routing_group_config=config,
|
||||
num_requests=5,
|
||||
concurrency=5,
|
||||
mock=True,
|
||||
)
|
||||
|
||||
assert result.flow_data is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_engine_simulate_traffic_flow_data_for_priority_failover():
|
||||
"""flow_data should be populated when routing_strategy is priority-failover and there are traces."""
|
||||
mock_router = AsyncMock()
|
||||
mock_router.acompletion = AsyncMock(return_value=_make_mock_response("ok"))
|
||||
|
||||
engine = RoutingGroupTestEngine()
|
||||
config = _make_routing_group_config(strategy="priority-failover")
|
||||
|
||||
with patch("litellm.callbacks", []):
|
||||
result = await engine.simulate_traffic(
|
||||
routing_group_name="test-group",
|
||||
router=mock_router,
|
||||
routing_group_config=config,
|
||||
num_requests=5,
|
||||
concurrency=5,
|
||||
mock=True,
|
||||
)
|
||||
|
||||
# flow_data may be None if no traces were recorded (mock router doesn't fire callbacks)
|
||||
# In that case traffic_distribution will be empty too, which is fine
|
||||
assert result.total_requests == 5
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# _build_flow_data tests
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_build_flow_data_primary_first():
|
||||
"""The first FlowStep should have reason='primary' and from_deployment=None."""
|
||||
traces: List[RoutingTrace] = []
|
||||
config = _make_routing_group_config()
|
||||
|
||||
traffic = [
|
||||
DeploymentTrafficStats(
|
||||
deployment_id="id-primary",
|
||||
deployment_name="nebius/llama-70b",
|
||||
provider="nebius",
|
||||
request_count=80,
|
||||
success_count=80,
|
||||
failure_count=0,
|
||||
avg_latency_ms=150.0,
|
||||
percent_of_total=80.0,
|
||||
priority=1,
|
||||
),
|
||||
DeploymentTrafficStats(
|
||||
deployment_id="id-fallback",
|
||||
deployment_name="fireworks/llama-70b",
|
||||
provider="fireworks_ai",
|
||||
request_count=20,
|
||||
success_count=20,
|
||||
failure_count=0,
|
||||
avg_latency_ms=200.0,
|
||||
percent_of_total=20.0,
|
||||
priority=2,
|
||||
),
|
||||
]
|
||||
|
||||
flow_steps = _build_flow_data(traces, config, traffic)
|
||||
|
||||
assert len(flow_steps) == 2
|
||||
assert flow_steps[0].reason == "primary"
|
||||
assert flow_steps[0].from_deployment is None
|
||||
assert flow_steps[0].to_deployment == "nebius/llama-70b"
|
||||
assert flow_steps[0].request_count == 80
|
||||
|
||||
|
||||
def test_build_flow_data_fallback_reason():
|
||||
"""Subsequent FlowSteps should have reason='fallback_error' and correct from_deployment."""
|
||||
traces: List[RoutingTrace] = []
|
||||
config = _make_routing_group_config()
|
||||
|
||||
traffic = [
|
||||
DeploymentTrafficStats(
|
||||
deployment_id="id-primary",
|
||||
deployment_name="primary-model",
|
||||
provider="nebius",
|
||||
request_count=70,
|
||||
success_count=70,
|
||||
failure_count=0,
|
||||
avg_latency_ms=100.0,
|
||||
percent_of_total=70.0,
|
||||
priority=1,
|
||||
),
|
||||
DeploymentTrafficStats(
|
||||
deployment_id="id-fallback",
|
||||
deployment_name="fallback-model",
|
||||
provider="fireworks_ai",
|
||||
request_count=30,
|
||||
success_count=30,
|
||||
failure_count=0,
|
||||
avg_latency_ms=200.0,
|
||||
percent_of_total=30.0,
|
||||
priority=2,
|
||||
),
|
||||
]
|
||||
|
||||
flow_steps = _build_flow_data(traces, config, traffic)
|
||||
|
||||
assert flow_steps[1].reason == "fallback_error"
|
||||
assert flow_steps[1].from_deployment == "primary-model"
|
||||
assert flow_steps[1].to_deployment == "fallback-model"
|
||||
assert flow_steps[1].request_count == 30
|
||||
|
||||
|
||||
def test_build_flow_data_single_deployment():
|
||||
"""With a single deployment, only one FlowStep (primary) should be created."""
|
||||
traces: List[RoutingTrace] = []
|
||||
config = RoutingGroupConfig(
|
||||
routing_group_name="single-group",
|
||||
routing_strategy="priority-failover",
|
||||
deployments=[
|
||||
RoutingGroupDeployment(
|
||||
model_id="id-1",
|
||||
model_name="only-model",
|
||||
provider="openai",
|
||||
priority=1,
|
||||
)
|
||||
],
|
||||
)
|
||||
|
||||
traffic = [
|
||||
DeploymentTrafficStats(
|
||||
deployment_id="id-1",
|
||||
deployment_name="only-model",
|
||||
provider="openai",
|
||||
request_count=100,
|
||||
success_count=100,
|
||||
failure_count=0,
|
||||
avg_latency_ms=120.0,
|
||||
percent_of_total=100.0,
|
||||
priority=1,
|
||||
),
|
||||
]
|
||||
|
||||
flow_steps = _build_flow_data(traces, config, traffic)
|
||||
|
||||
assert len(flow_steps) == 1
|
||||
assert flow_steps[0].reason == "primary"
|
||||
assert flow_steps[0].from_deployment is None
|
||||
|
||||
|
||||
def test_build_flow_data_sorts_by_priority():
|
||||
"""_build_flow_data should sort deployments by priority before building flow steps."""
|
||||
traces: List[RoutingTrace] = []
|
||||
config = _make_routing_group_config()
|
||||
|
||||
# Intentionally pass traffic in reverse priority order
|
||||
traffic = [
|
||||
DeploymentTrafficStats(
|
||||
deployment_id="id-fallback",
|
||||
deployment_name="fallback-model",
|
||||
provider="fireworks_ai",
|
||||
request_count=20,
|
||||
success_count=20,
|
||||
failure_count=0,
|
||||
avg_latency_ms=200.0,
|
||||
percent_of_total=20.0,
|
||||
priority=2,
|
||||
),
|
||||
DeploymentTrafficStats(
|
||||
deployment_id="id-primary",
|
||||
deployment_name="primary-model",
|
||||
provider="nebius",
|
||||
request_count=80,
|
||||
success_count=80,
|
||||
failure_count=0,
|
||||
avg_latency_ms=100.0,
|
||||
percent_of_total=80.0,
|
||||
priority=1,
|
||||
),
|
||||
]
|
||||
|
||||
flow_steps = _build_flow_data(traces, config, traffic)
|
||||
|
||||
# First step should be the priority=1 deployment regardless of input order
|
||||
assert flow_steps[0].to_deployment == "primary-model"
|
||||
assert flow_steps[0].reason == "primary"
|
||||
assert flow_steps[1].to_deployment == "fallback-model"
|
||||
assert flow_steps[1].reason == "fallback_error"
|
||||
122
tests/litellm/proxy/test_routing_group_types.py
Normal file
122
tests/litellm/proxy/test_routing_group_types.py
Normal file
|
|
@ -0,0 +1,122 @@
|
|||
"""
|
||||
Unit tests for Routing Group Pydantic types.
|
||||
"""
|
||||
from litellm.types.router import (
|
||||
DeploymentTrafficStats,
|
||||
FailureInjectionConfig,
|
||||
RoutingGroupConfig,
|
||||
RoutingGroupDeployment,
|
||||
RoutingGroupSimulationResult,
|
||||
RoutingTrace,
|
||||
)
|
||||
|
||||
|
||||
def test_routing_group_deployment_basic():
|
||||
d = RoutingGroupDeployment(
|
||||
model_id="test-id",
|
||||
model_name="nebius/meta-llama/Llama-3.3-70B",
|
||||
provider="nebius",
|
||||
)
|
||||
assert d.priority is None
|
||||
assert d.weight is None
|
||||
|
||||
|
||||
def test_routing_group_deployment_with_priority():
|
||||
d = RoutingGroupDeployment(
|
||||
model_id="test-id",
|
||||
model_name="nebius/meta-llama/Llama-3.3-70B",
|
||||
provider="nebius",
|
||||
priority=1,
|
||||
)
|
||||
assert d.priority == 1
|
||||
|
||||
|
||||
def test_routing_group_config_basic():
|
||||
config = RoutingGroupConfig(
|
||||
routing_group_name="test-group",
|
||||
routing_strategy="priority-failover",
|
||||
deployments=[
|
||||
RoutingGroupDeployment(
|
||||
model_id="id-1",
|
||||
model_name="nebius/meta-llama/Llama-3.3-70B",
|
||||
provider="nebius",
|
||||
priority=1,
|
||||
),
|
||||
RoutingGroupDeployment(
|
||||
model_id="id-2",
|
||||
model_name="fireworks_ai/llama-v3p3-70b",
|
||||
provider="fireworks_ai",
|
||||
priority=2,
|
||||
),
|
||||
],
|
||||
)
|
||||
assert config.routing_group_name == "test-group"
|
||||
assert len(config.deployments) == 2
|
||||
assert config.is_active is True
|
||||
assert config.routing_group_id is None
|
||||
|
||||
|
||||
def test_routing_group_config_weighted():
|
||||
config = RoutingGroupConfig(
|
||||
routing_group_name="weighted-group",
|
||||
routing_strategy="weighted",
|
||||
deployments=[
|
||||
RoutingGroupDeployment(
|
||||
model_id="id-1",
|
||||
model_name="nebius/meta-llama/Llama-3.3-70B",
|
||||
provider="nebius",
|
||||
weight=80,
|
||||
),
|
||||
RoutingGroupDeployment(
|
||||
model_id="id-2",
|
||||
model_name="azure/gpt-4o",
|
||||
provider="azure",
|
||||
weight=20,
|
||||
),
|
||||
],
|
||||
)
|
||||
assert config.routing_strategy == "weighted"
|
||||
|
||||
|
||||
def test_routing_trace():
|
||||
trace = RoutingTrace(
|
||||
deployment_id="id-1",
|
||||
deployment_name="nebius/meta-llama/Llama-3.3-70B",
|
||||
provider="nebius",
|
||||
latency_ms=149.5,
|
||||
was_fallback=False,
|
||||
status="success",
|
||||
)
|
||||
assert trace.fallback_depth == 0
|
||||
assert trace.error_message is None
|
||||
|
||||
|
||||
def test_routing_group_simulation_result():
|
||||
result = RoutingGroupSimulationResult(
|
||||
total_requests=100,
|
||||
successful_requests=98,
|
||||
failed_requests=2,
|
||||
avg_latency_ms=169.0,
|
||||
fallback_count=5,
|
||||
traffic_distribution=[
|
||||
DeploymentTrafficStats(
|
||||
deployment_id="id-1",
|
||||
deployment_name="Nebius",
|
||||
provider="nebius",
|
||||
request_count=85,
|
||||
success_count=85,
|
||||
failure_count=0,
|
||||
avg_latency_ms=149.0,
|
||||
percent_of_total=85.0,
|
||||
)
|
||||
],
|
||||
)
|
||||
assert result.total_requests == 100
|
||||
assert result.flow_data is None
|
||||
|
||||
|
||||
def test_failure_injection_config():
|
||||
config = FailureInjectionConfig(
|
||||
deployment_failure_rates={"id-1": 0.5, "id-2": 0.1}
|
||||
)
|
||||
assert config.deployment_failure_rates["id-1"] == 0.5
|
||||
|
|
@ -0,0 +1,488 @@
|
|||
"""
|
||||
Unit tests for routing group management endpoints.
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import types
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import pytest
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
import litellm.proxy.proxy_server as ps
|
||||
from litellm.proxy._types import UserAPIKeyAuth
|
||||
from litellm.proxy.proxy_server import app
|
||||
from litellm.types.router import (
|
||||
RoutingGroupConfig,
|
||||
RoutingGroupDeployment,
|
||||
)
|
||||
|
||||
sys.path.insert(0, os.path.abspath("../../../"))
|
||||
|
||||
|
||||
def _make_routing_group_record(
|
||||
routing_group_id: str = "rg-123",
|
||||
routing_group_name: str = "test-group",
|
||||
routing_strategy: str = "simple-shuffle",
|
||||
deployments: list | None = None,
|
||||
description: str | None = None,
|
||||
fallback_config: dict | None = None,
|
||||
retry_config: dict | None = None,
|
||||
cooldown_config: dict | None = None,
|
||||
settings: dict | None = None,
|
||||
assigned_team_ids: list | None = None,
|
||||
assigned_key_ids: list | None = None,
|
||||
is_active: bool = True,
|
||||
):
|
||||
data = {
|
||||
"routing_group_id": routing_group_id,
|
||||
"routing_group_name": routing_group_name,
|
||||
"routing_strategy": routing_strategy,
|
||||
"deployments": deployments or [],
|
||||
"description": description,
|
||||
"fallback_config": fallback_config or {},
|
||||
"retry_config": retry_config or {},
|
||||
"cooldown_config": cooldown_config or {},
|
||||
"settings": settings or {},
|
||||
"assigned_team_ids": assigned_team_ids or [],
|
||||
"assigned_key_ids": assigned_key_ids or [],
|
||||
"is_active": is_active,
|
||||
}
|
||||
record = MagicMock()
|
||||
for k, v in data.items():
|
||||
setattr(record, k, v)
|
||||
return record
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def client_and_mocks(monkeypatch):
|
||||
"""Setup mock prisma and auth for routing group endpoints."""
|
||||
mock_routing_group_table = MagicMock()
|
||||
mock_prisma = MagicMock()
|
||||
|
||||
def _create_side_effect(*, data):
|
||||
return _make_routing_group_record(
|
||||
routing_group_id=data.get("routing_group_id", "rg-new"),
|
||||
routing_group_name=data.get("routing_group_name", "new-group"),
|
||||
routing_strategy=data.get("routing_strategy", "simple-shuffle"),
|
||||
deployments=data.get("deployments", []),
|
||||
description=data.get("description"),
|
||||
assigned_team_ids=data.get("assigned_team_ids", []),
|
||||
assigned_key_ids=data.get("assigned_key_ids", []),
|
||||
is_active=data.get("is_active", True),
|
||||
)
|
||||
|
||||
mock_routing_group_table.create = AsyncMock(side_effect=_create_side_effect)
|
||||
mock_routing_group_table.find_unique = AsyncMock(return_value=None)
|
||||
mock_routing_group_table.find_many = AsyncMock(return_value=[])
|
||||
mock_routing_group_table.count = AsyncMock(return_value=0)
|
||||
mock_routing_group_table.update = AsyncMock(
|
||||
side_effect=lambda *, where, data: _make_routing_group_record(
|
||||
routing_group_id=where.get("routing_group_id", "rg-123"),
|
||||
routing_group_name=data.get("routing_group_name", "updated-group"),
|
||||
routing_strategy=data.get("routing_strategy", "simple-shuffle"),
|
||||
deployments=data.get("deployments", []),
|
||||
description=data.get("description"),
|
||||
assigned_team_ids=data.get("assigned_team_ids", []),
|
||||
assigned_key_ids=data.get("assigned_key_ids", []),
|
||||
is_active=data.get("is_active", True),
|
||||
)
|
||||
)
|
||||
mock_routing_group_table.delete = AsyncMock(return_value=None)
|
||||
|
||||
mock_db = types.SimpleNamespace(
|
||||
litellm_routinggrouptable=mock_routing_group_table,
|
||||
)
|
||||
mock_prisma.db = mock_db
|
||||
|
||||
monkeypatch.setattr(ps, "prisma_client", mock_prisma)
|
||||
|
||||
user = UserAPIKeyAuth(user_id="test-user")
|
||||
app.dependency_overrides[ps.user_api_key_auth] = lambda: user
|
||||
|
||||
# Mock the router sync so tests don't need a live LLM router
|
||||
mock_llm_router = MagicMock()
|
||||
mock_llm_router.fallbacks = []
|
||||
mock_llm_router.add_deployment = MagicMock()
|
||||
monkeypatch.setattr(ps, "llm_router", mock_llm_router)
|
||||
|
||||
client = TestClient(app)
|
||||
|
||||
yield client, mock_prisma, mock_routing_group_table, mock_llm_router
|
||||
|
||||
app.dependency_overrides.clear()
|
||||
monkeypatch.setattr(ps, "prisma_client", ps.prisma_client)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# CREATE
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_create_routing_group_success(client_and_mocks):
|
||||
client, _, mock_table, _ = client_and_mocks
|
||||
|
||||
payload = {
|
||||
"routing_group_name": "my-group",
|
||||
"routing_strategy": "simple-shuffle",
|
||||
"deployments": [
|
||||
{
|
||||
"model_id": "id-1",
|
||||
"model_name": "gpt-4o",
|
||||
"provider": "openai",
|
||||
}
|
||||
],
|
||||
}
|
||||
resp = client.post("/v1/routing_group", json=payload)
|
||||
assert resp.status_code == 201
|
||||
body = resp.json()
|
||||
assert body["routing_group_name"] == "my-group"
|
||||
mock_table.create.assert_awaited_once()
|
||||
|
||||
|
||||
def test_create_routing_group_invalid_strategy(client_and_mocks):
|
||||
client, _, _, _ = client_and_mocks
|
||||
|
||||
payload = {
|
||||
"routing_group_name": "bad-group",
|
||||
"routing_strategy": "not-a-real-strategy",
|
||||
"deployments": [
|
||||
{
|
||||
"model_id": "id-1",
|
||||
"model_name": "gpt-4o",
|
||||
"provider": "openai",
|
||||
}
|
||||
],
|
||||
}
|
||||
resp = client.post("/v1/routing_group", json=payload)
|
||||
assert resp.status_code == 400
|
||||
assert "Invalid routing_strategy" in resp.json()["detail"]
|
||||
|
||||
|
||||
def test_create_routing_group_empty_deployments(client_and_mocks):
|
||||
client, _, _, _ = client_and_mocks
|
||||
|
||||
payload = {
|
||||
"routing_group_name": "empty-group",
|
||||
"routing_strategy": "simple-shuffle",
|
||||
"deployments": [],
|
||||
}
|
||||
resp = client.post("/v1/routing_group", json=payload)
|
||||
assert resp.status_code == 400
|
||||
assert "deployments" in resp.json()["detail"]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# LIST
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_list_routing_groups_empty(client_and_mocks):
|
||||
client, _, _, _ = client_and_mocks
|
||||
|
||||
resp = client.get("/v1/routing_group")
|
||||
assert resp.status_code == 200
|
||||
body = resp.json()
|
||||
assert body["routing_groups"] == []
|
||||
assert body["total"] == 0
|
||||
|
||||
|
||||
def test_list_routing_groups_with_records(client_and_mocks):
|
||||
client, _, mock_table, _ = client_and_mocks
|
||||
|
||||
records = [
|
||||
_make_routing_group_record(
|
||||
routing_group_id="rg-1",
|
||||
routing_group_name="group-1",
|
||||
deployments=[
|
||||
{"model_id": "id-1", "model_name": "gpt-4o", "provider": "openai"}
|
||||
],
|
||||
),
|
||||
_make_routing_group_record(
|
||||
routing_group_id="rg-2",
|
||||
routing_group_name="group-2",
|
||||
deployments=[
|
||||
{
|
||||
"model_id": "id-2",
|
||||
"model_name": "claude-3-5-sonnet-20241022",
|
||||
"provider": "anthropic",
|
||||
}
|
||||
],
|
||||
),
|
||||
]
|
||||
mock_table.find_many = AsyncMock(return_value=records)
|
||||
mock_table.count = AsyncMock(return_value=2)
|
||||
|
||||
resp = client.get("/v1/routing_group")
|
||||
assert resp.status_code == 200
|
||||
body = resp.json()
|
||||
assert body["total"] == 2
|
||||
assert len(body["routing_groups"]) == 2
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# GET
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_get_routing_group_not_found(client_and_mocks):
|
||||
client, _, mock_table, _ = client_and_mocks
|
||||
mock_table.find_unique = AsyncMock(return_value=None)
|
||||
|
||||
resp = client.get("/v1/routing_group/nonexistent-id")
|
||||
assert resp.status_code == 404
|
||||
|
||||
|
||||
def test_get_routing_group_success(client_and_mocks):
|
||||
client, _, mock_table, _ = client_and_mocks
|
||||
|
||||
record = _make_routing_group_record(
|
||||
routing_group_id="rg-abc",
|
||||
routing_group_name="found-group",
|
||||
deployments=[
|
||||
{"model_id": "id-1", "model_name": "gpt-4o", "provider": "openai"}
|
||||
],
|
||||
)
|
||||
mock_table.find_unique = AsyncMock(return_value=record)
|
||||
|
||||
resp = client.get("/v1/routing_group/rg-abc")
|
||||
assert resp.status_code == 200
|
||||
body = resp.json()
|
||||
assert body["routing_group_id"] == "rg-abc"
|
||||
assert body["routing_group_name"] == "found-group"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# UPDATE
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_update_routing_group_success(client_and_mocks):
|
||||
client, _, mock_table, _ = client_and_mocks
|
||||
|
||||
payload = {
|
||||
"routing_group_name": "updated-group",
|
||||
"routing_strategy": "latency-based-routing",
|
||||
"deployments": [
|
||||
{"model_id": "id-1", "model_name": "gpt-4o", "provider": "openai"}
|
||||
],
|
||||
}
|
||||
resp = client.put("/v1/routing_group/rg-123", json=payload)
|
||||
assert resp.status_code == 200
|
||||
body = resp.json()
|
||||
assert body["routing_group_name"] == "updated-group"
|
||||
mock_table.update.assert_awaited_once()
|
||||
|
||||
|
||||
def test_update_routing_group_db_error_returns_404(client_and_mocks):
|
||||
client, _, mock_table, _ = client_and_mocks
|
||||
|
||||
mock_table.update = AsyncMock(side_effect=Exception("Record not found"))
|
||||
|
||||
payload = {
|
||||
"routing_group_name": "updated-group",
|
||||
"routing_strategy": "simple-shuffle",
|
||||
"deployments": [
|
||||
{"model_id": "id-1", "model_name": "gpt-4o", "provider": "openai"}
|
||||
],
|
||||
}
|
||||
resp = client.put("/v1/routing_group/bad-id", json=payload)
|
||||
assert resp.status_code == 404
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# DELETE
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_delete_routing_group_success(client_and_mocks):
|
||||
client, _, mock_table, _ = client_and_mocks
|
||||
|
||||
resp = client.delete("/v1/routing_group/rg-123")
|
||||
assert resp.status_code == 200
|
||||
body = resp.json()
|
||||
assert body["deleted"] is True
|
||||
assert body["routing_group_id"] == "rg-123"
|
||||
mock_table.delete.assert_awaited_once()
|
||||
|
||||
|
||||
def test_delete_routing_group_not_found(client_and_mocks):
|
||||
client, _, mock_table, _ = client_and_mocks
|
||||
mock_table.delete = AsyncMock(side_effect=Exception("Record does not exist"))
|
||||
|
||||
resp = client.delete("/v1/routing_group/missing-id")
|
||||
assert resp.status_code == 404
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# _sync_routing_group_to_router unit tests
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_sync_priority_failover_wires_fallbacks():
|
||||
"""priority-failover strategy should create tiered groups and wire fallback chain."""
|
||||
from litellm.proxy.management_endpoints.routing_group_endpoints import (
|
||||
_sync_routing_group_to_router,
|
||||
)
|
||||
|
||||
mock_router = MagicMock()
|
||||
mock_router.fallbacks = []
|
||||
mock_router.add_deployment = MagicMock()
|
||||
|
||||
config = RoutingGroupConfig(
|
||||
routing_group_name="my-failover-group",
|
||||
routing_strategy="priority-failover",
|
||||
deployments=[
|
||||
RoutingGroupDeployment(
|
||||
model_id="id-1",
|
||||
model_name="gpt-4o",
|
||||
provider="openai",
|
||||
priority=1,
|
||||
),
|
||||
RoutingGroupDeployment(
|
||||
model_id="id-2",
|
||||
model_name="claude-3-5-sonnet-20241022",
|
||||
provider="anthropic",
|
||||
priority=2,
|
||||
),
|
||||
],
|
||||
)
|
||||
|
||||
with patch(
|
||||
"litellm.proxy.management_endpoints.routing_group_endpoints._get_llm_router",
|
||||
return_value=mock_router,
|
||||
):
|
||||
await _sync_routing_group_to_router(config)
|
||||
|
||||
# add_deployment should be called twice (once per deployment)
|
||||
assert mock_router.add_deployment.call_count == 2
|
||||
|
||||
# Fallback chain should be wired: primary -> [fallback_p2]
|
||||
assert len(mock_router.fallbacks) == 1
|
||||
fallback_entry = mock_router.fallbacks[0]
|
||||
assert "my-failover-group" in fallback_entry
|
||||
assert fallback_entry["my-failover-group"] == ["my-failover-group__fallback_p2"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_sync_weighted_includes_weight_in_litellm_params():
|
||||
"""weighted strategy should pass weight into litellm_params."""
|
||||
from litellm.proxy.management_endpoints.routing_group_endpoints import (
|
||||
_sync_routing_group_to_router,
|
||||
)
|
||||
|
||||
captured_deployments = []
|
||||
|
||||
mock_router = MagicMock()
|
||||
mock_router.fallbacks = []
|
||||
|
||||
def capture_add_deployment(deployment):
|
||||
captured_deployments.append(deployment)
|
||||
|
||||
mock_router.add_deployment = MagicMock(side_effect=capture_add_deployment)
|
||||
|
||||
config = RoutingGroupConfig(
|
||||
routing_group_name="weighted-group",
|
||||
routing_strategy="weighted",
|
||||
deployments=[
|
||||
RoutingGroupDeployment(
|
||||
model_id="id-1",
|
||||
model_name="gpt-4o",
|
||||
provider="openai",
|
||||
weight=80,
|
||||
),
|
||||
RoutingGroupDeployment(
|
||||
model_id="id-2",
|
||||
model_name="claude-3-5-sonnet-20241022",
|
||||
provider="anthropic",
|
||||
weight=20,
|
||||
),
|
||||
],
|
||||
)
|
||||
|
||||
with patch(
|
||||
"litellm.proxy.management_endpoints.routing_group_endpoints._get_llm_router",
|
||||
return_value=mock_router,
|
||||
):
|
||||
await _sync_routing_group_to_router(config)
|
||||
|
||||
assert mock_router.add_deployment.call_count == 2
|
||||
|
||||
# Each captured deployment should have weight in its litellm_params
|
||||
for dep in captured_deployments:
|
||||
assert hasattr(dep, "litellm_params")
|
||||
params = dep.litellm_params
|
||||
# litellm_params is a pydantic model; access as dict or attribute
|
||||
params_dict = (
|
||||
params.model_dump() if hasattr(params, "model_dump") else dict(params)
|
||||
)
|
||||
assert "weight" in params_dict
|
||||
assert params_dict["weight"] in (80, 20)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_sync_no_router_logs_warning():
|
||||
"""When llm_router is None, _sync should log a warning and return gracefully."""
|
||||
from litellm.proxy.management_endpoints.routing_group_endpoints import (
|
||||
_sync_routing_group_to_router,
|
||||
)
|
||||
|
||||
config = RoutingGroupConfig(
|
||||
routing_group_name="no-router-group",
|
||||
routing_strategy="simple-shuffle",
|
||||
deployments=[
|
||||
RoutingGroupDeployment(
|
||||
model_id="id-1",
|
||||
model_name="gpt-4o",
|
||||
provider="openai",
|
||||
)
|
||||
],
|
||||
)
|
||||
|
||||
with patch(
|
||||
"litellm.proxy.management_endpoints.routing_group_endpoints._get_llm_router",
|
||||
return_value=None,
|
||||
):
|
||||
# Should not raise
|
||||
await _sync_routing_group_to_router(config)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_sync_simple_shuffle_no_fallbacks():
|
||||
"""Non-priority-failover strategy should not modify router.fallbacks."""
|
||||
from litellm.proxy.management_endpoints.routing_group_endpoints import (
|
||||
_sync_routing_group_to_router,
|
||||
)
|
||||
|
||||
mock_router = MagicMock()
|
||||
mock_router.fallbacks = []
|
||||
mock_router.add_deployment = MagicMock()
|
||||
|
||||
config = RoutingGroupConfig(
|
||||
routing_group_name="shuffle-group",
|
||||
routing_strategy="simple-shuffle",
|
||||
deployments=[
|
||||
RoutingGroupDeployment(
|
||||
model_id="id-1",
|
||||
model_name="gpt-4o",
|
||||
provider="openai",
|
||||
),
|
||||
RoutingGroupDeployment(
|
||||
model_id="id-2",
|
||||
model_name="gpt-4o-mini",
|
||||
provider="openai",
|
||||
),
|
||||
],
|
||||
)
|
||||
|
||||
with patch(
|
||||
"litellm.proxy.management_endpoints.routing_group_endpoints._get_llm_router",
|
||||
return_value=mock_router,
|
||||
):
|
||||
await _sync_routing_group_to_router(config)
|
||||
|
||||
assert mock_router.add_deployment.call_count == 2
|
||||
# fallbacks should remain unchanged (empty)
|
||||
assert mock_router.fallbacks == []
|
||||
|
|
@ -20,6 +20,7 @@ import {
|
|||
ToolOutlined,
|
||||
TagsOutlined,
|
||||
AuditOutlined,
|
||||
BranchesOutlined,
|
||||
} from "@ant-design/icons";
|
||||
// import {
|
||||
// all_admin_roles,
|
||||
|
|
@ -105,6 +106,8 @@ const routeFor = (slug: string): string => {
|
|||
return "guardrails";
|
||||
case "policies":
|
||||
return "policies";
|
||||
case "routing-groups":
|
||||
return "routing-groups";
|
||||
|
||||
// tools
|
||||
case "mcp-servers":
|
||||
|
|
@ -168,6 +171,13 @@ const menuItems: MenuItemCfg[] = [
|
|||
icon: <BlockOutlined style={{ fontSize: 18 }} />,
|
||||
roles: rolesWithWriteAccess,
|
||||
},
|
||||
{
|
||||
key: "29",
|
||||
page: "routing-groups",
|
||||
label: "Routing Groups",
|
||||
icon: <BranchesOutlined style={{ fontSize: 18 }} />,
|
||||
roles: rolesWithWriteAccess,
|
||||
},
|
||||
{
|
||||
key: "12",
|
||||
page: "new_usage",
|
||||
|
|
|
|||
|
|
@ -0,0 +1,18 @@
|
|||
"use client";
|
||||
|
||||
import RoutingGroupsView from "@/components/routing_groups/RoutingGroupsView";
|
||||
import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized";
|
||||
|
||||
const RoutingGroupsPage = () => {
|
||||
const { accessToken, userId, userRole } = useAuthorized();
|
||||
|
||||
return (
|
||||
<RoutingGroupsView
|
||||
accessToken={accessToken}
|
||||
userRole={userRole ?? ""}
|
||||
userId={userId ?? ""}
|
||||
/>
|
||||
);
|
||||
};
|
||||
|
||||
export default RoutingGroupsPage;
|
||||
|
|
@ -10035,3 +10035,172 @@ export const updateToolPolicy = async (
|
|||
}
|
||||
return response.json();
|
||||
};
|
||||
|
||||
export const routingGroupCreateCall = async (
|
||||
accessToken: string,
|
||||
formValues: Record<string, unknown>,
|
||||
): Promise<Record<string, unknown>> => {
|
||||
const url = proxyBaseUrl
|
||||
? `${proxyBaseUrl}/v1/routing_group`
|
||||
: `/v1/routing_group`;
|
||||
const response = await fetch(url, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
[globalLitellmHeaderName]: `Bearer ${accessToken}`,
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
body: JSON.stringify(formValues),
|
||||
});
|
||||
if (!response.ok) {
|
||||
const errorData = await response.text();
|
||||
handleError(errorData);
|
||||
throw new Error("Network response was not ok");
|
||||
}
|
||||
return response.json();
|
||||
};
|
||||
|
||||
export const routingGroupListCall = async (
|
||||
accessToken: string,
|
||||
page: number = 1,
|
||||
size: number = 50,
|
||||
): Promise<{ routing_groups: Record<string, unknown>[]; total: number; page: number; size: number }> => {
|
||||
const url = proxyBaseUrl
|
||||
? `${proxyBaseUrl}/v1/routing_group?page=${page}&size=${size}`
|
||||
: `/v1/routing_group?page=${page}&size=${size}`;
|
||||
const response = await fetch(url, {
|
||||
method: "GET",
|
||||
headers: {
|
||||
[globalLitellmHeaderName]: `Bearer ${accessToken}`,
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
});
|
||||
if (!response.ok) {
|
||||
const errorData = await response.text();
|
||||
handleError(errorData);
|
||||
throw new Error("Network response was not ok");
|
||||
}
|
||||
return response.json();
|
||||
};
|
||||
|
||||
export const routingGroupGetCall = async (
|
||||
accessToken: string,
|
||||
routingGroupId: string,
|
||||
): Promise<Record<string, unknown>> => {
|
||||
const url = proxyBaseUrl
|
||||
? `${proxyBaseUrl}/v1/routing_group/${routingGroupId}`
|
||||
: `/v1/routing_group/${routingGroupId}`;
|
||||
const response = await fetch(url, {
|
||||
method: "GET",
|
||||
headers: {
|
||||
[globalLitellmHeaderName]: `Bearer ${accessToken}`,
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
});
|
||||
if (!response.ok) {
|
||||
const errorData = await response.text();
|
||||
handleError(errorData);
|
||||
throw new Error("Network response was not ok");
|
||||
}
|
||||
return response.json();
|
||||
};
|
||||
|
||||
export const routingGroupUpdateCall = async (
|
||||
accessToken: string,
|
||||
routingGroupId: string,
|
||||
formValues: Record<string, unknown>,
|
||||
): Promise<Record<string, unknown>> => {
|
||||
const url = proxyBaseUrl
|
||||
? `${proxyBaseUrl}/v1/routing_group/${routingGroupId}`
|
||||
: `/v1/routing_group/${routingGroupId}`;
|
||||
const response = await fetch(url, {
|
||||
method: "PUT",
|
||||
headers: {
|
||||
[globalLitellmHeaderName]: `Bearer ${accessToken}`,
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
body: JSON.stringify(formValues),
|
||||
});
|
||||
if (!response.ok) {
|
||||
const errorData = await response.text();
|
||||
handleError(errorData);
|
||||
throw new Error("Network response was not ok");
|
||||
}
|
||||
return response.json();
|
||||
};
|
||||
|
||||
export const routingGroupDeleteCall = async (
|
||||
accessToken: string,
|
||||
routingGroupId: string,
|
||||
): Promise<Record<string, unknown>> => {
|
||||
const url = proxyBaseUrl
|
||||
? `${proxyBaseUrl}/v1/routing_group/${routingGroupId}`
|
||||
: `/v1/routing_group/${routingGroupId}`;
|
||||
const response = await fetch(url, {
|
||||
method: "DELETE",
|
||||
headers: {
|
||||
[globalLitellmHeaderName]: `Bearer ${accessToken}`,
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
});
|
||||
if (!response.ok) {
|
||||
const errorData = await response.text();
|
||||
handleError(errorData);
|
||||
throw new Error("Network response was not ok");
|
||||
}
|
||||
return response.json();
|
||||
};
|
||||
|
||||
export const routingGroupTestCall = async (
|
||||
accessToken: string,
|
||||
routingGroupId: string,
|
||||
messages?: Array<Record<string, string>>,
|
||||
mock: boolean = false,
|
||||
): Promise<Record<string, unknown>> => {
|
||||
const url = proxyBaseUrl
|
||||
? `${proxyBaseUrl}/v1/routing_group/${routingGroupId}/test`
|
||||
: `/v1/routing_group/${routingGroupId}/test`;
|
||||
const response = await fetch(url, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
[globalLitellmHeaderName]: `Bearer ${accessToken}`,
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
body: JSON.stringify({ messages, mock }),
|
||||
});
|
||||
if (!response.ok) {
|
||||
const errorData = await response.text();
|
||||
handleError(errorData);
|
||||
throw new Error("Network response was not ok");
|
||||
}
|
||||
return response.json();
|
||||
};
|
||||
|
||||
export const routingGroupSimulateCall = async (
|
||||
accessToken: string,
|
||||
routingGroupId: string,
|
||||
config: {
|
||||
num_requests: number;
|
||||
concurrency: number;
|
||||
mock: boolean;
|
||||
failure_injection?: { deployment_failure_rates: Record<string, number> };
|
||||
},
|
||||
): Promise<Record<string, unknown>> => {
|
||||
const url = proxyBaseUrl
|
||||
? `${proxyBaseUrl}/v1/routing_group/${routingGroupId}/simulate`
|
||||
: `/v1/routing_group/${routingGroupId}/simulate`;
|
||||
const response = await fetch(url, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
[globalLitellmHeaderName]: `Bearer ${accessToken}`,
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
body: JSON.stringify(config),
|
||||
});
|
||||
if (!response.ok) {
|
||||
const errorData = await response.text();
|
||||
handleError(errorData);
|
||||
throw new Error("Network response was not ok");
|
||||
}
|
||||
return response.json();
|
||||
};
|
||||
};
|
||||
|
|
|
|||
|
|
@ -0,0 +1,48 @@
|
|||
"use client";
|
||||
import React from "react";
|
||||
|
||||
interface AnimatedStreamProps {
|
||||
/** SVG path d string */
|
||||
path: string;
|
||||
color: string;
|
||||
/** stroke-width in px */
|
||||
thickness: number;
|
||||
animated?: boolean;
|
||||
}
|
||||
|
||||
export default function AnimatedStream({ path, color, thickness, animated = true }: AnimatedStreamProps) {
|
||||
const dashLen = Math.max(8, thickness * 3);
|
||||
const gapLen = Math.max(4, thickness * 2);
|
||||
|
||||
return (
|
||||
<g>
|
||||
{/* Dim base */}
|
||||
<path d={path} fill="none" stroke={color} strokeWidth={thickness} strokeOpacity={0.12} />
|
||||
{/* Glow */}
|
||||
<path d={path} fill="none" stroke={color} strokeWidth={thickness + 6} strokeOpacity={0.06} />
|
||||
{/* Animated dashes */}
|
||||
<path
|
||||
d={path}
|
||||
fill="none"
|
||||
stroke={color}
|
||||
strokeWidth={thickness}
|
||||
strokeOpacity={0.7}
|
||||
strokeDasharray={`${dashLen} ${gapLen}`}
|
||||
style={
|
||||
animated
|
||||
? {
|
||||
animation: `routingFlow 0.9s linear infinite`,
|
||||
strokeDashoffset: 0,
|
||||
}
|
||||
: undefined
|
||||
}
|
||||
/>
|
||||
<style>{`
|
||||
@keyframes routingFlow {
|
||||
from { stroke-dashoffset: ${dashLen + gapLen}; }
|
||||
to { stroke-dashoffset: 0; }
|
||||
}
|
||||
`}</style>
|
||||
</g>
|
||||
);
|
||||
}
|
||||
|
|
@ -0,0 +1,56 @@
|
|||
"use client";
|
||||
import React from "react";
|
||||
|
||||
interface DeploymentCardProps {
|
||||
provider: string;
|
||||
displayName: string;
|
||||
providerColor: string;
|
||||
label: string; // "Priority 1" or "Weight 83%"
|
||||
avgLatencyMs: number;
|
||||
percentage: number; // 0-100
|
||||
requestCount: number;
|
||||
width?: number; // px, default 220
|
||||
}
|
||||
|
||||
export default function DeploymentCard({
|
||||
provider,
|
||||
displayName,
|
||||
providerColor,
|
||||
label,
|
||||
avgLatencyMs,
|
||||
percentage,
|
||||
requestCount,
|
||||
width = 220,
|
||||
}: DeploymentCardProps) {
|
||||
const initial = displayName.charAt(0).toUpperCase();
|
||||
return (
|
||||
<div
|
||||
style={{ width, borderColor: providerColor + "33" }}
|
||||
className="flex items-center gap-3 rounded-xl border bg-[#1f2937] px-4 py-3"
|
||||
>
|
||||
{/* Provider icon */}
|
||||
<div
|
||||
style={{ backgroundColor: providerColor + "33", color: providerColor }}
|
||||
className="flex h-10 w-10 flex-shrink-0 items-center justify-center rounded-lg text-sm font-bold"
|
||||
>
|
||||
{initial}
|
||||
</div>
|
||||
|
||||
{/* Name + label */}
|
||||
<div className="min-w-0 flex-1">
|
||||
<div className="truncate text-sm font-semibold text-white">{displayName}</div>
|
||||
<div className="text-xs text-gray-400">
|
||||
{label} · {avgLatencyMs > 0 ? `${Math.round(avgLatencyMs)}ms avg` : "—"}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Percentage + req count */}
|
||||
<div className="flex flex-shrink-0 flex-col items-end">
|
||||
<span style={{ color: providerColor }} className="text-lg font-bold leading-none">
|
||||
{Math.round(percentage)}%
|
||||
</span>
|
||||
<span className="text-xs text-gray-500">{requestCount} req</span>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
|
@ -0,0 +1,208 @@
|
|||
"use client";
|
||||
import React, { useState } from "react";
|
||||
import { Button, InputNumber, Radio, Spin, Typography } from "antd";
|
||||
import { SwapOutlined } from "@ant-design/icons";
|
||||
import OrderedFallbackFlow from "./OrderedFallbackFlow";
|
||||
import WeightedRoundRobinFlow from "./WeightedRoundRobinFlow";
|
||||
import StatsBar from "./StatsBar";
|
||||
import { routingGroupSimulateCall } from "@/components/networking";
|
||||
|
||||
interface Deployment {
|
||||
deployment_id: string;
|
||||
provider: string;
|
||||
display_name: string;
|
||||
priority?: number;
|
||||
weight?: number;
|
||||
avg_latency_ms: number;
|
||||
percent_of_total: number;
|
||||
request_count: number;
|
||||
success_count: number;
|
||||
failure_count: number;
|
||||
}
|
||||
|
||||
interface LiveTesterProps {
|
||||
accessToken: string;
|
||||
routingGroupId: string;
|
||||
routingGroupName: string;
|
||||
routingStrategy: string; // "priority-failover" | "weighted" | etc.
|
||||
initialDeployments?: Deployment[];
|
||||
}
|
||||
|
||||
const isOrderedStrategy = (s: string) =>
|
||||
s === "priority-failover";
|
||||
|
||||
export default function LiveTester({
|
||||
accessToken,
|
||||
routingGroupId,
|
||||
routingGroupName,
|
||||
routingStrategy,
|
||||
initialDeployments = [],
|
||||
}: LiveTesterProps) {
|
||||
// Force mode based on strategy, but allow user to toggle
|
||||
const [viewMode, setViewMode] = useState<"ordered" | "weighted">(
|
||||
isOrderedStrategy(routingStrategy) ? "ordered" : "weighted"
|
||||
);
|
||||
const [loading, setLoading] = useState(false);
|
||||
const [numRequests, setNumRequests] = useState(100);
|
||||
const [concurrency, setConcurrency] = useState(10);
|
||||
const [mockMode, setMockMode] = useState(true);
|
||||
|
||||
// Live stats state
|
||||
const [deployments, setDeployments] = useState<Deployment[]>(
|
||||
initialDeployments.length > 0 ? initialDeployments : []
|
||||
);
|
||||
const [stats, setStats] = useState({
|
||||
totalRequests: 0,
|
||||
successRate: 100,
|
||||
avgLatencyMs: 0,
|
||||
fallbackCount: 0,
|
||||
});
|
||||
|
||||
const isOrdered = viewMode === "ordered";
|
||||
|
||||
const handleSimulate = async () => {
|
||||
if (!accessToken) return;
|
||||
setLoading(true);
|
||||
try {
|
||||
const result = await routingGroupSimulateCall(accessToken, routingGroupId, {
|
||||
num_requests: numRequests,
|
||||
concurrency,
|
||||
mock: mockMode,
|
||||
}) as Record<string, unknown>;
|
||||
|
||||
const dist = (result.traffic_distribution as Deployment[]) || [];
|
||||
setDeployments(dist);
|
||||
const total = (result.total_requests as number) || 0;
|
||||
const successful = (result.successful_requests as number) || 0;
|
||||
const avgLatency = (result.avg_latency_ms as number) || 0;
|
||||
const fallbacks = (result.fallback_count as number) || 0;
|
||||
setStats({
|
||||
totalRequests: total,
|
||||
successRate: total > 0 ? (successful / total) * 100 : 100,
|
||||
avgLatencyMs: avgLatency,
|
||||
fallbackCount: fallbacks,
|
||||
});
|
||||
} catch (e) {
|
||||
console.error("Simulation error:", e);
|
||||
} finally {
|
||||
setLoading(false);
|
||||
}
|
||||
};
|
||||
|
||||
const infoText = isOrdered
|
||||
? "Traffic flows to the primary provider first. When a request fails, it cascades down the priority chain. Thicker streams = more traffic. Dashed red lines show the fallback path."
|
||||
: "Traffic is distributed across providers based on weights. Thicker streams = more traffic. Adjust weights in the routing group settings.";
|
||||
|
||||
const badgeLabel = isOrdered ? "Ordered Fallback" : "Weighted Round-Robin";
|
||||
const badgeColor = isOrdered ? "#92400e" : "#134e4a";
|
||||
const badgeTextColor = isOrdered ? "#fbbf24" : "#2dd4bf";
|
||||
|
||||
return (
|
||||
<div className="flex flex-col gap-4">
|
||||
{/* Header */}
|
||||
<div className="flex items-start justify-between">
|
||||
<div>
|
||||
<div className="flex items-center gap-3">
|
||||
<Typography.Title level={3} style={{ margin: 0, color: "#f1f5f9" }}>
|
||||
Live Tester
|
||||
</Typography.Title>
|
||||
<span
|
||||
style={{ background: badgeColor, color: badgeTextColor }}
|
||||
className="rounded-md px-2.5 py-1 text-xs font-semibold"
|
||||
>
|
||||
{badgeLabel}
|
||||
</span>
|
||||
</div>
|
||||
<Typography.Text style={{ color: "#64748b" }}>
|
||||
Sending to{" "}
|
||||
<span style={{ color: "#38bdf8" }} className="font-mono">
|
||||
{routingGroupName}
|
||||
</span>
|
||||
</Typography.Text>
|
||||
</div>
|
||||
<Button
|
||||
icon={<SwapOutlined />}
|
||||
onClick={() => setViewMode(isOrdered ? "weighted" : "ordered")}
|
||||
style={{ background: "#1e293b", borderColor: "#334155", color: "#94a3b8" }}
|
||||
>
|
||||
Switch to {isOrdered ? "Weighted" : "Ordered"}
|
||||
</Button>
|
||||
</div>
|
||||
|
||||
{/* Stats bar */}
|
||||
<StatsBar
|
||||
requests={stats.totalRequests}
|
||||
successRate={stats.successRate}
|
||||
avgLatencyMs={stats.avgLatencyMs}
|
||||
fallbackCount={stats.fallbackCount}
|
||||
/>
|
||||
|
||||
{/* Flow diagram */}
|
||||
<Spin spinning={loading}>
|
||||
{deployments.length === 0 ? (
|
||||
<div className="flex h-48 items-center justify-center rounded-2xl bg-[#0f1117] text-gray-500">
|
||||
Run a simulation to see traffic flow
|
||||
</div>
|
||||
) : isOrdered ? (
|
||||
<OrderedFallbackFlow deployments={deployments} routingGroupName={routingGroupName} />
|
||||
) : (
|
||||
<WeightedRoundRobinFlow deployments={deployments} routingGroupName={routingGroupName} />
|
||||
)}
|
||||
</Spin>
|
||||
|
||||
{/* Info box */}
|
||||
<div className="rounded-xl border border-gray-700 bg-[#111827] px-4 py-3 text-sm text-gray-400">
|
||||
{infoText}
|
||||
</div>
|
||||
|
||||
{/* Simulation controls */}
|
||||
<div className="rounded-xl border border-gray-700 bg-[#1a1f2e] p-4">
|
||||
<Typography.Text strong style={{ color: "#94a3b8", display: "block", marginBottom: 12 }}>
|
||||
Simulation Controls
|
||||
</Typography.Text>
|
||||
<div className="flex flex-wrap items-end gap-4">
|
||||
<div>
|
||||
<div className="mb-1 text-xs text-gray-500">Requests</div>
|
||||
<InputNumber
|
||||
min={1}
|
||||
max={10000}
|
||||
value={numRequests}
|
||||
onChange={(v) => setNumRequests(v ?? 100)}
|
||||
style={{ width: 100, background: "#0f172a", borderColor: "#334155", color: "#f1f5f9" }}
|
||||
/>
|
||||
</div>
|
||||
<div>
|
||||
<div className="mb-1 text-xs text-gray-500">Concurrency</div>
|
||||
<InputNumber
|
||||
min={1}
|
||||
max={100}
|
||||
value={concurrency}
|
||||
onChange={(v) => setConcurrency(v ?? 10)}
|
||||
style={{ width: 90, background: "#0f172a", borderColor: "#334155", color: "#f1f5f9" }}
|
||||
/>
|
||||
</div>
|
||||
<div>
|
||||
<div className="mb-1 text-xs text-gray-500">Mode</div>
|
||||
<Radio.Group
|
||||
value={mockMode ? "mock" : "real"}
|
||||
onChange={(e) => setMockMode(e.target.value === "mock")}
|
||||
buttonStyle="solid"
|
||||
size="small"
|
||||
>
|
||||
<Radio.Button value="mock">Mock</Radio.Button>
|
||||
<Radio.Button value="real">Real</Radio.Button>
|
||||
</Radio.Group>
|
||||
</div>
|
||||
<Button
|
||||
type="primary"
|
||||
loading={loading}
|
||||
onClick={handleSimulate}
|
||||
style={{ background: "#0f766e", borderColor: "#0f766e" }}
|
||||
>
|
||||
Run Simulation
|
||||
</Button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
|
@ -0,0 +1,129 @@
|
|||
"use client";
|
||||
import React from "react";
|
||||
import AnimatedStream from "./AnimatedStream";
|
||||
import DeploymentCard from "./DeploymentCard";
|
||||
|
||||
const PROVIDER_COLORS: Record<string, string> = {
|
||||
nebius: "#14b8a6",
|
||||
fireworks_ai: "#f97316",
|
||||
azure: "#8b5cf6",
|
||||
openai: "#10b981",
|
||||
anthropic: "#d97706",
|
||||
google: "#3b82f6",
|
||||
aws: "#ef4444",
|
||||
};
|
||||
const getProviderColor = (p: string) => PROVIDER_COLORS[p.toLowerCase()] ?? "#6b7280";
|
||||
|
||||
interface Deployment {
|
||||
deployment_id: string;
|
||||
provider: string;
|
||||
display_name: string;
|
||||
priority?: number;
|
||||
avg_latency_ms: number;
|
||||
percent_of_total: number;
|
||||
request_count: number;
|
||||
}
|
||||
|
||||
interface OrderedFallbackFlowProps {
|
||||
deployments: Deployment[];
|
||||
routingGroupName: string;
|
||||
}
|
||||
|
||||
const CARD_HEIGHT = 76;
|
||||
const CARD_GAP = 48; // space between cards (for the "fail" arrow)
|
||||
const CARD_WIDTH = 220;
|
||||
const LEFT_PADDING = 40;
|
||||
const HUB_RADIUS = 28;
|
||||
|
||||
export default function OrderedFallbackFlow({ deployments, routingGroupName }: OrderedFallbackFlowProps) {
|
||||
const sorted = [...deployments].sort((a, b) => (a.priority ?? 999) - (b.priority ?? 999));
|
||||
const totalHeight = sorted.length * CARD_HEIGHT + (sorted.length - 1) * CARD_GAP + 40;
|
||||
const svgWidth = 600;
|
||||
|
||||
// Hub sits vertically centered, 1/3 from left
|
||||
const hubX = svgWidth * 0.32;
|
||||
const hubY = totalHeight / 2;
|
||||
|
||||
// Cards start at x = svgWidth - CARD_WIDTH - LEFT_PADDING
|
||||
const cardX = svgWidth - CARD_WIDTH - LEFT_PADDING;
|
||||
|
||||
// Y positions for each card (centered block)
|
||||
const blockHeight = sorted.length * CARD_HEIGHT + (sorted.length - 1) * CARD_GAP;
|
||||
const blockTop = (totalHeight - blockHeight) / 2;
|
||||
const cardYs = sorted.map((_, i) => blockTop + i * (CARD_HEIGHT + CARD_GAP));
|
||||
const cardCenterYs = cardYs.map((y) => y + CARD_HEIGHT / 2);
|
||||
|
||||
// Primary stream: from hub to first card center
|
||||
const primaryPath = `M ${hubX} ${hubY} L ${cardX} ${cardCenterYs[0]}`;
|
||||
|
||||
// Fallback dashed paths between cards
|
||||
const fallbackPaths = sorted.slice(0, -1).map((_, i) => {
|
||||
const x = cardX + CARD_WIDTH / 2;
|
||||
const fromY = cardYs[i] + CARD_HEIGHT;
|
||||
const toY = cardYs[i + 1];
|
||||
return { path: `M ${x} ${fromY} L ${x} ${toY}`, midY: (fromY + toY) / 2 };
|
||||
});
|
||||
|
||||
return (
|
||||
<div className="relative w-full overflow-hidden rounded-2xl bg-[#0f1117]" style={{ minHeight: totalHeight + 40 }}>
|
||||
{/* SVG layer */}
|
||||
<svg
|
||||
className="absolute inset-0"
|
||||
width="100%"
|
||||
height={totalHeight + 40}
|
||||
viewBox={`0 0 ${svgWidth} ${totalHeight + 40}`}
|
||||
preserveAspectRatio="xMidYMid meet"
|
||||
>
|
||||
{/* Client → hub stream */}
|
||||
<AnimatedStream
|
||||
path={`M 20 ${hubY} L ${hubX - HUB_RADIUS} ${hubY}`}
|
||||
color="#14b8a6"
|
||||
thickness={6}
|
||||
/>
|
||||
|
||||
{/* Hub circle */}
|
||||
<circle cx={hubX} cy={hubY} r={HUB_RADIUS} fill="#1a2535" stroke="#14b8a6" strokeWidth={2} strokeOpacity={0.6} />
|
||||
<text x={hubX} y={hubY + 4} textAnchor="middle" fill="#14b8a6" fontSize={8} fontWeight="600" fontFamily="monospace">
|
||||
LITELLM
|
||||
</text>
|
||||
|
||||
{/* Primary stream: hub → first card */}
|
||||
<AnimatedStream path={primaryPath} color="#14b8a6" thickness={10} />
|
||||
|
||||
{/* Fallback dashed arrows between cards */}
|
||||
{fallbackPaths.map(({ path, midY }, i) => (
|
||||
<g key={i}>
|
||||
<path d={path} fill="none" stroke="#ef4444" strokeWidth={1.5} strokeDasharray="5 4" strokeOpacity={0.7} />
|
||||
<text x={cardX + CARD_WIDTH / 2 + 6} y={midY + 4} fill="#ef4444" fontSize={9} fontFamily="monospace" opacity={0.8}>
|
||||
fail →
|
||||
</text>
|
||||
</g>
|
||||
))}
|
||||
</svg>
|
||||
|
||||
{/* "client" label */}
|
||||
<div className="absolute text-xs font-mono text-gray-500" style={{ left: 24, top: hubY + 40 - 8 }}>
|
||||
client
|
||||
</div>
|
||||
|
||||
{/* Deployment cards */}
|
||||
<div className="absolute" style={{ right: LEFT_PADDING, top: 20 }}>
|
||||
<div className="flex flex-col" style={{ gap: CARD_GAP }}>
|
||||
{sorted.map((dep, i) => (
|
||||
<DeploymentCard
|
||||
key={dep.deployment_id}
|
||||
provider={dep.provider}
|
||||
displayName={dep.display_name}
|
||||
providerColor={getProviderColor(dep.provider)}
|
||||
label={`Priority ${dep.priority ?? i + 1}`}
|
||||
avgLatencyMs={dep.avg_latency_ms}
|
||||
percentage={dep.percent_of_total}
|
||||
requestCount={dep.request_count}
|
||||
width={CARD_WIDTH}
|
||||
/>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
|
@ -0,0 +1,312 @@
|
|||
"use client";
|
||||
|
||||
import React, { useEffect, useState } from "react";
|
||||
import {
|
||||
Button,
|
||||
Collapse,
|
||||
Form,
|
||||
Input,
|
||||
InputNumber,
|
||||
Modal,
|
||||
Radio,
|
||||
Space,
|
||||
Typography,
|
||||
} from "antd";
|
||||
import { CloseOutlined, PlusOutlined } from "@ant-design/icons";
|
||||
import { routingGroupCreateCall, routingGroupUpdateCall } from "@/components/networking";
|
||||
|
||||
interface Deployment {
|
||||
model_name: string;
|
||||
litellm_provider: string;
|
||||
weight?: number;
|
||||
priority?: number;
|
||||
}
|
||||
|
||||
interface RoutingGroupBuilderProps {
|
||||
accessToken: string | null;
|
||||
visible: boolean;
|
||||
editTarget: Record<string, unknown> | null;
|
||||
onClose: () => void;
|
||||
onSuccess: () => void;
|
||||
}
|
||||
|
||||
const ROUTING_STRATEGIES = [
|
||||
{ value: "priority-failover", label: "Priority Failover: ordered, first success wins" },
|
||||
{ value: "weighted", label: "Weighted: split traffic by percentage" },
|
||||
{ value: "cost-based-routing", label: "Cost-Based: route to cheapest" },
|
||||
{ value: "latency-based-routing", label: "Latency-Based: route to fastest" },
|
||||
{ value: "least-busy", label: "Least Busy: fewest in-flight requests" },
|
||||
{ value: "usage-based-routing-v2", label: "Usage-Based: TPM/RPM aware" },
|
||||
{ value: "simple-shuffle", label: "Round Robin: even distribution" },
|
||||
];
|
||||
|
||||
export default function RoutingGroupBuilder({
|
||||
accessToken,
|
||||
visible,
|
||||
editTarget,
|
||||
onClose,
|
||||
onSuccess,
|
||||
}: RoutingGroupBuilderProps) {
|
||||
const [form] = Form.useForm();
|
||||
const [deployments, setDeployments] = useState<Deployment[]>([]);
|
||||
const [newModelName, setNewModelName] = useState("");
|
||||
const [newProvider, setNewProvider] = useState("");
|
||||
const [submitting, setSubmitting] = useState(false);
|
||||
const [strategy, setStrategy] = useState<string>("priority-failover");
|
||||
|
||||
const isEditing = editTarget !== null;
|
||||
|
||||
useEffect(() => {
|
||||
if (editTarget) {
|
||||
form.setFieldsValue({
|
||||
routing_group_name: editTarget.routing_group_name,
|
||||
description: editTarget.description,
|
||||
routing_strategy: editTarget.routing_strategy ?? "priority-failover",
|
||||
max_retries: editTarget.max_retries,
|
||||
cooldown_time: editTarget.cooldown_time,
|
||||
});
|
||||
setStrategy((editTarget.routing_strategy as string) ?? "priority-failover");
|
||||
if (Array.isArray(editTarget.deployments)) {
|
||||
setDeployments(editTarget.deployments as Deployment[]);
|
||||
}
|
||||
} else {
|
||||
form.resetFields();
|
||||
setDeployments([]);
|
||||
setStrategy("priority-failover");
|
||||
}
|
||||
}, [editTarget, form]);
|
||||
|
||||
const addDeployment = () => {
|
||||
const trimmedModel = newModelName.trim();
|
||||
if (!trimmedModel) return;
|
||||
const newDeployment: Deployment = {
|
||||
model_name: trimmedModel,
|
||||
litellm_provider: newProvider.trim(),
|
||||
weight: strategy === "weighted" ? 1 : undefined,
|
||||
priority: strategy === "priority-failover" ? deployments.length + 1 : undefined,
|
||||
};
|
||||
setDeployments((prev) => [...prev, newDeployment]);
|
||||
setNewModelName("");
|
||||
setNewProvider("");
|
||||
};
|
||||
|
||||
const removeDeployment = (index: number) => {
|
||||
setDeployments((prev) => {
|
||||
const updated = prev.filter((_, i) => i !== index);
|
||||
if (strategy === "priority-failover") {
|
||||
return updated.map((d, i) => ({ ...d, priority: i + 1 }));
|
||||
}
|
||||
return updated;
|
||||
});
|
||||
};
|
||||
|
||||
const updateDeploymentWeight = (index: number, weight: number | null) => {
|
||||
setDeployments((prev) =>
|
||||
prev.map((d, i) => (i === index ? { ...d, weight: weight ?? 1 } : d))
|
||||
);
|
||||
};
|
||||
|
||||
const handleStrategyChange = (value: string) => {
|
||||
setStrategy(value);
|
||||
setDeployments((prev) =>
|
||||
prev.map((d, i) => ({
|
||||
...d,
|
||||
weight: value === "weighted" ? (d.weight ?? 1) : undefined,
|
||||
priority: value === "priority-failover" ? i + 1 : undefined,
|
||||
}))
|
||||
);
|
||||
};
|
||||
|
||||
const handleSubmit = async () => {
|
||||
if (!accessToken) return;
|
||||
try {
|
||||
const values = await form.validateFields();
|
||||
setSubmitting(true);
|
||||
|
||||
const payload: Record<string, unknown> = {
|
||||
routing_group_name: values.routing_group_name,
|
||||
description: values.description,
|
||||
routing_strategy: values.routing_strategy,
|
||||
deployments,
|
||||
max_retries: values.max_retries,
|
||||
cooldown_time: values.cooldown_time,
|
||||
};
|
||||
|
||||
if (isEditing && editTarget?.routing_group_id) {
|
||||
await routingGroupUpdateCall(
|
||||
accessToken,
|
||||
editTarget.routing_group_id as string,
|
||||
payload
|
||||
);
|
||||
} else {
|
||||
await routingGroupCreateCall(accessToken, payload);
|
||||
}
|
||||
|
||||
onSuccess();
|
||||
} catch (err) {
|
||||
console.error("Failed to save routing group:", err);
|
||||
} finally {
|
||||
setSubmitting(false);
|
||||
}
|
||||
};
|
||||
|
||||
return (
|
||||
<Modal
|
||||
title={isEditing ? "Edit Routing Group" : "Create Routing Group"}
|
||||
open={visible}
|
||||
onCancel={onClose}
|
||||
width={700}
|
||||
footer={[
|
||||
<Button key="cancel" onClick={onClose}>
|
||||
Cancel
|
||||
</Button>,
|
||||
<Button
|
||||
key="submit"
|
||||
type="primary"
|
||||
loading={submitting}
|
||||
onClick={handleSubmit}
|
||||
>
|
||||
{isEditing ? "Save Changes" : "Create"}
|
||||
</Button>,
|
||||
]}
|
||||
>
|
||||
<Form
|
||||
form={form}
|
||||
layout="vertical"
|
||||
initialValues={{ routing_strategy: "priority-failover" }}
|
||||
>
|
||||
<Form.Item
|
||||
name="routing_group_name"
|
||||
label="Name"
|
||||
rules={[{ required: true, message: "Please enter a name" }]}
|
||||
>
|
||||
<Input placeholder="e.g. production-gpt4-pool" />
|
||||
</Form.Item>
|
||||
|
||||
<Form.Item name="description" label="Description">
|
||||
<Input.TextArea rows={2} placeholder="Optional description" />
|
||||
</Form.Item>
|
||||
|
||||
<Form.Item
|
||||
name="routing_strategy"
|
||||
label="Routing Strategy"
|
||||
rules={[{ required: true, message: "Please select a strategy" }]}
|
||||
>
|
||||
<Radio.Group
|
||||
onChange={(e) => handleStrategyChange(e.target.value)}
|
||||
>
|
||||
<Space direction="vertical">
|
||||
{ROUTING_STRATEGIES.map((s) => (
|
||||
<Radio key={s.value} value={s.value}>
|
||||
{s.label}
|
||||
</Radio>
|
||||
))}
|
||||
</Space>
|
||||
</Radio.Group>
|
||||
</Form.Item>
|
||||
|
||||
<div style={{ marginBottom: 16 }}>
|
||||
<Typography.Text strong>Deployments</Typography.Text>
|
||||
<div style={{ marginTop: 8, marginBottom: 8 }}>
|
||||
<Space>
|
||||
<Input
|
||||
placeholder="Model name (e.g. gpt-4)"
|
||||
value={newModelName}
|
||||
onChange={(e) => setNewModelName(e.target.value)}
|
||||
style={{ width: 200 }}
|
||||
onPressEnter={addDeployment}
|
||||
/>
|
||||
<Input
|
||||
placeholder="Provider (e.g. openai)"
|
||||
value={newProvider}
|
||||
onChange={(e) => setNewProvider(e.target.value)}
|
||||
style={{ width: 160 }}
|
||||
onPressEnter={addDeployment}
|
||||
/>
|
||||
<Button icon={<PlusOutlined />} onClick={addDeployment}>
|
||||
Add Deployment
|
||||
</Button>
|
||||
</Space>
|
||||
</div>
|
||||
|
||||
{deployments.length === 0 && (
|
||||
<Typography.Text type="secondary">
|
||||
No deployments added yet.
|
||||
</Typography.Text>
|
||||
)}
|
||||
|
||||
{deployments.map((dep, idx) => (
|
||||
<div
|
||||
key={idx}
|
||||
style={{
|
||||
display: "flex",
|
||||
alignItems: "center",
|
||||
gap: 8,
|
||||
marginBottom: 6,
|
||||
padding: "6px 10px",
|
||||
background: "#fafafa",
|
||||
border: "1px solid #f0f0f0",
|
||||
borderRadius: 4,
|
||||
}}
|
||||
>
|
||||
{strategy === "priority-failover" && (
|
||||
<Typography.Text type="secondary" style={{ minWidth: 24 }}>
|
||||
#{dep.priority}
|
||||
</Typography.Text>
|
||||
)}
|
||||
<Typography.Text style={{ flex: 1 }}>
|
||||
{dep.model_name}
|
||||
{dep.litellm_provider ? ` (${dep.litellm_provider})` : ""}
|
||||
</Typography.Text>
|
||||
{strategy === "weighted" && (
|
||||
<InputNumber
|
||||
size="small"
|
||||
min={1}
|
||||
max={100}
|
||||
value={dep.weight ?? 1}
|
||||
onChange={(val) => updateDeploymentWeight(idx, val)}
|
||||
addonAfter="%"
|
||||
style={{ width: 100 }}
|
||||
/>
|
||||
)}
|
||||
<Button
|
||||
size="small"
|
||||
type="text"
|
||||
danger
|
||||
icon={<CloseOutlined />}
|
||||
onClick={() => removeDeployment(idx)}
|
||||
/>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
|
||||
<Collapse ghost>
|
||||
<Collapse.Panel header="Advanced Settings" key="advanced">
|
||||
<Form.Item name="max_retries" label="Max Retries">
|
||||
<InputNumber min={0} max={100} style={{ width: 160 }} />
|
||||
</Form.Item>
|
||||
<Form.Item name="cooldown_time" label="Cooldown Time (seconds)">
|
||||
<InputNumber min={0} style={{ width: 160 }} />
|
||||
</Form.Item>
|
||||
</Collapse.Panel>
|
||||
</Collapse>
|
||||
|
||||
<div
|
||||
style={{
|
||||
marginTop: 12,
|
||||
padding: "8px 12px",
|
||||
background: "#f6f8fa",
|
||||
borderRadius: 4,
|
||||
}}
|
||||
>
|
||||
<Typography.Text type="secondary">
|
||||
Access is managed via Teams.{" "}
|
||||
<a href="/teams" target="_blank" rel="noreferrer">
|
||||
Manage Teams
|
||||
</a>
|
||||
</Typography.Text>
|
||||
</div>
|
||||
</Form>
|
||||
</Modal>
|
||||
);
|
||||
}
|
||||
|
|
@ -0,0 +1,168 @@
|
|||
"use client";
|
||||
|
||||
import React, { useEffect, useState } from "react";
|
||||
import { Button, Popconfirm, Table, Tag, Typography } from "antd";
|
||||
import { DeleteOutlined, EditOutlined, ThunderboltOutlined } from "@ant-design/icons";
|
||||
import { routingGroupDeleteCall, routingGroupListCall } from "@/components/networking";
|
||||
|
||||
interface RoutingGroupsTableProps {
|
||||
accessToken: string | null;
|
||||
refreshKey: number;
|
||||
onEdit: (group: Record<string, unknown>) => void;
|
||||
onTest?: (group: Record<string, unknown>) => void;
|
||||
}
|
||||
|
||||
const strategyColors: Record<string, string> = {
|
||||
"priority-failover": "blue",
|
||||
weighted: "green",
|
||||
"cost-based-routing": "gold",
|
||||
"latency-based-routing": "cyan",
|
||||
"least-busy": "purple",
|
||||
"usage-based-routing-v2": "orange",
|
||||
"simple-shuffle": "geekblue",
|
||||
};
|
||||
|
||||
const strategyLabels: Record<string, string> = {
|
||||
"priority-failover": "Priority Failover",
|
||||
weighted: "Weighted",
|
||||
"cost-based-routing": "Cost-Based",
|
||||
"latency-based-routing": "Latency-Based",
|
||||
"least-busy": "Least Busy",
|
||||
"usage-based-routing-v2": "Usage-Based",
|
||||
"simple-shuffle": "Round Robin",
|
||||
};
|
||||
|
||||
export default function RoutingGroupsTable({
|
||||
accessToken,
|
||||
refreshKey,
|
||||
onEdit,
|
||||
onTest,
|
||||
}: RoutingGroupsTableProps) {
|
||||
const [groups, setGroups] = useState<Record<string, unknown>[]>([]);
|
||||
const [loading, setLoading] = useState(false);
|
||||
|
||||
useEffect(() => {
|
||||
if (!accessToken) return;
|
||||
setLoading(true);
|
||||
routingGroupListCall(accessToken, 1, 50)
|
||||
.then((resp) => {
|
||||
setGroups(resp.routing_groups || []);
|
||||
})
|
||||
.catch(console.error)
|
||||
.finally(() => setLoading(false));
|
||||
}, [accessToken, refreshKey]);
|
||||
|
||||
const handleDelete = async (groupId: string) => {
|
||||
if (!accessToken) return;
|
||||
try {
|
||||
await routingGroupDeleteCall(accessToken, groupId);
|
||||
setGroups((prev) =>
|
||||
prev.filter((g) => (g.routing_group_id as string) !== groupId)
|
||||
);
|
||||
} catch (err) {
|
||||
console.error("Failed to delete routing group:", err);
|
||||
}
|
||||
};
|
||||
|
||||
const columns = [
|
||||
{
|
||||
title: "Name",
|
||||
dataIndex: "routing_group_name",
|
||||
key: "routing_group_name",
|
||||
render: (name: string) => (
|
||||
<Typography.Text strong>{name}</Typography.Text>
|
||||
),
|
||||
},
|
||||
{
|
||||
title: "Strategy",
|
||||
dataIndex: "routing_strategy",
|
||||
key: "routing_strategy",
|
||||
render: (strategy: string) => {
|
||||
const label = strategyLabels[strategy] ?? strategy;
|
||||
const color = strategyColors[strategy] ?? "default";
|
||||
return <Tag color={color}>{label}</Tag>;
|
||||
},
|
||||
},
|
||||
{
|
||||
title: "Deployments",
|
||||
dataIndex: "deployments",
|
||||
key: "deployments",
|
||||
render: (deployments: unknown) => {
|
||||
const count = Array.isArray(deployments) ? deployments.length : 0;
|
||||
return <Typography.Text>{count}</Typography.Text>;
|
||||
},
|
||||
},
|
||||
{
|
||||
title: "Status",
|
||||
dataIndex: "status",
|
||||
key: "status",
|
||||
render: (status: string) => {
|
||||
const isActive = !status || status === "active";
|
||||
return (
|
||||
<Tag color={isActive ? "success" : "default"}>
|
||||
{isActive ? "Active" : "Inactive"}
|
||||
</Tag>
|
||||
);
|
||||
},
|
||||
},
|
||||
{
|
||||
title: "Created",
|
||||
dataIndex: "created_at",
|
||||
key: "created_at",
|
||||
render: (created: string) => {
|
||||
if (!created) return "-";
|
||||
return new Date(created).toLocaleDateString();
|
||||
},
|
||||
},
|
||||
{
|
||||
title: "Actions",
|
||||
key: "actions",
|
||||
render: (_: unknown, record: Record<string, unknown>) => {
|
||||
const groupId = record.routing_group_id as string;
|
||||
return (
|
||||
<div style={{ display: "flex", gap: 8 }}>
|
||||
{onTest && (
|
||||
<Button
|
||||
size="small"
|
||||
icon={<ThunderboltOutlined />}
|
||||
onClick={() => onTest(record)}
|
||||
style={{ color: "#14b8a6", borderColor: "#14b8a6" }}
|
||||
>
|
||||
Test
|
||||
</Button>
|
||||
)}
|
||||
<Button
|
||||
size="small"
|
||||
icon={<EditOutlined />}
|
||||
onClick={() => onEdit(record)}
|
||||
>
|
||||
Edit
|
||||
</Button>
|
||||
<Popconfirm
|
||||
title="Delete routing group?"
|
||||
description="This action cannot be undone."
|
||||
onConfirm={() => handleDelete(groupId)}
|
||||
okText="Delete"
|
||||
okButtonProps={{ danger: true }}
|
||||
cancelText="Cancel"
|
||||
>
|
||||
<Button size="small" danger icon={<DeleteOutlined />}>
|
||||
Delete
|
||||
</Button>
|
||||
</Popconfirm>
|
||||
</div>
|
||||
);
|
||||
},
|
||||
},
|
||||
];
|
||||
|
||||
return (
|
||||
<Table
|
||||
dataSource={groups}
|
||||
columns={columns}
|
||||
loading={loading}
|
||||
rowKey={(record) => (record.routing_group_id as string) ?? Math.random().toString()}
|
||||
pagination={{ pageSize: 20 }}
|
||||
/>
|
||||
);
|
||||
}
|
||||
|
|
@ -0,0 +1,112 @@
|
|||
"use client";
|
||||
|
||||
import React, { useState } from "react";
|
||||
import { Button, Divider, Typography } from "antd";
|
||||
import { PlusOutlined, CloseOutlined } from "@ant-design/icons";
|
||||
import RoutingGroupsTable from "./RoutingGroupsTable";
|
||||
import RoutingGroupBuilder from "./RoutingGroupBuilder";
|
||||
import LiveTester from "./LiveTester";
|
||||
|
||||
interface RoutingGroupsViewProps {
|
||||
accessToken: string | null;
|
||||
userRole: string;
|
||||
userId: string;
|
||||
}
|
||||
|
||||
export default function RoutingGroupsView({
|
||||
accessToken,
|
||||
userRole,
|
||||
userId,
|
||||
}: RoutingGroupsViewProps) {
|
||||
const [createModalVisible, setCreateModalVisible] = useState(false);
|
||||
const [editTarget, setEditTarget] = useState<Record<string, unknown> | null>(null);
|
||||
const [refreshKey, setRefreshKey] = useState(0);
|
||||
const [selectedGroup, setSelectedGroup] = useState<Record<string, unknown> | null>(null);
|
||||
|
||||
const handleEdit = (group: Record<string, unknown>) => {
|
||||
setEditTarget(group);
|
||||
setCreateModalVisible(true);
|
||||
};
|
||||
|
||||
const handleTest = (group: Record<string, unknown>) => {
|
||||
setSelectedGroup(group);
|
||||
};
|
||||
|
||||
const handleClose = () => {
|
||||
setCreateModalVisible(false);
|
||||
setEditTarget(null);
|
||||
};
|
||||
|
||||
const handleSuccess = () => {
|
||||
setCreateModalVisible(false);
|
||||
setEditTarget(null);
|
||||
setRefreshKey((k) => k + 1);
|
||||
};
|
||||
|
||||
return (
|
||||
<div className="w-full mx-4">
|
||||
<div className="flex justify-between items-center mb-6">
|
||||
<div>
|
||||
<Typography.Title level={4} style={{ margin: 0 }}>
|
||||
Routing Groups
|
||||
</Typography.Title>
|
||||
<Typography.Text type="secondary">
|
||||
Configure named routing pipelines with fallback strategies
|
||||
</Typography.Text>
|
||||
</div>
|
||||
<Button
|
||||
type="primary"
|
||||
icon={<PlusOutlined />}
|
||||
onClick={() => setCreateModalVisible(true)}
|
||||
>
|
||||
Create Routing Group
|
||||
</Button>
|
||||
</div>
|
||||
|
||||
<RoutingGroupsTable
|
||||
accessToken={accessToken}
|
||||
refreshKey={refreshKey}
|
||||
onEdit={handleEdit}
|
||||
onTest={handleTest}
|
||||
/>
|
||||
|
||||
{selectedGroup && accessToken && (
|
||||
<>
|
||||
<Divider />
|
||||
<div className="flex items-center justify-between mb-4">
|
||||
<Typography.Text type="secondary" className="text-sm">
|
||||
Testing:{" "}
|
||||
<span className="font-mono font-semibold">
|
||||
{selectedGroup.routing_group_name as string}
|
||||
</span>
|
||||
</Typography.Text>
|
||||
<Button
|
||||
size="small"
|
||||
icon={<CloseOutlined />}
|
||||
onClick={() => setSelectedGroup(null)}
|
||||
style={{ color: "#64748b", borderColor: "#334155" }}
|
||||
>
|
||||
Close
|
||||
</Button>
|
||||
</div>
|
||||
<LiveTester
|
||||
accessToken={accessToken}
|
||||
routingGroupId={selectedGroup.routing_group_id as string}
|
||||
routingGroupName={selectedGroup.routing_group_name as string}
|
||||
routingStrategy={(selectedGroup.routing_strategy as string) ?? "weighted"}
|
||||
/>
|
||||
</>
|
||||
)}
|
||||
|
||||
{createModalVisible && (
|
||||
<RoutingGroupBuilder
|
||||
accessToken={accessToken}
|
||||
visible={createModalVisible}
|
||||
editTarget={editTarget}
|
||||
onClose={handleClose}
|
||||
onSuccess={handleSuccess}
|
||||
/>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
|
@ -0,0 +1,29 @@
|
|||
"use client";
|
||||
import React from "react";
|
||||
|
||||
interface StatsBarProps {
|
||||
requests: number;
|
||||
successRate: number; // 0-100
|
||||
avgLatencyMs: number;
|
||||
fallbackCount: number;
|
||||
}
|
||||
|
||||
export default function StatsBar({ requests, successRate, avgLatencyMs, fallbackCount }: StatsBarProps) {
|
||||
const stats = [
|
||||
{ label: "REQUESTS", value: requests.toLocaleString() },
|
||||
{ label: "SUCCESS", value: `${Math.round(successRate)}%` },
|
||||
{ label: "AVG LATENCY", value: avgLatencyMs > 0 ? `${Math.round(avgLatencyMs)}ms` : "—" },
|
||||
{ label: "FALLBACKS", value: fallbackCount.toLocaleString() },
|
||||
];
|
||||
|
||||
return (
|
||||
<div className="grid grid-cols-4 divide-x divide-gray-700 rounded-xl border border-gray-700 bg-[#1a1f2e] px-0">
|
||||
{stats.map(({ label, value }) => (
|
||||
<div key={label} className="flex flex-col items-start gap-1 px-6 py-4">
|
||||
<span className="text-xs font-medium uppercase tracking-widest text-gray-500">{label}</span>
|
||||
<span className="text-2xl font-bold text-white">{value}</span>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
|
@ -0,0 +1,122 @@
|
|||
"use client";
|
||||
import React from "react";
|
||||
import AnimatedStream from "./AnimatedStream";
|
||||
import DeploymentCard from "./DeploymentCard";
|
||||
|
||||
const PROVIDER_COLORS: Record<string, string> = {
|
||||
nebius: "#14b8a6",
|
||||
fireworks_ai: "#f97316",
|
||||
azure: "#8b5cf6",
|
||||
openai: "#10b981",
|
||||
anthropic: "#d97706",
|
||||
google: "#3b82f6",
|
||||
aws: "#ef4444",
|
||||
};
|
||||
const getProviderColor = (p: string) => PROVIDER_COLORS[p.toLowerCase()] ?? "#6b7280";
|
||||
|
||||
interface Deployment {
|
||||
deployment_id: string;
|
||||
provider: string;
|
||||
display_name: string;
|
||||
weight?: number;
|
||||
avg_latency_ms: number;
|
||||
percent_of_total: number;
|
||||
request_count: number;
|
||||
}
|
||||
|
||||
interface WeightedRoundRobinFlowProps {
|
||||
deployments: Deployment[];
|
||||
routingGroupName: string;
|
||||
}
|
||||
|
||||
const CARD_HEIGHT = 76;
|
||||
const CARD_GAP = 28;
|
||||
const CARD_WIDTH = 220;
|
||||
const LEFT_PADDING = 40;
|
||||
const HUB_RADIUS = 28;
|
||||
|
||||
export default function WeightedRoundRobinFlow({ deployments }: WeightedRoundRobinFlowProps) {
|
||||
// Sort by weight desc (highest traffic first = top position)
|
||||
const sorted = [...deployments].sort((a, b) => (b.weight ?? 0) - (a.weight ?? 0));
|
||||
const totalHeight = sorted.length * CARD_HEIGHT + (sorted.length - 1) * CARD_GAP + 80;
|
||||
const svgWidth = 600;
|
||||
|
||||
const hubX = svgWidth * 0.32;
|
||||
const hubY = totalHeight / 2;
|
||||
|
||||
const cardX = svgWidth - CARD_WIDTH - LEFT_PADDING;
|
||||
|
||||
const blockHeight = sorted.length * CARD_HEIGHT + (sorted.length - 1) * CARD_GAP;
|
||||
const blockTop = (totalHeight - blockHeight) / 2;
|
||||
const cardCenterYs = sorted.map((_, i) => blockTop + i * (CARD_HEIGHT + CARD_GAP) + CARD_HEIGHT / 2);
|
||||
|
||||
// Max thickness 14px for 100%, min 2px
|
||||
const maxPct = Math.max(...sorted.map((d) => d.percent_of_total), 1);
|
||||
const getThickness = (pct: number) => Math.max(2, Math.round((pct / maxPct) * 14));
|
||||
|
||||
// Bezier path from hub to each card center
|
||||
const getPath = (targetY: number) => {
|
||||
const cp1x = hubX + 80;
|
||||
const cp2x = cardX - 60;
|
||||
return `M ${hubX} ${hubY} C ${cp1x} ${hubY} ${cp2x} ${targetY} ${cardX} ${targetY}`;
|
||||
};
|
||||
|
||||
return (
|
||||
<div className="relative w-full overflow-hidden rounded-2xl bg-[#0f1117]" style={{ minHeight: totalHeight + 40 }}>
|
||||
<svg
|
||||
className="absolute inset-0"
|
||||
width="100%"
|
||||
height={totalHeight + 40}
|
||||
viewBox={`0 0 ${svgWidth} ${totalHeight + 40}`}
|
||||
preserveAspectRatio="xMidYMid meet"
|
||||
>
|
||||
{/* Client → hub */}
|
||||
<AnimatedStream
|
||||
path={`M 20 ${hubY} L ${hubX - HUB_RADIUS} ${hubY}`}
|
||||
color="#14b8a6"
|
||||
thickness={6}
|
||||
/>
|
||||
|
||||
{/* Hub */}
|
||||
<circle cx={hubX} cy={hubY} r={HUB_RADIUS} fill="#1a2535" stroke="#14b8a6" strokeWidth={2} strokeOpacity={0.6} />
|
||||
<text x={hubX} y={hubY + 4} textAnchor="middle" fill="#14b8a6" fontSize={8} fontWeight="600" fontFamily="monospace">
|
||||
LITELLM
|
||||
</text>
|
||||
|
||||
{/* Fan-out streams */}
|
||||
{sorted.map((dep, i) => (
|
||||
<AnimatedStream
|
||||
key={dep.deployment_id}
|
||||
path={getPath(cardCenterYs[i])}
|
||||
color={getProviderColor(dep.provider)}
|
||||
thickness={getThickness(dep.percent_of_total)}
|
||||
/>
|
||||
))}
|
||||
</svg>
|
||||
|
||||
{/* "client" label */}
|
||||
<div className="absolute text-xs font-mono text-gray-500" style={{ left: 24, top: hubY + 40 - 8 }}>
|
||||
client
|
||||
</div>
|
||||
|
||||
{/* Cards */}
|
||||
<div className="absolute" style={{ right: LEFT_PADDING, top: 20 }}>
|
||||
<div className="flex flex-col" style={{ gap: CARD_GAP }}>
|
||||
{sorted.map((dep) => (
|
||||
<DeploymentCard
|
||||
key={dep.deployment_id}
|
||||
provider={dep.provider}
|
||||
displayName={dep.display_name}
|
||||
providerColor={getProviderColor(dep.provider)}
|
||||
label={`Weight ${dep.weight ?? Math.round(dep.percent_of_total)}%`}
|
||||
avgLatencyMs={dep.avg_latency_ms}
|
||||
percentage={dep.percent_of_total}
|
||||
requestCount={dep.request_count}
|
||||
width={CARD_WIDTH}
|
||||
/>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
Loading…
Add table
Reference in a new issue