mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
feat(router): add zero-setup auto configuration
This commit is contained in:
parent
321636ef5d
commit
591f8aef1c
27 changed files with 9144 additions and 101 deletions
|
|
@ -860,6 +860,7 @@ class LiteLLMRoutes(enum.Enum):
|
|||
# proxy admin, or team admin naming their own team via team_id
|
||||
"/auto_router/test_routing",
|
||||
"/auto_router/validate_complexity_router_config",
|
||||
"/auto_router/recommendation",
|
||||
# Agent registry - reads are role-scoped and writes are proxy-admin-gated
|
||||
# inside agent_endpoints/endpoints.py
|
||||
*agent_management_routes,
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ AUTO ROUTER MANAGEMENT ENDPOINTS
|
|||
|
||||
POST /auto_router/test_routing - Route one request through an unsaved complexity-router config
|
||||
POST /auto_router/validate_complexity_router_config - Dry-run the complexity-router write gate without saving
|
||||
GET /auto_router/recommendation - Build an editable config from the caller's available model groups
|
||||
"""
|
||||
|
||||
from collections.abc import Mapping, Sequence
|
||||
|
|
@ -10,7 +11,7 @@ from datetime import datetime, timedelta, timezone
|
|||
from itertools import chain, groupby
|
||||
from operator import attrgetter
|
||||
from types import MappingProxyType
|
||||
from typing import TYPE_CHECKING, Annotated, Final, Protocol
|
||||
from typing import TYPE_CHECKING, Annotated, Final, Literal, Protocol, cast
|
||||
from uuid import uuid4
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, TypeAdapter, field_validator
|
||||
|
|
@ -52,6 +53,7 @@ from litellm.types.management_endpoints.auto_router_endpoints import (
|
|||
AutoRouterBenchmarkTotals,
|
||||
AutoRouterCacheBucket,
|
||||
AutoRouterCacheStats,
|
||||
AutoRouterRecommendationResponse,
|
||||
AutoRouterRoutingTestRequest,
|
||||
AutoRouterRoutingTestResponse,
|
||||
ComplexityRouterConfigValidationRequest,
|
||||
|
|
@ -334,6 +336,117 @@ async def validate_complexity_router_config(
|
|||
return ComplexityRouterConfigValidationResponse(valid=error is None, error=error)
|
||||
|
||||
|
||||
def _available_model_refs(
|
||||
llm_router: "Router", model_names: Sequence[str], team_id: str | None
|
||||
) -> Mapping[str, tuple[str, ...]]:
|
||||
refs: Final[dict[str, tuple[str, ...]]] = {}
|
||||
for model_name in model_names:
|
||||
deployments = llm_router.get_model_list(model_name=model_name, team_id=team_id) or ()
|
||||
resolved: list[str] = [model_name]
|
||||
for raw_deployment in deployments:
|
||||
deployment = cast(Mapping[str, object], raw_deployment)
|
||||
litellm_params = deployment.get("litellm_params")
|
||||
model_info = deployment.get("model_info")
|
||||
if isinstance(litellm_params, Mapping):
|
||||
typed_params = cast(Mapping[str, object], litellm_params)
|
||||
resolved.extend(
|
||||
value
|
||||
for key in ("model", "base_model")
|
||||
if isinstance(value := typed_params.get(key), str) and value
|
||||
)
|
||||
if isinstance(model_info, Mapping):
|
||||
base_model = cast(Mapping[str, object], model_info).get("base_model")
|
||||
if isinstance(base_model, str) and base_model:
|
||||
resolved.append(base_model)
|
||||
refs[model_name] = tuple(dict.fromkeys(resolved))
|
||||
return MappingProxyType(refs)
|
||||
|
||||
|
||||
@router.get(
|
||||
"/auto_router/recommendation",
|
||||
tags=["model management"], # mutable-ok: fastapi's decorator signature types tags as a list
|
||||
dependencies=[Depends(user_api_key_auth)], # mutable-ok: fastapi's decorator signature types dependencies as a list
|
||||
response_model=AutoRouterRecommendationResponse,
|
||||
status_code=status.HTTP_200_OK,
|
||||
)
|
||||
async def get_auto_router_recommendation(
|
||||
user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)],
|
||||
quality_level: Annotated[
|
||||
Literal["economy", "balanced", "high", "max"],
|
||||
Query(description="How close every admitted model must be to the best available quality score"),
|
||||
] = "balanced",
|
||||
optimize_for: Annotated[
|
||||
Literal["cost", "task_completion_speed", "balanced"],
|
||||
Query(description="What to minimize after the selected quality gate is applied"),
|
||||
] = "balanced",
|
||||
team_id: Annotated[str | None, Query(description="Team whose model access the recommendation must use")] = None,
|
||||
) -> AutoRouterRecommendationResponse:
|
||||
"""Build an editable four-tier router from only the model groups this caller can use."""
|
||||
|
||||
from litellm.proxy.proxy_server import (
|
||||
general_settings,
|
||||
llm_router,
|
||||
prisma_client,
|
||||
proxy_logging_obj,
|
||||
user_api_key_cache,
|
||||
user_model,
|
||||
)
|
||||
from litellm.proxy.utils import get_available_models_for_user
|
||||
from litellm.router_strategy.complexity_router.auto_setup import (
|
||||
build_auto_setup_config,
|
||||
load_auto_router_snapshot,
|
||||
)
|
||||
|
||||
await _authorize_router_dry_run(user_api_key_dict=user_api_key_dict, team_id=team_id)
|
||||
if llm_router is None:
|
||||
raise HTTPException(
|
||||
status_code=500,
|
||||
detail={"error": CommonProxyErrors.no_llm_router.value}, # mutable-ok: HTTPException detail is JSON-shaped
|
||||
)
|
||||
available_models: Final = await get_available_models_for_user(
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
llm_router=llm_router,
|
||||
general_settings=general_settings,
|
||||
user_model=user_model,
|
||||
prisma_client=prisma_client,
|
||||
proxy_logging_obj=proxy_logging_obj,
|
||||
team_id=team_id,
|
||||
user_api_key_cache=user_api_key_cache,
|
||||
)
|
||||
snapshot: Final = load_auto_router_snapshot()
|
||||
try:
|
||||
config: Final = build_auto_setup_config(
|
||||
snapshot=snapshot,
|
||||
available_model_refs=_available_model_refs(llm_router, available_models, team_id),
|
||||
quality_level=quality_level,
|
||||
optimize_for=optimize_for,
|
||||
)
|
||||
except ValueError as exc:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail={
|
||||
"error": f"Could not build an Auto setup: {exc}"
|
||||
}, # mutable-ok: HTTPException detail is JSON-shaped
|
||||
) from exc
|
||||
matched: Final = tuple(
|
||||
dict.fromkeys(
|
||||
model
|
||||
for tier_models in config.tiers.values()
|
||||
for model in ((tier_models,) if isinstance(tier_models, str) else tier_models)
|
||||
)
|
||||
)
|
||||
return AutoRouterRecommendationResponse(
|
||||
quality_level=quality_level,
|
||||
optimize_for=optimize_for,
|
||||
snapshot_id=snapshot.snapshot_id,
|
||||
snapshot_generated_at=snapshot.generated_at,
|
||||
matched_model_groups=matched,
|
||||
complexity_router_config=RequestComplexityRouterConfig.model_validate(
|
||||
config.model_dump(mode="json", exclude_none=True)
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@router.post(
|
||||
"/auto_router/test_routing",
|
||||
tags=["model management"], # mutable-ok: fastapi's decorator signature types tags as a list
|
||||
|
|
|
|||
|
|
@ -7865,6 +7865,12 @@ class Router:
|
|||
# WS session wrappers fire with result=None; per-turn costs tracked by inner calls.
|
||||
if kwargs.get("call_type") in ("_aresponses_websocket", "_arealtime"):
|
||||
return
|
||||
await self._record_auto_setup_response_latency(
|
||||
kwargs=kwargs,
|
||||
completion_response=completion_response,
|
||||
start_time=start_time,
|
||||
end_time=end_time,
|
||||
)
|
||||
standard_logging_object: Final[StandardLoggingPayload | None] = kwargs.get("standard_logging_object", None)
|
||||
if standard_logging_object is None:
|
||||
raise ValueError("standard_logging_object is None")
|
||||
|
|
@ -7976,6 +7982,55 @@ class Router:
|
|||
"litellm.router.Router::deployment_callback_on_success(): Exception occured - %s", e
|
||||
)
|
||||
|
||||
async def _record_auto_setup_response_latency(
|
||||
self,
|
||||
*,
|
||||
kwargs: Mapping[str, object],
|
||||
completion_response: object,
|
||||
start_time: object,
|
||||
end_time: object,
|
||||
) -> None:
|
||||
raw_metadata = cast(object, get_litellm_metadata_from_kwargs(dict(kwargs)))
|
||||
if not isinstance(raw_metadata, Mapping):
|
||||
return
|
||||
metadata = cast(Mapping[str, object], raw_metadata)
|
||||
raw_decision = metadata.get("routing_decision")
|
||||
if not isinstance(raw_decision, Mapping):
|
||||
return
|
||||
decision = cast(Mapping[str, object], raw_decision)
|
||||
if decision.get("auto_setup_selection_mode") != "runtime_response_latency":
|
||||
return
|
||||
router_model_name = decision.get("router_model_name")
|
||||
tier = decision.get("tier")
|
||||
routed_model = decision.get("routed_model")
|
||||
if not isinstance(router_model_name, str) or not router_model_name:
|
||||
return
|
||||
if not isinstance(tier, str) or not tier:
|
||||
return
|
||||
if not isinstance(routed_model, str) or not routed_model:
|
||||
return
|
||||
registered: Final = self.complexity_routers.get(router_model_name)
|
||||
if not registered or not any(tagged.strategy.config.auto_setup is not None for tagged in registered):
|
||||
return
|
||||
|
||||
from litellm.router_strategy.complexity_router.response_latency import (
|
||||
record_runtime_response_latency,
|
||||
)
|
||||
|
||||
try:
|
||||
await record_runtime_response_latency(
|
||||
router_cache=self.cache,
|
||||
router_model_name=router_model_name,
|
||||
tier=tier,
|
||||
routed_model=routed_model,
|
||||
kwargs=kwargs,
|
||||
response_obj=completion_response,
|
||||
start_time=start_time,
|
||||
end_time=end_time,
|
||||
)
|
||||
except Exception as exc: # noqa: BLE001 -- optional telemetry must never fail a successful completion
|
||||
verbose_router_logger.warning("Could not record Auto setup response latency: %s", exc)
|
||||
|
||||
def sync_deployment_callback_on_success(
|
||||
self,
|
||||
kwargs, # kwargs to completion
|
||||
|
|
|
|||
|
|
@ -68,6 +68,47 @@ still resolve to a deployment in `model_list`; this configuration does not creat
|
|||
- abc
|
||||
```
|
||||
|
||||
### Zero-setup Auto configuration
|
||||
|
||||
The dashboard's Add Router flow can generate the four tiers from the model groups the
|
||||
current admin or team can actually use. Auto setup asks for two product choices:
|
||||
|
||||
- Quality level: Economy, Balanced, High, or Max
|
||||
- Optimize for: Cost, Task completion speed, or Balanced
|
||||
|
||||
The quality level is a gate, not an optimization weight. For every complexity tier,
|
||||
LiteLLM finds the best available model's conservative LiveBench quality score and admits
|
||||
only models within the selected maximum regret: 15 percentage points for Economy, 7 for
|
||||
Balanced, 3 for High, and 1 for Max. The objective then ranks only those admitted models.
|
||||
This means Max still sends trivial work to a smaller model when that model is within one
|
||||
point of the best available quality for trivial work.
|
||||
|
||||
Cost is benchmark cost per completed task, not input or output token price. It is observed
|
||||
request dollars divided by quality-adjusted completed questions in the same complexity
|
||||
bucket, with partial benchmark credit counted as a fractional completion.
|
||||
|
||||
Task completion speed uses two forms of evidence:
|
||||
|
||||
- SIMPLE and MEDIUM tiers learn equal-output response time from this proxy's successful
|
||||
requests: time to first token plus the time to generate the expected number of visible
|
||||
output tokens. Reasoning tokens are excluded. The policy explores every quality-admitted
|
||||
model twice before using a rolling seven-day sample
|
||||
- COMPLEX and REASONING tiers use retry-adjusted Terminal-Bench wall-clock time to a
|
||||
successful task. Results are compared only inside the same agent and benchmark cohort
|
||||
and fall back to cost when comparable speed evidence is unavailable
|
||||
|
||||
Balanced minimizes the combined log-normalized cost and completion-time scores after the
|
||||
same quality gate. The recommendation stores its snapshot ID, objective, admitted models,
|
||||
and rank order directly in the router configuration, so the generated result is visible
|
||||
and editable. Changing a generated tier's model pool in the dashboard removes that Auto
|
||||
selection policy and returns the tier to normal manual pool selection.
|
||||
|
||||
These rules apply only when a router is created through Auto setup. Existing routers,
|
||||
manual configurations, and LiteLLM's deployment-level latency routing are unchanged. The
|
||||
bundled benchmark data is a versioned snapshot: creating or regenerating a router uses the
|
||||
snapshot shipped with that LiteLLM version, while an already saved router keeps its stored
|
||||
policy.
|
||||
|
||||
### Heuristic v2
|
||||
|
||||
Set `classifier_type: heuristic_v2` to classify with the bundled calibrated
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
378
litellm/router_strategy/complexity_router/auto_setup.py
Normal file
378
litellm/router_strategy/complexity_router/auto_setup.py
Normal file
|
|
@ -0,0 +1,378 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import math
|
||||
import re
|
||||
from collections.abc import Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from functools import lru_cache
|
||||
from importlib.resources import files
|
||||
from typing import Final, Literal, cast
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field, model_validator
|
||||
|
||||
from .config import (
|
||||
AutoSetupCandidate,
|
||||
AutoSetupConfig,
|
||||
AutoSetupObjective,
|
||||
AutoSetupQualityLevel,
|
||||
AutoSetupTierPolicy,
|
||||
ComplexityRouterConfig,
|
||||
)
|
||||
|
||||
SnapshotComplexity = Literal["trivial", "simple", "standard", "complex"]
|
||||
|
||||
_SNAPSHOT_FILENAME: Final = "auto_router_snapshot_v0.json"
|
||||
_TIER_COMPLEXITIES: Final[Mapping[str, SnapshotComplexity]] = {
|
||||
"SIMPLE": "trivial",
|
||||
"MEDIUM": "simple",
|
||||
"COMPLEX": "standard",
|
||||
"REASONING": "complex",
|
||||
}
|
||||
_EASY_COMPLEXITIES: Final = frozenset(("trivial", "simple"))
|
||||
_VERSION_SEPARATOR: Final = re.compile(r"(?<=\d)\.(?=\d)")
|
||||
|
||||
|
||||
class SnapshotQualityTier(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True)
|
||||
|
||||
label: str
|
||||
maximum_quality_regret: float = Field(ge=0, le=1)
|
||||
|
||||
|
||||
class SnapshotIdentity(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True)
|
||||
|
||||
benchmark_model_id: str
|
||||
family_id: str
|
||||
is_routable: bool
|
||||
litellm_model_keys: tuple[str, ...]
|
||||
reasoning_effort: str | None
|
||||
required_parameters: Mapping[str, object]
|
||||
|
||||
|
||||
class SnapshotQualityProfile(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True)
|
||||
|
||||
benchmark_model_id: str
|
||||
complexity: SnapshotComplexity
|
||||
cost_per_completed_task_usd: float = Field(ge=0)
|
||||
mean_request_cost_usd: float = Field(ge=0)
|
||||
quality_lower_bound: float = Field(ge=0, le=1)
|
||||
quality_score: float = Field(ge=0, le=1)
|
||||
question_count: int = Field(gt=0)
|
||||
source_id: str
|
||||
subtask_count: int = Field(gt=0)
|
||||
|
||||
|
||||
class SnapshotSpeedProfile(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True)
|
||||
|
||||
attempt_count: int = Field(gt=0)
|
||||
benchmark_model_id: str
|
||||
cohort_id: str
|
||||
complexity: SnapshotComplexity
|
||||
cost_per_completed_task_usd: float | None = Field(default=None, ge=0)
|
||||
excluded_infrastructure_trials: int = Field(ge=0)
|
||||
mean_attempt_cost_usd: float | None = Field(default=None, ge=0)
|
||||
observed_duration_p50_ms: float = Field(ge=0)
|
||||
observed_duration_p95_ms: float = Field(ge=0)
|
||||
quality_lower_bound: float = Field(ge=0, le=1)
|
||||
retry_adjusted_completion_ms: float | None = Field(default=None, ge=0)
|
||||
source_id: str
|
||||
success_probability: float = Field(ge=0, le=1)
|
||||
task_count: int = Field(gt=0)
|
||||
terminal_bench_model_id: str
|
||||
|
||||
|
||||
class SnapshotModel(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True)
|
||||
|
||||
benchmark_model_id: str
|
||||
identity: SnapshotIdentity
|
||||
quality_by_complexity: Mapping[SnapshotComplexity, SnapshotQualityProfile]
|
||||
task_completion_speed_by_complexity: Mapping[SnapshotComplexity, tuple[SnapshotSpeedProfile, ...]]
|
||||
|
||||
|
||||
class AutoRouterSnapshot(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True)
|
||||
|
||||
activation_scope: Mapping[str, object]
|
||||
artifact_sha256: str = Field(pattern=r"^[0-9a-f]{64}$")
|
||||
complexity_policy: Mapping[str, object]
|
||||
definitions: Mapping[str, str]
|
||||
generated_at: str
|
||||
limitations: tuple[str, ...]
|
||||
models: Mapping[str, SnapshotModel]
|
||||
provenance: Mapping[str, object]
|
||||
quality_tiers: Mapping[AutoSetupQualityLevel, SnapshotQualityTier]
|
||||
reference_routes: Mapping[str, object]
|
||||
schema_version: Literal["2.0"]
|
||||
snapshot_id: str
|
||||
task_complexities: Mapping[str, Mapping[str, SnapshotComplexity]]
|
||||
|
||||
@model_validator(mode="after")
|
||||
def _validate_policy(self) -> AutoRouterSnapshot:
|
||||
if frozenset(self.quality_tiers) != frozenset(("economy", "balanced", "high", "max")):
|
||||
raise ValueError("Auto Router snapshot must define economy, balanced, high, and max quality levels")
|
||||
if self.activation_scope.get("setup_mode") != "auto":
|
||||
raise ValueError("Auto Router snapshot is not scoped to Auto setup")
|
||||
return self
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _AvailableProfile:
|
||||
model_name: str
|
||||
benchmark_model_id: str
|
||||
quality: SnapshotQualityProfile
|
||||
speed: tuple[SnapshotSpeedProfile, ...]
|
||||
required_parameters: Mapping[str, object]
|
||||
|
||||
|
||||
def _canonical_json(value: object) -> str:
|
||||
return json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=False)
|
||||
|
||||
|
||||
@lru_cache(maxsize=1)
|
||||
def load_auto_router_snapshot() -> AutoRouterSnapshot:
|
||||
parsed = cast(
|
||||
object,
|
||||
json.loads(
|
||||
files("litellm.router_strategy.complexity_router")
|
||||
.joinpath("artifacts", _SNAPSHOT_FILENAME)
|
||||
.read_text(encoding="utf-8")
|
||||
),
|
||||
)
|
||||
if not isinstance(parsed, dict):
|
||||
raise ValueError("Auto Router snapshot must be a JSON object")
|
||||
untyped = cast(dict[object, object], parsed)
|
||||
if not all(isinstance(key, str) for key in untyped):
|
||||
raise ValueError("Auto Router snapshot keys must be strings")
|
||||
raw = cast(dict[str, object], untyped)
|
||||
expected = raw.get("artifact_sha256")
|
||||
if not isinstance(expected, str):
|
||||
raise ValueError("Auto Router snapshot must carry an artifact_sha256")
|
||||
unsigned: Final = {key: value for key, value in raw.items() if key != "artifact_sha256"}
|
||||
actual: Final = hashlib.sha256(_canonical_json(unsigned).encode("utf-8")).hexdigest()
|
||||
if expected != actual:
|
||||
raise ValueError(f"Auto Router snapshot hash mismatch: expected {expected}, calculated {actual}")
|
||||
return AutoRouterSnapshot.model_validate(raw)
|
||||
|
||||
|
||||
def _normalized_model_key(value: str) -> str:
|
||||
return _VERSION_SEPARATOR.sub("-", value.strip().casefold())
|
||||
|
||||
|
||||
def _matching_snapshot_models(snapshot: AutoRouterSnapshot, model_refs: Sequence[str]) -> tuple[SnapshotModel, ...]:
|
||||
normalized_refs: Final = frozenset(_normalized_model_key(ref) for ref in model_refs)
|
||||
return tuple(
|
||||
model
|
||||
for model in snapshot.models.values()
|
||||
if model.identity.is_routable
|
||||
and normalized_refs.intersection(_normalized_model_key(key) for key in model.identity.litellm_model_keys)
|
||||
)
|
||||
|
||||
|
||||
def _available_profiles(
|
||||
snapshot: AutoRouterSnapshot,
|
||||
available_model_refs: Mapping[str, Sequence[str]],
|
||||
complexity: SnapshotComplexity,
|
||||
) -> tuple[_AvailableProfile, ...]:
|
||||
profiles: Final[list[_AvailableProfile]] = []
|
||||
for model_name, refs in available_model_refs.items():
|
||||
matches = _matching_snapshot_models(snapshot, (model_name, *refs))
|
||||
if not matches:
|
||||
continue
|
||||
best = min(
|
||||
matches,
|
||||
key=lambda model: (
|
||||
-model.quality_by_complexity[complexity].quality_lower_bound,
|
||||
model.quality_by_complexity[complexity].cost_per_completed_task_usd,
|
||||
model.benchmark_model_id,
|
||||
),
|
||||
)
|
||||
profiles.append(
|
||||
_AvailableProfile(
|
||||
model_name=model_name,
|
||||
benchmark_model_id=best.benchmark_model_id,
|
||||
quality=best.quality_by_complexity[complexity],
|
||||
speed=best.task_completion_speed_by_complexity[complexity],
|
||||
required_parameters=best.identity.required_parameters,
|
||||
)
|
||||
)
|
||||
return tuple(profiles)
|
||||
|
||||
|
||||
def _cost_rank(profiles: Sequence[_AvailableProfile]) -> tuple[_AvailableProfile, ...]:
|
||||
return tuple(
|
||||
sorted(
|
||||
profiles,
|
||||
key=lambda profile: (
|
||||
profile.quality.cost_per_completed_task_usd,
|
||||
-profile.quality.quality_lower_bound,
|
||||
profile.model_name,
|
||||
),
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _normalize_log(value: float, values: Sequence[float]) -> float:
|
||||
logged: Final = tuple(math.log(max(item, 1e-12)) for item in values)
|
||||
low: Final = min(logged)
|
||||
high: Final = max(logged)
|
||||
if low == high:
|
||||
return 0.0
|
||||
return (math.log(max(value, 1e-12)) - low) / (high - low)
|
||||
|
||||
|
||||
def _hard_task_rank(
|
||||
profiles: Sequence[_AvailableProfile], objective: AutoSetupObjective
|
||||
) -> tuple[tuple[_AvailableProfile, ...], str]:
|
||||
cost_ranked: Final = _cost_rank(profiles)
|
||||
speed_profiles: Final = tuple(
|
||||
(profile, speed)
|
||||
for profile in profiles
|
||||
for speed in profile.speed
|
||||
if speed.retry_adjusted_completion_ms is not None
|
||||
)
|
||||
if not speed_profiles:
|
||||
return cost_ranked, "fallback_no_speed_evidence"
|
||||
cohorts: dict[str, dict[str, tuple[_AvailableProfile, SnapshotSpeedProfile]]] = {}
|
||||
for profile, speed in speed_profiles:
|
||||
by_model = cohorts.setdefault(speed.cohort_id, {})
|
||||
current = by_model.get(profile.model_name)
|
||||
if current is None or speed.attempt_count > current[1].attempt_count:
|
||||
by_model[profile.model_name] = (profile, speed)
|
||||
measured_profiles: Final = tuple({profile.model_name: profile for profile, _ in speed_profiles}.values())
|
||||
anchor: Final = max(
|
||||
measured_profiles,
|
||||
key=lambda profile: (profile.quality.quality_lower_bound, -profile.quality.cost_per_completed_task_usd),
|
||||
)
|
||||
anchor_cohorts: Final = tuple(cohort_id for cohort_id, by_model in cohorts.items() if anchor.model_name in by_model)
|
||||
selected_cohort = max(
|
||||
anchor_cohorts,
|
||||
key=lambda cohort_id: (
|
||||
len(cohorts[cohort_id]),
|
||||
sum(speed.attempt_count for _, speed in cohorts[cohort_id].values()),
|
||||
cohort_id,
|
||||
),
|
||||
)
|
||||
comparable = tuple(cohorts[selected_cohort].values())
|
||||
if objective == "task_completion_speed":
|
||||
return tuple(
|
||||
profile
|
||||
for profile, _ in sorted(
|
||||
comparable,
|
||||
key=lambda item: (
|
||||
item[1].retry_adjusted_completion_ms or math.inf,
|
||||
-item[0].quality.quality_lower_bound,
|
||||
item[0].model_name,
|
||||
),
|
||||
)
|
||||
), "cohort_scoped"
|
||||
balanced_comparable = tuple(item for item in comparable if item[1].cost_per_completed_task_usd is not None)
|
||||
if not balanced_comparable:
|
||||
return cost_ranked, "fallback_no_comparable_cost_evidence"
|
||||
costs: Final = tuple(cast(float, speed.cost_per_completed_task_usd) for _, speed in balanced_comparable)
|
||||
times: Final = tuple(cast(float, speed.retry_adjusted_completion_ms) for _, speed in balanced_comparable)
|
||||
return tuple(
|
||||
profile
|
||||
for profile, _ in sorted(
|
||||
balanced_comparable,
|
||||
key=lambda item: (
|
||||
math.hypot(
|
||||
_normalize_log(cast(float, item[1].cost_per_completed_task_usd), costs),
|
||||
_normalize_log(cast(float, item[1].retry_adjusted_completion_ms), times),
|
||||
),
|
||||
-item[0].quality.quality_lower_bound,
|
||||
item[0].model_name,
|
||||
),
|
||||
)
|
||||
), "cohort_scoped"
|
||||
|
||||
|
||||
def _policy_candidate(profile: _AvailableProfile) -> AutoSetupCandidate:
|
||||
return AutoSetupCandidate(
|
||||
model_name=profile.model_name,
|
||||
benchmark_model_id=profile.benchmark_model_id,
|
||||
quality_lower_bound=profile.quality.quality_lower_bound,
|
||||
cost_per_completed_task_usd=profile.quality.cost_per_completed_task_usd,
|
||||
)
|
||||
|
||||
|
||||
def _tier_policy(
|
||||
profiles: Sequence[_AvailableProfile],
|
||||
complexity: SnapshotComplexity,
|
||||
objective: AutoSetupObjective,
|
||||
) -> AutoSetupTierPolicy:
|
||||
cost_ranked: Final = _cost_rank(profiles)
|
||||
if objective == "cost":
|
||||
return AutoSetupTierPolicy(
|
||||
selection_mode="snapshot_ranked",
|
||||
candidates=tuple(_policy_candidate(profile) for profile in cost_ranked),
|
||||
evidence_status="direct",
|
||||
)
|
||||
if complexity in _EASY_COMPLEXITIES:
|
||||
return AutoSetupTierPolicy(
|
||||
selection_mode="runtime_response_latency",
|
||||
candidates=tuple(_policy_candidate(profile) for profile in cost_ranked),
|
||||
cold_start_model=cost_ranked[0].model_name,
|
||||
evidence_status="runtime_response_speed_required",
|
||||
)
|
||||
ranked, evidence_status = _hard_task_rank(profiles, objective)
|
||||
return AutoSetupTierPolicy(
|
||||
selection_mode="snapshot_ranked",
|
||||
candidates=tuple(_policy_candidate(profile) for profile in ranked),
|
||||
evidence_status=evidence_status,
|
||||
)
|
||||
|
||||
|
||||
def build_auto_setup_config(
|
||||
*,
|
||||
snapshot: AutoRouterSnapshot,
|
||||
available_model_refs: Mapping[str, Sequence[str]],
|
||||
quality_level: AutoSetupQualityLevel,
|
||||
optimize_for: AutoSetupObjective,
|
||||
) -> ComplexityRouterConfig:
|
||||
"""Recompute quality gates and rankings over only the caller's available groups."""
|
||||
|
||||
maximum_regret: Final = snapshot.quality_tiers[quality_level].maximum_quality_regret
|
||||
policies: Final[dict[str, AutoSetupTierPolicy]] = {}
|
||||
tiers: Final[dict[str, list[str]]] = {}
|
||||
tier_model_configs: Final[dict[str, list[dict[str, object]]]] = {}
|
||||
for tier_name, complexity in _TIER_COMPLEXITIES.items():
|
||||
profiles = _available_profiles(snapshot, available_model_refs, complexity)
|
||||
if not profiles:
|
||||
raise ValueError(f"None of the available model groups has {complexity} evidence in this snapshot")
|
||||
best_quality = max(profile.quality.quality_lower_bound for profile in profiles)
|
||||
floor = max(0.0, best_quality - maximum_regret)
|
||||
admitted = tuple(profile for profile in profiles if profile.quality.quality_lower_bound >= floor)
|
||||
policy = _tier_policy(admitted, complexity, optimize_for)
|
||||
policies[tier_name] = policy
|
||||
tiers[tier_name] = [candidate.model_name for candidate in policy.candidates]
|
||||
parameterized: list[dict[str, object]] = [
|
||||
{
|
||||
"model_name": profile.model_name,
|
||||
"litellm_params": dict(profile.required_parameters),
|
||||
}
|
||||
for profile in admitted
|
||||
if profile.model_name in frozenset(tiers[tier_name]) and profile.required_parameters
|
||||
]
|
||||
if parameterized:
|
||||
tier_model_configs[tier_name] = parameterized
|
||||
|
||||
return ComplexityRouterConfig.model_validate(
|
||||
{
|
||||
"tiers": tiers,
|
||||
"tier_model_configs": tier_model_configs,
|
||||
"classifier_type": "heuristic_v2",
|
||||
"auto_setup": AutoSetupConfig(
|
||||
snapshot_id=snapshot.snapshot_id,
|
||||
snapshot_sha256=snapshot.artifact_sha256,
|
||||
quality_level=quality_level,
|
||||
optimize_for=optimize_for,
|
||||
tier_policies=policies,
|
||||
).model_dump(mode="json"),
|
||||
}
|
||||
)
|
||||
|
|
@ -1322,6 +1322,13 @@ class ComplexityRouter(CustomLogger):
|
|||
if tier is not None:
|
||||
tier_name: Final = _tier_name(tier)
|
||||
decision["tier"] = tier_name
|
||||
if self.config.auto_setup is not None:
|
||||
auto_policy: Final = self.config.auto_setup.tier_policies.get(tier_name)
|
||||
if auto_policy is not None:
|
||||
decision["auto_setup_snapshot_id"] = self.config.auto_setup.snapshot_id
|
||||
decision["auto_setup_quality_level"] = self.config.auto_setup.quality_level
|
||||
decision["auto_setup_optimize_for"] = self.config.auto_setup.optimize_for
|
||||
decision["auto_setup_selection_mode"] = auto_policy.selection_mode
|
||||
if not self.config.has_custom_tiers:
|
||||
label = self.config.tier_label(ComplexityTier(tier_name))
|
||||
if label != tier_name:
|
||||
|
|
@ -1815,6 +1822,37 @@ class ComplexityRouter(CustomLogger):
|
|||
raise ValueError(f"Empty model pool for tier {tier_key}")
|
||||
return random.choice(model)
|
||||
|
||||
async def _pick_from_auto_setup_policy(self, tier_key: str, pool: Sequence[str]) -> str | None:
|
||||
setup: Final = self.config.auto_setup
|
||||
if setup is None:
|
||||
return None
|
||||
policy: Final = setup.tier_policies.get(tier_key)
|
||||
if policy is None:
|
||||
return None
|
||||
available: Final = tuple(dict.fromkeys(pool))
|
||||
if not available:
|
||||
raise ValueError(f"No models configured for tier {tier_key}")
|
||||
ranked: Final = tuple(
|
||||
candidate.model_name for candidate in policy.candidates if candidate.model_name in frozenset(available)
|
||||
)
|
||||
if policy.selection_mode == "snapshot_ranked":
|
||||
return ranked[0] if ranked else available[0]
|
||||
|
||||
from .response_latency import select_runtime_response_model
|
||||
|
||||
cold_start: Final = policy.cold_start_model
|
||||
if cold_start is None:
|
||||
raise ValueError(f"Auto setup runtime response-latency policy for {tier_key} has no cold-start model")
|
||||
return await select_runtime_response_model(
|
||||
router_cache=self.litellm_router_instance.cache,
|
||||
router_model_name=self.model_name,
|
||||
tier=tier_key,
|
||||
candidates=policy.candidates,
|
||||
available_models=available,
|
||||
cold_start_model=cold_start,
|
||||
objective=setup.optimize_for,
|
||||
)
|
||||
|
||||
def _tier_pools(self) -> dict[str, list[str]]:
|
||||
return {tier: (models if isinstance(models, list) else [models]) for tier, models in self.config.tiers.items()}
|
||||
|
||||
|
|
@ -1826,17 +1864,25 @@ class ComplexityRouter(CustomLogger):
|
|||
request_kwargs: dict,
|
||||
allowed_models: tuple[str, ...] | None = None,
|
||||
) -> str:
|
||||
tier_key = _tier_name(tier)
|
||||
if not self.config.plugins:
|
||||
if allowed_models is not None:
|
||||
return self._pick_from_tier_value(allowed_models, _tier_name(tier))
|
||||
return self.get_model_for_tier(tier)
|
||||
configured = self.config.tiers.get(tier_key)
|
||||
if configured is None:
|
||||
return self.get_model_for_tier(tier)
|
||||
full_pool = (configured,) if isinstance(configured, str) else tuple(configured)
|
||||
pool = (
|
||||
tuple(model for model in full_pool if model in allowed_models)
|
||||
if allowed_models is not None
|
||||
else full_pool
|
||||
)
|
||||
auto_pick: Final = await self._pick_from_auto_setup_policy(tier_key, pool)
|
||||
return auto_pick if auto_pick is not None else self._pick_from_tier_value(pool, tier_key)
|
||||
|
||||
from litellm.types.router import RoutingContext
|
||||
|
||||
tier_key: Final = _tier_name(tier)
|
||||
metadata_key: Final = get_metadata_variable_name_from_kwargs(request_kwargs)
|
||||
full_pool: Final = tuple(self._tier_pools().get(tier_key, ()))
|
||||
pool: Final = (
|
||||
full_pool = tuple(self._tier_pools().get(tier_key, ()))
|
||||
pool = (
|
||||
tuple(model for model in full_pool if model in allowed_models) if allowed_models is not None else full_pool
|
||||
)
|
||||
if not pool:
|
||||
|
|
@ -1860,7 +1906,8 @@ class ComplexityRouter(CustomLogger):
|
|||
# silently bypassed. Raise instead, matching the Router-level plugin
|
||||
# pipeline's own fail-closed behavior for the same situation.
|
||||
raise ValueError(f"No candidate models left for tier {tier_key} after routing-plugin filtering")
|
||||
return self._pick_from_tier_value(context.candidate_models, tier_key)
|
||||
auto_pick: Final = await self._pick_from_auto_setup_policy(tier_key, context.candidate_models)
|
||||
return auto_pick if auto_pick is not None else self._pick_from_tier_value(context.candidate_models, tier_key)
|
||||
|
||||
def _ensure_adaptive_router(self) -> Any | None:
|
||||
if not self.config.adaptive:
|
||||
|
|
|
|||
|
|
@ -27,6 +27,81 @@ class ComplexityTier(str, Enum):
|
|||
REASONING = "REASONING"
|
||||
|
||||
|
||||
AutoSetupQualityLevel = Literal["economy", "balanced", "high", "max"]
|
||||
AutoSetupObjective = Literal["cost", "task_completion_speed", "balanced"]
|
||||
AutoSetupSelectionMode = Literal["snapshot_ranked", "runtime_response_latency"]
|
||||
|
||||
|
||||
class AutoSetupCandidate(BaseModel):
|
||||
"""One benchmark-backed model group in an Auto-generated tier policy."""
|
||||
|
||||
model_config = ConfigDict(extra="forbid", frozen=True)
|
||||
|
||||
model_name: str = Field(min_length=1)
|
||||
benchmark_model_id: str = Field(min_length=1)
|
||||
quality_lower_bound: float = Field(ge=0, le=1)
|
||||
cost_per_completed_task_usd: float = Field(ge=0)
|
||||
|
||||
|
||||
class AutoSetupTierPolicy(BaseModel):
|
||||
"""How an Auto-generated tier chooses from its quality-admitted model groups."""
|
||||
|
||||
model_config = ConfigDict(extra="forbid", frozen=True)
|
||||
|
||||
selection_mode: AutoSetupSelectionMode
|
||||
candidates: tuple[AutoSetupCandidate, ...] = Field(min_length=1)
|
||||
cold_start_model: str | None = None
|
||||
evidence_status: str = Field(min_length=1)
|
||||
|
||||
@model_validator(mode="after")
|
||||
def _validate_candidates(self) -> "AutoSetupTierPolicy":
|
||||
names: Final = tuple(candidate.model_name for candidate in self.candidates)
|
||||
if len(names) != len(frozenset(names)):
|
||||
raise ValueError("Auto setup tier candidates must use distinct model_name values")
|
||||
if self.selection_mode == "runtime_response_latency":
|
||||
if self.cold_start_model not in names:
|
||||
raise ValueError("A runtime response-latency policy needs a cold_start_model in candidates")
|
||||
elif self.cold_start_model is not None:
|
||||
raise ValueError("cold_start_model is only valid for runtime_response_latency")
|
||||
return self
|
||||
|
||||
|
||||
class AutoSetupConfig(BaseModel):
|
||||
"""Provenance and selection policy stamped only by zero-setup Auto creation."""
|
||||
|
||||
model_config = ConfigDict(extra="forbid", frozen=True)
|
||||
|
||||
schema_version: Literal["1"] = "1"
|
||||
snapshot_id: str = Field(min_length=1)
|
||||
snapshot_sha256: str = Field(pattern=r"^[0-9a-f]{64}$")
|
||||
quality_level: AutoSetupQualityLevel
|
||||
optimize_for: AutoSetupObjective
|
||||
tier_policies: Mapping[str, AutoSetupTierPolicy]
|
||||
|
||||
@field_validator("tier_policies", mode="before")
|
||||
@classmethod
|
||||
def _freeze_tier_policies(cls, value: Mapping[str, object]) -> Mapping[str, object]:
|
||||
return MappingProxyType(dict(value))
|
||||
|
||||
@field_serializer("tier_policies")
|
||||
def _serialize_tier_policies(self, value: Mapping[str, AutoSetupTierPolicy]) -> Mapping[str, AutoSetupTierPolicy]:
|
||||
return dict(value) # mutable-ok: Pydantic JSON serialization requires a concrete mapping
|
||||
|
||||
@model_validator(mode="after")
|
||||
def _validate_tier_policies(self) -> "AutoSetupConfig":
|
||||
expected: Final = frozenset(tier.value for tier in TIER_SEVERITY_ORDER)
|
||||
actual: Final = frozenset(self.tier_policies)
|
||||
if actual != expected:
|
||||
raise ValueError(
|
||||
f"Auto setup tier_policies must define exactly {', '.join(tier.value for tier in TIER_SEVERITY_ORDER)}"
|
||||
)
|
||||
if self.optimize_for == "cost" and any(
|
||||
policy.selection_mode != "snapshot_ranked" for policy in self.tier_policies.values()
|
||||
):
|
||||
raise ValueError("Cost-optimized Auto setup cannot enable runtime response-latency selection")
|
||||
return self
|
||||
|
||||
|
||||
class ClassificationRubric(str, Enum):
|
||||
"""Which calibration examples, and for BUSINESS which tier criteria, the built-in classifier rubric carries."""
|
||||
|
||||
|
|
@ -517,6 +592,14 @@ class ComplexityRouterConfig(BaseModel):
|
|||
default_factory=dict,
|
||||
)
|
||||
|
||||
auto_setup: AutoSetupConfig | None = Field(
|
||||
default=None,
|
||||
description=(
|
||||
"Benchmark snapshot and per-tier selection policy generated by the zero-setup Auto flow. "
|
||||
"Absent on manually configured routers, whose pool selection behavior is unchanged"
|
||||
),
|
||||
)
|
||||
|
||||
tier_definitions: tuple[TierDefinition, ...] | None = Field(
|
||||
default=None,
|
||||
description=(
|
||||
|
|
@ -1422,6 +1505,24 @@ class ComplexityRouterConfig(BaseModel):
|
|||
)
|
||||
return self
|
||||
|
||||
@model_validator(mode="after")
|
||||
def _validate_auto_setup(self) -> "ComplexityRouterConfig":
|
||||
setup: Final = self.auto_setup
|
||||
if setup is None:
|
||||
return self
|
||||
if self.adaptive:
|
||||
raise ValueError("auto_setup and adaptive=True cannot both select from the same tier pool")
|
||||
if self.tier_definitions is not None:
|
||||
raise ValueError("auto_setup requires the built-in four-tier complexity ladder")
|
||||
for tier_name, policy in setup.tier_policies.items():
|
||||
configured = self.tiers.get(tier_name)
|
||||
configured_names = frozenset((configured,)) if isinstance(configured, str) else frozenset(configured or ())
|
||||
policy_names = frozenset(candidate.model_name for candidate in policy.candidates)
|
||||
if not policy_names.issubset(configured_names):
|
||||
missing = ", ".join(sorted(policy_names - configured_names))
|
||||
raise ValueError(f"auto_setup tier {tier_name} references models not present in tiers: {missing}")
|
||||
return self
|
||||
|
||||
@model_validator(mode="after")
|
||||
def _validate_tier_param_placement(self) -> "ComplexityRouterConfig":
|
||||
"""Reject a router setting written into a tier entry's request params.
|
||||
|
|
|
|||
232
litellm/router_strategy/complexity_router/response_latency.py
Normal file
232
litellm/router_strategy/complexity_router/response_latency.py
Normal file
|
|
@ -0,0 +1,232 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import math
|
||||
import statistics
|
||||
from collections.abc import Mapping, Sequence
|
||||
from datetime import datetime, timedelta
|
||||
from typing import Final, Protocol, cast
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field, ValidationError
|
||||
|
||||
from litellm._logging import verbose_router_logger
|
||||
|
||||
from .config import AutoSetupCandidate, AutoSetupObjective
|
||||
|
||||
_CACHE_PREFIX: Final = "auto_setup_response_latency:v1"
|
||||
_CACHE_TTL_SECONDS: Final = 7 * 24 * 60 * 60
|
||||
_MAX_SAMPLES_PER_MODEL: Final = 20
|
||||
_MAX_EXPECTED_OUTPUT_SAMPLES: Final = 100
|
||||
_MIN_SAMPLES_PER_MODEL: Final = 2
|
||||
|
||||
|
||||
class ResponseLatencyCache(Protocol):
|
||||
async def async_get_cache(
|
||||
self,
|
||||
key: str,
|
||||
parent_otel_span: object | None = None,
|
||||
local_only: bool = False,
|
||||
**kwargs: object,
|
||||
) -> object: ...
|
||||
|
||||
async def async_set_cache(
|
||||
self,
|
||||
key: str,
|
||||
value: object,
|
||||
local_only: bool = False,
|
||||
**kwargs: object,
|
||||
) -> None: ...
|
||||
|
||||
|
||||
class ResponseLatencySample(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True)
|
||||
|
||||
ttft_ms: float = Field(ge=0)
|
||||
output_tokens_per_ms: float = Field(gt=0)
|
||||
|
||||
|
||||
class ResponseLatencyState(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
samples_by_model: dict[str, list[ResponseLatencySample]] = Field(default_factory=dict)
|
||||
visible_output_tokens: list[int] = Field(default_factory=list)
|
||||
|
||||
|
||||
def _cache_key(router_model_name: str, tier: str) -> str:
|
||||
namespace: Final = hashlib.sha256(router_model_name.encode("utf-8")).hexdigest()[:24]
|
||||
return f"{_CACHE_PREFIX}:{namespace}:{tier}"
|
||||
|
||||
|
||||
def _state(value: object) -> ResponseLatencyState:
|
||||
try:
|
||||
return ResponseLatencyState.model_validate(value or {})
|
||||
except ValidationError:
|
||||
return ResponseLatencyState()
|
||||
|
||||
|
||||
def _candidate_order(
|
||||
candidates: Sequence[AutoSetupCandidate],
|
||||
available_models: Sequence[str],
|
||||
cold_start_model: str,
|
||||
) -> tuple[str, ...]:
|
||||
available: Final = frozenset(available_models)
|
||||
ranked: Final = tuple(candidate.model_name for candidate in candidates if candidate.model_name in available)
|
||||
extras: Final = tuple(model for model in available_models if model not in frozenset(ranked))
|
||||
ordered: Final = tuple(dict.fromkeys((*ranked, *extras)))
|
||||
if cold_start_model not in ordered:
|
||||
return ordered
|
||||
return (cold_start_model, *(model for model in ordered if model != cold_start_model))
|
||||
|
||||
|
||||
def _median_prediction_ms(samples: Sequence[ResponseLatencySample], expected_output_tokens: float) -> float:
|
||||
return statistics.median(
|
||||
sample.ttft_ms + expected_output_tokens / sample.output_tokens_per_ms for sample in samples
|
||||
)
|
||||
|
||||
|
||||
def _normalized_log(value: float, values: Sequence[float]) -> float:
|
||||
logged: Final = tuple(math.log(max(item, 1e-12)) for item in values)
|
||||
low: Final = min(logged)
|
||||
high: Final = max(logged)
|
||||
if low == high:
|
||||
return 0.0
|
||||
return (math.log(max(value, 1e-12)) - low) / (high - low)
|
||||
|
||||
|
||||
def _best_observed_model(
|
||||
state: ResponseLatencyState,
|
||||
candidates: Sequence[AutoSetupCandidate],
|
||||
ordered_models: Sequence[str],
|
||||
objective: AutoSetupObjective,
|
||||
) -> str:
|
||||
expected_tokens: Final = statistics.median(state.visible_output_tokens) if state.visible_output_tokens else 1.0
|
||||
latencies: Final = {
|
||||
model: _median_prediction_ms(state.samples_by_model[model], expected_tokens) for model in ordered_models
|
||||
}
|
||||
if objective == "task_completion_speed":
|
||||
return min(ordered_models, key=lambda model: (latencies[model], ordered_models.index(model)))
|
||||
|
||||
configured_costs: Final = {candidate.model_name: candidate.cost_per_completed_task_usd for candidate in candidates}
|
||||
known_costs: Final = tuple(configured_costs.values())
|
||||
conservative_unknown_cost: Final = max(known_costs) if known_costs else 1.0
|
||||
costs: Final = {model: configured_costs.get(model, conservative_unknown_cost) for model in ordered_models}
|
||||
latency_values: Final = tuple(latencies.values())
|
||||
cost_values: Final = tuple(costs.values())
|
||||
return min(
|
||||
ordered_models,
|
||||
key=lambda model: (
|
||||
math.hypot(
|
||||
_normalized_log(latencies[model], latency_values),
|
||||
_normalized_log(costs[model], cost_values),
|
||||
),
|
||||
ordered_models.index(model),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
async def select_runtime_response_model(
|
||||
*,
|
||||
router_cache: ResponseLatencyCache,
|
||||
router_model_name: str,
|
||||
tier: str,
|
||||
candidates: Sequence[AutoSetupCandidate],
|
||||
available_models: Sequence[str],
|
||||
cold_start_model: str,
|
||||
objective: AutoSetupObjective,
|
||||
) -> str:
|
||||
"""Explore quality-admitted groups, then minimize equal-output response time."""
|
||||
|
||||
ordered: Final = _candidate_order(candidates, available_models, cold_start_model)
|
||||
if not ordered:
|
||||
raise ValueError(f"No Auto setup candidates remain for tier {tier}")
|
||||
try:
|
||||
cached = await router_cache.async_get_cache(key=_cache_key(router_model_name, tier))
|
||||
state: Final = _state(cached)
|
||||
except Exception as exc: # noqa: BLE001 -- optional observations must never make a request unroutable
|
||||
verbose_router_logger.warning("Auto setup response-latency cache read failed: %s", exc)
|
||||
return ordered[0]
|
||||
|
||||
counts: Final = {model: len(state.samples_by_model.get(model, ())) for model in ordered}
|
||||
least_observed: Final = min(counts.values())
|
||||
if least_observed < _MIN_SAMPLES_PER_MODEL:
|
||||
return next(model for model in ordered if counts[model] == least_observed)
|
||||
return _best_observed_model(state, candidates, ordered, objective)
|
||||
|
||||
|
||||
def _seconds(value: object) -> float | None:
|
||||
if isinstance(value, datetime):
|
||||
return value.timestamp()
|
||||
if isinstance(value, timedelta):
|
||||
return value.total_seconds()
|
||||
if isinstance(value, bool) or not isinstance(value, int | float):
|
||||
return None
|
||||
return float(value)
|
||||
|
||||
|
||||
def _field(value: object, key: str) -> object | None:
|
||||
if isinstance(value, Mapping):
|
||||
return cast(Mapping[object, object], value).get(key)
|
||||
try:
|
||||
return cast(object, object.__getattribute__(value, key))
|
||||
except AttributeError:
|
||||
return None
|
||||
|
||||
|
||||
def _visible_output_tokens(response_obj: object) -> int | None:
|
||||
usage = _field(response_obj, "usage")
|
||||
if usage is None:
|
||||
return None
|
||||
total = _field(usage, "completion_tokens")
|
||||
if total is None:
|
||||
total = _field(usage, "output_tokens")
|
||||
if isinstance(total, bool) or not isinstance(total, int | float) or total <= 0:
|
||||
return None
|
||||
details = _field(usage, "completion_tokens_details")
|
||||
if details is None:
|
||||
details = _field(usage, "output_tokens_details")
|
||||
reasoning = _field(details, "reasoning_tokens") if details is not None else None
|
||||
reasoning_tokens = float(reasoning) if isinstance(reasoning, int | float) and not isinstance(reasoning, bool) else 0
|
||||
return max(1, round(float(total) - reasoning_tokens))
|
||||
|
||||
|
||||
def response_latency_sample(
|
||||
kwargs: Mapping[str, object], response_obj: object, start_time: object, end_time: object
|
||||
) -> tuple[ResponseLatencySample, int] | None:
|
||||
start: Final = _seconds(start_time)
|
||||
end: Final = _seconds(end_time)
|
||||
visible_tokens: Final = _visible_output_tokens(response_obj)
|
||||
if start is None or end is None or end <= start or visible_tokens is None:
|
||||
return None
|
||||
total_ms: Final = (end - start) * 1000
|
||||
completion_start: Final = _seconds(kwargs.get("completion_start_time"))
|
||||
ttft_ms = (
|
||||
(completion_start - start) * 1000 if completion_start is not None and start < completion_start < end else 0.0
|
||||
)
|
||||
generation_ms: Final = max(total_ms - ttft_ms, 1e-6)
|
||||
return ResponseLatencySample(
|
||||
ttft_ms=ttft_ms,
|
||||
output_tokens_per_ms=visible_tokens / generation_ms,
|
||||
), visible_tokens
|
||||
|
||||
|
||||
async def record_runtime_response_latency(
|
||||
*,
|
||||
router_cache: ResponseLatencyCache,
|
||||
router_model_name: str,
|
||||
tier: str,
|
||||
routed_model: str,
|
||||
kwargs: Mapping[str, object],
|
||||
response_obj: object,
|
||||
start_time: object,
|
||||
end_time: object,
|
||||
) -> None:
|
||||
measured: Final = response_latency_sample(kwargs, response_obj, start_time, end_time)
|
||||
if measured is None:
|
||||
return
|
||||
sample, visible_tokens = measured
|
||||
key: Final = _cache_key(router_model_name, tier)
|
||||
state: Final = _state(await router_cache.async_get_cache(key=key))
|
||||
samples: Final = [*state.samples_by_model.get(routed_model, ()), sample][-_MAX_SAMPLES_PER_MODEL:]
|
||||
state.samples_by_model[routed_model] = samples
|
||||
state.visible_output_tokens = [*state.visible_output_tokens, visible_tokens][-_MAX_EXPECTED_OUTPUT_SAMPLES:]
|
||||
await router_cache.async_set_cache(key=key, value=state.model_dump(mode="json"), ttl=_CACHE_TTL_SECONDS)
|
||||
|
|
@ -9,7 +9,11 @@ from typing import Final, Literal, TypeAlias
|
|||
|
||||
from pydantic import BaseModel, Field, computed_field, field_validator, model_validator
|
||||
|
||||
from litellm.router_strategy.complexity_router.config import ComplexityRouterConfig
|
||||
from litellm.router_strategy.complexity_router.config import (
|
||||
AutoSetupObjective,
|
||||
AutoSetupQualityLevel,
|
||||
ComplexityRouterConfig,
|
||||
)
|
||||
from litellm.types.utils import StandardLoggingRoutingDecision
|
||||
|
||||
DEFAULT_ROUTING_TEST_ROUTER_NAME: Final[str] = "auto_router_routing_test"
|
||||
|
|
@ -44,6 +48,17 @@ class ComplexityRouterConfigValidationResponse(BaseModel):
|
|||
error: str | None = None
|
||||
|
||||
|
||||
class AutoRouterRecommendationResponse(BaseModel):
|
||||
"""An editable complexity-router config generated from the caller's usable model groups."""
|
||||
|
||||
quality_level: AutoSetupQualityLevel
|
||||
optimize_for: AutoSetupObjective
|
||||
snapshot_id: str
|
||||
snapshot_generated_at: str
|
||||
matched_model_groups: tuple[str, ...]
|
||||
complexity_router_config: RequestComplexityRouterConfig
|
||||
|
||||
|
||||
class AutoRouterRoutingTestRequest(BaseModel):
|
||||
"""A single request to classify against a complexity-router config that need not be saved yet.
|
||||
|
||||
|
|
|
|||
|
|
@ -2938,6 +2938,10 @@ class StandardLoggingRoutingDecision(TypedDict, total=False):
|
|||
savings_baseline_model: str
|
||||
savings_baseline_deployment_id: str
|
||||
tier_litellm_params: Mapping[str, object] # writable-ok: Pydantic warns on ReadOnly TypedDict fields
|
||||
auto_setup_snapshot_id: str
|
||||
auto_setup_quality_level: Literal["economy", "balanced", "high", "max"]
|
||||
auto_setup_optimize_for: Literal["cost", "task_completion_speed", "balanced"]
|
||||
auto_setup_selection_mode: Literal["snapshot_ranked", "runtime_response_latency"]
|
||||
|
||||
|
||||
# Fields whose values quote the caller's prompt. Dropped when an operator turns message
|
||||
|
|
|
|||
|
|
@ -18,6 +18,7 @@ from litellm.proxy._types import (
|
|||
UserAPIKeyAuth,
|
||||
)
|
||||
from litellm.proxy.management_endpoints.auto_router_endpoints import (
|
||||
get_auto_router_recommendation,
|
||||
preview_auto_router_routing,
|
||||
)
|
||||
from litellm.router import Router
|
||||
|
|
@ -104,6 +105,51 @@ async def _route(prompt: str, monkeypatch: pytest.MonkeyPatch, **config_override
|
|||
return await _route_body({"prompt": prompt}, monkeypatch, **config_overrides)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_recommendation_uses_caller_groups_and_returns_an_editable_complexity_config(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
import litellm.proxy.management_endpoints.auto_router_endpoints as endpoint_module
|
||||
import litellm.proxy.utils as proxy_utils
|
||||
from litellm.proxy import proxy_server
|
||||
|
||||
model_list = [
|
||||
{
|
||||
"model_name": "small-group",
|
||||
"litellm_params": {"model": "gpt-5.4-nano", "api_key": "fake"},
|
||||
},
|
||||
{
|
||||
"model_name": "smart-group",
|
||||
"litellm_params": {"model": "gpt-5.6-sol", "api_key": "fake"},
|
||||
},
|
||||
{
|
||||
"model_name": "unavailable-group",
|
||||
"litellm_params": {"model": "gpt-5.5", "api_key": "fake"},
|
||||
},
|
||||
]
|
||||
monkeypatch.setattr(proxy_server, "llm_router", Router(model_list=model_list))
|
||||
available = AsyncMock(return_value=["small-group", "smart-group"])
|
||||
monkeypatch.setattr(proxy_utils, "get_available_models_for_user", available)
|
||||
authorize = AsyncMock()
|
||||
monkeypatch.setattr(endpoint_module, "_authorize_router_dry_run", authorize)
|
||||
|
||||
response = await get_auto_router_recommendation(
|
||||
user_api_key_dict=ADMIN,
|
||||
quality_level="economy",
|
||||
optimize_for="task_completion_speed",
|
||||
)
|
||||
|
||||
authorize.assert_awaited_once_with(user_api_key_dict=ADMIN, team_id=None)
|
||||
assert set(response.matched_model_groups) <= {"small-group", "smart-group"}
|
||||
assert "unavailable-group" not in response.matched_model_groups
|
||||
assert response.complexity_router_config.classifier_type == "heuristic_v2"
|
||||
assert response.complexity_router_config.auto_setup is not None
|
||||
assert response.complexity_router_config.auto_setup.optimize_for == "task_completion_speed"
|
||||
assert response.complexity_router_config.auto_setup.tier_policies["SIMPLE"].selection_mode == (
|
||||
"runtime_response_latency"
|
||||
)
|
||||
|
||||
|
||||
AGENTIC_MESSAGES = [
|
||||
{"role": "system", "content": "You are a database migration assistant for a payments ledger"},
|
||||
{"role": "user", "content": "duplicate ledger postings since the celery upgrade, same event_id twice"},
|
||||
|
|
|
|||
|
|
@ -0,0 +1,276 @@
|
|||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import pytest
|
||||
from pydantic import ValidationError
|
||||
|
||||
from litellm import Router
|
||||
from litellm.router_strategy.complexity_router.auto_setup import (
|
||||
build_auto_setup_config,
|
||||
load_auto_router_snapshot,
|
||||
)
|
||||
from litellm.router_strategy.complexity_router.complexity_router import ComplexityRouter
|
||||
from litellm.router_strategy.complexity_router.config import (
|
||||
AutoSetupCandidate,
|
||||
ComplexityRouterConfig,
|
||||
ComplexityTier,
|
||||
)
|
||||
from litellm.router_strategy.complexity_router.response_latency import (
|
||||
record_runtime_response_latency,
|
||||
response_latency_sample,
|
||||
select_runtime_response_model,
|
||||
)
|
||||
|
||||
|
||||
def _available_refs(*model_names: str) -> dict[str, tuple[str, ...]]:
|
||||
return {model_name: (model_name,) for model_name in model_names}
|
||||
|
||||
|
||||
def test_snapshot_is_verified_and_quality_gate_is_recomputed_for_available_models() -> None:
|
||||
snapshot = load_auto_router_snapshot()
|
||||
available = _available_refs("gpt-5.6-sol", "gpt-5.4-nano")
|
||||
|
||||
economy = build_auto_setup_config(
|
||||
snapshot=snapshot,
|
||||
available_model_refs=available,
|
||||
quality_level="economy",
|
||||
optimize_for="cost",
|
||||
)
|
||||
maximum = build_auto_setup_config(
|
||||
snapshot=snapshot,
|
||||
available_model_refs=available,
|
||||
quality_level="max",
|
||||
optimize_for="cost",
|
||||
)
|
||||
nano_only = build_auto_setup_config(
|
||||
snapshot=snapshot,
|
||||
available_model_refs=_available_refs("gpt-5.4-nano"),
|
||||
quality_level="max",
|
||||
optimize_for="cost",
|
||||
)
|
||||
|
||||
assert snapshot.artifact_sha256 == "d0ea7e584dbb9ea37eb15ce20ae342612e0dfaa5801b553e71fc76818d7cef64"
|
||||
assert economy.tiers["SIMPLE"] == ["gpt-5.4-nano", "gpt-5.6-sol"]
|
||||
assert maximum.tiers["SIMPLE"] == ["gpt-5.6-sol"]
|
||||
assert nano_only.tiers["SIMPLE"] == ["gpt-5.4-nano"]
|
||||
|
||||
|
||||
def test_speed_only_changes_auto_generated_selection_policy() -> None:
|
||||
config = build_auto_setup_config(
|
||||
snapshot=load_auto_router_snapshot(),
|
||||
available_model_refs=_available_refs("gpt-5.6-sol", "gpt-5.4-nano"),
|
||||
quality_level="economy",
|
||||
optimize_for="task_completion_speed",
|
||||
)
|
||||
|
||||
assert config.auto_setup is not None
|
||||
assert config.auto_setup.tier_policies["SIMPLE"].selection_mode == "runtime_response_latency"
|
||||
assert config.auto_setup.tier_policies["MEDIUM"].selection_mode == "runtime_response_latency"
|
||||
assert config.auto_setup.tier_policies["COMPLEX"].selection_mode == "snapshot_ranked"
|
||||
assert config.auto_setup.tier_policies["REASONING"].selection_mode == "snapshot_ranked"
|
||||
assert ComplexityRouterConfig(tiers=config.tiers).auto_setup is None
|
||||
|
||||
|
||||
def test_auto_setup_cannot_change_adaptive_or_custom_tier_routers() -> None:
|
||||
generated = build_auto_setup_config(
|
||||
snapshot=load_auto_router_snapshot(),
|
||||
available_model_refs=_available_refs("gpt-5.6-sol"),
|
||||
quality_level="max",
|
||||
optimize_for="cost",
|
||||
)
|
||||
payload = generated.model_dump(mode="json")
|
||||
|
||||
with pytest.raises(ValidationError, match="auto_setup and adaptive"):
|
||||
ComplexityRouterConfig.model_validate({**payload, "adaptive": True})
|
||||
with pytest.raises(ValidationError, match="built-in four-tier"):
|
||||
ComplexityRouterConfig.model_validate(
|
||||
{
|
||||
**payload,
|
||||
"tier_definitions": [
|
||||
{"name": "fast", "description": "Fast"},
|
||||
{"name": "strong", "description": "Strong"},
|
||||
],
|
||||
"tiers": {"fast": ["gpt-5.6-sol"], "strong": ["gpt-5.6-sol"]},
|
||||
"fallback_tier": "fast",
|
||||
"classifier_type": "llm",
|
||||
"classifier_llm_config": {"model": "gpt-5.6-sol"},
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
class _Cache:
|
||||
def __init__(self) -> None:
|
||||
self.values: dict[str, object] = {}
|
||||
|
||||
async def async_get_cache(
|
||||
self,
|
||||
key: str,
|
||||
parent_otel_span: object | None = None,
|
||||
local_only: bool = False,
|
||||
**kwargs: object,
|
||||
) -> object:
|
||||
return self.values.get(key)
|
||||
|
||||
async def async_set_cache(
|
||||
self,
|
||||
key: str,
|
||||
value: object,
|
||||
local_only: bool = False,
|
||||
**kwargs: object,
|
||||
) -> None:
|
||||
self.values[key] = value
|
||||
|
||||
|
||||
def _candidate(model_name: str, cost: float) -> AutoSetupCandidate:
|
||||
return AutoSetupCandidate(
|
||||
model_name=model_name,
|
||||
benchmark_model_id=f"benchmark-{model_name}",
|
||||
quality_lower_bound=0.95,
|
||||
cost_per_completed_task_usd=cost,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_easy_task_speed_learns_equal_output_response_time() -> None:
|
||||
cache = _Cache()
|
||||
candidates = (_candidate("cheap-slow", 0.01), _candidate("fast", 0.04))
|
||||
|
||||
assert (
|
||||
await select_runtime_response_model(
|
||||
router_cache=cache,
|
||||
router_model_name="auto",
|
||||
tier="SIMPLE",
|
||||
candidates=candidates,
|
||||
available_models=("cheap-slow", "fast"),
|
||||
cold_start_model="cheap-slow",
|
||||
objective="task_completion_speed",
|
||||
)
|
||||
== "cheap-slow"
|
||||
)
|
||||
|
||||
for model, end in (("cheap-slow", 0.30), ("cheap-slow", 0.30), ("fast", 0.15), ("fast", 0.15)):
|
||||
await record_runtime_response_latency(
|
||||
router_cache=cache,
|
||||
router_model_name="auto",
|
||||
tier="SIMPLE",
|
||||
routed_model=model,
|
||||
kwargs={"completion_start_time": 0.10},
|
||||
response_obj={
|
||||
"usage": {
|
||||
"completion_tokens": 20,
|
||||
"completion_tokens_details": {"reasoning_tokens": 10},
|
||||
}
|
||||
},
|
||||
start_time=0.0,
|
||||
end_time=end,
|
||||
)
|
||||
|
||||
measured = response_latency_sample(
|
||||
{"completion_start_time": 0.10},
|
||||
{"usage": {"completion_tokens": 20, "completion_tokens_details": {"reasoning_tokens": 10}}},
|
||||
0.0,
|
||||
0.30,
|
||||
)
|
||||
assert measured is not None and measured[1] == 10
|
||||
assert (
|
||||
await select_runtime_response_model(
|
||||
router_cache=cache,
|
||||
router_model_name="auto",
|
||||
tier="SIMPLE",
|
||||
candidates=candidates,
|
||||
available_models=("cheap-slow", "fast"),
|
||||
cold_start_model="cheap-slow",
|
||||
objective="task_completion_speed",
|
||||
)
|
||||
== "fast"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_snapshot_policy_is_deterministic_but_manual_pool_keeps_random_selection() -> None:
|
||||
generated = build_auto_setup_config(
|
||||
snapshot=load_auto_router_snapshot(),
|
||||
available_model_refs=_available_refs("gpt-5.6-sol", "gpt-5.4-nano"),
|
||||
quality_level="economy",
|
||||
optimize_for="cost",
|
||||
)
|
||||
router_instance = MagicMock()
|
||||
router_instance.cache = AsyncMock()
|
||||
automatic = ComplexityRouter(
|
||||
model_name="auto",
|
||||
litellm_router_instance=router_instance,
|
||||
complexity_router_config=generated.model_dump(mode="json"),
|
||||
)
|
||||
manual = ComplexityRouter(
|
||||
model_name="manual",
|
||||
litellm_router_instance=router_instance,
|
||||
complexity_router_config={"tiers": {"SIMPLE": ["first", "second"]}},
|
||||
)
|
||||
|
||||
assert await automatic._pick_model_for_tier(ComplexityTier.SIMPLE, None, None, {}) == "gpt-5.4-nano"
|
||||
assert (
|
||||
await automatic._pick_model_for_tier(
|
||||
ComplexityTier.SIMPLE,
|
||||
None,
|
||||
None,
|
||||
{},
|
||||
allowed_models=("gpt-5.6-sol",),
|
||||
)
|
||||
== "gpt-5.6-sol"
|
||||
)
|
||||
with patch(
|
||||
"litellm.router_strategy.complexity_router.complexity_router.random.choice", return_value="second"
|
||||
) as pick:
|
||||
assert await manual._pick_model_for_tier(ComplexityTier.SIMPLE, None, None, {}) == "second"
|
||||
pick.assert_called_once_with(("first", "second"))
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_live_latency_telemetry_is_recorded_only_for_an_auto_runtime_policy() -> None:
|
||||
generated = build_auto_setup_config(
|
||||
snapshot=load_auto_router_snapshot(),
|
||||
available_model_refs=_available_refs("gpt-5.6-sol", "gpt-5.4-nano"),
|
||||
quality_level="economy",
|
||||
optimize_for="task_completion_speed",
|
||||
)
|
||||
router = Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "smart",
|
||||
"litellm_params": {
|
||||
"model": "auto_router/complexity_router",
|
||||
"complexity_router_config": generated.model_dump(mode="json"),
|
||||
"complexity_router_default_model": "gpt-5.6-sol",
|
||||
},
|
||||
},
|
||||
{"model_name": "gpt-5.6-sol", "litellm_params": {"model": "gpt-5.6-sol"}},
|
||||
{"model_name": "gpt-5.4-nano", "litellm_params": {"model": "gpt-5.4-nano"}},
|
||||
]
|
||||
)
|
||||
metadata = {
|
||||
"routing_decision": {
|
||||
"router_model_name": "smart",
|
||||
"tier": "SIMPLE",
|
||||
"routed_model": "gpt-5.4-nano",
|
||||
"auto_setup_selection_mode": "runtime_response_latency",
|
||||
}
|
||||
}
|
||||
|
||||
with patch(
|
||||
"litellm.router_strategy.complexity_router.response_latency.record_runtime_response_latency",
|
||||
new_callable=AsyncMock,
|
||||
) as recorder:
|
||||
await router._record_auto_setup_response_latency(
|
||||
kwargs={"litellm_params": {"metadata": metadata}},
|
||||
completion_response={"usage": {"completion_tokens": 1}},
|
||||
start_time=0.0,
|
||||
end_time=0.1,
|
||||
)
|
||||
metadata["routing_decision"]["auto_setup_selection_mode"] = "snapshot_ranked"
|
||||
await router._record_auto_setup_response_latency(
|
||||
kwargs={"litellm_params": {"metadata": metadata}},
|
||||
completion_response={"usage": {"completion_tokens": 1}},
|
||||
start_time=0.0,
|
||||
end_time=0.1,
|
||||
)
|
||||
|
||||
recorder.assert_awaited_once()
|
||||
|
|
@ -0,0 +1,34 @@
|
|||
import { useQuery } from "@tanstack/react-query";
|
||||
|
||||
import { getAutoRouterRecommendation, type AutoRouterRecommendationResponse } from "@/components/networking";
|
||||
import type { AutoSetupObjective, AutoSetupQualityLevel } from "@/components/add_model/build_complexity_router_config";
|
||||
import { createQueryKeys } from "../common/queryKeysFactory";
|
||||
|
||||
const recommendationKeys = createQueryKeys("autoRouterRecommendation");
|
||||
|
||||
interface UseAutoRouterRecommendationParams {
|
||||
accessToken: string;
|
||||
qualityLevel: AutoSetupQualityLevel;
|
||||
optimizeFor: AutoSetupObjective;
|
||||
teamId?: string;
|
||||
enabled: boolean;
|
||||
}
|
||||
|
||||
export const useAutoRouterRecommendation = ({
|
||||
accessToken,
|
||||
qualityLevel,
|
||||
optimizeFor,
|
||||
teamId,
|
||||
enabled,
|
||||
}: UseAutoRouterRecommendationParams) => {
|
||||
const filters = { qualityLevel, optimizeFor, ...(teamId ? { teamId } : {}) };
|
||||
const queryOptions = {
|
||||
queryKey: recommendationKeys.list({
|
||||
filters,
|
||||
}),
|
||||
queryFn: () => getAutoRouterRecommendation(accessToken, qualityLevel, optimizeFor, teamId),
|
||||
enabled: enabled && Boolean(accessToken),
|
||||
staleTime: 30 * 1000,
|
||||
};
|
||||
return useQuery<AutoRouterRecommendationResponse>(queryOptions);
|
||||
};
|
||||
|
|
@ -0,0 +1,134 @@
|
|||
import type { AutoRouterRecommendationResponse } from "@/components/networking";
|
||||
import { Button } from "@/components/ui/button";
|
||||
import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select";
|
||||
import { UiLoadingSpinner } from "@/components/ui/ui-loading-spinner";
|
||||
import type { AutoSetupObjective, AutoSetupQualityLevel } from "./build_complexity_router_config";
|
||||
|
||||
const OBJECTIVE_DESCRIPTION: Record<AutoSetupObjective, string> = {
|
||||
cost: "Uses expected cost per completed benchmark task, including failures",
|
||||
task_completion_speed: "Uses live response speed for easy requests and benchmark completion time for hard ones",
|
||||
balanced: "Balances cost per completed task with those same completion-time signals",
|
||||
};
|
||||
|
||||
interface AutoSetupControlsProps {
|
||||
qualityLevel: AutoSetupQualityLevel;
|
||||
optimizeFor: AutoSetupObjective;
|
||||
onQualityLevelChange: (value: AutoSetupQualityLevel) => void;
|
||||
onOptimizeForChange: (value: AutoSetupObjective) => void;
|
||||
recommendation?: AutoRouterRecommendationResponse;
|
||||
recommendationEnabled: boolean;
|
||||
recommendationPending: boolean;
|
||||
recommendationFailed: boolean;
|
||||
recommendationError: unknown;
|
||||
requiresTeamScope: boolean;
|
||||
hasAutoPolicy: boolean;
|
||||
onRetry: () => void;
|
||||
onRegenerate: () => void;
|
||||
onUseManual: () => void;
|
||||
}
|
||||
|
||||
const recommendationErrorMessage = (error: unknown): string =>
|
||||
error instanceof Error ? error.message : "Could not build Auto setup";
|
||||
|
||||
const readyMessage = (recommendation: AutoRouterRecommendationResponse): string => {
|
||||
const count = recommendation.matched_model_groups.length;
|
||||
return `Ready with ${count} available model${count === 1 ? "" : "s"}. Review or edit below`;
|
||||
};
|
||||
|
||||
export default function AutoSetupControls({
|
||||
qualityLevel,
|
||||
optimizeFor,
|
||||
onQualityLevelChange,
|
||||
onOptimizeForChange,
|
||||
recommendation,
|
||||
recommendationEnabled,
|
||||
recommendationPending,
|
||||
recommendationFailed,
|
||||
recommendationError,
|
||||
requiresTeamScope,
|
||||
hasAutoPolicy,
|
||||
onRetry,
|
||||
onRegenerate,
|
||||
onUseManual,
|
||||
}: AutoSetupControlsProps) {
|
||||
return (
|
||||
<div className="space-y-4 rounded-lg border border-border p-4" data-testid="auto-setup-controls">
|
||||
<div>
|
||||
<div className="font-medium text-foreground">Auto setup</div>
|
||||
<div className="text-xs text-muted-foreground">
|
||||
Builds an editable complexity router from the models available to you
|
||||
</div>
|
||||
</div>
|
||||
<div className="grid grid-cols-1 gap-4 sm:grid-cols-2">
|
||||
<div>
|
||||
<label className="mb-2 block text-sm font-medium text-foreground">Quality level</label>
|
||||
<Select
|
||||
value={qualityLevel}
|
||||
onValueChange={(value: AutoSetupQualityLevel | null) => value && onQualityLevelChange(value)}
|
||||
>
|
||||
<SelectTrigger data-testid="auto-quality-selector" className="w-full">
|
||||
<SelectValue />
|
||||
</SelectTrigger>
|
||||
<SelectContent>
|
||||
<SelectItem value="economy">Economy</SelectItem>
|
||||
<SelectItem value="balanced">Balanced</SelectItem>
|
||||
<SelectItem value="high">High</SelectItem>
|
||||
<SelectItem value="max">Max</SelectItem>
|
||||
</SelectContent>
|
||||
</Select>
|
||||
<div className="mt-1 text-xs text-muted-foreground">
|
||||
Higher levels keep each tier closer to the smartest available model
|
||||
</div>
|
||||
</div>
|
||||
<div>
|
||||
<label className="mb-2 block text-sm font-medium text-foreground">Optimize for</label>
|
||||
<Select
|
||||
value={optimizeFor}
|
||||
onValueChange={(value: AutoSetupObjective | null) => value && onOptimizeForChange(value)}
|
||||
>
|
||||
<SelectTrigger data-testid="auto-objective-selector" className="w-full">
|
||||
<SelectValue />
|
||||
</SelectTrigger>
|
||||
<SelectContent>
|
||||
<SelectItem value="cost">Cost</SelectItem>
|
||||
<SelectItem value="task_completion_speed">Task completion speed</SelectItem>
|
||||
<SelectItem value="balanced">Balanced</SelectItem>
|
||||
</SelectContent>
|
||||
</Select>
|
||||
<div className="mt-1 text-xs text-muted-foreground">{OBJECTIVE_DESCRIPTION[optimizeFor]}</div>
|
||||
</div>
|
||||
</div>
|
||||
{recommendationPending && recommendation === undefined && (
|
||||
<div className="flex items-center gap-2 text-xs text-muted-foreground">
|
||||
<UiLoadingSpinner className="size-3" /> Building your router...
|
||||
</div>
|
||||
)}
|
||||
{!recommendationEnabled && requiresTeamScope && (
|
||||
<div className="text-xs text-muted-foreground">Select a team to build its Auto setup</div>
|
||||
)}
|
||||
{recommendationFailed && recommendation === undefined && (
|
||||
<div className="text-xs text-destructive">
|
||||
{recommendationErrorMessage(recommendationError)}.{" "}
|
||||
<button type="button" className="underline" onClick={onRetry}>
|
||||
Retry
|
||||
</button>
|
||||
</div>
|
||||
)}
|
||||
{recommendation && (
|
||||
<div className="space-y-1">
|
||||
<div className={`text-xs ${hasAutoPolicy ? "text-success" : "text-warning"}`}>
|
||||
{hasAutoPolicy ? readyMessage(recommendation) : "Customized model pools now use standard manual selection"}
|
||||
</div>
|
||||
{!hasAutoPolicy && (
|
||||
<button type="button" className="text-xs text-primary underline" onClick={onRegenerate}>
|
||||
Regenerate Auto setup
|
||||
</button>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
<Button type="button" variant="ghost" className="h-auto px-0" onClick={onUseManual}>
|
||||
Choose a template or set up manually
|
||||
</Button>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
|
@ -2,6 +2,7 @@ import { fireEvent, renderWithProviders, screen, within } from "../../../tests/t
|
|||
import userEvent from "@testing-library/user-event";
|
||||
import { vi } from "vitest";
|
||||
import ComplexityRouterConfig, { ComplexityRouterConfigValue } from "./ComplexityRouterConfig";
|
||||
import { detachAutoSetupAfterPoolEdit } from "./auto_setup_policy";
|
||||
vi.mock(
|
||||
"@/app/(dashboard)/hooks/autoRouter/useComplexityScorerDefaults",
|
||||
async () => await import("../../../tests/mocks/complexityScorerDefaults"),
|
||||
|
|
@ -44,6 +45,30 @@ const baseProps = {
|
|||
};
|
||||
|
||||
describe("ComplexityRouterConfig", () => {
|
||||
it("keeps Auto policy for classifier edits and detaches it for model-pool edits", () => {
|
||||
const automatic: ComplexityRouterConfigValue = {
|
||||
...defaultValue,
|
||||
auto_setup: {
|
||||
schema_version: "1",
|
||||
snapshot_id: "snapshot-test",
|
||||
snapshot_sha256: "a".repeat(64),
|
||||
quality_level: "max",
|
||||
optimize_for: "cost",
|
||||
tier_policies: {},
|
||||
},
|
||||
};
|
||||
|
||||
expect(detachAutoSetupAfterPoolEdit(automatic, { ...automatic, classifier_type: "heuristic_v2" }).auto_setup).toBe(
|
||||
automatic.auto_setup,
|
||||
);
|
||||
expect(
|
||||
detachAutoSetupAfterPoolEdit(automatic, {
|
||||
...automatic,
|
||||
tiers: { ...automatic.tiers, SIMPLE: ["gpt-4"] },
|
||||
}).auto_setup,
|
||||
).toBeUndefined();
|
||||
});
|
||||
|
||||
it("should render", () => {
|
||||
renderWithProviders(<ComplexityRouterConfig {...baseProps} />);
|
||||
expect(screen.getByText("Complexity Tier Configuration")).toBeInTheDocument();
|
||||
|
|
|
|||
|
|
@ -48,6 +48,7 @@ import EscalationKeywords from "./EscalationKeywords";
|
|||
import KeywordTierRules, { KeywordTierRule } from "./KeywordTierRules";
|
||||
import SemanticKeywordMatching from "./SemanticKeywordMatching";
|
||||
import { type DimensionWeights, type TierBoundaries, type TokenThresholds } from "./heuristic_scoring_knobs";
|
||||
import type { AutoSetupConfigPayload } from "./build_complexity_router_config";
|
||||
|
||||
export type { DimensionWeights, TierBoundaries, TokenThresholds };
|
||||
export type { CustomTierSet, TierRow } from "./tier_rows";
|
||||
|
|
@ -446,6 +447,8 @@ export interface ComplexityRouterConfigValue {
|
|||
* edit round-trip.
|
||||
*/
|
||||
tier_model_params?: TierModelParamsByTier;
|
||||
/** Present only when the zero-setup Auto flow generated this editable config. */
|
||||
auto_setup?: AutoSetupConfigPayload;
|
||||
}
|
||||
|
||||
/** Session affinity wins where a hand-authored config sets both, matching the backend's own `or`. */
|
||||
|
|
|
|||
|
|
@ -4,13 +4,14 @@ import { vi } from "vitest";
|
|||
import AddAutoRouterTab from "./add_auto_router_tab";
|
||||
import { toast } from "@/lib/toast";
|
||||
import { handleAddAutoRouterSubmit } from "./handle_add_auto_router_submit";
|
||||
import { getMissingTiersError } from "./build_complexity_router_config";
|
||||
import { getMissingTiersError, type AutoSetupConfigPayload } from "./build_complexity_router_config";
|
||||
import { getSubmitBlockedReason } from "./add_auto_router_tab";
|
||||
import { buildModelAvailability } from "@/lib/autorouter_presets";
|
||||
import { testAutoRouterRouting } from "../networking";
|
||||
import { ModelGroup } from "@/components/llm_calls/fetch_models";
|
||||
import { AutoRouterPreset, getRequiredModelsInPreset } from "@/lib/autorouter_presets";
|
||||
import { BUNDLED_PRESETS, LOADED_PRESETS_QUERY, useAutoRouterPresets } from "../../../tests/mocks/autoRouterPresets";
|
||||
import { EMPTY_RECOMMENDATION_QUERY, useAutoRouterRecommendation } from "../../../tests/mocks/autoRouterRecommendation";
|
||||
vi.mock(
|
||||
"@/app/(dashboard)/hooks/autoRouter/useComplexityScorerDefaults",
|
||||
async () => await import("../../../tests/mocks/complexityScorerDefaults"),
|
||||
|
|
@ -19,6 +20,10 @@ vi.mock(
|
|||
"@/app/(dashboard)/hooks/autoRouter/useAutoRouterPresets",
|
||||
async () => await import("../../../tests/mocks/autoRouterPresets"),
|
||||
);
|
||||
vi.mock(
|
||||
"@/app/(dashboard)/hooks/autoRouter/useAutoRouterRecommendation",
|
||||
async () => await import("../../../tests/mocks/autoRouterRecommendation"),
|
||||
);
|
||||
|
||||
const getAllPresets = (): AutoRouterPreset[] => BUNDLED_PRESETS;
|
||||
const getPresetByKey = (key: string): AutoRouterPreset | undefined => BUNDLED_PRESETS.find((p) => p.key === key);
|
||||
|
|
@ -37,6 +42,9 @@ const ALL_FAMILY_MODELS: ModelGroup[] = [
|
|||
const ANTHROPIC_ONLY_MODEL = ANTHROPIC_TIERS.COMPLEX[0];
|
||||
|
||||
const openTemplateDropdown = (): void => {
|
||||
if (!screen.queryByTestId("template-selector")) {
|
||||
fireEvent.click(screen.getByRole("button", { name: /choose a template or set up manually/i }));
|
||||
}
|
||||
fireEvent.click(screen.getByTestId("template-selector"));
|
||||
};
|
||||
|
||||
|
|
@ -141,6 +149,86 @@ describe("AddAutoRouterTab", () => {
|
|||
testQueryClient.clear();
|
||||
mockFetchAvailableModels.mockResolvedValue([]);
|
||||
mockFetchAllModelDeployments.mockResolvedValue([]);
|
||||
vi.mocked(useAutoRouterRecommendation).mockReturnValue(EMPTY_RECOMMENDATION_QUERY);
|
||||
});
|
||||
|
||||
it("starts with only quality and optimization choices in Auto setup", () => {
|
||||
renderWithProviders(<Harness />);
|
||||
|
||||
expect(screen.getByTestId("auto-quality-selector")).toBeInTheDocument();
|
||||
expect(screen.getByTestId("auto-objective-selector")).toBeInTheDocument();
|
||||
expect(screen.queryByTestId("template-selector")).not.toBeInTheDocument();
|
||||
expect(useAutoRouterRecommendation).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ qualityLevel: "balanced", optimizeFor: "balanced", enabled: true }),
|
||||
);
|
||||
});
|
||||
|
||||
it("submits the editable policy generated for the caller's available model groups", async () => {
|
||||
const user = userEvent.setup();
|
||||
const autoSetup: AutoSetupConfigPayload = {
|
||||
schema_version: "1" as const,
|
||||
snapshot_id: "snapshot-test",
|
||||
snapshot_sha256: "a".repeat(64),
|
||||
quality_level: "max" as const,
|
||||
optimize_for: "task_completion_speed" as const,
|
||||
tier_policies: Object.fromEntries(
|
||||
["SIMPLE", "MEDIUM", "COMPLEX", "REASONING"].map((tier) => [
|
||||
tier,
|
||||
{
|
||||
selection_mode: tier === "SIMPLE" || tier === "MEDIUM" ? "runtime_response_latency" : "snapshot_ranked",
|
||||
candidates: [
|
||||
{
|
||||
model_name: "smart-group",
|
||||
benchmark_model_id: "benchmark-smart",
|
||||
quality_lower_bound: 0.98,
|
||||
cost_per_completed_task_usd: 0.04,
|
||||
},
|
||||
],
|
||||
...(tier === "SIMPLE" || tier === "MEDIUM" ? { cold_start_model: "smart-group" } : {}),
|
||||
evidence_status: "test",
|
||||
},
|
||||
]),
|
||||
),
|
||||
};
|
||||
mockFetchAvailableModels.mockResolvedValue([{ model_group: "smart-group", mode: "chat" }]);
|
||||
vi.mocked(useAutoRouterRecommendation).mockReturnValue({
|
||||
data: {
|
||||
quality_level: "max",
|
||||
optimize_for: "task_completion_speed",
|
||||
snapshot_id: "snapshot-test",
|
||||
snapshot_generated_at: "2026-09-03T00:00:00Z",
|
||||
matched_model_groups: ["smart-group"],
|
||||
complexity_router_config: {
|
||||
tiers: {
|
||||
SIMPLE: ["smart-group"],
|
||||
MEDIUM: ["smart-group"],
|
||||
COMPLEX: ["smart-group"],
|
||||
REASONING: ["smart-group"],
|
||||
},
|
||||
classifier_type: "heuristic_v2",
|
||||
classification_mode: "per_request",
|
||||
session_affinity: false,
|
||||
deployment_affinity: false,
|
||||
modality_routing: false,
|
||||
modality_pin_override: false,
|
||||
auto_setup: autoSetup,
|
||||
},
|
||||
},
|
||||
isPending: false,
|
||||
isError: false,
|
||||
error: null,
|
||||
refetch: vi.fn(),
|
||||
});
|
||||
|
||||
renderWithProviders(<Harness />);
|
||||
await screen.findByText(/Ready with 1 available model/);
|
||||
await user.type(screen.getByPlaceholderText(/smart_router/i), "max-speed-router");
|
||||
await user.click(screen.getByRole("button", { name: /add auto router/i }));
|
||||
|
||||
await waitFor(() => expect(handleAddAutoRouterSubmit).toHaveBeenCalled());
|
||||
expect(vi.mocked(handleAddAutoRouterSubmit).mock.calls.at(-1)?.[0].complexity_router_config).toMatchObject({
|
||||
auto_setup: autoSetup,
|
||||
});
|
||||
});
|
||||
|
||||
// Detailed Configuration starts collapsed so the modal opens onto just Name + Template; a caller
|
||||
|
|
@ -163,6 +251,23 @@ describe("AddAutoRouterTab", () => {
|
|||
expect(screen.getByRole("button", { name: /add auto router/i })).toBeDisabled();
|
||||
});
|
||||
|
||||
it("blocks an implicit form submit while Auto setup is still building", async () => {
|
||||
vi.mocked(getMissingTiersError).mockReturnValue(null);
|
||||
vi.mocked(useAutoRouterRecommendation).mockReturnValue({
|
||||
...EMPTY_RECOMMENDATION_QUERY,
|
||||
isPending: true,
|
||||
});
|
||||
const { container } = renderWithProviders(<Harness />);
|
||||
|
||||
fireEvent.change(screen.getByPlaceholderText(/smart_router/i), { target: { value: "early-router" } });
|
||||
fireEvent.submit(container.querySelector("form")!);
|
||||
|
||||
await waitFor(() =>
|
||||
expect(toast.fromError).toHaveBeenCalledWith("Building the best setup for your available models"),
|
||||
);
|
||||
expect(handleAddAutoRouterSubmit).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("still flags the router name once the config no longer blocks the submit", async () => {
|
||||
const user = userEvent.setup();
|
||||
vi.mocked(getMissingTiersError).mockReturnValue(null);
|
||||
|
|
@ -354,22 +459,19 @@ describe("AddAutoRouterTab", () => {
|
|||
});
|
||||
|
||||
it("blocks the submit when a team admin has not picked a team", async () => {
|
||||
const user = userEvent.setup();
|
||||
vi.mocked(getMissingTiersError).mockReturnValue(null);
|
||||
|
||||
renderWithProviders(
|
||||
<AddAutoRouterTab handleOk={vi.fn()} accessToken="token" userRole="Internal User" createScope="team-required" />,
|
||||
);
|
||||
|
||||
await user.type(screen.getByPlaceholderText(/smart_router/i), "team-scoped-router");
|
||||
await user.click(screen.getByRole("button", { name: /add auto router/i }));
|
||||
|
||||
expect(await screen.findByText("Please select a team to continue")).toBeInTheDocument();
|
||||
expect(screen.getByText("Select a team to build its Auto setup")).toBeInTheDocument();
|
||||
expect(screen.getByRole("button", { name: /add auto router/i })).toBeDisabled();
|
||||
expect(handleAddAutoRouterSubmit).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
// The shared dropdown emits null on clear while this form's schema wants a string, so the
|
||||
// form maps null back to "": the user sees the pick-a-team message, not a zod type error.
|
||||
// form maps null back to "": Auto setup returns to its explicit pick-a-team state.
|
||||
it("treats a team picked and then cleared like no team at all", async () => {
|
||||
const user = userEvent.setup();
|
||||
vi.mocked(getMissingTiersError).mockReturnValue(null);
|
||||
|
|
@ -381,9 +483,8 @@ describe("AddAutoRouterTab", () => {
|
|||
await user.type(screen.getByPlaceholderText(/smart_router/i), "team-scoped-router");
|
||||
await user.selectOptions(screen.getByTestId("team-dropdown"), "team-1");
|
||||
await user.click(screen.getByTestId("team-dropdown-clear"));
|
||||
await user.click(screen.getByRole("button", { name: /add auto router/i }));
|
||||
|
||||
expect(await screen.findByText("Please select a team to continue")).toBeInTheDocument();
|
||||
expect(screen.getByText("Select a team to build its Auto setup")).toBeInTheDocument();
|
||||
expect(screen.getByRole("button", { name: /add auto router/i })).toBeDisabled();
|
||||
expect(handleAddAutoRouterSubmit).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
|
|
@ -750,8 +851,8 @@ describe("AddAutoRouterTab", () => {
|
|||
|
||||
renderWithProviders(<Harness />);
|
||||
|
||||
expect(await screen.findByText("Could not load available models.")).toBeInTheDocument();
|
||||
openTemplateDropdown();
|
||||
expect(await screen.findByText("Could not load available models.")).toBeInTheDocument();
|
||||
const anthropicOption = optionByLabel("Anthropic Family")!;
|
||||
expect(isOptionDisabled(anthropicOption)).toBe(true);
|
||||
expect(anthropicOption).toHaveTextContent(/Cannot verify these models are available/);
|
||||
|
|
@ -1227,6 +1328,7 @@ describe("preset catalog fetch states", () => {
|
|||
} as never);
|
||||
renderWithProviders(<Harness />);
|
||||
|
||||
openTemplateDropdown();
|
||||
expect(screen.getByText("Loading templates...")).toBeInTheDocument();
|
||||
});
|
||||
|
||||
|
|
@ -1240,9 +1342,8 @@ describe("preset catalog fetch states", () => {
|
|||
} as never);
|
||||
renderWithProviders(<Harness />);
|
||||
|
||||
expect(await screen.findByText(/Could not load templates/)).toBeInTheDocument();
|
||||
|
||||
openTemplateDropdown();
|
||||
expect(await screen.findByText(/Could not load templates/)).toBeInTheDocument();
|
||||
const options = screen.queryAllByRole("option");
|
||||
expect(options).toHaveLength(1);
|
||||
expect(options[0]).toHaveTextContent("Custom Configuration");
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
import React, { useEffect, useState } from "react";
|
||||
import React, { useEffect, useRef, useState } from "react";
|
||||
import { useQuery } from "@tanstack/react-query";
|
||||
import { useWatch } from "react-hook-form";
|
||||
import { ChevronDown, ChevronRight } from "lucide-react";
|
||||
|
|
@ -30,10 +30,13 @@ import ComplexityRouterConfig, {
|
|||
DEFAULT_DEPLOYMENT_AFFINITY,
|
||||
DEFAULT_TIER_DISTANCE_PENALTY,
|
||||
} from "./ComplexityRouterConfig";
|
||||
import { detachAutoSetupAfterPoolEdit, getRecommendationBlockedReason, type SetupMode } from "./auto_setup_policy";
|
||||
import { KeywordTierRule } from "./KeywordTierRules";
|
||||
import { DEFAULT_ESCALATION_KEYWORDS } from "./EscalationKeywords";
|
||||
import { DEFAULT_MATCH_THRESHOLD } from "./SemanticKeywordMatching";
|
||||
import {
|
||||
AutoSetupObjective,
|
||||
AutoSetupQualityLevel,
|
||||
BuildComplexityRouterConfigParams,
|
||||
buildComplexityRouterConfig,
|
||||
getKeywordTierRulesError,
|
||||
|
|
@ -63,7 +66,9 @@ import {
|
|||
AutoRouterPreset,
|
||||
} from "@/lib/autorouter_presets";
|
||||
import { useAutoRouterPresets } from "@/app/(dashboard)/hooks/autoRouter/useAutoRouterPresets";
|
||||
import { useAutoRouterRecommendation } from "@/app/(dashboard)/hooks/autoRouter/useAutoRouterRecommendation";
|
||||
import { Dialog, DialogContent, DialogFooter, DialogHeader, DialogTitle } from "@/components/ui/dialog";
|
||||
import AutoSetupControls from "./AutoSetupControls";
|
||||
|
||||
interface AddAutoRouterTabProps {
|
||||
handleOk: () => void;
|
||||
|
|
@ -181,6 +186,10 @@ const AddAutoRouterTab: React.FC<AddAutoRouterTabProps> = ({
|
|||
const watchedName = useWatch({ control: form.control, name: "auto_router_name" });
|
||||
const watchedTeamId = useWatch({ control: form.control, name: "team_id" });
|
||||
const [modelAccessGroups, setModelAccessGroups] = useState<string[]>([]);
|
||||
const [setupMode, setSetupMode] = useState<SetupMode>("auto");
|
||||
const [qualityLevel, setQualityLevel] = useState<AutoSetupQualityLevel>("balanced");
|
||||
const [optimizeFor, setOptimizeFor] = useState<AutoSetupObjective>("balanced");
|
||||
const appliedRecommendationKey = useRef<string | null>(null);
|
||||
|
||||
const [complexityRouterConfig, setComplexityRouterConfig] = useState<ComplexityRouterConfigValue>({
|
||||
tiers: { SIMPLE: [], MEDIUM: [], COMPLEX: [], REASONING: [] },
|
||||
|
|
@ -267,6 +276,22 @@ const AddAutoRouterTab: React.FC<AddAutoRouterTabProps> = ({
|
|||
[modelInfo],
|
||||
);
|
||||
|
||||
const recommendationEnabled = setupMode === "auto" && (!requiresTeamScope || Boolean(watchedTeamId));
|
||||
const recommendationParams = {
|
||||
accessToken,
|
||||
qualityLevel,
|
||||
optimizeFor,
|
||||
teamId: requiresTeamScope ? watchedTeamId : undefined,
|
||||
enabled: recommendationEnabled,
|
||||
};
|
||||
const {
|
||||
data: recommendation,
|
||||
isPending: recommendationPending,
|
||||
isError: recommendationFailed,
|
||||
error: recommendationError,
|
||||
refetch: refetchRecommendation,
|
||||
} = useAutoRouterRecommendation(recommendationParams);
|
||||
|
||||
// A preset's models can only be trusted against a successfully loaded list. Selection and the
|
||||
// greyed-out state derive from this one function, so a preset that cannot be selected can never
|
||||
// have been applied: while loading we withhold selection rather than let a caller pick a preset
|
||||
|
|
@ -302,7 +327,7 @@ const AddAutoRouterTab: React.FC<AddAutoRouterTabProps> = ({
|
|||
[sortedPresetOptions],
|
||||
);
|
||||
|
||||
const applyPrefill = (prefill: PresetPrefill) => {
|
||||
const applyPrefill = React.useCallback((prefill: PresetPrefill) => {
|
||||
setEditingTiers(false);
|
||||
setComplexityRouterConfig(prefill.complexityRouterConfig);
|
||||
setCustomTechnicalKeywords(prefill.customTechnicalKeywords);
|
||||
|
|
@ -311,7 +336,23 @@ const AddAutoRouterTab: React.FC<AddAutoRouterTabProps> = ({
|
|||
setEmbeddingModel(prefill.embeddingModel);
|
||||
setMatchThreshold(prefill.matchThreshold);
|
||||
setEscalationKeywords(prefill.escalationKeywords);
|
||||
};
|
||||
}, []);
|
||||
|
||||
useEffect(() => {
|
||||
if (!recommendation || setupMode !== "auto") return;
|
||||
const key = [
|
||||
recommendation.snapshot_id,
|
||||
qualityLevel,
|
||||
optimizeFor,
|
||||
watchedTeamId ?? "",
|
||||
...recommendation.matched_model_groups,
|
||||
].join(":");
|
||||
if (appliedRecommendationKey.current === key) return;
|
||||
appliedRecommendationKey.current = key;
|
||||
setSelectedPreset(undefined);
|
||||
applyPrefill(buildPresetPrefill(recommendation.complexity_router_config, availability));
|
||||
setDetailsExpanded(false);
|
||||
}, [recommendation, setupMode, qualityLevel, optimizeFor, watchedTeamId, availability, applyPrefill]);
|
||||
|
||||
const handlePresetChange = (presetKey: string | undefined) => {
|
||||
if (!presetKey || presetKey === "custom") {
|
||||
|
|
@ -342,13 +383,28 @@ const AddAutoRouterTab: React.FC<AddAutoRouterTabProps> = ({
|
|||
defaultModel: complexityRouterConfig.default_model,
|
||||
};
|
||||
|
||||
const submitBlockedReason = getSubmitBlockedReason(
|
||||
complexityRouterConfig,
|
||||
keywordTierRules,
|
||||
referencedModelsParams,
|
||||
groupsOnlyAvailability,
|
||||
modelInfo,
|
||||
);
|
||||
const recommendationStatus = {
|
||||
setupMode,
|
||||
recommendationEnabled,
|
||||
recommendationPending,
|
||||
recommendationFailed,
|
||||
hasRecommendation: recommendation !== undefined,
|
||||
};
|
||||
const recommendationBlockedReason = getRecommendationBlockedReason(recommendationStatus);
|
||||
const regenerateAutoSetup = () => {
|
||||
if (!recommendation) return;
|
||||
applyPrefill(buildPresetPrefill(recommendation.complexity_router_config, availability));
|
||||
setDetailsExpanded(false);
|
||||
};
|
||||
const submitBlockedReason =
|
||||
recommendationBlockedReason ??
|
||||
getSubmitBlockedReason(
|
||||
complexityRouterConfig,
|
||||
keywordTierRules,
|
||||
referencedModelsParams,
|
||||
groupsOnlyAvailability,
|
||||
modelInfo,
|
||||
);
|
||||
|
||||
const complexityRouterConfigParams: BuildComplexityRouterConfigParams = {
|
||||
tiers: complexityRouterConfig.tiers,
|
||||
|
|
@ -388,6 +444,7 @@ const AddAutoRouterTab: React.FC<AddAutoRouterTabProps> = ({
|
|||
reasoningOverrideMinScore: complexityRouterConfig.reasoning_override_min_score,
|
||||
enableContextWindowEscalation: complexityRouterConfig.enable_context_window_escalation,
|
||||
contextWindowEscalationBuffer: complexityRouterConfig.context_window_escalation_buffer,
|
||||
autoSetup: complexityRouterConfig.auto_setup,
|
||||
};
|
||||
|
||||
const submitRecommendedRouter = async (name: string) => {
|
||||
|
|
@ -395,13 +452,15 @@ const AddAutoRouterTab: React.FC<AddAutoRouterTabProps> = ({
|
|||
// disagree about why. The handler needs it in its own right: the form fires this on Enter
|
||||
// regardless of the button's disabled state.
|
||||
const blockedReason =
|
||||
recommendationBlockedReason ??
|
||||
getSubmitBlockedReason(
|
||||
complexityRouterConfig,
|
||||
keywordTierRules,
|
||||
referencedModelsParams,
|
||||
groupsOnlyAvailability,
|
||||
modelInfo,
|
||||
) ?? getSemanticConfigError({ semanticMatchingEnabled, embeddingModel, keywordTierRules });
|
||||
) ??
|
||||
getSemanticConfigError({ semanticMatchingEnabled, embeddingModel, keywordTierRules });
|
||||
if (blockedReason) {
|
||||
setShowValidationErrors(true);
|
||||
toast.fromError(blockedReason);
|
||||
|
|
@ -507,71 +566,6 @@ const AddAutoRouterTab: React.FC<AddAutoRouterTabProps> = ({
|
|||
{({ ref, ...field }) => <Input {...field} ref={ref} placeholder="e.g., smart_router, auto_router_1" />}
|
||||
</FormField>
|
||||
|
||||
<div>
|
||||
<label className="block text-sm font-medium text-foreground mb-2">Template</label>
|
||||
<Select
|
||||
items={templateItems}
|
||||
value={selectedPreset ?? null}
|
||||
onValueChange={(presetKey: string | null) => handlePresetChange(presetKey ?? undefined)}
|
||||
>
|
||||
<SelectTrigger data-testid="template-selector" className="w-full">
|
||||
<SelectValue placeholder="Choose a template or select Custom to define your own" />
|
||||
</SelectTrigger>
|
||||
<SelectContent>
|
||||
{sortedPresetOptions.map(({ preset, availability: presetState }) => {
|
||||
const disabledHint = presetDisabledHint(presetState);
|
||||
const hintClass = isPresetHintAlarming(presetState)
|
||||
? "text-destructive"
|
||||
: "text-muted-foreground";
|
||||
const matchedHint =
|
||||
presetState.kind === "available" && presetState.viaDeployments
|
||||
? "Matches your deployments"
|
||||
: null;
|
||||
|
||||
return (
|
||||
<SelectItem
|
||||
key={preset.key}
|
||||
value={preset.key}
|
||||
label={preset.label}
|
||||
disabled={disabledHint !== null}
|
||||
title={disabledHint ?? preset.description}
|
||||
>
|
||||
<div>
|
||||
<div className="font-medium">{preset.label}</div>
|
||||
<div className="text-xs text-muted-foreground">{preset.description}</div>
|
||||
{disabledHint && <div className={`text-xs mt-1 ${hintClass}`}>{disabledHint}</div>}
|
||||
{matchedHint && <div className="text-xs mt-1 text-success">{matchedHint}</div>}
|
||||
</div>
|
||||
</SelectItem>
|
||||
);
|
||||
})}
|
||||
<SelectItem value="custom" label="Custom Configuration">
|
||||
<div>
|
||||
<div className="font-medium">Custom Configuration</div>
|
||||
<div className="text-xs text-muted-foreground">Define your auto router from scratch</div>
|
||||
</div>
|
||||
</SelectItem>
|
||||
</SelectContent>
|
||||
</Select>
|
||||
{modelsUnverifiable && (
|
||||
<div className="text-xs mt-1 text-destructive">
|
||||
Could not load available models.{" "}
|
||||
<button type="button" className="underline" onClick={() => refetchModels()}>
|
||||
Retry
|
||||
</button>
|
||||
</div>
|
||||
)}
|
||||
{presetsPending && <div className="text-xs mt-1 text-muted-foreground">Loading templates...</div>}
|
||||
{presetsUnavailable && (
|
||||
<div className="text-xs mt-1 text-destructive">
|
||||
Could not load templates, so only Custom Configuration is shown.{" "}
|
||||
<button type="button" className="underline" onClick={() => void refetchPresets()}>
|
||||
Retry
|
||||
</button>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
|
||||
{requiresTeamScope && (
|
||||
<FormField
|
||||
control={form.control}
|
||||
|
|
@ -587,6 +581,103 @@ const AddAutoRouterTab: React.FC<AddAutoRouterTabProps> = ({
|
|||
</FormField>
|
||||
)}
|
||||
|
||||
{setupMode === "auto" ? (
|
||||
<AutoSetupControls
|
||||
qualityLevel={qualityLevel}
|
||||
optimizeFor={optimizeFor}
|
||||
onQualityLevelChange={setQualityLevel}
|
||||
onOptimizeForChange={setOptimizeFor}
|
||||
recommendation={recommendation}
|
||||
recommendationEnabled={recommendationEnabled}
|
||||
recommendationPending={recommendationPending}
|
||||
recommendationFailed={recommendationFailed}
|
||||
recommendationError={recommendationError}
|
||||
requiresTeamScope={requiresTeamScope}
|
||||
hasAutoPolicy={Boolean(complexityRouterConfig.auto_setup)}
|
||||
onRetry={() => void refetchRecommendation()}
|
||||
onRegenerate={regenerateAutoSetup}
|
||||
onUseManual={() => setSetupMode("manual")}
|
||||
/>
|
||||
) : (
|
||||
<div>
|
||||
<div className="mb-2 flex items-center justify-between gap-3">
|
||||
<label className="block text-sm font-medium text-foreground">Template</label>
|
||||
<Button
|
||||
type="button"
|
||||
variant="ghost"
|
||||
className="h-auto px-0"
|
||||
onClick={() => {
|
||||
appliedRecommendationKey.current = null;
|
||||
setSetupMode("auto");
|
||||
}}
|
||||
>
|
||||
Use Auto setup
|
||||
</Button>
|
||||
</div>
|
||||
<Select
|
||||
items={templateItems}
|
||||
value={selectedPreset ?? null}
|
||||
onValueChange={(presetKey: string | null) => handlePresetChange(presetKey ?? undefined)}
|
||||
>
|
||||
<SelectTrigger data-testid="template-selector" className="w-full">
|
||||
<SelectValue placeholder="Choose a template or select Custom to define your own" />
|
||||
</SelectTrigger>
|
||||
<SelectContent>
|
||||
{sortedPresetOptions.map(({ preset, availability: presetState }) => {
|
||||
const disabledHint = presetDisabledHint(presetState);
|
||||
const hintClass = isPresetHintAlarming(presetState)
|
||||
? "text-destructive"
|
||||
: "text-muted-foreground";
|
||||
const matchedHint =
|
||||
presetState.kind === "available" && presetState.viaDeployments
|
||||
? "Matches your deployments"
|
||||
: null;
|
||||
|
||||
return (
|
||||
<SelectItem
|
||||
key={preset.key}
|
||||
value={preset.key}
|
||||
label={preset.label}
|
||||
disabled={disabledHint !== null}
|
||||
title={disabledHint ?? preset.description}
|
||||
>
|
||||
<div>
|
||||
<div className="font-medium">{preset.label}</div>
|
||||
<div className="text-xs text-muted-foreground">{preset.description}</div>
|
||||
{disabledHint && <div className={`text-xs mt-1 ${hintClass}`}>{disabledHint}</div>}
|
||||
{matchedHint && <div className="text-xs mt-1 text-success">{matchedHint}</div>}
|
||||
</div>
|
||||
</SelectItem>
|
||||
);
|
||||
})}
|
||||
<SelectItem value="custom" label="Custom Configuration">
|
||||
<div>
|
||||
<div className="font-medium">Custom Configuration</div>
|
||||
<div className="text-xs text-muted-foreground">Define your auto router from scratch</div>
|
||||
</div>
|
||||
</SelectItem>
|
||||
</SelectContent>
|
||||
</Select>
|
||||
{modelsUnverifiable && (
|
||||
<div className="text-xs mt-1 text-destructive">
|
||||
Could not load available models.{" "}
|
||||
<button type="button" className="underline" onClick={() => refetchModels()}>
|
||||
Retry
|
||||
</button>
|
||||
</div>
|
||||
)}
|
||||
{presetsPending && <div className="text-xs mt-1 text-muted-foreground">Loading templates...</div>}
|
||||
{presetsUnavailable && (
|
||||
<div className="text-xs mt-1 text-destructive">
|
||||
Could not load templates, so only Custom Configuration is shown.{" "}
|
||||
<button type="button" className="underline" onClick={() => void refetchPresets()}>
|
||||
Retry
|
||||
</button>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
|
||||
<div className="border border-border rounded-lg">
|
||||
<button
|
||||
type="button"
|
||||
|
|
@ -615,7 +706,9 @@ const AddAutoRouterTab: React.FC<AddAutoRouterTabProps> = ({
|
|||
onEditingTiersChange={setEditingTiers}
|
||||
modelInfo={modelInfo}
|
||||
value={complexityRouterConfig}
|
||||
onChange={setComplexityRouterConfig}
|
||||
onChange={(next) =>
|
||||
setComplexityRouterConfig((previous) => detachAutoSetupAfterPoolEdit(previous, next))
|
||||
}
|
||||
customTechnicalKeywords={customTechnicalKeywords}
|
||||
onCustomTechnicalKeywordsChange={setCustomTechnicalKeywords}
|
||||
keywordTierRules={keywordTierRules}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,42 @@
|
|||
import type { ComplexityRouterConfigValue } from "./ComplexityRouterConfig";
|
||||
|
||||
export type SetupMode = "auto" | "manual";
|
||||
|
||||
export interface RecommendationStatus {
|
||||
setupMode: SetupMode;
|
||||
recommendationEnabled: boolean;
|
||||
recommendationPending: boolean;
|
||||
recommendationFailed: boolean;
|
||||
hasRecommendation: boolean;
|
||||
}
|
||||
|
||||
export const getRecommendationBlockedReason = ({
|
||||
setupMode,
|
||||
recommendationEnabled,
|
||||
recommendationPending,
|
||||
recommendationFailed,
|
||||
hasRecommendation,
|
||||
}: RecommendationStatus): string | null => {
|
||||
if (setupMode !== "auto") return null;
|
||||
if (!recommendationEnabled) return "Select a team so Auto can use that team's available models";
|
||||
if (recommendationPending && !hasRecommendation) return "Building the best setup for your available models";
|
||||
if (recommendationFailed && !hasRecommendation) {
|
||||
return "Auto setup could not build a router for your available models";
|
||||
}
|
||||
return null;
|
||||
};
|
||||
|
||||
const autoSetupPoolShape = (value: ComplexityRouterConfigValue): string =>
|
||||
JSON.stringify({
|
||||
tiers: value.tiers,
|
||||
customTierSet: value.custom_tier_set,
|
||||
tierModelParams: value.tier_model_params,
|
||||
});
|
||||
|
||||
export const detachAutoSetupAfterPoolEdit = (
|
||||
previous: ComplexityRouterConfigValue,
|
||||
next: ComplexityRouterConfigValue,
|
||||
): ComplexityRouterConfigValue =>
|
||||
previous.auto_setup && autoSetupPoolShape(previous) !== autoSetupPoolShape(next)
|
||||
? { ...next, auto_setup: undefined }
|
||||
: next;
|
||||
|
|
@ -138,6 +138,7 @@ export interface BuildComplexityRouterConfigParams {
|
|||
tierModelParams?: TierModelParamsByTier;
|
||||
enableContextWindowEscalation?: boolean;
|
||||
contextWindowEscalationBuffer?: number;
|
||||
autoSetup?: AutoSetupConfigPayload;
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -196,6 +197,33 @@ export interface ComplexityRouterConfigPayload {
|
|||
enable_context_window_escalation?: boolean;
|
||||
context_window_escalation_buffer?: number;
|
||||
tier_model_configs?: Record<string, { model_name: string; litellm_params: TierModelParams }[]>;
|
||||
auto_setup?: AutoSetupConfigPayload;
|
||||
}
|
||||
|
||||
export type AutoSetupQualityLevel = "economy" | "balanced" | "high" | "max";
|
||||
export type AutoSetupObjective = "cost" | "task_completion_speed" | "balanced";
|
||||
|
||||
export interface AutoSetupCandidatePayload {
|
||||
model_name: string;
|
||||
benchmark_model_id: string;
|
||||
quality_lower_bound: number;
|
||||
cost_per_completed_task_usd: number;
|
||||
}
|
||||
|
||||
export interface AutoSetupTierPolicyPayload {
|
||||
selection_mode: "snapshot_ranked" | "runtime_response_latency";
|
||||
candidates: AutoSetupCandidatePayload[];
|
||||
cold_start_model?: string | null;
|
||||
evidence_status: string;
|
||||
}
|
||||
|
||||
export interface AutoSetupConfigPayload {
|
||||
schema_version: "1";
|
||||
snapshot_id: string;
|
||||
snapshot_sha256: string;
|
||||
quality_level: AutoSetupQualityLevel;
|
||||
optimize_for: AutoSetupObjective;
|
||||
tier_policies: Record<string, AutoSetupTierPolicyPayload>;
|
||||
}
|
||||
|
||||
export const serializeTierLabels = (tierLabels: ComplexityTierLabels | undefined): ComplexityTierLabels | undefined => {
|
||||
|
|
@ -456,6 +484,7 @@ export const buildComplexityRouterConfig = ({
|
|||
tierModelParams,
|
||||
enableContextWindowEscalation,
|
||||
contextWindowEscalationBuffer,
|
||||
autoSetup,
|
||||
}: BuildComplexityRouterConfigParams): ComplexityRouterConfigPayload => {
|
||||
const serializedTierModelConfigs = customTierSet
|
||||
? serializeTierModelConfigs(
|
||||
|
|
@ -522,6 +551,7 @@ export const buildComplexityRouterConfig = ({
|
|||
...(contextWindowEscalationBuffer !== undefined && {
|
||||
context_window_escalation_buffer: contextWindowEscalationBuffer,
|
||||
}),
|
||||
...(autoSetup && { auto_setup: autoSetup }),
|
||||
...scorerKnobs,
|
||||
};
|
||||
if (!customTierSet) return payload;
|
||||
|
|
|
|||
|
|
@ -553,6 +553,7 @@ describe("managed keys survive an untouched open-and-save", () => {
|
|||
"fallback_tier",
|
||||
"classification_prompt",
|
||||
"hybrid_boundary_margin",
|
||||
"auto_setup",
|
||||
]);
|
||||
|
||||
it("carries every managed key a built-in router can hold through hydrate then save", () => {
|
||||
|
|
@ -565,6 +566,21 @@ describe("managed keys survive an untouched open-and-save", () => {
|
|||
expect(dropped).toEqual([]);
|
||||
});
|
||||
|
||||
it("round-trips the stored Auto policy while its model pools stay unchanged", () => {
|
||||
const autoSetup = {
|
||||
schema_version: "1" as const,
|
||||
snapshot_id: "snapshot-test",
|
||||
snapshot_sha256: "a".repeat(64),
|
||||
quality_level: "max" as const,
|
||||
optimize_for: "cost" as const,
|
||||
tier_policies: {},
|
||||
};
|
||||
const stored = { ...STORED, classifier_type: "heuristic" as const, auto_setup: autoSetup };
|
||||
const hydrated = hydrateComplexityRouterConfig(stored, undefined);
|
||||
|
||||
expect(buildUpdatedComplexityRouterConfig(stored, hydrated).auto_setup).toEqual(autoSetup);
|
||||
});
|
||||
|
||||
it("drops a stored local-scorer threshold when the operator converts the router to custom tiers", () => {
|
||||
const hydrated = hydrateComplexityRouterConfig(STORED_ALL_MANAGED, undefined);
|
||||
const converted = {
|
||||
|
|
|
|||
|
|
@ -38,6 +38,7 @@ import {
|
|||
hydratePlanModeMinTier,
|
||||
hydrateTierLabels,
|
||||
dryRunRejection,
|
||||
type AutoSetupConfigPayload,
|
||||
} from "../add_model/build_complexity_router_config";
|
||||
import { KeywordTierRule } from "../add_model/KeywordTierRules";
|
||||
import { DEFAULT_MATCH_THRESHOLD } from "../add_model/SemanticKeywordMatching";
|
||||
|
|
@ -60,6 +61,7 @@ import ComplexityRouterConfig, {
|
|||
DEFAULT_DEPLOYMENT_AFFINITY,
|
||||
DEFAULT_TIER_DISTANCE_PENALTY,
|
||||
} from "../add_model/ComplexityRouterConfig";
|
||||
import { detachAutoSetupAfterPoolEdit } from "../add_model/auto_setup_policy";
|
||||
import {
|
||||
Dialog,
|
||||
DialogContent,
|
||||
|
|
@ -114,6 +116,7 @@ export interface StoredComplexityRouterConfig {
|
|||
return_raw_model_name?: boolean;
|
||||
enable_context_window_escalation?: unknown;
|
||||
context_window_escalation_buffer?: unknown;
|
||||
auto_setup?: AutoSetupConfigPayload;
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -202,6 +205,7 @@ export const hydrateComplexityRouterConfig = (
|
|||
typeof parsedConfig.context_window_escalation_buffer === "number"
|
||||
? parsedConfig.context_window_escalation_buffer
|
||||
: undefined,
|
||||
auto_setup: parsedConfig.auto_setup,
|
||||
};
|
||||
};
|
||||
|
||||
|
|
@ -238,6 +242,7 @@ export const MANAGED_COMPLEXITY_ROUTER_KEYS = new Set([
|
|||
"reasoning_override_min_score",
|
||||
"enable_context_window_escalation",
|
||||
"context_window_escalation_buffer",
|
||||
"auto_setup",
|
||||
]);
|
||||
|
||||
// Managed only when the caller passes the corresponding state. A caller that does not render
|
||||
|
|
@ -343,6 +348,7 @@ export const buildUpdatedComplexityRouterConfig = (
|
|||
tierModelParams: value.tier_model_params,
|
||||
enableContextWindowEscalation: value.enable_context_window_escalation,
|
||||
contextWindowEscalationBuffer: value.context_window_escalation_buffer,
|
||||
autoSetup: value.auto_setup,
|
||||
};
|
||||
const built = buildComplexityRouterConfig(builderParams);
|
||||
|
||||
|
|
@ -723,7 +729,7 @@ const EditAutoRouterModal: React.FC<EditAutoRouterModalProps> = ({
|
|||
modelInfo={modelInfo}
|
||||
value={complexityRouterConfig}
|
||||
onChange={(config) => {
|
||||
setComplexityRouterConfig(config);
|
||||
setComplexityRouterConfig((previous) => detachAutoSetupAfterPoolEdit(previous, config));
|
||||
}}
|
||||
customTechnicalKeywords={customTechnicalKeywords}
|
||||
onCustomTechnicalKeywordsChange={setCustomTechnicalKeywords}
|
||||
|
|
|
|||
|
|
@ -89,7 +89,11 @@ import type {
|
|||
CoordinationRedisTestResponse,
|
||||
} from "@/app/(dashboard)/caching/_components/coordination_redis_settings/types";
|
||||
import { MCP_TOOLS_PREVIEW_FORBIDDEN_MESSAGE } from "./mcp_tools/constants";
|
||||
import type { ComplexityRouterConfigPayload } from "./add_model/build_complexity_router_config";
|
||||
import type {
|
||||
AutoSetupObjective,
|
||||
AutoSetupQualityLevel,
|
||||
ComplexityRouterConfigPayload,
|
||||
} from "./add_model/build_complexity_router_config";
|
||||
import type { AutoRouterPresetsResponse } from "@/lib/autorouter_presets";
|
||||
import type { VectorStoreIndex } from "@/app/(dashboard)/vector-stores/_components/IndexesTab";
|
||||
import type { RoutingDecision } from "./view_logs/LogDetailsDrawer/RoutingDecisionCard";
|
||||
|
|
@ -2416,6 +2420,30 @@ export interface AutoRouterRoutingTestRequest {
|
|||
team_id?: string;
|
||||
}
|
||||
|
||||
export interface AutoRouterRecommendationResponse {
|
||||
quality_level: AutoSetupQualityLevel;
|
||||
optimize_for: AutoSetupObjective;
|
||||
snapshot_id: string;
|
||||
snapshot_generated_at: string;
|
||||
matched_model_groups: string[];
|
||||
complexity_router_config: ComplexityRouterConfigPayload;
|
||||
}
|
||||
|
||||
export const getAutoRouterRecommendation = async (
|
||||
accessToken: string,
|
||||
qualityLevel: AutoSetupQualityLevel,
|
||||
optimizeFor: AutoSetupObjective,
|
||||
teamId?: string,
|
||||
): Promise<AutoRouterRecommendationResponse> =>
|
||||
await apiClient.get<AutoRouterRecommendationResponse>("/auto_router/recommendation", {
|
||||
accessToken,
|
||||
query: {
|
||||
quality_level: qualityLevel,
|
||||
optimize_for: optimizeFor,
|
||||
...(teamId ? { team_id: teamId } : {}),
|
||||
},
|
||||
});
|
||||
|
||||
export interface AutoRouterRoutingTestResult {
|
||||
routed_model: string;
|
||||
routed_model_configured: boolean;
|
||||
|
|
|
|||
|
|
@ -294,6 +294,7 @@ export const buildPresetPrefill = (
|
|||
return_raw_model_name: config.return_raw_model_name,
|
||||
enable_context_window_escalation: config.enable_context_window_escalation,
|
||||
context_window_escalation_buffer: config.context_window_escalation_buffer,
|
||||
auto_setup: config.auto_setup,
|
||||
},
|
||||
customTechnicalKeywords: config.custom_technical_keywords ?? [],
|
||||
keywordTierRules: hydrateKeywordTierRules(config.keyword_tier_rules ?? []),
|
||||
|
|
|
|||
159
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
159
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
|
|
@ -1225,6 +1225,26 @@ export interface paths {
|
|||
patch?: never;
|
||||
trace?: never;
|
||||
};
|
||||
"/auto_router/recommendation": {
|
||||
parameters: {
|
||||
query?: never;
|
||||
header?: never;
|
||||
path?: never;
|
||||
cookie?: never;
|
||||
};
|
||||
/**
|
||||
* Get Auto Router Recommendation
|
||||
* @description Build an editable four-tier router from only the model groups this caller can use.
|
||||
*/
|
||||
get: operations["get_auto_router_recommendation_auto_router_recommendation_get"];
|
||||
put?: never;
|
||||
post?: never;
|
||||
delete?: never;
|
||||
options?: never;
|
||||
head?: never;
|
||||
patch?: never;
|
||||
trace?: never;
|
||||
};
|
||||
"/auto_router/shadow_eval": {
|
||||
parameters: {
|
||||
query?: never;
|
||||
|
|
@ -23461,6 +23481,29 @@ export interface components {
|
|||
/** Simple */
|
||||
SIMPLE: string[];
|
||||
};
|
||||
/**
|
||||
* AutoRouterRecommendationResponse
|
||||
* @description An editable complexity-router config generated from the caller's usable model groups.
|
||||
*/
|
||||
AutoRouterRecommendationResponse: {
|
||||
complexity_router_config: components["schemas"]["RequestComplexityRouterConfig"];
|
||||
/** Matched Model Groups */
|
||||
matched_model_groups: string[];
|
||||
/**
|
||||
* Optimize For
|
||||
* @enum {string}
|
||||
*/
|
||||
optimize_for: "cost" | "task_completion_speed" | "balanced";
|
||||
/**
|
||||
* Quality Level
|
||||
* @enum {string}
|
||||
*/
|
||||
quality_level: "economy" | "balanced" | "high" | "max";
|
||||
/** Snapshot Generated At */
|
||||
snapshot_generated_at: string;
|
||||
/** Snapshot Id */
|
||||
snapshot_id: string;
|
||||
};
|
||||
/**
|
||||
* AutoRouterRoutingTestRequest
|
||||
* @description A single request to classify against a complexity-router config that need not be saved yet.
|
||||
|
|
@ -23534,6 +23577,67 @@ export interface components {
|
|||
/** @description The decision record this request would have written to its log row */
|
||||
routing_decision: components["schemas"]["StandardLoggingRoutingDecision"];
|
||||
};
|
||||
/**
|
||||
* AutoSetupCandidate
|
||||
* @description One benchmark-backed model group in an Auto-generated tier policy.
|
||||
*/
|
||||
AutoSetupCandidate: {
|
||||
/** Benchmark Model Id */
|
||||
benchmark_model_id: string;
|
||||
/** Cost Per Completed Task Usd */
|
||||
cost_per_completed_task_usd: number;
|
||||
/** Model Name */
|
||||
model_name: string;
|
||||
/** Quality Lower Bound */
|
||||
quality_lower_bound: number;
|
||||
};
|
||||
/**
|
||||
* AutoSetupConfig
|
||||
* @description Provenance and selection policy stamped only by zero-setup Auto creation.
|
||||
*/
|
||||
AutoSetupConfig: {
|
||||
/**
|
||||
* Optimize For
|
||||
* @enum {string}
|
||||
*/
|
||||
optimize_for: "cost" | "task_completion_speed" | "balanced";
|
||||
/**
|
||||
* Quality Level
|
||||
* @enum {string}
|
||||
*/
|
||||
quality_level: "economy" | "balanced" | "high" | "max";
|
||||
/**
|
||||
* Schema Version
|
||||
* @default 1
|
||||
* @constant
|
||||
*/
|
||||
schema_version: "1";
|
||||
/** Snapshot Id */
|
||||
snapshot_id: string;
|
||||
/** Snapshot Sha256 */
|
||||
snapshot_sha256: string;
|
||||
/** Tier Policies */
|
||||
tier_policies: {
|
||||
[key: string]: components["schemas"]["AutoSetupTierPolicy"];
|
||||
};
|
||||
};
|
||||
/**
|
||||
* AutoSetupTierPolicy
|
||||
* @description How an Auto-generated tier chooses from its quality-admitted model groups.
|
||||
*/
|
||||
AutoSetupTierPolicy: {
|
||||
/** Candidates */
|
||||
candidates: components["schemas"]["AutoSetupCandidate"][];
|
||||
/** Cold Start Model */
|
||||
cold_start_model?: string | null;
|
||||
/** Evidence Status */
|
||||
evidence_status: string;
|
||||
/**
|
||||
* Selection Mode
|
||||
* @enum {string}
|
||||
*/
|
||||
selection_mode: "snapshot_ranked" | "runtime_response_latency";
|
||||
};
|
||||
/** BaseLitellmParams */
|
||||
BaseLitellmParams: {
|
||||
/**
|
||||
|
|
@ -34598,6 +34702,8 @@ export interface components {
|
|||
adaptive_eligible: "all" | "classified_tier";
|
||||
/** @description Quality vs cost weights for adaptive selection (used when adaptive=True) */
|
||||
adaptive_weights?: components["schemas"]["AdaptiveRouterWeights"];
|
||||
/** @description Benchmark snapshot and per-tier selection policy generated by the zero-setup Auto flow. Absent on manually configured routers, whose pool selection behavior is unchanged */
|
||||
auto_setup?: components["schemas"]["AutoSetupConfig"] | null;
|
||||
/**
|
||||
* Classification Mode
|
||||
* @description When to run the complexity classifier. 'every_request' (the default) classifies every inference request, including the tool-result continuation turns of an agentic loop. 'user_turn' classifies only requests whose newest turn is a new human ask and replays the session's held routing decision on continuation turns, which cuts classifier spend and eliminates mid-loop model switches. Continuations with no held decision to replay (no resolvable session_id, expired pin, fresh restart) still classify. Unlike session_affinity, a new human ask always re-classifies, so a session can still move tiers between asks. Suppressed when plugins are configured, for the same reason session_affinity is: a replayed decision would bypass the plugin pipeline.
|
||||
|
|
@ -35954,6 +36060,23 @@ export interface components {
|
|||
* @description Per-request provenance for a pre-routing strategy (auto-router) decision.
|
||||
*/
|
||||
StandardLoggingRoutingDecision: {
|
||||
/**
|
||||
* Auto Setup Optimize For
|
||||
* @enum {string}
|
||||
*/
|
||||
auto_setup_optimize_for?: "cost" | "task_completion_speed" | "balanced";
|
||||
/**
|
||||
* Auto Setup Quality Level
|
||||
* @enum {string}
|
||||
*/
|
||||
auto_setup_quality_level?: "economy" | "balanced" | "high" | "max";
|
||||
/**
|
||||
* Auto Setup Selection Mode
|
||||
* @enum {string}
|
||||
*/
|
||||
auto_setup_selection_mode?: "snapshot_ranked" | "runtime_response_latency";
|
||||
/** Auto Setup Snapshot Id */
|
||||
auto_setup_snapshot_id?: string;
|
||||
/**
|
||||
* Cause
|
||||
* @enum {string}
|
||||
|
|
@ -41152,6 +41275,42 @@ export interface operations {
|
|||
};
|
||||
};
|
||||
};
|
||||
get_auto_router_recommendation_auto_router_recommendation_get: {
|
||||
parameters: {
|
||||
query?: {
|
||||
/** @description How close every admitted model must be to the best available quality score */
|
||||
quality_level?: "economy" | "balanced" | "high" | "max";
|
||||
/** @description What to minimize after the selected quality gate is applied */
|
||||
optimize_for?: "cost" | "task_completion_speed" | "balanced";
|
||||
/** @description Team whose model access the recommendation must use */
|
||||
team_id?: string | null;
|
||||
};
|
||||
header?: never;
|
||||
path?: never;
|
||||
cookie?: never;
|
||||
};
|
||||
requestBody?: never;
|
||||
responses: {
|
||||
/** @description Successful Response */
|
||||
200: {
|
||||
headers: {
|
||||
[name: string]: unknown;
|
||||
};
|
||||
content: {
|
||||
"application/json": components["schemas"]["AutoRouterRecommendationResponse"];
|
||||
};
|
||||
};
|
||||
/** @description Validation Error */
|
||||
422: {
|
||||
headers: {
|
||||
[name: string]: unknown;
|
||||
};
|
||||
content: {
|
||||
"application/json": components["schemas"]["HTTPValidationError"];
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
list_shadow_eval_jobs_auto_router_shadow_eval_get: {
|
||||
parameters: {
|
||||
query?: {
|
||||
|
|
|
|||
20
ui/litellm-dashboard/tests/mocks/autoRouterRecommendation.ts
Normal file
20
ui/litellm-dashboard/tests/mocks/autoRouterRecommendation.ts
Normal file
|
|
@ -0,0 +1,20 @@
|
|||
import { vi } from "vitest";
|
||||
import type { AutoRouterRecommendationResponse } from "@/components/networking";
|
||||
|
||||
interface AutoRouterRecommendationQuery {
|
||||
data: AutoRouterRecommendationResponse | undefined;
|
||||
isPending: boolean;
|
||||
isError: boolean;
|
||||
error: unknown;
|
||||
refetch: ReturnType<typeof vi.fn>;
|
||||
}
|
||||
|
||||
export const EMPTY_RECOMMENDATION_QUERY: AutoRouterRecommendationQuery = {
|
||||
data: undefined,
|
||||
isPending: false,
|
||||
isError: false,
|
||||
error: null,
|
||||
refetch: vi.fn(),
|
||||
};
|
||||
|
||||
export const useAutoRouterRecommendation = vi.fn((): AutoRouterRecommendationQuery => EMPTY_RECOMMENDATION_QUERY);
|
||||
Loading…
Add table
Reference in a new issue