feat(router): add zero-setup auto configuration

This commit is contained in:
moe-berri 2026-09-03 14:26:25 -07:00
parent 321636ef5d
commit 591f8aef1c
27 changed files with 9144 additions and 101 deletions

View file

@ -860,6 +860,7 @@ class LiteLLMRoutes(enum.Enum):
# proxy admin, or team admin naming their own team via team_id
"/auto_router/test_routing",
"/auto_router/validate_complexity_router_config",
"/auto_router/recommendation",
# Agent registry - reads are role-scoped and writes are proxy-admin-gated
# inside agent_endpoints/endpoints.py
*agent_management_routes,

View file

@ -3,6 +3,7 @@ AUTO ROUTER MANAGEMENT ENDPOINTS
POST /auto_router/test_routing - Route one request through an unsaved complexity-router config
POST /auto_router/validate_complexity_router_config - Dry-run the complexity-router write gate without saving
GET /auto_router/recommendation - Build an editable config from the caller's available model groups
"""
from collections.abc import Mapping, Sequence
@ -10,7 +11,7 @@ from datetime import datetime, timedelta, timezone
from itertools import chain, groupby
from operator import attrgetter
from types import MappingProxyType
from typing import TYPE_CHECKING, Annotated, Final, Protocol
from typing import TYPE_CHECKING, Annotated, Final, Literal, Protocol, cast
from uuid import uuid4
from pydantic import BaseModel, ConfigDict, TypeAdapter, field_validator
@ -52,6 +53,7 @@ from litellm.types.management_endpoints.auto_router_endpoints import (
AutoRouterBenchmarkTotals,
AutoRouterCacheBucket,
AutoRouterCacheStats,
AutoRouterRecommendationResponse,
AutoRouterRoutingTestRequest,
AutoRouterRoutingTestResponse,
ComplexityRouterConfigValidationRequest,
@ -334,6 +336,117 @@ async def validate_complexity_router_config(
return ComplexityRouterConfigValidationResponse(valid=error is None, error=error)
def _available_model_refs(
llm_router: "Router", model_names: Sequence[str], team_id: str | None
) -> Mapping[str, tuple[str, ...]]:
refs: Final[dict[str, tuple[str, ...]]] = {}
for model_name in model_names:
deployments = llm_router.get_model_list(model_name=model_name, team_id=team_id) or ()
resolved: list[str] = [model_name]
for raw_deployment in deployments:
deployment = cast(Mapping[str, object], raw_deployment)
litellm_params = deployment.get("litellm_params")
model_info = deployment.get("model_info")
if isinstance(litellm_params, Mapping):
typed_params = cast(Mapping[str, object], litellm_params)
resolved.extend(
value
for key in ("model", "base_model")
if isinstance(value := typed_params.get(key), str) and value
)
if isinstance(model_info, Mapping):
base_model = cast(Mapping[str, object], model_info).get("base_model")
if isinstance(base_model, str) and base_model:
resolved.append(base_model)
refs[model_name] = tuple(dict.fromkeys(resolved))
return MappingProxyType(refs)
@router.get(
"/auto_router/recommendation",
tags=["model management"], # mutable-ok: fastapi's decorator signature types tags as a list
dependencies=[Depends(user_api_key_auth)], # mutable-ok: fastapi's decorator signature types dependencies as a list
response_model=AutoRouterRecommendationResponse,
status_code=status.HTTP_200_OK,
)
async def get_auto_router_recommendation(
user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)],
quality_level: Annotated[
Literal["economy", "balanced", "high", "max"],
Query(description="How close every admitted model must be to the best available quality score"),
] = "balanced",
optimize_for: Annotated[
Literal["cost", "task_completion_speed", "balanced"],
Query(description="What to minimize after the selected quality gate is applied"),
] = "balanced",
team_id: Annotated[str | None, Query(description="Team whose model access the recommendation must use")] = None,
) -> AutoRouterRecommendationResponse:
"""Build an editable four-tier router from only the model groups this caller can use."""
from litellm.proxy.proxy_server import (
general_settings,
llm_router,
prisma_client,
proxy_logging_obj,
user_api_key_cache,
user_model,
)
from litellm.proxy.utils import get_available_models_for_user
from litellm.router_strategy.complexity_router.auto_setup import (
build_auto_setup_config,
load_auto_router_snapshot,
)
await _authorize_router_dry_run(user_api_key_dict=user_api_key_dict, team_id=team_id)
if llm_router is None:
raise HTTPException(
status_code=500,
detail={"error": CommonProxyErrors.no_llm_router.value}, # mutable-ok: HTTPException detail is JSON-shaped
)
available_models: Final = await get_available_models_for_user(
user_api_key_dict=user_api_key_dict,
llm_router=llm_router,
general_settings=general_settings,
user_model=user_model,
prisma_client=prisma_client,
proxy_logging_obj=proxy_logging_obj,
team_id=team_id,
user_api_key_cache=user_api_key_cache,
)
snapshot: Final = load_auto_router_snapshot()
try:
config: Final = build_auto_setup_config(
snapshot=snapshot,
available_model_refs=_available_model_refs(llm_router, available_models, team_id),
quality_level=quality_level,
optimize_for=optimize_for,
)
except ValueError as exc:
raise HTTPException(
status_code=400,
detail={
"error": f"Could not build an Auto setup: {exc}"
}, # mutable-ok: HTTPException detail is JSON-shaped
) from exc
matched: Final = tuple(
dict.fromkeys(
model
for tier_models in config.tiers.values()
for model in ((tier_models,) if isinstance(tier_models, str) else tier_models)
)
)
return AutoRouterRecommendationResponse(
quality_level=quality_level,
optimize_for=optimize_for,
snapshot_id=snapshot.snapshot_id,
snapshot_generated_at=snapshot.generated_at,
matched_model_groups=matched,
complexity_router_config=RequestComplexityRouterConfig.model_validate(
config.model_dump(mode="json", exclude_none=True)
),
)
@router.post(
"/auto_router/test_routing",
tags=["model management"], # mutable-ok: fastapi's decorator signature types tags as a list

View file

@ -7865,6 +7865,12 @@ class Router:
# WS session wrappers fire with result=None; per-turn costs tracked by inner calls.
if kwargs.get("call_type") in ("_aresponses_websocket", "_arealtime"):
return
await self._record_auto_setup_response_latency(
kwargs=kwargs,
completion_response=completion_response,
start_time=start_time,
end_time=end_time,
)
standard_logging_object: Final[StandardLoggingPayload | None] = kwargs.get("standard_logging_object", None)
if standard_logging_object is None:
raise ValueError("standard_logging_object is None")
@ -7976,6 +7982,55 @@ class Router:
"litellm.router.Router::deployment_callback_on_success(): Exception occured - %s", e
)
async def _record_auto_setup_response_latency(
self,
*,
kwargs: Mapping[str, object],
completion_response: object,
start_time: object,
end_time: object,
) -> None:
raw_metadata = cast(object, get_litellm_metadata_from_kwargs(dict(kwargs)))
if not isinstance(raw_metadata, Mapping):
return
metadata = cast(Mapping[str, object], raw_metadata)
raw_decision = metadata.get("routing_decision")
if not isinstance(raw_decision, Mapping):
return
decision = cast(Mapping[str, object], raw_decision)
if decision.get("auto_setup_selection_mode") != "runtime_response_latency":
return
router_model_name = decision.get("router_model_name")
tier = decision.get("tier")
routed_model = decision.get("routed_model")
if not isinstance(router_model_name, str) or not router_model_name:
return
if not isinstance(tier, str) or not tier:
return
if not isinstance(routed_model, str) or not routed_model:
return
registered: Final = self.complexity_routers.get(router_model_name)
if not registered or not any(tagged.strategy.config.auto_setup is not None for tagged in registered):
return
from litellm.router_strategy.complexity_router.response_latency import (
record_runtime_response_latency,
)
try:
await record_runtime_response_latency(
router_cache=self.cache,
router_model_name=router_model_name,
tier=tier,
routed_model=routed_model,
kwargs=kwargs,
response_obj=completion_response,
start_time=start_time,
end_time=end_time,
)
except Exception as exc: # noqa: BLE001 -- optional telemetry must never fail a successful completion
verbose_router_logger.warning("Could not record Auto setup response latency: %s", exc)
def sync_deployment_callback_on_success(
self,
kwargs, # kwargs to completion

View file

@ -68,6 +68,47 @@ still resolve to a deployment in `model_list`; this configuration does not creat
- abc
```
### Zero-setup Auto configuration
The dashboard's Add Router flow can generate the four tiers from the model groups the
current admin or team can actually use. Auto setup asks for two product choices:
- Quality level: Economy, Balanced, High, or Max
- Optimize for: Cost, Task completion speed, or Balanced
The quality level is a gate, not an optimization weight. For every complexity tier,
LiteLLM finds the best available model's conservative LiveBench quality score and admits
only models within the selected maximum regret: 15 percentage points for Economy, 7 for
Balanced, 3 for High, and 1 for Max. The objective then ranks only those admitted models.
This means Max still sends trivial work to a smaller model when that model is within one
point of the best available quality for trivial work.
Cost is benchmark cost per completed task, not input or output token price. It is observed
request dollars divided by quality-adjusted completed questions in the same complexity
bucket, with partial benchmark credit counted as a fractional completion.
Task completion speed uses two forms of evidence:
- SIMPLE and MEDIUM tiers learn equal-output response time from this proxy's successful
requests: time to first token plus the time to generate the expected number of visible
output tokens. Reasoning tokens are excluded. The policy explores every quality-admitted
model twice before using a rolling seven-day sample
- COMPLEX and REASONING tiers use retry-adjusted Terminal-Bench wall-clock time to a
successful task. Results are compared only inside the same agent and benchmark cohort
and fall back to cost when comparable speed evidence is unavailable
Balanced minimizes the combined log-normalized cost and completion-time scores after the
same quality gate. The recommendation stores its snapshot ID, objective, admitted models,
and rank order directly in the router configuration, so the generated result is visible
and editable. Changing a generated tier's model pool in the dashboard removes that Auto
selection policy and returns the tier to normal manual pool selection.
These rules apply only when a router is created through Auto setup. Existing routers,
manual configurations, and LiteLLM's deployment-level latency routing are unchanged. The
bundled benchmark data is a versioned snapshot: creating or regenerating a router uses the
snapshot shipped with that LiteLLM version, while an already saved router keeps its stored
policy.
### Heuristic v2
Set `classifier_type: heuristic_v2` to classify with the bundled calibrated

File diff suppressed because it is too large Load diff

View file

@ -0,0 +1,378 @@
from __future__ import annotations
import hashlib
import json
import math
import re
from collections.abc import Mapping, Sequence
from dataclasses import dataclass
from functools import lru_cache
from importlib.resources import files
from typing import Final, Literal, cast
from pydantic import BaseModel, ConfigDict, Field, model_validator
from .config import (
AutoSetupCandidate,
AutoSetupConfig,
AutoSetupObjective,
AutoSetupQualityLevel,
AutoSetupTierPolicy,
ComplexityRouterConfig,
)
SnapshotComplexity = Literal["trivial", "simple", "standard", "complex"]
_SNAPSHOT_FILENAME: Final = "auto_router_snapshot_v0.json"
_TIER_COMPLEXITIES: Final[Mapping[str, SnapshotComplexity]] = {
"SIMPLE": "trivial",
"MEDIUM": "simple",
"COMPLEX": "standard",
"REASONING": "complex",
}
_EASY_COMPLEXITIES: Final = frozenset(("trivial", "simple"))
_VERSION_SEPARATOR: Final = re.compile(r"(?<=\d)\.(?=\d)")
class SnapshotQualityTier(BaseModel):
model_config = ConfigDict(extra="forbid", frozen=True)
label: str
maximum_quality_regret: float = Field(ge=0, le=1)
class SnapshotIdentity(BaseModel):
model_config = ConfigDict(extra="forbid", frozen=True)
benchmark_model_id: str
family_id: str
is_routable: bool
litellm_model_keys: tuple[str, ...]
reasoning_effort: str | None
required_parameters: Mapping[str, object]
class SnapshotQualityProfile(BaseModel):
model_config = ConfigDict(extra="forbid", frozen=True)
benchmark_model_id: str
complexity: SnapshotComplexity
cost_per_completed_task_usd: float = Field(ge=0)
mean_request_cost_usd: float = Field(ge=0)
quality_lower_bound: float = Field(ge=0, le=1)
quality_score: float = Field(ge=0, le=1)
question_count: int = Field(gt=0)
source_id: str
subtask_count: int = Field(gt=0)
class SnapshotSpeedProfile(BaseModel):
model_config = ConfigDict(extra="forbid", frozen=True)
attempt_count: int = Field(gt=0)
benchmark_model_id: str
cohort_id: str
complexity: SnapshotComplexity
cost_per_completed_task_usd: float | None = Field(default=None, ge=0)
excluded_infrastructure_trials: int = Field(ge=0)
mean_attempt_cost_usd: float | None = Field(default=None, ge=0)
observed_duration_p50_ms: float = Field(ge=0)
observed_duration_p95_ms: float = Field(ge=0)
quality_lower_bound: float = Field(ge=0, le=1)
retry_adjusted_completion_ms: float | None = Field(default=None, ge=0)
source_id: str
success_probability: float = Field(ge=0, le=1)
task_count: int = Field(gt=0)
terminal_bench_model_id: str
class SnapshotModel(BaseModel):
model_config = ConfigDict(extra="forbid", frozen=True)
benchmark_model_id: str
identity: SnapshotIdentity
quality_by_complexity: Mapping[SnapshotComplexity, SnapshotQualityProfile]
task_completion_speed_by_complexity: Mapping[SnapshotComplexity, tuple[SnapshotSpeedProfile, ...]]
class AutoRouterSnapshot(BaseModel):
model_config = ConfigDict(extra="forbid", frozen=True)
activation_scope: Mapping[str, object]
artifact_sha256: str = Field(pattern=r"^[0-9a-f]{64}$")
complexity_policy: Mapping[str, object]
definitions: Mapping[str, str]
generated_at: str
limitations: tuple[str, ...]
models: Mapping[str, SnapshotModel]
provenance: Mapping[str, object]
quality_tiers: Mapping[AutoSetupQualityLevel, SnapshotQualityTier]
reference_routes: Mapping[str, object]
schema_version: Literal["2.0"]
snapshot_id: str
task_complexities: Mapping[str, Mapping[str, SnapshotComplexity]]
@model_validator(mode="after")
def _validate_policy(self) -> AutoRouterSnapshot:
if frozenset(self.quality_tiers) != frozenset(("economy", "balanced", "high", "max")):
raise ValueError("Auto Router snapshot must define economy, balanced, high, and max quality levels")
if self.activation_scope.get("setup_mode") != "auto":
raise ValueError("Auto Router snapshot is not scoped to Auto setup")
return self
@dataclass(frozen=True, slots=True)
class _AvailableProfile:
model_name: str
benchmark_model_id: str
quality: SnapshotQualityProfile
speed: tuple[SnapshotSpeedProfile, ...]
required_parameters: Mapping[str, object]
def _canonical_json(value: object) -> str:
return json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=False)
@lru_cache(maxsize=1)
def load_auto_router_snapshot() -> AutoRouterSnapshot:
parsed = cast(
object,
json.loads(
files("litellm.router_strategy.complexity_router")
.joinpath("artifacts", _SNAPSHOT_FILENAME)
.read_text(encoding="utf-8")
),
)
if not isinstance(parsed, dict):
raise ValueError("Auto Router snapshot must be a JSON object")
untyped = cast(dict[object, object], parsed)
if not all(isinstance(key, str) for key in untyped):
raise ValueError("Auto Router snapshot keys must be strings")
raw = cast(dict[str, object], untyped)
expected = raw.get("artifact_sha256")
if not isinstance(expected, str):
raise ValueError("Auto Router snapshot must carry an artifact_sha256")
unsigned: Final = {key: value for key, value in raw.items() if key != "artifact_sha256"}
actual: Final = hashlib.sha256(_canonical_json(unsigned).encode("utf-8")).hexdigest()
if expected != actual:
raise ValueError(f"Auto Router snapshot hash mismatch: expected {expected}, calculated {actual}")
return AutoRouterSnapshot.model_validate(raw)
def _normalized_model_key(value: str) -> str:
return _VERSION_SEPARATOR.sub("-", value.strip().casefold())
def _matching_snapshot_models(snapshot: AutoRouterSnapshot, model_refs: Sequence[str]) -> tuple[SnapshotModel, ...]:
normalized_refs: Final = frozenset(_normalized_model_key(ref) for ref in model_refs)
return tuple(
model
for model in snapshot.models.values()
if model.identity.is_routable
and normalized_refs.intersection(_normalized_model_key(key) for key in model.identity.litellm_model_keys)
)
def _available_profiles(
snapshot: AutoRouterSnapshot,
available_model_refs: Mapping[str, Sequence[str]],
complexity: SnapshotComplexity,
) -> tuple[_AvailableProfile, ...]:
profiles: Final[list[_AvailableProfile]] = []
for model_name, refs in available_model_refs.items():
matches = _matching_snapshot_models(snapshot, (model_name, *refs))
if not matches:
continue
best = min(
matches,
key=lambda model: (
-model.quality_by_complexity[complexity].quality_lower_bound,
model.quality_by_complexity[complexity].cost_per_completed_task_usd,
model.benchmark_model_id,
),
)
profiles.append(
_AvailableProfile(
model_name=model_name,
benchmark_model_id=best.benchmark_model_id,
quality=best.quality_by_complexity[complexity],
speed=best.task_completion_speed_by_complexity[complexity],
required_parameters=best.identity.required_parameters,
)
)
return tuple(profiles)
def _cost_rank(profiles: Sequence[_AvailableProfile]) -> tuple[_AvailableProfile, ...]:
return tuple(
sorted(
profiles,
key=lambda profile: (
profile.quality.cost_per_completed_task_usd,
-profile.quality.quality_lower_bound,
profile.model_name,
),
)
)
def _normalize_log(value: float, values: Sequence[float]) -> float:
logged: Final = tuple(math.log(max(item, 1e-12)) for item in values)
low: Final = min(logged)
high: Final = max(logged)
if low == high:
return 0.0
return (math.log(max(value, 1e-12)) - low) / (high - low)
def _hard_task_rank(
profiles: Sequence[_AvailableProfile], objective: AutoSetupObjective
) -> tuple[tuple[_AvailableProfile, ...], str]:
cost_ranked: Final = _cost_rank(profiles)
speed_profiles: Final = tuple(
(profile, speed)
for profile in profiles
for speed in profile.speed
if speed.retry_adjusted_completion_ms is not None
)
if not speed_profiles:
return cost_ranked, "fallback_no_speed_evidence"
cohorts: dict[str, dict[str, tuple[_AvailableProfile, SnapshotSpeedProfile]]] = {}
for profile, speed in speed_profiles:
by_model = cohorts.setdefault(speed.cohort_id, {})
current = by_model.get(profile.model_name)
if current is None or speed.attempt_count > current[1].attempt_count:
by_model[profile.model_name] = (profile, speed)
measured_profiles: Final = tuple({profile.model_name: profile for profile, _ in speed_profiles}.values())
anchor: Final = max(
measured_profiles,
key=lambda profile: (profile.quality.quality_lower_bound, -profile.quality.cost_per_completed_task_usd),
)
anchor_cohorts: Final = tuple(cohort_id for cohort_id, by_model in cohorts.items() if anchor.model_name in by_model)
selected_cohort = max(
anchor_cohorts,
key=lambda cohort_id: (
len(cohorts[cohort_id]),
sum(speed.attempt_count for _, speed in cohorts[cohort_id].values()),
cohort_id,
),
)
comparable = tuple(cohorts[selected_cohort].values())
if objective == "task_completion_speed":
return tuple(
profile
for profile, _ in sorted(
comparable,
key=lambda item: (
item[1].retry_adjusted_completion_ms or math.inf,
-item[0].quality.quality_lower_bound,
item[0].model_name,
),
)
), "cohort_scoped"
balanced_comparable = tuple(item for item in comparable if item[1].cost_per_completed_task_usd is not None)
if not balanced_comparable:
return cost_ranked, "fallback_no_comparable_cost_evidence"
costs: Final = tuple(cast(float, speed.cost_per_completed_task_usd) for _, speed in balanced_comparable)
times: Final = tuple(cast(float, speed.retry_adjusted_completion_ms) for _, speed in balanced_comparable)
return tuple(
profile
for profile, _ in sorted(
balanced_comparable,
key=lambda item: (
math.hypot(
_normalize_log(cast(float, item[1].cost_per_completed_task_usd), costs),
_normalize_log(cast(float, item[1].retry_adjusted_completion_ms), times),
),
-item[0].quality.quality_lower_bound,
item[0].model_name,
),
)
), "cohort_scoped"
def _policy_candidate(profile: _AvailableProfile) -> AutoSetupCandidate:
return AutoSetupCandidate(
model_name=profile.model_name,
benchmark_model_id=profile.benchmark_model_id,
quality_lower_bound=profile.quality.quality_lower_bound,
cost_per_completed_task_usd=profile.quality.cost_per_completed_task_usd,
)
def _tier_policy(
profiles: Sequence[_AvailableProfile],
complexity: SnapshotComplexity,
objective: AutoSetupObjective,
) -> AutoSetupTierPolicy:
cost_ranked: Final = _cost_rank(profiles)
if objective == "cost":
return AutoSetupTierPolicy(
selection_mode="snapshot_ranked",
candidates=tuple(_policy_candidate(profile) for profile in cost_ranked),
evidence_status="direct",
)
if complexity in _EASY_COMPLEXITIES:
return AutoSetupTierPolicy(
selection_mode="runtime_response_latency",
candidates=tuple(_policy_candidate(profile) for profile in cost_ranked),
cold_start_model=cost_ranked[0].model_name,
evidence_status="runtime_response_speed_required",
)
ranked, evidence_status = _hard_task_rank(profiles, objective)
return AutoSetupTierPolicy(
selection_mode="snapshot_ranked",
candidates=tuple(_policy_candidate(profile) for profile in ranked),
evidence_status=evidence_status,
)
def build_auto_setup_config(
*,
snapshot: AutoRouterSnapshot,
available_model_refs: Mapping[str, Sequence[str]],
quality_level: AutoSetupQualityLevel,
optimize_for: AutoSetupObjective,
) -> ComplexityRouterConfig:
"""Recompute quality gates and rankings over only the caller's available groups."""
maximum_regret: Final = snapshot.quality_tiers[quality_level].maximum_quality_regret
policies: Final[dict[str, AutoSetupTierPolicy]] = {}
tiers: Final[dict[str, list[str]]] = {}
tier_model_configs: Final[dict[str, list[dict[str, object]]]] = {}
for tier_name, complexity in _TIER_COMPLEXITIES.items():
profiles = _available_profiles(snapshot, available_model_refs, complexity)
if not profiles:
raise ValueError(f"None of the available model groups has {complexity} evidence in this snapshot")
best_quality = max(profile.quality.quality_lower_bound for profile in profiles)
floor = max(0.0, best_quality - maximum_regret)
admitted = tuple(profile for profile in profiles if profile.quality.quality_lower_bound >= floor)
policy = _tier_policy(admitted, complexity, optimize_for)
policies[tier_name] = policy
tiers[tier_name] = [candidate.model_name for candidate in policy.candidates]
parameterized: list[dict[str, object]] = [
{
"model_name": profile.model_name,
"litellm_params": dict(profile.required_parameters),
}
for profile in admitted
if profile.model_name in frozenset(tiers[tier_name]) and profile.required_parameters
]
if parameterized:
tier_model_configs[tier_name] = parameterized
return ComplexityRouterConfig.model_validate(
{
"tiers": tiers,
"tier_model_configs": tier_model_configs,
"classifier_type": "heuristic_v2",
"auto_setup": AutoSetupConfig(
snapshot_id=snapshot.snapshot_id,
snapshot_sha256=snapshot.artifact_sha256,
quality_level=quality_level,
optimize_for=optimize_for,
tier_policies=policies,
).model_dump(mode="json"),
}
)

View file

@ -1322,6 +1322,13 @@ class ComplexityRouter(CustomLogger):
if tier is not None:
tier_name: Final = _tier_name(tier)
decision["tier"] = tier_name
if self.config.auto_setup is not None:
auto_policy: Final = self.config.auto_setup.tier_policies.get(tier_name)
if auto_policy is not None:
decision["auto_setup_snapshot_id"] = self.config.auto_setup.snapshot_id
decision["auto_setup_quality_level"] = self.config.auto_setup.quality_level
decision["auto_setup_optimize_for"] = self.config.auto_setup.optimize_for
decision["auto_setup_selection_mode"] = auto_policy.selection_mode
if not self.config.has_custom_tiers:
label = self.config.tier_label(ComplexityTier(tier_name))
if label != tier_name:
@ -1815,6 +1822,37 @@ class ComplexityRouter(CustomLogger):
raise ValueError(f"Empty model pool for tier {tier_key}")
return random.choice(model)
async def _pick_from_auto_setup_policy(self, tier_key: str, pool: Sequence[str]) -> str | None:
setup: Final = self.config.auto_setup
if setup is None:
return None
policy: Final = setup.tier_policies.get(tier_key)
if policy is None:
return None
available: Final = tuple(dict.fromkeys(pool))
if not available:
raise ValueError(f"No models configured for tier {tier_key}")
ranked: Final = tuple(
candidate.model_name for candidate in policy.candidates if candidate.model_name in frozenset(available)
)
if policy.selection_mode == "snapshot_ranked":
return ranked[0] if ranked else available[0]
from .response_latency import select_runtime_response_model
cold_start: Final = policy.cold_start_model
if cold_start is None:
raise ValueError(f"Auto setup runtime response-latency policy for {tier_key} has no cold-start model")
return await select_runtime_response_model(
router_cache=self.litellm_router_instance.cache,
router_model_name=self.model_name,
tier=tier_key,
candidates=policy.candidates,
available_models=available,
cold_start_model=cold_start,
objective=setup.optimize_for,
)
def _tier_pools(self) -> dict[str, list[str]]:
return {tier: (models if isinstance(models, list) else [models]) for tier, models in self.config.tiers.items()}
@ -1826,17 +1864,25 @@ class ComplexityRouter(CustomLogger):
request_kwargs: dict,
allowed_models: tuple[str, ...] | None = None,
) -> str:
tier_key = _tier_name(tier)
if not self.config.plugins:
if allowed_models is not None:
return self._pick_from_tier_value(allowed_models, _tier_name(tier))
return self.get_model_for_tier(tier)
configured = self.config.tiers.get(tier_key)
if configured is None:
return self.get_model_for_tier(tier)
full_pool = (configured,) if isinstance(configured, str) else tuple(configured)
pool = (
tuple(model for model in full_pool if model in allowed_models)
if allowed_models is not None
else full_pool
)
auto_pick: Final = await self._pick_from_auto_setup_policy(tier_key, pool)
return auto_pick if auto_pick is not None else self._pick_from_tier_value(pool, tier_key)
from litellm.types.router import RoutingContext
tier_key: Final = _tier_name(tier)
metadata_key: Final = get_metadata_variable_name_from_kwargs(request_kwargs)
full_pool: Final = tuple(self._tier_pools().get(tier_key, ()))
pool: Final = (
full_pool = tuple(self._tier_pools().get(tier_key, ()))
pool = (
tuple(model for model in full_pool if model in allowed_models) if allowed_models is not None else full_pool
)
if not pool:
@ -1860,7 +1906,8 @@ class ComplexityRouter(CustomLogger):
# silently bypassed. Raise instead, matching the Router-level plugin
# pipeline's own fail-closed behavior for the same situation.
raise ValueError(f"No candidate models left for tier {tier_key} after routing-plugin filtering")
return self._pick_from_tier_value(context.candidate_models, tier_key)
auto_pick: Final = await self._pick_from_auto_setup_policy(tier_key, context.candidate_models)
return auto_pick if auto_pick is not None else self._pick_from_tier_value(context.candidate_models, tier_key)
def _ensure_adaptive_router(self) -> Any | None:
if not self.config.adaptive:

View file

@ -27,6 +27,81 @@ class ComplexityTier(str, Enum):
REASONING = "REASONING"
AutoSetupQualityLevel = Literal["economy", "balanced", "high", "max"]
AutoSetupObjective = Literal["cost", "task_completion_speed", "balanced"]
AutoSetupSelectionMode = Literal["snapshot_ranked", "runtime_response_latency"]
class AutoSetupCandidate(BaseModel):
"""One benchmark-backed model group in an Auto-generated tier policy."""
model_config = ConfigDict(extra="forbid", frozen=True)
model_name: str = Field(min_length=1)
benchmark_model_id: str = Field(min_length=1)
quality_lower_bound: float = Field(ge=0, le=1)
cost_per_completed_task_usd: float = Field(ge=0)
class AutoSetupTierPolicy(BaseModel):
"""How an Auto-generated tier chooses from its quality-admitted model groups."""
model_config = ConfigDict(extra="forbid", frozen=True)
selection_mode: AutoSetupSelectionMode
candidates: tuple[AutoSetupCandidate, ...] = Field(min_length=1)
cold_start_model: str | None = None
evidence_status: str = Field(min_length=1)
@model_validator(mode="after")
def _validate_candidates(self) -> "AutoSetupTierPolicy":
names: Final = tuple(candidate.model_name for candidate in self.candidates)
if len(names) != len(frozenset(names)):
raise ValueError("Auto setup tier candidates must use distinct model_name values")
if self.selection_mode == "runtime_response_latency":
if self.cold_start_model not in names:
raise ValueError("A runtime response-latency policy needs a cold_start_model in candidates")
elif self.cold_start_model is not None:
raise ValueError("cold_start_model is only valid for runtime_response_latency")
return self
class AutoSetupConfig(BaseModel):
"""Provenance and selection policy stamped only by zero-setup Auto creation."""
model_config = ConfigDict(extra="forbid", frozen=True)
schema_version: Literal["1"] = "1"
snapshot_id: str = Field(min_length=1)
snapshot_sha256: str = Field(pattern=r"^[0-9a-f]{64}$")
quality_level: AutoSetupQualityLevel
optimize_for: AutoSetupObjective
tier_policies: Mapping[str, AutoSetupTierPolicy]
@field_validator("tier_policies", mode="before")
@classmethod
def _freeze_tier_policies(cls, value: Mapping[str, object]) -> Mapping[str, object]:
return MappingProxyType(dict(value))
@field_serializer("tier_policies")
def _serialize_tier_policies(self, value: Mapping[str, AutoSetupTierPolicy]) -> Mapping[str, AutoSetupTierPolicy]:
return dict(value) # mutable-ok: Pydantic JSON serialization requires a concrete mapping
@model_validator(mode="after")
def _validate_tier_policies(self) -> "AutoSetupConfig":
expected: Final = frozenset(tier.value for tier in TIER_SEVERITY_ORDER)
actual: Final = frozenset(self.tier_policies)
if actual != expected:
raise ValueError(
f"Auto setup tier_policies must define exactly {', '.join(tier.value for tier in TIER_SEVERITY_ORDER)}"
)
if self.optimize_for == "cost" and any(
policy.selection_mode != "snapshot_ranked" for policy in self.tier_policies.values()
):
raise ValueError("Cost-optimized Auto setup cannot enable runtime response-latency selection")
return self
class ClassificationRubric(str, Enum):
"""Which calibration examples, and for BUSINESS which tier criteria, the built-in classifier rubric carries."""
@ -517,6 +592,14 @@ class ComplexityRouterConfig(BaseModel):
default_factory=dict,
)
auto_setup: AutoSetupConfig | None = Field(
default=None,
description=(
"Benchmark snapshot and per-tier selection policy generated by the zero-setup Auto flow. "
"Absent on manually configured routers, whose pool selection behavior is unchanged"
),
)
tier_definitions: tuple[TierDefinition, ...] | None = Field(
default=None,
description=(
@ -1422,6 +1505,24 @@ class ComplexityRouterConfig(BaseModel):
)
return self
@model_validator(mode="after")
def _validate_auto_setup(self) -> "ComplexityRouterConfig":
setup: Final = self.auto_setup
if setup is None:
return self
if self.adaptive:
raise ValueError("auto_setup and adaptive=True cannot both select from the same tier pool")
if self.tier_definitions is not None:
raise ValueError("auto_setup requires the built-in four-tier complexity ladder")
for tier_name, policy in setup.tier_policies.items():
configured = self.tiers.get(tier_name)
configured_names = frozenset((configured,)) if isinstance(configured, str) else frozenset(configured or ())
policy_names = frozenset(candidate.model_name for candidate in policy.candidates)
if not policy_names.issubset(configured_names):
missing = ", ".join(sorted(policy_names - configured_names))
raise ValueError(f"auto_setup tier {tier_name} references models not present in tiers: {missing}")
return self
@model_validator(mode="after")
def _validate_tier_param_placement(self) -> "ComplexityRouterConfig":
"""Reject a router setting written into a tier entry's request params.

View file

@ -0,0 +1,232 @@
from __future__ import annotations
import hashlib
import math
import statistics
from collections.abc import Mapping, Sequence
from datetime import datetime, timedelta
from typing import Final, Protocol, cast
from pydantic import BaseModel, ConfigDict, Field, ValidationError
from litellm._logging import verbose_router_logger
from .config import AutoSetupCandidate, AutoSetupObjective
_CACHE_PREFIX: Final = "auto_setup_response_latency:v1"
_CACHE_TTL_SECONDS: Final = 7 * 24 * 60 * 60
_MAX_SAMPLES_PER_MODEL: Final = 20
_MAX_EXPECTED_OUTPUT_SAMPLES: Final = 100
_MIN_SAMPLES_PER_MODEL: Final = 2
class ResponseLatencyCache(Protocol):
async def async_get_cache(
self,
key: str,
parent_otel_span: object | None = None,
local_only: bool = False,
**kwargs: object,
) -> object: ...
async def async_set_cache(
self,
key: str,
value: object,
local_only: bool = False,
**kwargs: object,
) -> None: ...
class ResponseLatencySample(BaseModel):
model_config = ConfigDict(extra="forbid", frozen=True)
ttft_ms: float = Field(ge=0)
output_tokens_per_ms: float = Field(gt=0)
class ResponseLatencyState(BaseModel):
model_config = ConfigDict(extra="forbid")
samples_by_model: dict[str, list[ResponseLatencySample]] = Field(default_factory=dict)
visible_output_tokens: list[int] = Field(default_factory=list)
def _cache_key(router_model_name: str, tier: str) -> str:
namespace: Final = hashlib.sha256(router_model_name.encode("utf-8")).hexdigest()[:24]
return f"{_CACHE_PREFIX}:{namespace}:{tier}"
def _state(value: object) -> ResponseLatencyState:
try:
return ResponseLatencyState.model_validate(value or {})
except ValidationError:
return ResponseLatencyState()
def _candidate_order(
candidates: Sequence[AutoSetupCandidate],
available_models: Sequence[str],
cold_start_model: str,
) -> tuple[str, ...]:
available: Final = frozenset(available_models)
ranked: Final = tuple(candidate.model_name for candidate in candidates if candidate.model_name in available)
extras: Final = tuple(model for model in available_models if model not in frozenset(ranked))
ordered: Final = tuple(dict.fromkeys((*ranked, *extras)))
if cold_start_model not in ordered:
return ordered
return (cold_start_model, *(model for model in ordered if model != cold_start_model))
def _median_prediction_ms(samples: Sequence[ResponseLatencySample], expected_output_tokens: float) -> float:
return statistics.median(
sample.ttft_ms + expected_output_tokens / sample.output_tokens_per_ms for sample in samples
)
def _normalized_log(value: float, values: Sequence[float]) -> float:
logged: Final = tuple(math.log(max(item, 1e-12)) for item in values)
low: Final = min(logged)
high: Final = max(logged)
if low == high:
return 0.0
return (math.log(max(value, 1e-12)) - low) / (high - low)
def _best_observed_model(
state: ResponseLatencyState,
candidates: Sequence[AutoSetupCandidate],
ordered_models: Sequence[str],
objective: AutoSetupObjective,
) -> str:
expected_tokens: Final = statistics.median(state.visible_output_tokens) if state.visible_output_tokens else 1.0
latencies: Final = {
model: _median_prediction_ms(state.samples_by_model[model], expected_tokens) for model in ordered_models
}
if objective == "task_completion_speed":
return min(ordered_models, key=lambda model: (latencies[model], ordered_models.index(model)))
configured_costs: Final = {candidate.model_name: candidate.cost_per_completed_task_usd for candidate in candidates}
known_costs: Final = tuple(configured_costs.values())
conservative_unknown_cost: Final = max(known_costs) if known_costs else 1.0
costs: Final = {model: configured_costs.get(model, conservative_unknown_cost) for model in ordered_models}
latency_values: Final = tuple(latencies.values())
cost_values: Final = tuple(costs.values())
return min(
ordered_models,
key=lambda model: (
math.hypot(
_normalized_log(latencies[model], latency_values),
_normalized_log(costs[model], cost_values),
),
ordered_models.index(model),
),
)
async def select_runtime_response_model(
*,
router_cache: ResponseLatencyCache,
router_model_name: str,
tier: str,
candidates: Sequence[AutoSetupCandidate],
available_models: Sequence[str],
cold_start_model: str,
objective: AutoSetupObjective,
) -> str:
"""Explore quality-admitted groups, then minimize equal-output response time."""
ordered: Final = _candidate_order(candidates, available_models, cold_start_model)
if not ordered:
raise ValueError(f"No Auto setup candidates remain for tier {tier}")
try:
cached = await router_cache.async_get_cache(key=_cache_key(router_model_name, tier))
state: Final = _state(cached)
except Exception as exc: # noqa: BLE001 -- optional observations must never make a request unroutable
verbose_router_logger.warning("Auto setup response-latency cache read failed: %s", exc)
return ordered[0]
counts: Final = {model: len(state.samples_by_model.get(model, ())) for model in ordered}
least_observed: Final = min(counts.values())
if least_observed < _MIN_SAMPLES_PER_MODEL:
return next(model for model in ordered if counts[model] == least_observed)
return _best_observed_model(state, candidates, ordered, objective)
def _seconds(value: object) -> float | None:
if isinstance(value, datetime):
return value.timestamp()
if isinstance(value, timedelta):
return value.total_seconds()
if isinstance(value, bool) or not isinstance(value, int | float):
return None
return float(value)
def _field(value: object, key: str) -> object | None:
if isinstance(value, Mapping):
return cast(Mapping[object, object], value).get(key)
try:
return cast(object, object.__getattribute__(value, key))
except AttributeError:
return None
def _visible_output_tokens(response_obj: object) -> int | None:
usage = _field(response_obj, "usage")
if usage is None:
return None
total = _field(usage, "completion_tokens")
if total is None:
total = _field(usage, "output_tokens")
if isinstance(total, bool) or not isinstance(total, int | float) or total <= 0:
return None
details = _field(usage, "completion_tokens_details")
if details is None:
details = _field(usage, "output_tokens_details")
reasoning = _field(details, "reasoning_tokens") if details is not None else None
reasoning_tokens = float(reasoning) if isinstance(reasoning, int | float) and not isinstance(reasoning, bool) else 0
return max(1, round(float(total) - reasoning_tokens))
def response_latency_sample(
kwargs: Mapping[str, object], response_obj: object, start_time: object, end_time: object
) -> tuple[ResponseLatencySample, int] | None:
start: Final = _seconds(start_time)
end: Final = _seconds(end_time)
visible_tokens: Final = _visible_output_tokens(response_obj)
if start is None or end is None or end <= start or visible_tokens is None:
return None
total_ms: Final = (end - start) * 1000
completion_start: Final = _seconds(kwargs.get("completion_start_time"))
ttft_ms = (
(completion_start - start) * 1000 if completion_start is not None and start < completion_start < end else 0.0
)
generation_ms: Final = max(total_ms - ttft_ms, 1e-6)
return ResponseLatencySample(
ttft_ms=ttft_ms,
output_tokens_per_ms=visible_tokens / generation_ms,
), visible_tokens
async def record_runtime_response_latency(
*,
router_cache: ResponseLatencyCache,
router_model_name: str,
tier: str,
routed_model: str,
kwargs: Mapping[str, object],
response_obj: object,
start_time: object,
end_time: object,
) -> None:
measured: Final = response_latency_sample(kwargs, response_obj, start_time, end_time)
if measured is None:
return
sample, visible_tokens = measured
key: Final = _cache_key(router_model_name, tier)
state: Final = _state(await router_cache.async_get_cache(key=key))
samples: Final = [*state.samples_by_model.get(routed_model, ()), sample][-_MAX_SAMPLES_PER_MODEL:]
state.samples_by_model[routed_model] = samples
state.visible_output_tokens = [*state.visible_output_tokens, visible_tokens][-_MAX_EXPECTED_OUTPUT_SAMPLES:]
await router_cache.async_set_cache(key=key, value=state.model_dump(mode="json"), ttl=_CACHE_TTL_SECONDS)

View file

@ -9,7 +9,11 @@ from typing import Final, Literal, TypeAlias
from pydantic import BaseModel, Field, computed_field, field_validator, model_validator
from litellm.router_strategy.complexity_router.config import ComplexityRouterConfig
from litellm.router_strategy.complexity_router.config import (
AutoSetupObjective,
AutoSetupQualityLevel,
ComplexityRouterConfig,
)
from litellm.types.utils import StandardLoggingRoutingDecision
DEFAULT_ROUTING_TEST_ROUTER_NAME: Final[str] = "auto_router_routing_test"
@ -44,6 +48,17 @@ class ComplexityRouterConfigValidationResponse(BaseModel):
error: str | None = None
class AutoRouterRecommendationResponse(BaseModel):
"""An editable complexity-router config generated from the caller's usable model groups."""
quality_level: AutoSetupQualityLevel
optimize_for: AutoSetupObjective
snapshot_id: str
snapshot_generated_at: str
matched_model_groups: tuple[str, ...]
complexity_router_config: RequestComplexityRouterConfig
class AutoRouterRoutingTestRequest(BaseModel):
"""A single request to classify against a complexity-router config that need not be saved yet.

View file

@ -2938,6 +2938,10 @@ class StandardLoggingRoutingDecision(TypedDict, total=False):
savings_baseline_model: str
savings_baseline_deployment_id: str
tier_litellm_params: Mapping[str, object] # writable-ok: Pydantic warns on ReadOnly TypedDict fields
auto_setup_snapshot_id: str
auto_setup_quality_level: Literal["economy", "balanced", "high", "max"]
auto_setup_optimize_for: Literal["cost", "task_completion_speed", "balanced"]
auto_setup_selection_mode: Literal["snapshot_ranked", "runtime_response_latency"]
# Fields whose values quote the caller's prompt. Dropped when an operator turns message

View file

@ -18,6 +18,7 @@ from litellm.proxy._types import (
UserAPIKeyAuth,
)
from litellm.proxy.management_endpoints.auto_router_endpoints import (
get_auto_router_recommendation,
preview_auto_router_routing,
)
from litellm.router import Router
@ -104,6 +105,51 @@ async def _route(prompt: str, monkeypatch: pytest.MonkeyPatch, **config_override
return await _route_body({"prompt": prompt}, monkeypatch, **config_overrides)
@pytest.mark.asyncio
async def test_recommendation_uses_caller_groups_and_returns_an_editable_complexity_config(
monkeypatch: pytest.MonkeyPatch,
) -> None:
import litellm.proxy.management_endpoints.auto_router_endpoints as endpoint_module
import litellm.proxy.utils as proxy_utils
from litellm.proxy import proxy_server
model_list = [
{
"model_name": "small-group",
"litellm_params": {"model": "gpt-5.4-nano", "api_key": "fake"},
},
{
"model_name": "smart-group",
"litellm_params": {"model": "gpt-5.6-sol", "api_key": "fake"},
},
{
"model_name": "unavailable-group",
"litellm_params": {"model": "gpt-5.5", "api_key": "fake"},
},
]
monkeypatch.setattr(proxy_server, "llm_router", Router(model_list=model_list))
available = AsyncMock(return_value=["small-group", "smart-group"])
monkeypatch.setattr(proxy_utils, "get_available_models_for_user", available)
authorize = AsyncMock()
monkeypatch.setattr(endpoint_module, "_authorize_router_dry_run", authorize)
response = await get_auto_router_recommendation(
user_api_key_dict=ADMIN,
quality_level="economy",
optimize_for="task_completion_speed",
)
authorize.assert_awaited_once_with(user_api_key_dict=ADMIN, team_id=None)
assert set(response.matched_model_groups) <= {"small-group", "smart-group"}
assert "unavailable-group" not in response.matched_model_groups
assert response.complexity_router_config.classifier_type == "heuristic_v2"
assert response.complexity_router_config.auto_setup is not None
assert response.complexity_router_config.auto_setup.optimize_for == "task_completion_speed"
assert response.complexity_router_config.auto_setup.tier_policies["SIMPLE"].selection_mode == (
"runtime_response_latency"
)
AGENTIC_MESSAGES = [
{"role": "system", "content": "You are a database migration assistant for a payments ledger"},
{"role": "user", "content": "duplicate ledger postings since the celery upgrade, same event_id twice"},

View file

@ -0,0 +1,276 @@
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from pydantic import ValidationError
from litellm import Router
from litellm.router_strategy.complexity_router.auto_setup import (
build_auto_setup_config,
load_auto_router_snapshot,
)
from litellm.router_strategy.complexity_router.complexity_router import ComplexityRouter
from litellm.router_strategy.complexity_router.config import (
AutoSetupCandidate,
ComplexityRouterConfig,
ComplexityTier,
)
from litellm.router_strategy.complexity_router.response_latency import (
record_runtime_response_latency,
response_latency_sample,
select_runtime_response_model,
)
def _available_refs(*model_names: str) -> dict[str, tuple[str, ...]]:
return {model_name: (model_name,) for model_name in model_names}
def test_snapshot_is_verified_and_quality_gate_is_recomputed_for_available_models() -> None:
snapshot = load_auto_router_snapshot()
available = _available_refs("gpt-5.6-sol", "gpt-5.4-nano")
economy = build_auto_setup_config(
snapshot=snapshot,
available_model_refs=available,
quality_level="economy",
optimize_for="cost",
)
maximum = build_auto_setup_config(
snapshot=snapshot,
available_model_refs=available,
quality_level="max",
optimize_for="cost",
)
nano_only = build_auto_setup_config(
snapshot=snapshot,
available_model_refs=_available_refs("gpt-5.4-nano"),
quality_level="max",
optimize_for="cost",
)
assert snapshot.artifact_sha256 == "d0ea7e584dbb9ea37eb15ce20ae342612e0dfaa5801b553e71fc76818d7cef64"
assert economy.tiers["SIMPLE"] == ["gpt-5.4-nano", "gpt-5.6-sol"]
assert maximum.tiers["SIMPLE"] == ["gpt-5.6-sol"]
assert nano_only.tiers["SIMPLE"] == ["gpt-5.4-nano"]
def test_speed_only_changes_auto_generated_selection_policy() -> None:
config = build_auto_setup_config(
snapshot=load_auto_router_snapshot(),
available_model_refs=_available_refs("gpt-5.6-sol", "gpt-5.4-nano"),
quality_level="economy",
optimize_for="task_completion_speed",
)
assert config.auto_setup is not None
assert config.auto_setup.tier_policies["SIMPLE"].selection_mode == "runtime_response_latency"
assert config.auto_setup.tier_policies["MEDIUM"].selection_mode == "runtime_response_latency"
assert config.auto_setup.tier_policies["COMPLEX"].selection_mode == "snapshot_ranked"
assert config.auto_setup.tier_policies["REASONING"].selection_mode == "snapshot_ranked"
assert ComplexityRouterConfig(tiers=config.tiers).auto_setup is None
def test_auto_setup_cannot_change_adaptive_or_custom_tier_routers() -> None:
generated = build_auto_setup_config(
snapshot=load_auto_router_snapshot(),
available_model_refs=_available_refs("gpt-5.6-sol"),
quality_level="max",
optimize_for="cost",
)
payload = generated.model_dump(mode="json")
with pytest.raises(ValidationError, match="auto_setup and adaptive"):
ComplexityRouterConfig.model_validate({**payload, "adaptive": True})
with pytest.raises(ValidationError, match="built-in four-tier"):
ComplexityRouterConfig.model_validate(
{
**payload,
"tier_definitions": [
{"name": "fast", "description": "Fast"},
{"name": "strong", "description": "Strong"},
],
"tiers": {"fast": ["gpt-5.6-sol"], "strong": ["gpt-5.6-sol"]},
"fallback_tier": "fast",
"classifier_type": "llm",
"classifier_llm_config": {"model": "gpt-5.6-sol"},
}
)
class _Cache:
def __init__(self) -> None:
self.values: dict[str, object] = {}
async def async_get_cache(
self,
key: str,
parent_otel_span: object | None = None,
local_only: bool = False,
**kwargs: object,
) -> object:
return self.values.get(key)
async def async_set_cache(
self,
key: str,
value: object,
local_only: bool = False,
**kwargs: object,
) -> None:
self.values[key] = value
def _candidate(model_name: str, cost: float) -> AutoSetupCandidate:
return AutoSetupCandidate(
model_name=model_name,
benchmark_model_id=f"benchmark-{model_name}",
quality_lower_bound=0.95,
cost_per_completed_task_usd=cost,
)
@pytest.mark.asyncio
async def test_easy_task_speed_learns_equal_output_response_time() -> None:
cache = _Cache()
candidates = (_candidate("cheap-slow", 0.01), _candidate("fast", 0.04))
assert (
await select_runtime_response_model(
router_cache=cache,
router_model_name="auto",
tier="SIMPLE",
candidates=candidates,
available_models=("cheap-slow", "fast"),
cold_start_model="cheap-slow",
objective="task_completion_speed",
)
== "cheap-slow"
)
for model, end in (("cheap-slow", 0.30), ("cheap-slow", 0.30), ("fast", 0.15), ("fast", 0.15)):
await record_runtime_response_latency(
router_cache=cache,
router_model_name="auto",
tier="SIMPLE",
routed_model=model,
kwargs={"completion_start_time": 0.10},
response_obj={
"usage": {
"completion_tokens": 20,
"completion_tokens_details": {"reasoning_tokens": 10},
}
},
start_time=0.0,
end_time=end,
)
measured = response_latency_sample(
{"completion_start_time": 0.10},
{"usage": {"completion_tokens": 20, "completion_tokens_details": {"reasoning_tokens": 10}}},
0.0,
0.30,
)
assert measured is not None and measured[1] == 10
assert (
await select_runtime_response_model(
router_cache=cache,
router_model_name="auto",
tier="SIMPLE",
candidates=candidates,
available_models=("cheap-slow", "fast"),
cold_start_model="cheap-slow",
objective="task_completion_speed",
)
== "fast"
)
@pytest.mark.asyncio
async def test_snapshot_policy_is_deterministic_but_manual_pool_keeps_random_selection() -> None:
generated = build_auto_setup_config(
snapshot=load_auto_router_snapshot(),
available_model_refs=_available_refs("gpt-5.6-sol", "gpt-5.4-nano"),
quality_level="economy",
optimize_for="cost",
)
router_instance = MagicMock()
router_instance.cache = AsyncMock()
automatic = ComplexityRouter(
model_name="auto",
litellm_router_instance=router_instance,
complexity_router_config=generated.model_dump(mode="json"),
)
manual = ComplexityRouter(
model_name="manual",
litellm_router_instance=router_instance,
complexity_router_config={"tiers": {"SIMPLE": ["first", "second"]}},
)
assert await automatic._pick_model_for_tier(ComplexityTier.SIMPLE, None, None, {}) == "gpt-5.4-nano"
assert (
await automatic._pick_model_for_tier(
ComplexityTier.SIMPLE,
None,
None,
{},
allowed_models=("gpt-5.6-sol",),
)
== "gpt-5.6-sol"
)
with patch(
"litellm.router_strategy.complexity_router.complexity_router.random.choice", return_value="second"
) as pick:
assert await manual._pick_model_for_tier(ComplexityTier.SIMPLE, None, None, {}) == "second"
pick.assert_called_once_with(("first", "second"))
@pytest.mark.asyncio
async def test_live_latency_telemetry_is_recorded_only_for_an_auto_runtime_policy() -> None:
generated = build_auto_setup_config(
snapshot=load_auto_router_snapshot(),
available_model_refs=_available_refs("gpt-5.6-sol", "gpt-5.4-nano"),
quality_level="economy",
optimize_for="task_completion_speed",
)
router = Router(
model_list=[
{
"model_name": "smart",
"litellm_params": {
"model": "auto_router/complexity_router",
"complexity_router_config": generated.model_dump(mode="json"),
"complexity_router_default_model": "gpt-5.6-sol",
},
},
{"model_name": "gpt-5.6-sol", "litellm_params": {"model": "gpt-5.6-sol"}},
{"model_name": "gpt-5.4-nano", "litellm_params": {"model": "gpt-5.4-nano"}},
]
)
metadata = {
"routing_decision": {
"router_model_name": "smart",
"tier": "SIMPLE",
"routed_model": "gpt-5.4-nano",
"auto_setup_selection_mode": "runtime_response_latency",
}
}
with patch(
"litellm.router_strategy.complexity_router.response_latency.record_runtime_response_latency",
new_callable=AsyncMock,
) as recorder:
await router._record_auto_setup_response_latency(
kwargs={"litellm_params": {"metadata": metadata}},
completion_response={"usage": {"completion_tokens": 1}},
start_time=0.0,
end_time=0.1,
)
metadata["routing_decision"]["auto_setup_selection_mode"] = "snapshot_ranked"
await router._record_auto_setup_response_latency(
kwargs={"litellm_params": {"metadata": metadata}},
completion_response={"usage": {"completion_tokens": 1}},
start_time=0.0,
end_time=0.1,
)
recorder.assert_awaited_once()

View file

@ -0,0 +1,34 @@
import { useQuery } from "@tanstack/react-query";
import { getAutoRouterRecommendation, type AutoRouterRecommendationResponse } from "@/components/networking";
import type { AutoSetupObjective, AutoSetupQualityLevel } from "@/components/add_model/build_complexity_router_config";
import { createQueryKeys } from "../common/queryKeysFactory";
const recommendationKeys = createQueryKeys("autoRouterRecommendation");
interface UseAutoRouterRecommendationParams {
accessToken: string;
qualityLevel: AutoSetupQualityLevel;
optimizeFor: AutoSetupObjective;
teamId?: string;
enabled: boolean;
}
export const useAutoRouterRecommendation = ({
accessToken,
qualityLevel,
optimizeFor,
teamId,
enabled,
}: UseAutoRouterRecommendationParams) => {
const filters = { qualityLevel, optimizeFor, ...(teamId ? { teamId } : {}) };
const queryOptions = {
queryKey: recommendationKeys.list({
filters,
}),
queryFn: () => getAutoRouterRecommendation(accessToken, qualityLevel, optimizeFor, teamId),
enabled: enabled && Boolean(accessToken),
staleTime: 30 * 1000,
};
return useQuery<AutoRouterRecommendationResponse>(queryOptions);
};

View file

@ -0,0 +1,134 @@
import type { AutoRouterRecommendationResponse } from "@/components/networking";
import { Button } from "@/components/ui/button";
import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select";
import { UiLoadingSpinner } from "@/components/ui/ui-loading-spinner";
import type { AutoSetupObjective, AutoSetupQualityLevel } from "./build_complexity_router_config";
const OBJECTIVE_DESCRIPTION: Record<AutoSetupObjective, string> = {
cost: "Uses expected cost per completed benchmark task, including failures",
task_completion_speed: "Uses live response speed for easy requests and benchmark completion time for hard ones",
balanced: "Balances cost per completed task with those same completion-time signals",
};
interface AutoSetupControlsProps {
qualityLevel: AutoSetupQualityLevel;
optimizeFor: AutoSetupObjective;
onQualityLevelChange: (value: AutoSetupQualityLevel) => void;
onOptimizeForChange: (value: AutoSetupObjective) => void;
recommendation?: AutoRouterRecommendationResponse;
recommendationEnabled: boolean;
recommendationPending: boolean;
recommendationFailed: boolean;
recommendationError: unknown;
requiresTeamScope: boolean;
hasAutoPolicy: boolean;
onRetry: () => void;
onRegenerate: () => void;
onUseManual: () => void;
}
const recommendationErrorMessage = (error: unknown): string =>
error instanceof Error ? error.message : "Could not build Auto setup";
const readyMessage = (recommendation: AutoRouterRecommendationResponse): string => {
const count = recommendation.matched_model_groups.length;
return `Ready with ${count} available model${count === 1 ? "" : "s"}. Review or edit below`;
};
export default function AutoSetupControls({
qualityLevel,
optimizeFor,
onQualityLevelChange,
onOptimizeForChange,
recommendation,
recommendationEnabled,
recommendationPending,
recommendationFailed,
recommendationError,
requiresTeamScope,
hasAutoPolicy,
onRetry,
onRegenerate,
onUseManual,
}: AutoSetupControlsProps) {
return (
<div className="space-y-4 rounded-lg border border-border p-4" data-testid="auto-setup-controls">
<div>
<div className="font-medium text-foreground">Auto setup</div>
<div className="text-xs text-muted-foreground">
Builds an editable complexity router from the models available to you
</div>
</div>
<div className="grid grid-cols-1 gap-4 sm:grid-cols-2">
<div>
<label className="mb-2 block text-sm font-medium text-foreground">Quality level</label>
<Select
value={qualityLevel}
onValueChange={(value: AutoSetupQualityLevel | null) => value && onQualityLevelChange(value)}
>
<SelectTrigger data-testid="auto-quality-selector" className="w-full">
<SelectValue />
</SelectTrigger>
<SelectContent>
<SelectItem value="economy">Economy</SelectItem>
<SelectItem value="balanced">Balanced</SelectItem>
<SelectItem value="high">High</SelectItem>
<SelectItem value="max">Max</SelectItem>
</SelectContent>
</Select>
<div className="mt-1 text-xs text-muted-foreground">
Higher levels keep each tier closer to the smartest available model
</div>
</div>
<div>
<label className="mb-2 block text-sm font-medium text-foreground">Optimize for</label>
<Select
value={optimizeFor}
onValueChange={(value: AutoSetupObjective | null) => value && onOptimizeForChange(value)}
>
<SelectTrigger data-testid="auto-objective-selector" className="w-full">
<SelectValue />
</SelectTrigger>
<SelectContent>
<SelectItem value="cost">Cost</SelectItem>
<SelectItem value="task_completion_speed">Task completion speed</SelectItem>
<SelectItem value="balanced">Balanced</SelectItem>
</SelectContent>
</Select>
<div className="mt-1 text-xs text-muted-foreground">{OBJECTIVE_DESCRIPTION[optimizeFor]}</div>
</div>
</div>
{recommendationPending && recommendation === undefined && (
<div className="flex items-center gap-2 text-xs text-muted-foreground">
<UiLoadingSpinner className="size-3" /> Building your router...
</div>
)}
{!recommendationEnabled && requiresTeamScope && (
<div className="text-xs text-muted-foreground">Select a team to build its Auto setup</div>
)}
{recommendationFailed && recommendation === undefined && (
<div className="text-xs text-destructive">
{recommendationErrorMessage(recommendationError)}.{" "}
<button type="button" className="underline" onClick={onRetry}>
Retry
</button>
</div>
)}
{recommendation && (
<div className="space-y-1">
<div className={`text-xs ${hasAutoPolicy ? "text-success" : "text-warning"}`}>
{hasAutoPolicy ? readyMessage(recommendation) : "Customized model pools now use standard manual selection"}
</div>
{!hasAutoPolicy && (
<button type="button" className="text-xs text-primary underline" onClick={onRegenerate}>
Regenerate Auto setup
</button>
)}
</div>
)}
<Button type="button" variant="ghost" className="h-auto px-0" onClick={onUseManual}>
Choose a template or set up manually
</Button>
</div>
);
}

View file

@ -2,6 +2,7 @@ import { fireEvent, renderWithProviders, screen, within } from "../../../tests/t
import userEvent from "@testing-library/user-event";
import { vi } from "vitest";
import ComplexityRouterConfig, { ComplexityRouterConfigValue } from "./ComplexityRouterConfig";
import { detachAutoSetupAfterPoolEdit } from "./auto_setup_policy";
vi.mock(
"@/app/(dashboard)/hooks/autoRouter/useComplexityScorerDefaults",
async () => await import("../../../tests/mocks/complexityScorerDefaults"),
@ -44,6 +45,30 @@ const baseProps = {
};
describe("ComplexityRouterConfig", () => {
it("keeps Auto policy for classifier edits and detaches it for model-pool edits", () => {
const automatic: ComplexityRouterConfigValue = {
...defaultValue,
auto_setup: {
schema_version: "1",
snapshot_id: "snapshot-test",
snapshot_sha256: "a".repeat(64),
quality_level: "max",
optimize_for: "cost",
tier_policies: {},
},
};
expect(detachAutoSetupAfterPoolEdit(automatic, { ...automatic, classifier_type: "heuristic_v2" }).auto_setup).toBe(
automatic.auto_setup,
);
expect(
detachAutoSetupAfterPoolEdit(automatic, {
...automatic,
tiers: { ...automatic.tiers, SIMPLE: ["gpt-4"] },
}).auto_setup,
).toBeUndefined();
});
it("should render", () => {
renderWithProviders(<ComplexityRouterConfig {...baseProps} />);
expect(screen.getByText("Complexity Tier Configuration")).toBeInTheDocument();

View file

@ -48,6 +48,7 @@ import EscalationKeywords from "./EscalationKeywords";
import KeywordTierRules, { KeywordTierRule } from "./KeywordTierRules";
import SemanticKeywordMatching from "./SemanticKeywordMatching";
import { type DimensionWeights, type TierBoundaries, type TokenThresholds } from "./heuristic_scoring_knobs";
import type { AutoSetupConfigPayload } from "./build_complexity_router_config";
export type { DimensionWeights, TierBoundaries, TokenThresholds };
export type { CustomTierSet, TierRow } from "./tier_rows";
@ -446,6 +447,8 @@ export interface ComplexityRouterConfigValue {
* edit round-trip.
*/
tier_model_params?: TierModelParamsByTier;
/** Present only when the zero-setup Auto flow generated this editable config. */
auto_setup?: AutoSetupConfigPayload;
}
/** Session affinity wins where a hand-authored config sets both, matching the backend's own `or`. */

View file

@ -4,13 +4,14 @@ import { vi } from "vitest";
import AddAutoRouterTab from "./add_auto_router_tab";
import { toast } from "@/lib/toast";
import { handleAddAutoRouterSubmit } from "./handle_add_auto_router_submit";
import { getMissingTiersError } from "./build_complexity_router_config";
import { getMissingTiersError, type AutoSetupConfigPayload } from "./build_complexity_router_config";
import { getSubmitBlockedReason } from "./add_auto_router_tab";
import { buildModelAvailability } from "@/lib/autorouter_presets";
import { testAutoRouterRouting } from "../networking";
import { ModelGroup } from "@/components/llm_calls/fetch_models";
import { AutoRouterPreset, getRequiredModelsInPreset } from "@/lib/autorouter_presets";
import { BUNDLED_PRESETS, LOADED_PRESETS_QUERY, useAutoRouterPresets } from "../../../tests/mocks/autoRouterPresets";
import { EMPTY_RECOMMENDATION_QUERY, useAutoRouterRecommendation } from "../../../tests/mocks/autoRouterRecommendation";
vi.mock(
"@/app/(dashboard)/hooks/autoRouter/useComplexityScorerDefaults",
async () => await import("../../../tests/mocks/complexityScorerDefaults"),
@ -19,6 +20,10 @@ vi.mock(
"@/app/(dashboard)/hooks/autoRouter/useAutoRouterPresets",
async () => await import("../../../tests/mocks/autoRouterPresets"),
);
vi.mock(
"@/app/(dashboard)/hooks/autoRouter/useAutoRouterRecommendation",
async () => await import("../../../tests/mocks/autoRouterRecommendation"),
);
const getAllPresets = (): AutoRouterPreset[] => BUNDLED_PRESETS;
const getPresetByKey = (key: string): AutoRouterPreset | undefined => BUNDLED_PRESETS.find((p) => p.key === key);
@ -37,6 +42,9 @@ const ALL_FAMILY_MODELS: ModelGroup[] = [
const ANTHROPIC_ONLY_MODEL = ANTHROPIC_TIERS.COMPLEX[0];
const openTemplateDropdown = (): void => {
if (!screen.queryByTestId("template-selector")) {
fireEvent.click(screen.getByRole("button", { name: /choose a template or set up manually/i }));
}
fireEvent.click(screen.getByTestId("template-selector"));
};
@ -141,6 +149,86 @@ describe("AddAutoRouterTab", () => {
testQueryClient.clear();
mockFetchAvailableModels.mockResolvedValue([]);
mockFetchAllModelDeployments.mockResolvedValue([]);
vi.mocked(useAutoRouterRecommendation).mockReturnValue(EMPTY_RECOMMENDATION_QUERY);
});
it("starts with only quality and optimization choices in Auto setup", () => {
renderWithProviders(<Harness />);
expect(screen.getByTestId("auto-quality-selector")).toBeInTheDocument();
expect(screen.getByTestId("auto-objective-selector")).toBeInTheDocument();
expect(screen.queryByTestId("template-selector")).not.toBeInTheDocument();
expect(useAutoRouterRecommendation).toHaveBeenCalledWith(
expect.objectContaining({ qualityLevel: "balanced", optimizeFor: "balanced", enabled: true }),
);
});
it("submits the editable policy generated for the caller's available model groups", async () => {
const user = userEvent.setup();
const autoSetup: AutoSetupConfigPayload = {
schema_version: "1" as const,
snapshot_id: "snapshot-test",
snapshot_sha256: "a".repeat(64),
quality_level: "max" as const,
optimize_for: "task_completion_speed" as const,
tier_policies: Object.fromEntries(
["SIMPLE", "MEDIUM", "COMPLEX", "REASONING"].map((tier) => [
tier,
{
selection_mode: tier === "SIMPLE" || tier === "MEDIUM" ? "runtime_response_latency" : "snapshot_ranked",
candidates: [
{
model_name: "smart-group",
benchmark_model_id: "benchmark-smart",
quality_lower_bound: 0.98,
cost_per_completed_task_usd: 0.04,
},
],
...(tier === "SIMPLE" || tier === "MEDIUM" ? { cold_start_model: "smart-group" } : {}),
evidence_status: "test",
},
]),
),
};
mockFetchAvailableModels.mockResolvedValue([{ model_group: "smart-group", mode: "chat" }]);
vi.mocked(useAutoRouterRecommendation).mockReturnValue({
data: {
quality_level: "max",
optimize_for: "task_completion_speed",
snapshot_id: "snapshot-test",
snapshot_generated_at: "2026-09-03T00:00:00Z",
matched_model_groups: ["smart-group"],
complexity_router_config: {
tiers: {
SIMPLE: ["smart-group"],
MEDIUM: ["smart-group"],
COMPLEX: ["smart-group"],
REASONING: ["smart-group"],
},
classifier_type: "heuristic_v2",
classification_mode: "per_request",
session_affinity: false,
deployment_affinity: false,
modality_routing: false,
modality_pin_override: false,
auto_setup: autoSetup,
},
},
isPending: false,
isError: false,
error: null,
refetch: vi.fn(),
});
renderWithProviders(<Harness />);
await screen.findByText(/Ready with 1 available model/);
await user.type(screen.getByPlaceholderText(/smart_router/i), "max-speed-router");
await user.click(screen.getByRole("button", { name: /add auto router/i }));
await waitFor(() => expect(handleAddAutoRouterSubmit).toHaveBeenCalled());
expect(vi.mocked(handleAddAutoRouterSubmit).mock.calls.at(-1)?.[0].complexity_router_config).toMatchObject({
auto_setup: autoSetup,
});
});
// Detailed Configuration starts collapsed so the modal opens onto just Name + Template; a caller
@ -163,6 +251,23 @@ describe("AddAutoRouterTab", () => {
expect(screen.getByRole("button", { name: /add auto router/i })).toBeDisabled();
});
it("blocks an implicit form submit while Auto setup is still building", async () => {
vi.mocked(getMissingTiersError).mockReturnValue(null);
vi.mocked(useAutoRouterRecommendation).mockReturnValue({
...EMPTY_RECOMMENDATION_QUERY,
isPending: true,
});
const { container } = renderWithProviders(<Harness />);
fireEvent.change(screen.getByPlaceholderText(/smart_router/i), { target: { value: "early-router" } });
fireEvent.submit(container.querySelector("form")!);
await waitFor(() =>
expect(toast.fromError).toHaveBeenCalledWith("Building the best setup for your available models"),
);
expect(handleAddAutoRouterSubmit).not.toHaveBeenCalled();
});
it("still flags the router name once the config no longer blocks the submit", async () => {
const user = userEvent.setup();
vi.mocked(getMissingTiersError).mockReturnValue(null);
@ -354,22 +459,19 @@ describe("AddAutoRouterTab", () => {
});
it("blocks the submit when a team admin has not picked a team", async () => {
const user = userEvent.setup();
vi.mocked(getMissingTiersError).mockReturnValue(null);
renderWithProviders(
<AddAutoRouterTab handleOk={vi.fn()} accessToken="token" userRole="Internal User" createScope="team-required" />,
);
await user.type(screen.getByPlaceholderText(/smart_router/i), "team-scoped-router");
await user.click(screen.getByRole("button", { name: /add auto router/i }));
expect(await screen.findByText("Please select a team to continue")).toBeInTheDocument();
expect(screen.getByText("Select a team to build its Auto setup")).toBeInTheDocument();
expect(screen.getByRole("button", { name: /add auto router/i })).toBeDisabled();
expect(handleAddAutoRouterSubmit).not.toHaveBeenCalled();
});
// The shared dropdown emits null on clear while this form's schema wants a string, so the
// form maps null back to "": the user sees the pick-a-team message, not a zod type error.
// form maps null back to "": Auto setup returns to its explicit pick-a-team state.
it("treats a team picked and then cleared like no team at all", async () => {
const user = userEvent.setup();
vi.mocked(getMissingTiersError).mockReturnValue(null);
@ -381,9 +483,8 @@ describe("AddAutoRouterTab", () => {
await user.type(screen.getByPlaceholderText(/smart_router/i), "team-scoped-router");
await user.selectOptions(screen.getByTestId("team-dropdown"), "team-1");
await user.click(screen.getByTestId("team-dropdown-clear"));
await user.click(screen.getByRole("button", { name: /add auto router/i }));
expect(await screen.findByText("Please select a team to continue")).toBeInTheDocument();
expect(screen.getByText("Select a team to build its Auto setup")).toBeInTheDocument();
expect(screen.getByRole("button", { name: /add auto router/i })).toBeDisabled();
expect(handleAddAutoRouterSubmit).not.toHaveBeenCalled();
});
@ -750,8 +851,8 @@ describe("AddAutoRouterTab", () => {
renderWithProviders(<Harness />);
expect(await screen.findByText("Could not load available models.")).toBeInTheDocument();
openTemplateDropdown();
expect(await screen.findByText("Could not load available models.")).toBeInTheDocument();
const anthropicOption = optionByLabel("Anthropic Family")!;
expect(isOptionDisabled(anthropicOption)).toBe(true);
expect(anthropicOption).toHaveTextContent(/Cannot verify these models are available/);
@ -1227,6 +1328,7 @@ describe("preset catalog fetch states", () => {
} as never);
renderWithProviders(<Harness />);
openTemplateDropdown();
expect(screen.getByText("Loading templates...")).toBeInTheDocument();
});
@ -1240,9 +1342,8 @@ describe("preset catalog fetch states", () => {
} as never);
renderWithProviders(<Harness />);
expect(await screen.findByText(/Could not load templates/)).toBeInTheDocument();
openTemplateDropdown();
expect(await screen.findByText(/Could not load templates/)).toBeInTheDocument();
const options = screen.queryAllByRole("option");
expect(options).toHaveLength(1);
expect(options[0]).toHaveTextContent("Custom Configuration");

View file

@ -1,4 +1,4 @@
import React, { useEffect, useState } from "react";
import React, { useEffect, useRef, useState } from "react";
import { useQuery } from "@tanstack/react-query";
import { useWatch } from "react-hook-form";
import { ChevronDown, ChevronRight } from "lucide-react";
@ -30,10 +30,13 @@ import ComplexityRouterConfig, {
DEFAULT_DEPLOYMENT_AFFINITY,
DEFAULT_TIER_DISTANCE_PENALTY,
} from "./ComplexityRouterConfig";
import { detachAutoSetupAfterPoolEdit, getRecommendationBlockedReason, type SetupMode } from "./auto_setup_policy";
import { KeywordTierRule } from "./KeywordTierRules";
import { DEFAULT_ESCALATION_KEYWORDS } from "./EscalationKeywords";
import { DEFAULT_MATCH_THRESHOLD } from "./SemanticKeywordMatching";
import {
AutoSetupObjective,
AutoSetupQualityLevel,
BuildComplexityRouterConfigParams,
buildComplexityRouterConfig,
getKeywordTierRulesError,
@ -63,7 +66,9 @@ import {
AutoRouterPreset,
} from "@/lib/autorouter_presets";
import { useAutoRouterPresets } from "@/app/(dashboard)/hooks/autoRouter/useAutoRouterPresets";
import { useAutoRouterRecommendation } from "@/app/(dashboard)/hooks/autoRouter/useAutoRouterRecommendation";
import { Dialog, DialogContent, DialogFooter, DialogHeader, DialogTitle } from "@/components/ui/dialog";
import AutoSetupControls from "./AutoSetupControls";
interface AddAutoRouterTabProps {
handleOk: () => void;
@ -181,6 +186,10 @@ const AddAutoRouterTab: React.FC<AddAutoRouterTabProps> = ({
const watchedName = useWatch({ control: form.control, name: "auto_router_name" });
const watchedTeamId = useWatch({ control: form.control, name: "team_id" });
const [modelAccessGroups, setModelAccessGroups] = useState<string[]>([]);
const [setupMode, setSetupMode] = useState<SetupMode>("auto");
const [qualityLevel, setQualityLevel] = useState<AutoSetupQualityLevel>("balanced");
const [optimizeFor, setOptimizeFor] = useState<AutoSetupObjective>("balanced");
const appliedRecommendationKey = useRef<string | null>(null);
const [complexityRouterConfig, setComplexityRouterConfig] = useState<ComplexityRouterConfigValue>({
tiers: { SIMPLE: [], MEDIUM: [], COMPLEX: [], REASONING: [] },
@ -267,6 +276,22 @@ const AddAutoRouterTab: React.FC<AddAutoRouterTabProps> = ({
[modelInfo],
);
const recommendationEnabled = setupMode === "auto" && (!requiresTeamScope || Boolean(watchedTeamId));
const recommendationParams = {
accessToken,
qualityLevel,
optimizeFor,
teamId: requiresTeamScope ? watchedTeamId : undefined,
enabled: recommendationEnabled,
};
const {
data: recommendation,
isPending: recommendationPending,
isError: recommendationFailed,
error: recommendationError,
refetch: refetchRecommendation,
} = useAutoRouterRecommendation(recommendationParams);
// A preset's models can only be trusted against a successfully loaded list. Selection and the
// greyed-out state derive from this one function, so a preset that cannot be selected can never
// have been applied: while loading we withhold selection rather than let a caller pick a preset
@ -302,7 +327,7 @@ const AddAutoRouterTab: React.FC<AddAutoRouterTabProps> = ({
[sortedPresetOptions],
);
const applyPrefill = (prefill: PresetPrefill) => {
const applyPrefill = React.useCallback((prefill: PresetPrefill) => {
setEditingTiers(false);
setComplexityRouterConfig(prefill.complexityRouterConfig);
setCustomTechnicalKeywords(prefill.customTechnicalKeywords);
@ -311,7 +336,23 @@ const AddAutoRouterTab: React.FC<AddAutoRouterTabProps> = ({
setEmbeddingModel(prefill.embeddingModel);
setMatchThreshold(prefill.matchThreshold);
setEscalationKeywords(prefill.escalationKeywords);
};
}, []);
useEffect(() => {
if (!recommendation || setupMode !== "auto") return;
const key = [
recommendation.snapshot_id,
qualityLevel,
optimizeFor,
watchedTeamId ?? "",
...recommendation.matched_model_groups,
].join(":");
if (appliedRecommendationKey.current === key) return;
appliedRecommendationKey.current = key;
setSelectedPreset(undefined);
applyPrefill(buildPresetPrefill(recommendation.complexity_router_config, availability));
setDetailsExpanded(false);
}, [recommendation, setupMode, qualityLevel, optimizeFor, watchedTeamId, availability, applyPrefill]);
const handlePresetChange = (presetKey: string | undefined) => {
if (!presetKey || presetKey === "custom") {
@ -342,13 +383,28 @@ const AddAutoRouterTab: React.FC<AddAutoRouterTabProps> = ({
defaultModel: complexityRouterConfig.default_model,
};
const submitBlockedReason = getSubmitBlockedReason(
complexityRouterConfig,
keywordTierRules,
referencedModelsParams,
groupsOnlyAvailability,
modelInfo,
);
const recommendationStatus = {
setupMode,
recommendationEnabled,
recommendationPending,
recommendationFailed,
hasRecommendation: recommendation !== undefined,
};
const recommendationBlockedReason = getRecommendationBlockedReason(recommendationStatus);
const regenerateAutoSetup = () => {
if (!recommendation) return;
applyPrefill(buildPresetPrefill(recommendation.complexity_router_config, availability));
setDetailsExpanded(false);
};
const submitBlockedReason =
recommendationBlockedReason ??
getSubmitBlockedReason(
complexityRouterConfig,
keywordTierRules,
referencedModelsParams,
groupsOnlyAvailability,
modelInfo,
);
const complexityRouterConfigParams: BuildComplexityRouterConfigParams = {
tiers: complexityRouterConfig.tiers,
@ -388,6 +444,7 @@ const AddAutoRouterTab: React.FC<AddAutoRouterTabProps> = ({
reasoningOverrideMinScore: complexityRouterConfig.reasoning_override_min_score,
enableContextWindowEscalation: complexityRouterConfig.enable_context_window_escalation,
contextWindowEscalationBuffer: complexityRouterConfig.context_window_escalation_buffer,
autoSetup: complexityRouterConfig.auto_setup,
};
const submitRecommendedRouter = async (name: string) => {
@ -395,13 +452,15 @@ const AddAutoRouterTab: React.FC<AddAutoRouterTabProps> = ({
// disagree about why. The handler needs it in its own right: the form fires this on Enter
// regardless of the button's disabled state.
const blockedReason =
recommendationBlockedReason ??
getSubmitBlockedReason(
complexityRouterConfig,
keywordTierRules,
referencedModelsParams,
groupsOnlyAvailability,
modelInfo,
) ?? getSemanticConfigError({ semanticMatchingEnabled, embeddingModel, keywordTierRules });
) ??
getSemanticConfigError({ semanticMatchingEnabled, embeddingModel, keywordTierRules });
if (blockedReason) {
setShowValidationErrors(true);
toast.fromError(blockedReason);
@ -507,71 +566,6 @@ const AddAutoRouterTab: React.FC<AddAutoRouterTabProps> = ({
{({ ref, ...field }) => <Input {...field} ref={ref} placeholder="e.g., smart_router, auto_router_1" />}
</FormField>
<div>
<label className="block text-sm font-medium text-foreground mb-2">Template</label>
<Select
items={templateItems}
value={selectedPreset ?? null}
onValueChange={(presetKey: string | null) => handlePresetChange(presetKey ?? undefined)}
>
<SelectTrigger data-testid="template-selector" className="w-full">
<SelectValue placeholder="Choose a template or select Custom to define your own" />
</SelectTrigger>
<SelectContent>
{sortedPresetOptions.map(({ preset, availability: presetState }) => {
const disabledHint = presetDisabledHint(presetState);
const hintClass = isPresetHintAlarming(presetState)
? "text-destructive"
: "text-muted-foreground";
const matchedHint =
presetState.kind === "available" && presetState.viaDeployments
? "Matches your deployments"
: null;
return (
<SelectItem
key={preset.key}
value={preset.key}
label={preset.label}
disabled={disabledHint !== null}
title={disabledHint ?? preset.description}
>
<div>
<div className="font-medium">{preset.label}</div>
<div className="text-xs text-muted-foreground">{preset.description}</div>
{disabledHint && <div className={`text-xs mt-1 ${hintClass}`}>{disabledHint}</div>}
{matchedHint && <div className="text-xs mt-1 text-success">{matchedHint}</div>}
</div>
</SelectItem>
);
})}
<SelectItem value="custom" label="Custom Configuration">
<div>
<div className="font-medium">Custom Configuration</div>
<div className="text-xs text-muted-foreground">Define your auto router from scratch</div>
</div>
</SelectItem>
</SelectContent>
</Select>
{modelsUnverifiable && (
<div className="text-xs mt-1 text-destructive">
Could not load available models.{" "}
<button type="button" className="underline" onClick={() => refetchModels()}>
Retry
</button>
</div>
)}
{presetsPending && <div className="text-xs mt-1 text-muted-foreground">Loading templates...</div>}
{presetsUnavailable && (
<div className="text-xs mt-1 text-destructive">
Could not load templates, so only Custom Configuration is shown.{" "}
<button type="button" className="underline" onClick={() => void refetchPresets()}>
Retry
</button>
</div>
)}
</div>
{requiresTeamScope && (
<FormField
control={form.control}
@ -587,6 +581,103 @@ const AddAutoRouterTab: React.FC<AddAutoRouterTabProps> = ({
</FormField>
)}
{setupMode === "auto" ? (
<AutoSetupControls
qualityLevel={qualityLevel}
optimizeFor={optimizeFor}
onQualityLevelChange={setQualityLevel}
onOptimizeForChange={setOptimizeFor}
recommendation={recommendation}
recommendationEnabled={recommendationEnabled}
recommendationPending={recommendationPending}
recommendationFailed={recommendationFailed}
recommendationError={recommendationError}
requiresTeamScope={requiresTeamScope}
hasAutoPolicy={Boolean(complexityRouterConfig.auto_setup)}
onRetry={() => void refetchRecommendation()}
onRegenerate={regenerateAutoSetup}
onUseManual={() => setSetupMode("manual")}
/>
) : (
<div>
<div className="mb-2 flex items-center justify-between gap-3">
<label className="block text-sm font-medium text-foreground">Template</label>
<Button
type="button"
variant="ghost"
className="h-auto px-0"
onClick={() => {
appliedRecommendationKey.current = null;
setSetupMode("auto");
}}
>
Use Auto setup
</Button>
</div>
<Select
items={templateItems}
value={selectedPreset ?? null}
onValueChange={(presetKey: string | null) => handlePresetChange(presetKey ?? undefined)}
>
<SelectTrigger data-testid="template-selector" className="w-full">
<SelectValue placeholder="Choose a template or select Custom to define your own" />
</SelectTrigger>
<SelectContent>
{sortedPresetOptions.map(({ preset, availability: presetState }) => {
const disabledHint = presetDisabledHint(presetState);
const hintClass = isPresetHintAlarming(presetState)
? "text-destructive"
: "text-muted-foreground";
const matchedHint =
presetState.kind === "available" && presetState.viaDeployments
? "Matches your deployments"
: null;
return (
<SelectItem
key={preset.key}
value={preset.key}
label={preset.label}
disabled={disabledHint !== null}
title={disabledHint ?? preset.description}
>
<div>
<div className="font-medium">{preset.label}</div>
<div className="text-xs text-muted-foreground">{preset.description}</div>
{disabledHint && <div className={`text-xs mt-1 ${hintClass}`}>{disabledHint}</div>}
{matchedHint && <div className="text-xs mt-1 text-success">{matchedHint}</div>}
</div>
</SelectItem>
);
})}
<SelectItem value="custom" label="Custom Configuration">
<div>
<div className="font-medium">Custom Configuration</div>
<div className="text-xs text-muted-foreground">Define your auto router from scratch</div>
</div>
</SelectItem>
</SelectContent>
</Select>
{modelsUnverifiable && (
<div className="text-xs mt-1 text-destructive">
Could not load available models.{" "}
<button type="button" className="underline" onClick={() => refetchModels()}>
Retry
</button>
</div>
)}
{presetsPending && <div className="text-xs mt-1 text-muted-foreground">Loading templates...</div>}
{presetsUnavailable && (
<div className="text-xs mt-1 text-destructive">
Could not load templates, so only Custom Configuration is shown.{" "}
<button type="button" className="underline" onClick={() => void refetchPresets()}>
Retry
</button>
</div>
)}
</div>
)}
<div className="border border-border rounded-lg">
<button
type="button"
@ -615,7 +706,9 @@ const AddAutoRouterTab: React.FC<AddAutoRouterTabProps> = ({
onEditingTiersChange={setEditingTiers}
modelInfo={modelInfo}
value={complexityRouterConfig}
onChange={setComplexityRouterConfig}
onChange={(next) =>
setComplexityRouterConfig((previous) => detachAutoSetupAfterPoolEdit(previous, next))
}
customTechnicalKeywords={customTechnicalKeywords}
onCustomTechnicalKeywordsChange={setCustomTechnicalKeywords}
keywordTierRules={keywordTierRules}

View file

@ -0,0 +1,42 @@
import type { ComplexityRouterConfigValue } from "./ComplexityRouterConfig";
export type SetupMode = "auto" | "manual";
export interface RecommendationStatus {
setupMode: SetupMode;
recommendationEnabled: boolean;
recommendationPending: boolean;
recommendationFailed: boolean;
hasRecommendation: boolean;
}
export const getRecommendationBlockedReason = ({
setupMode,
recommendationEnabled,
recommendationPending,
recommendationFailed,
hasRecommendation,
}: RecommendationStatus): string | null => {
if (setupMode !== "auto") return null;
if (!recommendationEnabled) return "Select a team so Auto can use that team's available models";
if (recommendationPending && !hasRecommendation) return "Building the best setup for your available models";
if (recommendationFailed && !hasRecommendation) {
return "Auto setup could not build a router for your available models";
}
return null;
};
const autoSetupPoolShape = (value: ComplexityRouterConfigValue): string =>
JSON.stringify({
tiers: value.tiers,
customTierSet: value.custom_tier_set,
tierModelParams: value.tier_model_params,
});
export const detachAutoSetupAfterPoolEdit = (
previous: ComplexityRouterConfigValue,
next: ComplexityRouterConfigValue,
): ComplexityRouterConfigValue =>
previous.auto_setup && autoSetupPoolShape(previous) !== autoSetupPoolShape(next)
? { ...next, auto_setup: undefined }
: next;

View file

@ -138,6 +138,7 @@ export interface BuildComplexityRouterConfigParams {
tierModelParams?: TierModelParamsByTier;
enableContextWindowEscalation?: boolean;
contextWindowEscalationBuffer?: number;
autoSetup?: AutoSetupConfigPayload;
}
/**
@ -196,6 +197,33 @@ export interface ComplexityRouterConfigPayload {
enable_context_window_escalation?: boolean;
context_window_escalation_buffer?: number;
tier_model_configs?: Record<string, { model_name: string; litellm_params: TierModelParams }[]>;
auto_setup?: AutoSetupConfigPayload;
}
export type AutoSetupQualityLevel = "economy" | "balanced" | "high" | "max";
export type AutoSetupObjective = "cost" | "task_completion_speed" | "balanced";
export interface AutoSetupCandidatePayload {
model_name: string;
benchmark_model_id: string;
quality_lower_bound: number;
cost_per_completed_task_usd: number;
}
export interface AutoSetupTierPolicyPayload {
selection_mode: "snapshot_ranked" | "runtime_response_latency";
candidates: AutoSetupCandidatePayload[];
cold_start_model?: string | null;
evidence_status: string;
}
export interface AutoSetupConfigPayload {
schema_version: "1";
snapshot_id: string;
snapshot_sha256: string;
quality_level: AutoSetupQualityLevel;
optimize_for: AutoSetupObjective;
tier_policies: Record<string, AutoSetupTierPolicyPayload>;
}
export const serializeTierLabels = (tierLabels: ComplexityTierLabels | undefined): ComplexityTierLabels | undefined => {
@ -456,6 +484,7 @@ export const buildComplexityRouterConfig = ({
tierModelParams,
enableContextWindowEscalation,
contextWindowEscalationBuffer,
autoSetup,
}: BuildComplexityRouterConfigParams): ComplexityRouterConfigPayload => {
const serializedTierModelConfigs = customTierSet
? serializeTierModelConfigs(
@ -522,6 +551,7 @@ export const buildComplexityRouterConfig = ({
...(contextWindowEscalationBuffer !== undefined && {
context_window_escalation_buffer: contextWindowEscalationBuffer,
}),
...(autoSetup && { auto_setup: autoSetup }),
...scorerKnobs,
};
if (!customTierSet) return payload;

View file

@ -553,6 +553,7 @@ describe("managed keys survive an untouched open-and-save", () => {
"fallback_tier",
"classification_prompt",
"hybrid_boundary_margin",
"auto_setup",
]);
it("carries every managed key a built-in router can hold through hydrate then save", () => {
@ -565,6 +566,21 @@ describe("managed keys survive an untouched open-and-save", () => {
expect(dropped).toEqual([]);
});
it("round-trips the stored Auto policy while its model pools stay unchanged", () => {
const autoSetup = {
schema_version: "1" as const,
snapshot_id: "snapshot-test",
snapshot_sha256: "a".repeat(64),
quality_level: "max" as const,
optimize_for: "cost" as const,
tier_policies: {},
};
const stored = { ...STORED, classifier_type: "heuristic" as const, auto_setup: autoSetup };
const hydrated = hydrateComplexityRouterConfig(stored, undefined);
expect(buildUpdatedComplexityRouterConfig(stored, hydrated).auto_setup).toEqual(autoSetup);
});
it("drops a stored local-scorer threshold when the operator converts the router to custom tiers", () => {
const hydrated = hydrateComplexityRouterConfig(STORED_ALL_MANAGED, undefined);
const converted = {

View file

@ -38,6 +38,7 @@ import {
hydratePlanModeMinTier,
hydrateTierLabels,
dryRunRejection,
type AutoSetupConfigPayload,
} from "../add_model/build_complexity_router_config";
import { KeywordTierRule } from "../add_model/KeywordTierRules";
import { DEFAULT_MATCH_THRESHOLD } from "../add_model/SemanticKeywordMatching";
@ -60,6 +61,7 @@ import ComplexityRouterConfig, {
DEFAULT_DEPLOYMENT_AFFINITY,
DEFAULT_TIER_DISTANCE_PENALTY,
} from "../add_model/ComplexityRouterConfig";
import { detachAutoSetupAfterPoolEdit } from "../add_model/auto_setup_policy";
import {
Dialog,
DialogContent,
@ -114,6 +116,7 @@ export interface StoredComplexityRouterConfig {
return_raw_model_name?: boolean;
enable_context_window_escalation?: unknown;
context_window_escalation_buffer?: unknown;
auto_setup?: AutoSetupConfigPayload;
}
/**
@ -202,6 +205,7 @@ export const hydrateComplexityRouterConfig = (
typeof parsedConfig.context_window_escalation_buffer === "number"
? parsedConfig.context_window_escalation_buffer
: undefined,
auto_setup: parsedConfig.auto_setup,
};
};
@ -238,6 +242,7 @@ export const MANAGED_COMPLEXITY_ROUTER_KEYS = new Set([
"reasoning_override_min_score",
"enable_context_window_escalation",
"context_window_escalation_buffer",
"auto_setup",
]);
// Managed only when the caller passes the corresponding state. A caller that does not render
@ -343,6 +348,7 @@ export const buildUpdatedComplexityRouterConfig = (
tierModelParams: value.tier_model_params,
enableContextWindowEscalation: value.enable_context_window_escalation,
contextWindowEscalationBuffer: value.context_window_escalation_buffer,
autoSetup: value.auto_setup,
};
const built = buildComplexityRouterConfig(builderParams);
@ -723,7 +729,7 @@ const EditAutoRouterModal: React.FC<EditAutoRouterModalProps> = ({
modelInfo={modelInfo}
value={complexityRouterConfig}
onChange={(config) => {
setComplexityRouterConfig(config);
setComplexityRouterConfig((previous) => detachAutoSetupAfterPoolEdit(previous, config));
}}
customTechnicalKeywords={customTechnicalKeywords}
onCustomTechnicalKeywordsChange={setCustomTechnicalKeywords}

View file

@ -89,7 +89,11 @@ import type {
CoordinationRedisTestResponse,
} from "@/app/(dashboard)/caching/_components/coordination_redis_settings/types";
import { MCP_TOOLS_PREVIEW_FORBIDDEN_MESSAGE } from "./mcp_tools/constants";
import type { ComplexityRouterConfigPayload } from "./add_model/build_complexity_router_config";
import type {
AutoSetupObjective,
AutoSetupQualityLevel,
ComplexityRouterConfigPayload,
} from "./add_model/build_complexity_router_config";
import type { AutoRouterPresetsResponse } from "@/lib/autorouter_presets";
import type { VectorStoreIndex } from "@/app/(dashboard)/vector-stores/_components/IndexesTab";
import type { RoutingDecision } from "./view_logs/LogDetailsDrawer/RoutingDecisionCard";
@ -2416,6 +2420,30 @@ export interface AutoRouterRoutingTestRequest {
team_id?: string;
}
export interface AutoRouterRecommendationResponse {
quality_level: AutoSetupQualityLevel;
optimize_for: AutoSetupObjective;
snapshot_id: string;
snapshot_generated_at: string;
matched_model_groups: string[];
complexity_router_config: ComplexityRouterConfigPayload;
}
export const getAutoRouterRecommendation = async (
accessToken: string,
qualityLevel: AutoSetupQualityLevel,
optimizeFor: AutoSetupObjective,
teamId?: string,
): Promise<AutoRouterRecommendationResponse> =>
await apiClient.get<AutoRouterRecommendationResponse>("/auto_router/recommendation", {
accessToken,
query: {
quality_level: qualityLevel,
optimize_for: optimizeFor,
...(teamId ? { team_id: teamId } : {}),
},
});
export interface AutoRouterRoutingTestResult {
routed_model: string;
routed_model_configured: boolean;

View file

@ -294,6 +294,7 @@ export const buildPresetPrefill = (
return_raw_model_name: config.return_raw_model_name,
enable_context_window_escalation: config.enable_context_window_escalation,
context_window_escalation_buffer: config.context_window_escalation_buffer,
auto_setup: config.auto_setup,
},
customTechnicalKeywords: config.custom_technical_keywords ?? [],
keywordTierRules: hydrateKeywordTierRules(config.keyword_tier_rules ?? []),

View file

@ -1225,6 +1225,26 @@ export interface paths {
patch?: never;
trace?: never;
};
"/auto_router/recommendation": {
parameters: {
query?: never;
header?: never;
path?: never;
cookie?: never;
};
/**
* Get Auto Router Recommendation
* @description Build an editable four-tier router from only the model groups this caller can use.
*/
get: operations["get_auto_router_recommendation_auto_router_recommendation_get"];
put?: never;
post?: never;
delete?: never;
options?: never;
head?: never;
patch?: never;
trace?: never;
};
"/auto_router/shadow_eval": {
parameters: {
query?: never;
@ -23461,6 +23481,29 @@ export interface components {
/** Simple */
SIMPLE: string[];
};
/**
* AutoRouterRecommendationResponse
* @description An editable complexity-router config generated from the caller's usable model groups.
*/
AutoRouterRecommendationResponse: {
complexity_router_config: components["schemas"]["RequestComplexityRouterConfig"];
/** Matched Model Groups */
matched_model_groups: string[];
/**
* Optimize For
* @enum {string}
*/
optimize_for: "cost" | "task_completion_speed" | "balanced";
/**
* Quality Level
* @enum {string}
*/
quality_level: "economy" | "balanced" | "high" | "max";
/** Snapshot Generated At */
snapshot_generated_at: string;
/** Snapshot Id */
snapshot_id: string;
};
/**
* AutoRouterRoutingTestRequest
* @description A single request to classify against a complexity-router config that need not be saved yet.
@ -23534,6 +23577,67 @@ export interface components {
/** @description The decision record this request would have written to its log row */
routing_decision: components["schemas"]["StandardLoggingRoutingDecision"];
};
/**
* AutoSetupCandidate
* @description One benchmark-backed model group in an Auto-generated tier policy.
*/
AutoSetupCandidate: {
/** Benchmark Model Id */
benchmark_model_id: string;
/** Cost Per Completed Task Usd */
cost_per_completed_task_usd: number;
/** Model Name */
model_name: string;
/** Quality Lower Bound */
quality_lower_bound: number;
};
/**
* AutoSetupConfig
* @description Provenance and selection policy stamped only by zero-setup Auto creation.
*/
AutoSetupConfig: {
/**
* Optimize For
* @enum {string}
*/
optimize_for: "cost" | "task_completion_speed" | "balanced";
/**
* Quality Level
* @enum {string}
*/
quality_level: "economy" | "balanced" | "high" | "max";
/**
* Schema Version
* @default 1
* @constant
*/
schema_version: "1";
/** Snapshot Id */
snapshot_id: string;
/** Snapshot Sha256 */
snapshot_sha256: string;
/** Tier Policies */
tier_policies: {
[key: string]: components["schemas"]["AutoSetupTierPolicy"];
};
};
/**
* AutoSetupTierPolicy
* @description How an Auto-generated tier chooses from its quality-admitted model groups.
*/
AutoSetupTierPolicy: {
/** Candidates */
candidates: components["schemas"]["AutoSetupCandidate"][];
/** Cold Start Model */
cold_start_model?: string | null;
/** Evidence Status */
evidence_status: string;
/**
* Selection Mode
* @enum {string}
*/
selection_mode: "snapshot_ranked" | "runtime_response_latency";
};
/** BaseLitellmParams */
BaseLitellmParams: {
/**
@ -34598,6 +34702,8 @@ export interface components {
adaptive_eligible: "all" | "classified_tier";
/** @description Quality vs cost weights for adaptive selection (used when adaptive=True) */
adaptive_weights?: components["schemas"]["AdaptiveRouterWeights"];
/** @description Benchmark snapshot and per-tier selection policy generated by the zero-setup Auto flow. Absent on manually configured routers, whose pool selection behavior is unchanged */
auto_setup?: components["schemas"]["AutoSetupConfig"] | null;
/**
* Classification Mode
* @description When to run the complexity classifier. 'every_request' (the default) classifies every inference request, including the tool-result continuation turns of an agentic loop. 'user_turn' classifies only requests whose newest turn is a new human ask and replays the session's held routing decision on continuation turns, which cuts classifier spend and eliminates mid-loop model switches. Continuations with no held decision to replay (no resolvable session_id, expired pin, fresh restart) still classify. Unlike session_affinity, a new human ask always re-classifies, so a session can still move tiers between asks. Suppressed when plugins are configured, for the same reason session_affinity is: a replayed decision would bypass the plugin pipeline.
@ -35954,6 +36060,23 @@ export interface components {
* @description Per-request provenance for a pre-routing strategy (auto-router) decision.
*/
StandardLoggingRoutingDecision: {
/**
* Auto Setup Optimize For
* @enum {string}
*/
auto_setup_optimize_for?: "cost" | "task_completion_speed" | "balanced";
/**
* Auto Setup Quality Level
* @enum {string}
*/
auto_setup_quality_level?: "economy" | "balanced" | "high" | "max";
/**
* Auto Setup Selection Mode
* @enum {string}
*/
auto_setup_selection_mode?: "snapshot_ranked" | "runtime_response_latency";
/** Auto Setup Snapshot Id */
auto_setup_snapshot_id?: string;
/**
* Cause
* @enum {string}
@ -41152,6 +41275,42 @@ export interface operations {
};
};
};
get_auto_router_recommendation_auto_router_recommendation_get: {
parameters: {
query?: {
/** @description How close every admitted model must be to the best available quality score */
quality_level?: "economy" | "balanced" | "high" | "max";
/** @description What to minimize after the selected quality gate is applied */
optimize_for?: "cost" | "task_completion_speed" | "balanced";
/** @description Team whose model access the recommendation must use */
team_id?: string | null;
};
header?: never;
path?: never;
cookie?: never;
};
requestBody?: never;
responses: {
/** @description Successful Response */
200: {
headers: {
[name: string]: unknown;
};
content: {
"application/json": components["schemas"]["AutoRouterRecommendationResponse"];
};
};
/** @description Validation Error */
422: {
headers: {
[name: string]: unknown;
};
content: {
"application/json": components["schemas"]["HTTPValidationError"];
};
};
};
};
list_shadow_eval_jobs_auto_router_shadow_eval_get: {
parameters: {
query?: {

View file

@ -0,0 +1,20 @@
import { vi } from "vitest";
import type { AutoRouterRecommendationResponse } from "@/components/networking";
interface AutoRouterRecommendationQuery {
data: AutoRouterRecommendationResponse | undefined;
isPending: boolean;
isError: boolean;
error: unknown;
refetch: ReturnType<typeof vi.fn>;
}
export const EMPTY_RECOMMENDATION_QUERY: AutoRouterRecommendationQuery = {
data: undefined,
isPending: false,
isError: false,
error: null,
refetch: vi.fn(),
};
export const useAutoRouterRecommendation = vi.fn((): AutoRouterRecommendationQuery => EMPTY_RECOMMENDATION_QUERY);