mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-24 00:52:24 +00:00
feat(ui): simplify auto-router setup and clarify feature limits (#42625)
* feat(ui): simplify auto-router setup and clarify feature limits * fix(ui): validate auto-router drafts before saving * fix: keep auto-router allowances consistent after deletes and refreshes
This commit is contained in:
parent
630c4624f6
commit
2157351004
42 changed files with 3364 additions and 1353 deletions
|
|
@ -937,6 +937,7 @@ class LiteLLMRoutes(enum.Enum):
|
|||
# proxy admin, or team admin naming their own team via team_id
|
||||
"/auto_router/test_routing",
|
||||
"/auto_router/validate_complexity_router_config",
|
||||
"/auto_router/availability",
|
||||
# Per-session auto-router read - the endpoint scopes the row to the caller's own key hash
|
||||
"/auto_router/session",
|
||||
"/cost/predict-cache",
|
||||
|
|
|
|||
|
|
@ -58,6 +58,8 @@ from litellm.router_utils.auto_router_model_naming import (
|
|||
)
|
||||
from litellm.types.management_endpoints.auto_router_endpoints import (
|
||||
SHADOW_EVAL_TURN_VALVE,
|
||||
AutoRouterAvailabilityRequest,
|
||||
AutoRouterAvailabilityResponse,
|
||||
AutoRouterBenchmarkGroup,
|
||||
AutoRouterBenchmarksResponse,
|
||||
AutoRouterBenchmarkTotals,
|
||||
|
|
@ -391,6 +393,54 @@ async def validate_complexity_router_config(
|
|||
return ComplexityRouterConfigValidationResponse(valid=error is None, error=error)
|
||||
|
||||
|
||||
@router.post(
|
||||
"/auto_router/availability",
|
||||
tags=["model management"], # mutable-ok: FastAPI requires a list
|
||||
response_model=AutoRouterAvailabilityResponse,
|
||||
)
|
||||
async def get_auto_router_availability(
|
||||
data: AutoRouterAvailabilityRequest,
|
||||
user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)],
|
||||
) -> AutoRouterAvailabilityResponse:
|
||||
from litellm.proxy.management_helpers.auto_router_availability import auto_router_availability
|
||||
from litellm.proxy.proxy_server import (
|
||||
_license_check, # pyright: ignore[reportPrivateUsage] # same entitlement owner as the model write gate
|
||||
heuristic_v1_tuning_baselines,
|
||||
llm_router,
|
||||
proxy_config,
|
||||
)
|
||||
|
||||
member_team: Final = await _authorize_router_dry_run(user_api_key_dict, data.team_id)
|
||||
rows: Final = proxy_config.auto_router_db_catalog
|
||||
if rows is None or llm_router is None:
|
||||
raise HTTPException(status_code=503, detail="Auto-router availability is unavailable")
|
||||
saved: Final = next((row for row in rows if row.model_id == data.saved_model_id), None)
|
||||
if data.saved_model_id is not None:
|
||||
if saved is None:
|
||||
raise HTTPException(status_code=404, detail="Saved auto router is unavailable")
|
||||
if user_api_key_dict.user_role != LitellmUserRoles.PROXY_ADMIN and (
|
||||
saved.team_id != data.team_id or (member_team is not None and saved.created_by != user_api_key_dict.user_id)
|
||||
):
|
||||
raise HTTPException(status_code=403, detail="Cannot check another user's auto router")
|
||||
existing: Final = saved.deployment if saved is not None else None
|
||||
others: Final = tuple(row.deployment for row in rows if row is not saved) + tuple(llm_router.config_deployments())
|
||||
candidate: Final = MappingProxyType(
|
||||
{
|
||||
"litellm_params": MappingProxyType(
|
||||
{"model": "auto_router/complexity_router", "complexity_router_config": data.complexity_router_config}
|
||||
),
|
||||
"model_info": MappingProxyType({"id": data.saved_model_id or "availability-new-router", "db_model": True}),
|
||||
}
|
||||
)
|
||||
return auto_router_availability(
|
||||
others=others,
|
||||
existing=existing,
|
||||
candidate=candidate,
|
||||
baselines=heuristic_v1_tuning_baselines,
|
||||
limit=_license_check.auto_router_capability_limit(),
|
||||
)
|
||||
|
||||
|
||||
async def _resolve_saved_routing_test(
|
||||
data: AutoRouterRoutingTestRequest,
|
||||
user_api_key_dict: UserAPIKeyAuth,
|
||||
|
|
|
|||
|
|
@ -1782,10 +1782,11 @@ async def delete_team_models(
|
|||
# Under MODEL_RECONCILE_LOCK, for the same reason as delete_model: the rows are
|
||||
# gone, but a reconcile holding a pre-delete snapshot would upsert these ids back
|
||||
# onto this pod. The lock orders the eviction after any in-flight reconcile.
|
||||
if llm_router is not None:
|
||||
from litellm.proxy.proxy_server import MODEL_RECONCILE_LOCK
|
||||
from litellm.proxy.proxy_server import MODEL_RECONCILE_LOCK, proxy_config
|
||||
|
||||
async with MODEL_RECONCILE_LOCK:
|
||||
async with MODEL_RECONCILE_LOCK:
|
||||
proxy_config.remove_auto_router_catalog_entries(frozenset(deleted_model_ids))
|
||||
if llm_router is not None:
|
||||
for model_id in deleted_model_ids:
|
||||
llm_router.delete_deployment(id=model_id)
|
||||
|
||||
|
|
@ -2194,6 +2195,7 @@ async def delete_model(
|
|||
llm_router,
|
||||
premium_user,
|
||||
prisma_client,
|
||||
proxy_config,
|
||||
proxy_logging_obj,
|
||||
store_model_in_db,
|
||||
user_api_key_cache,
|
||||
|
|
@ -2245,8 +2247,9 @@ async def delete_model(
|
|||
# this pod serving a model the database no longer has, until the next
|
||||
# reconcile. Taking the lock orders this eviction after any such in-flight
|
||||
# reconcile's re-add, so the eviction is the last word.
|
||||
if llm_router is not None:
|
||||
async with MODEL_RECONCILE_LOCK:
|
||||
async with MODEL_RECONCILE_LOCK:
|
||||
proxy_config.remove_auto_router_catalog_entries(frozenset({model_info.id}))
|
||||
if llm_router is not None:
|
||||
llm_router.delete_deployment(id=model_info.id)
|
||||
|
||||
# Runs after the row delete so the sibling check sees post-delete state.
|
||||
|
|
|
|||
148
litellm/proxy/management_helpers/auto_router_availability.py
Normal file
148
litellm/proxy/management_helpers/auto_router_availability.py
Normal file
|
|
@ -0,0 +1,148 @@
|
|||
from collections.abc import Mapping, Sequence
|
||||
from copy import deepcopy
|
||||
from dataclasses import dataclass
|
||||
from types import MappingProxyType
|
||||
from typing import Final
|
||||
|
||||
from pydantic import BaseModel, Json, TypeAdapter, ValidationError
|
||||
|
||||
from litellm.proxy.common_utils.encrypt_decrypt_utils import decrypt_value_helper
|
||||
from litellm.router_utils.auto_router_model_naming import (
|
||||
GATED_AUTO_ROUTER_CAPABILITIES,
|
||||
capability_limit_violation,
|
||||
classify_strategy_router_model,
|
||||
count_capability_routers,
|
||||
gated_capability_of,
|
||||
)
|
||||
from litellm.router_utils.auto_router_tuning_baseline import (
|
||||
is_mutable_tuned_candidate,
|
||||
mutable_tuned_identities,
|
||||
tuning_quota_violation,
|
||||
)
|
||||
from litellm.types.management_endpoints.auto_router_endpoints import (
|
||||
AutoRouterAllowance,
|
||||
AutoRouterAvailabilityResponse,
|
||||
)
|
||||
|
||||
|
||||
class _CatalogModelInfo(BaseModel):
|
||||
team_id: str | None = None
|
||||
|
||||
|
||||
class _CatalogSource(BaseModel):
|
||||
model_id: str
|
||||
created_by: str | None = None
|
||||
litellm_params: Json[dict[str, object]] | dict[str, object]
|
||||
model_info: Json[_CatalogModelInfo] | _CatalogModelInfo | None = None
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class AutoRouterCatalogEntry:
|
||||
model_id: str
|
||||
team_id: str | None
|
||||
created_by: str | None
|
||||
deployment: Mapping[str, object]
|
||||
|
||||
|
||||
def _catalog_field(value: object, key: str) -> object:
|
||||
if not isinstance(value, str):
|
||||
return deepcopy(value)
|
||||
return decrypt_value_helper(value, key=key, exception_type="debug", return_original_value=True)
|
||||
|
||||
|
||||
def build_auto_router_catalog(rows: Sequence[object]) -> tuple[AutoRouterCatalogEntry, ...] | None:
|
||||
try:
|
||||
sources: Final = TypeAdapter(tuple[_CatalogSource, ...]).validate_python(rows, from_attributes=True)
|
||||
except ValidationError:
|
||||
return None
|
||||
return tuple(
|
||||
AutoRouterCatalogEntry(
|
||||
model_id=row.model_id,
|
||||
team_id=row.model_info.team_id if row.model_info is not None else None,
|
||||
created_by=row.created_by,
|
||||
deployment=MappingProxyType(
|
||||
{
|
||||
"litellm_params": MappingProxyType(
|
||||
{
|
||||
"model": model,
|
||||
"complexity_router_config": _catalog_field(
|
||||
row.litellm_params.get("complexity_router_config"), "complexity_router_config"
|
||||
),
|
||||
}
|
||||
),
|
||||
"model_info": MappingProxyType({"id": row.model_id, "db_model": True}),
|
||||
}
|
||||
),
|
||||
)
|
||||
for row in sources
|
||||
if isinstance(model := _catalog_field(row.litellm_params.get("model"), "model"), str)
|
||||
and classify_strategy_router_model(model) == "complexity"
|
||||
)
|
||||
|
||||
|
||||
def auto_router_availability(
|
||||
*,
|
||||
others: Sequence[Mapping[str, object]],
|
||||
existing: Mapping[str, object] | None,
|
||||
candidate: Mapping[str, object],
|
||||
baselines: Mapping[str, str] | None,
|
||||
limit: int | None,
|
||||
) -> AutoRouterAvailabilityResponse:
|
||||
existing_params: Final = None if existing is None else existing.get("litellm_params")
|
||||
candidate_params: Final = candidate.get("litellm_params")
|
||||
owned: Final = gated_capability_of(existing_params) if isinstance(existing_params, Mapping) else None
|
||||
claimed: Final = gated_capability_of(candidate_params) if isinstance(candidate_params, Mapping) else None
|
||||
counts: Final = tuple(
|
||||
(capability, count_capability_routers(others, capability=capability))
|
||||
for capability in GATED_AUTO_ROUTER_CAPABILITIES
|
||||
)
|
||||
tuned_count: Final = len(mutable_tuned_identities(others, baselines)) if baselines is not None else 0
|
||||
allowances: Final = tuple(
|
||||
AutoRouterAllowance(
|
||||
key=capability.key,
|
||||
limit=limit,
|
||||
remaining=None if limit is None else max(0, limit - held),
|
||||
used_by_this_router=owned is capability,
|
||||
)
|
||||
for capability, held in counts
|
||||
)
|
||||
capability_error: Final = next(
|
||||
(
|
||||
capability_limit_violation(capability=capability, held=held + 1, limit=limit)
|
||||
for capability, held in counts
|
||||
if capability is claimed
|
||||
),
|
||||
None,
|
||||
)
|
||||
tuning_error: Final = (
|
||||
tuning_quota_violation(candidate=candidate, others=others, baselines=baselines, limit=limit)
|
||||
if baselines is not None
|
||||
else None
|
||||
)
|
||||
capability_labels: Final = {
|
||||
"heuristic_v2": "Heuristic v2",
|
||||
"capability": "Capability",
|
||||
"llm_v2": "Fuse v2",
|
||||
"tier_or_classifier_prompt": "Custom tiers or classifier instructions",
|
||||
}
|
||||
return AutoRouterAvailabilityResponse(
|
||||
allowances=(
|
||||
*allowances,
|
||||
AutoRouterAllowance(
|
||||
key="heuristic_tuning",
|
||||
limit=limit,
|
||||
remaining=None if limit is None or baselines is None else max(0, limit - tuned_count),
|
||||
available=limit is None or baselines is not None,
|
||||
used_by_this_router=bool(
|
||||
existing is not None and baselines is not None and is_mutable_tuned_candidate(existing, baselines)
|
||||
),
|
||||
),
|
||||
),
|
||||
error=(
|
||||
f"{capability_labels[claimed.key]} has no available allowance. Choose another option or free an existing allowance."
|
||||
if capability_error is not None and claimed is not None
|
||||
else "These scoring rules need an available Rule-based tuning allowance. Check the weights, thresholds, keywords, and custom dimensions in Advanced settings. Model choices do not use this allowance."
|
||||
if tuning_error is not None
|
||||
else None
|
||||
),
|
||||
)
|
||||
|
|
@ -132,6 +132,7 @@ from litellm.proxy.common_utils.callback_utils import (
|
|||
strip_callback_config,
|
||||
)
|
||||
from litellm.proxy.common_utils.realtime_utils import _realtime_request_body
|
||||
from litellm.proxy.management_helpers.auto_router_availability import AutoRouterCatalogEntry, build_auto_router_catalog
|
||||
from litellm.router_utils.access_windows import access_windows_config_error
|
||||
from litellm.router_utils.add_retry_fallback_headers import (
|
||||
get_fallback_errors_from_headers,
|
||||
|
|
@ -5068,6 +5069,7 @@ class ProxyConfig:
|
|||
|
||||
def __init__(self) -> None:
|
||||
self.config: Mapping[str, object] = MappingProxyType({})
|
||||
self.auto_router_db_catalog: tuple[AutoRouterCatalogEntry, ...] | None = None
|
||||
self._last_semantic_filter_config: dict[str, object] | None = None
|
||||
self._last_websearch_interception_config: dict[str, object] | None = None
|
||||
self._last_hashicorp_vault_config: dict[str, object] | None = None
|
||||
|
|
@ -7691,6 +7693,12 @@ class ProxyConfig:
|
|||
def _should_load_db_object(self, object_type: str | SupportedDBObjectType) -> bool:
|
||||
return should_load_db_object(object_type=object_type)
|
||||
|
||||
def remove_auto_router_catalog_entries(self, model_ids: frozenset[str]) -> None:
|
||||
if self.auto_router_db_catalog is not None:
|
||||
self.auto_router_db_catalog = tuple(
|
||||
row for row in self.auto_router_db_catalog if row.model_id not in model_ids
|
||||
)
|
||||
|
||||
async def _get_models_from_db(self, prisma_client: PrismaClient) -> Sequence[_ProxyModelRow] | None:
|
||||
"""
|
||||
Fetch all model deployments from the DB.
|
||||
|
|
@ -7711,6 +7719,7 @@ class ProxyConfig:
|
|||
new_models: Final[Sequence[_ProxyModelRow]] = await ModelRepository(
|
||||
WriterPinnedClient(prisma_client.db)
|
||||
).table.find_many()
|
||||
self.auto_router_db_catalog = build_auto_router_catalog(new_models)
|
||||
return new_models
|
||||
except Exception as e:
|
||||
verbose_proxy_logger.exception(
|
||||
|
|
|
|||
|
|
@ -1,18 +1,18 @@
|
|||
{
|
||||
"1m_context": {
|
||||
"label": "1M Context",
|
||||
"description": "Routes across models with 1M-token context windows: Luna for simple queries, Terra for medium, Sol for complex, Opus 5 at high thinking for reasoning.",
|
||||
"description": "Routes across models with 1M-token context windows: GPT-6 Luna for simple queries, GPT-5.6 Terra for medium, GPT-6 Sol for complex, Opus 5.5 at high thinking for reasoning.",
|
||||
"complexity_router_config": {
|
||||
"tiers": {
|
||||
"SIMPLE": ["gpt-5.6-luna"],
|
||||
"SIMPLE": ["gpt-6-luna"],
|
||||
"MEDIUM": ["gpt-5.6-terra"],
|
||||
"COMPLEX": ["gpt-5.6-sol"],
|
||||
"REASONING": ["claude-opus-5"]
|
||||
"COMPLEX": ["gpt-6-sol"],
|
||||
"REASONING": ["claude-opus-5-5"]
|
||||
},
|
||||
"tier_model_configs": {
|
||||
"REASONING": [
|
||||
{
|
||||
"model_name": "claude-opus-5",
|
||||
"model_name": "claude-opus-5-5",
|
||||
"litellm_params": { "reasoning_effort": "high" }
|
||||
}
|
||||
]
|
||||
|
|
@ -28,12 +28,12 @@
|
|||
},
|
||||
"anthropic_family": {
|
||||
"label": "Anthropic Family",
|
||||
"description": "Routes across the Claude model family: Haiku for simple queries, Sonnet for medium, Opus for complex, Fable 5.1 at high thinking for reasoning.",
|
||||
"description": "Routes across the Claude model family: Haiku for simple queries, Sonnet for medium, Opus 5.5 for complex, Fable 5.1 at high thinking for reasoning.",
|
||||
"complexity_router_config": {
|
||||
"tiers": {
|
||||
"SIMPLE": ["claude-haiku-4-5"],
|
||||
"MEDIUM": ["claude-sonnet-5"],
|
||||
"COMPLEX": ["claude-opus-5"],
|
||||
"COMPLEX": ["claude-opus-5-5"],
|
||||
"REASONING": ["claude-fable-5-1"]
|
||||
},
|
||||
"tier_model_configs": {
|
||||
|
|
@ -55,12 +55,12 @@
|
|||
},
|
||||
"gemini_family": {
|
||||
"label": "Gemini Family",
|
||||
"description": "Routes across the Gemini model family: Flash Lite 2.5 for simple queries, Flash Lite 3.1 for medium, Flash 3.7 for complex, Pro 3.1 for reasoning-heavy requests.",
|
||||
"description": "Routes across the Gemini model family: Flash Lite 3.5 for simple queries, Flash 3.8 for medium and complex queries, Pro 3.1 for reasoning-heavy requests.",
|
||||
"complexity_router_config": {
|
||||
"tiers": {
|
||||
"SIMPLE": ["gemini-2.5-flash-lite"],
|
||||
"MEDIUM": ["gemini-3.1-flash-lite"],
|
||||
"COMPLEX": ["gemini-3.7-flash"],
|
||||
"SIMPLE": ["gemini-3.5-flash-lite"],
|
||||
"MEDIUM": ["gemini-3.8-flash"],
|
||||
"COMPLEX": ["gemini-3.8-flash"],
|
||||
"REASONING": ["gemini-3.1-pro-preview"]
|
||||
},
|
||||
"classifier_type": "heuristic",
|
||||
|
|
@ -74,18 +74,18 @@
|
|||
},
|
||||
"lite": {
|
||||
"label": "Lite",
|
||||
"description": "Cost-optimized routing across providers: DeepSeek V4 Flash for simple queries, Muse Spark 1.2 at xhigh for medium, Kimi K3 at max for complex, Claude Opus 5 for reasoning. An LLM classifier with the agentic rubric assigns tiers.",
|
||||
"description": "Cost-optimized routing across providers: DeepSeek V4 Flash for simple queries, Muse Spark 1.3 at xhigh for medium, Kimi K3 at max for complex, Claude Opus 5.5 for reasoning. An LLM classifier with the agentic rubric assigns tiers.",
|
||||
"complexity_router_config": {
|
||||
"tiers": {
|
||||
"SIMPLE": ["deepseek-v4-flash"],
|
||||
"MEDIUM": ["muse-spark-1.2"],
|
||||
"MEDIUM": ["muse-spark-1.3"],
|
||||
"COMPLEX": ["kimi-k3"],
|
||||
"REASONING": ["claude-opus-5"]
|
||||
"REASONING": ["claude-opus-5-5"]
|
||||
},
|
||||
"tier_model_configs": {
|
||||
"MEDIUM": [
|
||||
{
|
||||
"model_name": "muse-spark-1.2",
|
||||
"model_name": "muse-spark-1.3",
|
||||
"litellm_params": { "reasoning_effort": "xhigh" }
|
||||
}
|
||||
],
|
||||
|
|
@ -113,12 +113,12 @@
|
|||
},
|
||||
"openai_family": {
|
||||
"label": "OpenAI Family",
|
||||
"description": "Routes across the GPT model family: Luna for simple queries, Terra for medium, Sol for complex, Astra at xhigh thinking for reasoning.",
|
||||
"description": "Routes across the GPT model family: GPT-6 Luna for simple queries, GPT-5.6 Terra for medium, GPT-6 Sol for complex, GPT-6 Astra at xhigh thinking for reasoning.",
|
||||
"complexity_router_config": {
|
||||
"tiers": {
|
||||
"SIMPLE": ["gpt-5.6-luna"],
|
||||
"SIMPLE": ["gpt-6-luna"],
|
||||
"MEDIUM": ["gpt-5.6-terra"],
|
||||
"COMPLEX": ["gpt-5.6-sol"],
|
||||
"COMPLEX": ["gpt-6-sol"],
|
||||
"REASONING": ["gpt-6-astra"]
|
||||
},
|
||||
"tier_model_configs": {
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
{
|
||||
"version": "2026-09-17-v1",
|
||||
"version": "2026-09-22-v1",
|
||||
"models": [
|
||||
{
|
||||
"id": "gpt-6-astra-v1",
|
||||
|
|
@ -8,6 +8,20 @@
|
|||
"text": "OpenAI model for demanding end-to-end work, including reasoning, coding, research, and document tasks",
|
||||
"sources": ["https://developers.openai.com/api/docs/models/gpt-6-astra"]
|
||||
},
|
||||
{
|
||||
"id": "gpt-6-sol-v1",
|
||||
"label": "GPT-6 Sol",
|
||||
"model": "gpt-6-sol",
|
||||
"text": "OpenAI model for complex coding and agentic workflows, supporting reasoning and tool calling through the Responses API",
|
||||
"sources": ["https://developers.openai.com/api/docs/models/gpt-6-sol"]
|
||||
},
|
||||
{
|
||||
"id": "gpt-6-luna-v1",
|
||||
"label": "GPT-6 Luna",
|
||||
"model": "gpt-6-luna",
|
||||
"text": "OpenAI model for efficient, high-volume workloads, supporting reasoning and tool calling through the Responses API",
|
||||
"sources": ["https://developers.openai.com/api/docs/models/gpt-6-luna"]
|
||||
},
|
||||
{
|
||||
"id": "gpt-5.6-sol-v1",
|
||||
"label": "GPT-5.6 Sol",
|
||||
|
|
@ -63,6 +77,13 @@
|
|||
"model": "claude-fable-5-1",
|
||||
"text": "Anthropic model for demanding reasoning, long-running agentic coding, and multistep research, with always-on adaptive thinking",
|
||||
"sources": ["https://platform.claude.com/docs/en/models/fable-5-1/overview"]
|
||||
},
|
||||
{
|
||||
"id": "claude-opus-5-5-v1",
|
||||
"label": "Claude Opus 5.5",
|
||||
"model": "claude-opus-5-5",
|
||||
"text": "Anthropic model for complex reasoning and agentic work, supporting adaptive thinking and tool use",
|
||||
"sources": ["https://platform.claude.com/docs/en/models/opus-5-5/overview"]
|
||||
}
|
||||
],
|
||||
"harnesses": [
|
||||
|
|
|
|||
|
|
@ -10,12 +10,10 @@ from pydantic import ValidationError
|
|||
|
||||
from litellm.router_strategy.complexity_router.config import ComplexityRouterConfig
|
||||
|
||||
TUNING_BASELINE_PARAM_NAME: Final = "auto_router_tuning_baseline_v2"
|
||||
# v2 hashes combine models and scoring rules; a new snapshot is required to separate them.
|
||||
TUNING_BASELINE_PARAM_NAME: Final = "auto_router_tuning_baseline_v3"
|
||||
|
||||
HEURISTIC_V1_TUNING_FIELDS: Final = (
|
||||
"tiers",
|
||||
"tier_model_configs",
|
||||
"classifier_type",
|
||||
"tier_boundaries",
|
||||
"reasoning_override_min_score",
|
||||
"token_thresholds",
|
||||
|
|
@ -49,8 +47,10 @@ def tuning_fingerprint(complexity_router_config: object) -> str | None:
|
|||
validated: Final = ComplexityRouterConfig.model_validate(raw)
|
||||
except ValidationError:
|
||||
return None
|
||||
supplied: Final = ((_TUNING_FIELD_SET - frozenset(("tier_model_configs",))) & frozenset(raw)) | (
|
||||
frozenset(("tier_model_configs",)) if validated.tier_model_configs else frozenset()
|
||||
# The UI always writes this built-in marker. Freeze its spelling so future defaults cannot change recorded hashes.
|
||||
default_escalation: Final = validated.escalation_keywords in (None, ["LITELLM ESCALATE"])
|
||||
supplied: Final = (_TUNING_FIELD_SET & frozenset(raw)) - (
|
||||
frozenset(("escalation_keywords",)) if default_escalation else frozenset()
|
||||
)
|
||||
payload: Final = validated.model_dump(
|
||||
mode="json",
|
||||
|
|
@ -148,9 +148,9 @@ def tuning_limit_violation(*, held: int, limit: int | None) -> str | None:
|
|||
if limit is None or held <= limit:
|
||||
return None
|
||||
return (
|
||||
f"At most {limit} auto-router(s) with changed heuristic scorer settings or tier models can be modified "
|
||||
f"At most {limit} auto-router(s) with changed heuristic scoring rules can be modified "
|
||||
"without an auto-router license. Keep this router on its recorded settings, or revert the other changed "
|
||||
"router to its baseline, or remove one of them."
|
||||
"router to its baseline, or remove one of them. Selecting models does not use this allowance."
|
||||
)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -44,6 +44,25 @@ class ComplexityRouterConfigValidationResponse(BaseModel):
|
|||
error: str | None = None
|
||||
|
||||
|
||||
class AutoRouterAvailabilityRequest(BaseModel):
|
||||
team_id: str | None = None
|
||||
saved_model_id: str | None = None
|
||||
complexity_router_config: Mapping[str, object] | None = None
|
||||
|
||||
|
||||
class AutoRouterAllowance(BaseModel):
|
||||
key: str
|
||||
limit: int | None
|
||||
remaining: int | None
|
||||
used_by_this_router: bool = False
|
||||
available: bool = True
|
||||
|
||||
|
||||
class AutoRouterAvailabilityResponse(BaseModel):
|
||||
allowances: tuple[AutoRouterAllowance, ...]
|
||||
error: str | None = None
|
||||
|
||||
|
||||
class AutoRouterRoutingTestRequest(BaseModel):
|
||||
"""A single request to classify against a complexity-router config that need not be saved yet.
|
||||
|
||||
|
|
|
|||
|
|
@ -723,11 +723,13 @@ class TestAutoRouterBenchmarks:
|
|||
def test_savings_compare_only_the_current_estimated_cohort(self, estimated_turns: int) -> None:
|
||||
from litellm.proxy.management_endpoints.auto_router_endpoints import _benchmark_totals
|
||||
|
||||
row: Final = self.ROW.model_copy(update={
|
||||
"savings_estimated_turns": estimated_turns,
|
||||
"savings_estimated_actual_spend": 2.0 if estimated_turns else 0.0,
|
||||
"savings_estimated_saved_spend": -0.5 if estimated_turns else 0.0,
|
||||
})
|
||||
row: Final = self.ROW.model_copy(
|
||||
update={
|
||||
"savings_estimated_turns": estimated_turns,
|
||||
"savings_estimated_actual_spend": 2.0 if estimated_turns else 0.0,
|
||||
"savings_estimated_saved_spend": -0.5 if estimated_turns else 0.0,
|
||||
}
|
||||
)
|
||||
totals: Final = _benchmark_totals(row)
|
||||
assert totals.spend == 10.0
|
||||
assert totals.savings_estimated_turns == estimated_turns
|
||||
|
|
@ -755,10 +757,16 @@ class TestAutoRouterBenchmarks:
|
|||
_summed_agg_row,
|
||||
)
|
||||
|
||||
other = self.ROW.model_copy(update={
|
||||
"router_name": "auto-2", "sessions": 1, "turns": 10, "spend": 0.0,
|
||||
"savings_estimated_turns": 10, "savings_estimated_actual_spend": 0.0,
|
||||
})
|
||||
other = self.ROW.model_copy(
|
||||
update={
|
||||
"router_name": "auto-2",
|
||||
"sessions": 1,
|
||||
"turns": 10,
|
||||
"spend": 0.0,
|
||||
"savings_estimated_turns": 10,
|
||||
"savings_estimated_actual_spend": 0.0,
|
||||
}
|
||||
)
|
||||
summed = _summed_agg_row([self.ROW, other])
|
||||
totals = _benchmark_totals(summed)
|
||||
assert summed.sessions == 5
|
||||
|
|
@ -1091,18 +1099,27 @@ class TestAutoRouterSession:
|
|||
return lookups
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("turns, estimated", [(3, True), (10, True), (10, False)], ids=["full", "partial", "legacy"])
|
||||
@pytest.mark.parametrize(
|
||||
"turns, estimated", [(3, True), (10, True), (10, False)], ids=["full", "partial", "legacy"]
|
||||
)
|
||||
async def test_a_key_reads_its_own_session_with_the_baseline_its_turns_were_priced_against(
|
||||
self, monkeypatch: pytest.MonkeyPatch, turns: int, estimated: bool,
|
||||
self,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
turns: int,
|
||||
estimated: bool,
|
||||
) -> None:
|
||||
from litellm.proxy.management_endpoints.auto_router_endpoints import get_auto_router_session
|
||||
|
||||
caller = UserAPIKeyAuth(api_key="sk-caller")
|
||||
row: Final = {key: value for key, value in self.ROW.items() if estimated or not key.startswith("savings_estimated_")}
|
||||
row: Final = {
|
||||
key: value for key, value in self.ROW.items() if estimated or not key.startswith("savings_estimated_")
|
||||
}
|
||||
spend: Final = 0.14 if turns == 3 else 10.0
|
||||
if estimated and turns != 3:
|
||||
row["savings_estimated_saved_spend"] = -0.04
|
||||
self._rig(monkeypatch, [{**row, "api_key": caller.api_key, "session_id": "sess-1", "turns": turns, "spend": spend}])
|
||||
self._rig(
|
||||
monkeypatch, [{**row, "api_key": caller.api_key, "session_id": "sess-1", "turns": turns, "spend": spend}]
|
||||
)
|
||||
response = await get_auto_router_session(user_api_key_dict=caller, session_id="sess-1")
|
||||
assert response.model_dump() == {
|
||||
"session_id": "sess-1",
|
||||
|
|
@ -1159,10 +1176,18 @@ class TestAutoRouterSession:
|
|||
from litellm.proxy.management_endpoints.auto_router_endpoints import get_auto_router_session
|
||||
|
||||
priced = {"anthropic/claude-opus-5": 2, "anthropic/claude-sonnet-5": 1}
|
||||
self._rig(monkeypatch, [{
|
||||
**self.ROW, "api_key": ADMIN.api_key, "session_id": "s",
|
||||
"baseline_models": {"old-baseline": 100}, "savings_estimated_baseline_models": priced,
|
||||
}])
|
||||
self._rig(
|
||||
monkeypatch,
|
||||
[
|
||||
{
|
||||
**self.ROW,
|
||||
"api_key": ADMIN.api_key,
|
||||
"session_id": "s",
|
||||
"baseline_models": {"old-baseline": 100},
|
||||
"savings_estimated_baseline_models": priced,
|
||||
}
|
||||
],
|
||||
)
|
||||
response = await get_auto_router_session(user_api_key_dict=ADMIN, session_id="s")
|
||||
assert response.baseline_model == "anthropic/claude-opus-5"
|
||||
assert response.baseline_models == priced
|
||||
|
|
@ -3562,3 +3587,97 @@ async def test_start_shadow_eval_seeds_a_zero_funnel_row_per_leg(monkeypatch: py
|
|||
if "group_id" in call.kwargs.get("where", {})
|
||||
]
|
||||
assert group_reads == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_availability_counts_db_and_yaml_without_disclosing_router_names(monkeypatch):
|
||||
from litellm.models.model import LiteLLM_ProxyModelTable
|
||||
from litellm.proxy.management_helpers.auto_router_availability import build_auto_router_catalog
|
||||
from litellm.types.management_endpoints.auto_router_endpoints import AutoRouterAvailabilityRequest
|
||||
|
||||
row = LiteLLM_ProxyModelTable(
|
||||
model_id="db-router",
|
||||
model_name="private-team-router",
|
||||
created_by="someone-else",
|
||||
litellm_params={
|
||||
"model": "auto_router/complexity_router",
|
||||
"complexity_router_config": {"classifier_type": "heuristic_v2"},
|
||||
},
|
||||
)
|
||||
yaml_row = {
|
||||
"model_name": "private-yaml-router",
|
||||
"litellm_params": {
|
||||
"model": "auto_router/complexity_router",
|
||||
"complexity_router_config": {"classifier_type": "capability"},
|
||||
},
|
||||
}
|
||||
find_many = AsyncMock(side_effect=AssertionError("Availability must not query the model table"))
|
||||
monkeypatch.setattr(
|
||||
proxy_server,
|
||||
"prisma_client",
|
||||
SimpleNamespace(db=SimpleNamespace(litellm_proxymodeltable=SimpleNamespace(find_many=find_many))),
|
||||
)
|
||||
monkeypatch.setattr(proxy_server.proxy_config, "auto_router_db_catalog", build_auto_router_catalog((row,)))
|
||||
monkeypatch.setattr(proxy_server, "llm_router", SimpleNamespace(config_deployments=lambda: (yaml_row,)))
|
||||
monkeypatch.setattr(proxy_server, "_license_check", SimpleNamespace(auto_router_capability_limit=lambda: 1))
|
||||
monkeypatch.setattr(proxy_server, "heuristic_v1_tuning_baselines", {})
|
||||
result = await auto_router_endpoints.get_auto_router_availability(AutoRouterAvailabilityRequest(), ADMIN)
|
||||
assert {slot.key: slot.remaining for slot in result.allowances} == {
|
||||
"heuristic_v2": 0,
|
||||
"capability": 0,
|
||||
"llm_v2": 1,
|
||||
"tier_or_classifier_prompt": 1,
|
||||
"heuristic_tuning": 1,
|
||||
}
|
||||
assert "private" not in result.model_dump_json()
|
||||
edit = await auto_router_endpoints.get_auto_router_availability(
|
||||
AutoRouterAvailabilityRequest(
|
||||
saved_model_id="db-router", complexity_router_config={"classifier_type": "heuristic_v2"}
|
||||
),
|
||||
ADMIN,
|
||||
)
|
||||
assert edit.allowances[0].used_by_this_router
|
||||
assert edit.allowances[0].remaining == 1
|
||||
assert edit.error is None
|
||||
find_many.assert_not_awaited()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_availability_denies_another_teams_edit_exemption(monkeypatch):
|
||||
from litellm.models.model import LiteLLM_ProxyModelTable
|
||||
from litellm.proxy.management_helpers.auto_router_availability import build_auto_router_catalog
|
||||
from litellm.types.management_endpoints.auto_router_endpoints import AutoRouterAvailabilityRequest
|
||||
|
||||
row = LiteLLM_ProxyModelTable(
|
||||
model_id="other-router",
|
||||
model_name="other",
|
||||
created_by="other",
|
||||
model_info={"team_id": "other-team"},
|
||||
litellm_params={"model": "auto_router/complexity_router"},
|
||||
)
|
||||
find_many = AsyncMock(side_effect=AssertionError("Availability must not query the model table"))
|
||||
monkeypatch.setattr(
|
||||
proxy_server,
|
||||
"prisma_client",
|
||||
SimpleNamespace(db=SimpleNamespace(litellm_proxymodeltable=SimpleNamespace(find_many=find_many))),
|
||||
)
|
||||
monkeypatch.setattr(proxy_server.proxy_config, "auto_router_db_catalog", build_auto_router_catalog((row,)))
|
||||
monkeypatch.setattr(proxy_server, "llm_router", SimpleNamespace(config_deployments=lambda: ()))
|
||||
monkeypatch.setattr(auto_router_endpoints, "_authorize_router_dry_run", AsyncMock(return_value=None))
|
||||
with pytest.raises(HTTPException) as error:
|
||||
await auto_router_endpoints.get_auto_router_availability(
|
||||
AutoRouterAvailabilityRequest(team_id="own-team", saved_model_id="other-router"),
|
||||
UserAPIKeyAuth(user_role=LitellmUserRoles.INTERNAL_USER, user_id="owner"),
|
||||
)
|
||||
assert error.value.status_code == 403
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_availability_waits_for_the_first_complete_catalog(monkeypatch):
|
||||
from litellm.types.management_endpoints.auto_router_endpoints import AutoRouterAvailabilityRequest
|
||||
|
||||
monkeypatch.setattr(proxy_server.proxy_config, "auto_router_db_catalog", None)
|
||||
monkeypatch.setattr(proxy_server, "llm_router", SimpleNamespace(config_deployments=lambda: ()))
|
||||
with pytest.raises(HTTPException) as error:
|
||||
await auto_router_endpoints.get_auto_router_availability(AutoRouterAvailabilityRequest(), ADMIN)
|
||||
assert error.value.status_code == 503
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ import asyncio
|
|||
import contextlib
|
||||
import json
|
||||
from collections.abc import Iterator, Mapping
|
||||
from types import SimpleNamespace
|
||||
from typing import Dict, Final, Optional
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
|
|
@ -1222,6 +1223,117 @@ class TestDeleteModelClearsRouterRegistry:
|
|||
assert mock_router.complexity_routers.get("shared-name") is config_router
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def deleted_auto_router_catalog(monkeypatch):
|
||||
from litellm.proxy import proxy_server
|
||||
from litellm.proxy.management_helpers.auto_router_availability import build_auto_router_catalog
|
||||
|
||||
rows = tuple(
|
||||
LiteLLM_ProxyModelTable(
|
||||
model_id=model_id,
|
||||
model_name=f"model_name_{team_id}_{model_id}",
|
||||
litellm_params={
|
||||
"model": "auto_router/complexity_router",
|
||||
"complexity_router_config": {"classifier_type": classifier},
|
||||
},
|
||||
model_info={"id": model_id, "team_id": team_id},
|
||||
created_by="admin",
|
||||
updated_by="admin",
|
||||
blocked=True,
|
||||
)
|
||||
for model_id, team_id, classifier in (
|
||||
("deleted-router", "deleted-team", "heuristic_v2"),
|
||||
("surviving-router", "surviving-team", "llm_v2"),
|
||||
)
|
||||
)
|
||||
config = proxy_server.ProxyConfig()
|
||||
config.auto_router_db_catalog = build_auto_router_catalog(rows)
|
||||
monkeypatch.setattr(proxy_server, "proxy_config", config)
|
||||
monkeypatch.setattr(proxy_server, "MODEL_RECONCILE_LOCK", asyncio.Lock())
|
||||
monkeypatch.setattr(proxy_server, "llm_router", Router(model_list=[]))
|
||||
monkeypatch.setattr(proxy_server, "_license_check", SimpleNamespace(auto_router_capability_limit=lambda: 1))
|
||||
monkeypatch.setattr(proxy_server, "heuristic_v1_tuning_baselines", {})
|
||||
return config, rows
|
||||
|
||||
|
||||
class TestDeletedAutoRouterAvailability:
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("delete_succeeds,has_router", ((True, True), (True, False), (False, True)))
|
||||
async def test_single_delete_releases_allowance_only_after_success(
|
||||
self, monkeypatch, deleted_auto_router_catalog, delete_succeeds, has_router
|
||||
):
|
||||
from litellm.proxy import proxy_server
|
||||
from litellm.proxy.management_endpoints.auto_router_endpoints import get_auto_router_availability
|
||||
from litellm.proxy.management_endpoints.model_management_endpoints import ModelInfoDelete, delete_model
|
||||
from litellm.types.management_endpoints.auto_router_endpoints import AutoRouterAvailabilityRequest
|
||||
|
||||
config, rows = deleted_auto_router_catalog
|
||||
original = config.auto_router_db_catalog
|
||||
row = rows[0].model_copy(update={"model_info": {"id": rows[0].model_id}})
|
||||
table = SimpleNamespace(
|
||||
find_unique=AsyncMock(return_value=row),
|
||||
delete=AsyncMock(return_value=row, side_effect=None if delete_succeeds else RuntimeError("delete failed")),
|
||||
)
|
||||
prisma = SimpleNamespace(
|
||||
db=SimpleNamespace(litellm_proxymodeltable=table, query_raw=AsyncMock(return_value=[]))
|
||||
)
|
||||
monkeypatch.setattr(proxy_server, "prisma_client", prisma)
|
||||
monkeypatch.setattr(proxy_server, "store_model_in_db", True)
|
||||
admin = UserAPIKeyAuth(user_id="admin", user_role=LitellmUserRoles.PROXY_ADMIN)
|
||||
request = AutoRouterAvailabilityRequest(complexity_router_config={"classifier_type": "heuristic_v2"})
|
||||
before = await get_auto_router_availability(request, admin)
|
||||
assert before.error is not None
|
||||
if not has_router:
|
||||
monkeypatch.setattr(proxy_server, "llm_router", None)
|
||||
|
||||
if not delete_succeeds:
|
||||
with pytest.raises(ProxyException, match="delete failed"):
|
||||
await delete_model(ModelInfoDelete(id=row.model_id), admin)
|
||||
assert config.auto_router_db_catalog == original
|
||||
return
|
||||
|
||||
await delete_model(ModelInfoDelete(id=row.model_id), admin)
|
||||
monkeypatch.setattr(proxy_server, "llm_router", Router(model_list=[]))
|
||||
after = await get_auto_router_availability(request, admin)
|
||||
assert after.error is None
|
||||
assert {slot.key: slot.remaining for slot in after.allowances} == {
|
||||
"heuristic_v2": 1,
|
||||
"capability": 1,
|
||||
"llm_v2": 0,
|
||||
"tier_or_classifier_prompt": 1,
|
||||
"heuristic_tuning": 1,
|
||||
}
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("has_router", (True, False))
|
||||
async def test_team_delete_releases_only_its_routers_allowance(
|
||||
self, monkeypatch, deleted_auto_router_catalog, has_router
|
||||
):
|
||||
from litellm.proxy import proxy_server
|
||||
from litellm.proxy.management_endpoints.auto_router_endpoints import get_auto_router_availability
|
||||
from litellm.types.management_endpoints.auto_router_endpoints import AutoRouterAvailabilityRequest
|
||||
|
||||
_, rows = deleted_auto_router_catalog
|
||||
prisma = _TxPrismaClient(rows)
|
||||
deleted = await delete_team_models(
|
||||
team_ids=["deleted-team"], prisma_client=prisma, llm_router=proxy_server.llm_router if has_router else None
|
||||
)
|
||||
|
||||
assert deleted == ["deleted-router"]
|
||||
after = await get_auto_router_availability(
|
||||
AutoRouterAvailabilityRequest(complexity_router_config={"classifier_type": "heuristic_v2"}),
|
||||
UserAPIKeyAuth(user_id="admin", user_role=LitellmUserRoles.PROXY_ADMIN),
|
||||
)
|
||||
assert after.error is None
|
||||
assert {slot.key: slot.remaining for slot in after.allowances} == {
|
||||
"heuristic_v2": 1,
|
||||
"capability": 1,
|
||||
"llm_v2": 0,
|
||||
"tier_or_classifier_prompt": 1,
|
||||
"heuristic_tuning": 1,
|
||||
}
|
||||
|
||||
|
||||
class TestUpdateModel:
|
||||
"""
|
||||
Tests for the update_model (POST /model/update) handler.
|
||||
|
|
@ -5274,7 +5386,7 @@ class TestDeleteEvictionsHoldTheReconcileLock:
|
|||
"""
|
||||
|
||||
@staticmethod
|
||||
async def _assert_evicts_under_lock(monkeypatch, call_endpoint, model_id: str) -> None:
|
||||
async def _assert_evicts_under_lock(monkeypatch, call_endpoint, model_id: str, config) -> None:
|
||||
"""Run ``call_endpoint`` with the lock already held and assert it blocks.
|
||||
|
||||
Holding MODEL_RECONCILE_LOCK stands in for a reconcile that is mid-flight. If
|
||||
|
|
@ -5291,6 +5403,7 @@ class TestDeleteEvictionsHoldTheReconcileLock:
|
|||
"""
|
||||
lock = asyncio.Lock()
|
||||
monkeypatch.setattr("litellm.proxy.proxy_server.MODEL_RECONCILE_LOCK", lock)
|
||||
stale_catalog = config.auto_router_db_catalog
|
||||
|
||||
async with lock:
|
||||
task = asyncio.create_task(call_endpoint())
|
||||
|
|
@ -5301,16 +5414,19 @@ class TestDeleteEvictionsHoldTheReconcileLock:
|
|||
f"deleting {model_id} did not wait for MODEL_RECONCILE_LOCK -- an "
|
||||
f"in-flight reconcile can resurrect the deployment it just evicted"
|
||||
)
|
||||
config.auto_router_db_catalog = stale_catalog
|
||||
await asyncio.wait_for(task, timeout=5)
|
||||
assert tuple(row.model_id for row in config.auto_router_db_catalog) == ("surviving-router",)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_delete_model_waits_for_an_in_flight_reconcile(self, monkeypatch):
|
||||
async def test_delete_model_waits_for_an_in_flight_reconcile(self, monkeypatch, deleted_auto_router_catalog):
|
||||
from litellm.proxy.management_endpoints.model_management_endpoints import (
|
||||
ModelInfoDelete,
|
||||
delete_model,
|
||||
)
|
||||
|
||||
model_id = "m-doomed"
|
||||
config, rows = deleted_auto_router_catalog
|
||||
model_id = rows[0].model_id
|
||||
row = MagicMock()
|
||||
row.model_dump.return_value = {
|
||||
"model_name": "gpt-4o",
|
||||
|
|
@ -5347,16 +5463,17 @@ class TestDeleteEvictionsHoldTheReconcileLock:
|
|||
),
|
||||
)
|
||||
|
||||
await self._assert_evicts_under_lock(monkeypatch, call, model_id)
|
||||
await self._assert_evicts_under_lock(monkeypatch, call, model_id, config)
|
||||
router.delete_deployment.assert_called_once_with(id=model_id)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_delete_team_models_waits_for_an_in_flight_reconcile(self, monkeypatch):
|
||||
async def test_delete_team_models_waits_for_an_in_flight_reconcile(self, monkeypatch, deleted_auto_router_catalog):
|
||||
from litellm.proxy.management_endpoints.model_management_endpoints import (
|
||||
delete_team_models,
|
||||
)
|
||||
|
||||
model_id = "m-team-doomed"
|
||||
config, rows = deleted_auto_router_catalog
|
||||
model_id = rows[0].model_id
|
||||
router = MagicMock()
|
||||
router.delete_deployment = MagicMock(return_value=True)
|
||||
|
||||
|
|
@ -5392,7 +5509,7 @@ class TestDeleteEvictionsHoldTheReconcileLock:
|
|||
team_ids=["team-1"], prisma_client=prisma, llm_router=router
|
||||
)
|
||||
|
||||
await self._assert_evicts_under_lock(monkeypatch, call, model_id)
|
||||
await self._assert_evicts_under_lock(monkeypatch, call, model_id, config)
|
||||
router.delete_deployment.assert_called_once_with(id=model_id)
|
||||
|
||||
|
||||
|
|
@ -6092,7 +6209,8 @@ class TestStrategyRouterWriteValidation:
|
|||
_TUNED_A = {"classifier_type": "heuristic", "tiers": {"SIMPLE": "gpt-4o-mini", "MEDIUM": "gpt-4o"}}
|
||||
_TUNED_A_EDITED = {**_TUNED_A, "dimension_weights": {"codePresence": 0.9}}
|
||||
_TUNED_B = {"classifier_type": "heuristic", "tiers": {"SIMPLE": "gpt-4o-mini", "MEDIUM": "gpt-4.1"}}
|
||||
_TUNED_B_EDITED = {**_TUNED_B, "tiers": {"SIMPLE": "gpt-4o", "MEDIUM": "gpt-4.1"}}
|
||||
_TUNED_B_EDITED = {**_TUNED_B, "code_keywords": ["internal-api"]}
|
||||
_MODELS_ONLY_B = {**_TUNED_B, "tiers": {"SIMPLE": "fast-model", "MEDIUM": "capable-model"}}
|
||||
|
||||
@staticmethod
|
||||
def _db_router_row(model_id: str, config: Mapping[str, object]) -> dict[str, object]:
|
||||
|
|
@ -6110,7 +6228,9 @@ class TestStrategyRouterWriteValidation:
|
|||
(1, ["a", "b"], {"a": "_TUNED_A", "b": "_TUNED_B"}, "a", "_TUNED_A_EDITED", "allowed"),
|
||||
(1, ["a", "b"], {"a": "_TUNED_A_EDITED", "b": "_TUNED_B"}, "a", "_TUNED_A_EDITED", "allowed"),
|
||||
(1, ["a", "b"], {"a": "_TUNED_A_EDITED", "b": "_TUNED_B"}, "b", "_TUNED_B_EDITED", "refused"),
|
||||
(1, ["a", "b"], {"a": "_TUNED_A_EDITED", "b": "_TUNED_B"}, "c", "_TUNED_B", "refused"),
|
||||
(1, ["a", "b"], {"a": "_TUNED_A_EDITED", "b": "_TUNED_B"}, "c", "_TUNED_B_EDITED", "refused"),
|
||||
(1, ["a", "b"], {"a": "_TUNED_A_EDITED", "b": "_TUNED_B"}, "c", "_TUNED_B", "allowed"),
|
||||
(1, ["a", "b"], {"a": "_TUNED_A_EDITED", "b": "_TUNED_B"}, "b", "_MODELS_ONLY_B", "allowed"),
|
||||
(1, ["a", "b"], {"a": "_TUNED_A_EDITED", "b": "_TUNED_B"}, "b", "_TUNED_B", "allowed"),
|
||||
(None, ["a", "b"], {"a": "_TUNED_A_EDITED", "b": "_TUNED_B"}, "b", "_TUNED_B_EDITED", "allowed"),
|
||||
(1, [], {}, "c", "_TUNED_B", "allowed"),
|
||||
|
|
@ -6138,6 +6258,7 @@ class TestStrategyRouterWriteValidation:
|
|||
"_TUNED_A_EDITED": self._TUNED_A_EDITED,
|
||||
"_TUNED_B": self._TUNED_B,
|
||||
"_TUNED_B_EDITED": self._TUNED_B_EDITED,
|
||||
"_MODELS_ONLY_B": self._MODELS_ONLY_B,
|
||||
}
|
||||
baselines = snapshot_tuning_baselines(
|
||||
[self._db_router_row(row_id, configs["_TUNED_A" if row_id == "a" else "_TUNED_B"]) for row_id in baseline_rows]
|
||||
|
|
@ -6172,7 +6293,7 @@ class TestStrategyRouterWriteValidation:
|
|||
async with _auto_router_capability_slot(fake, effective_params=effective_params, model_id=candidate_id):
|
||||
pass
|
||||
assert exc_info.value.status_code == 403
|
||||
assert "changed heuristic scorer settings or tier models" in str(exc_info.value.detail)
|
||||
assert "changed heuristic scoring rules" in str(exc_info.value.detail)
|
||||
assert "'auto_router' feature lifts the limit" in str(exc_info.value.detail)
|
||||
return
|
||||
async with _auto_router_capability_slot(fake, effective_params=effective_params, model_id=candidate_id) as table:
|
||||
|
|
@ -6222,13 +6343,13 @@ class TestStrategyRouterWriteValidation:
|
|||
model_params=Deployment(
|
||||
model_name="second-tuned",
|
||||
litellm_params=LiteLLM_Params(
|
||||
model="auto_router/complexity_router", complexity_router_config=self._TUNED_B
|
||||
model="auto_router/complexity_router", complexity_router_config=self._TUNED_B_EDITED
|
||||
),
|
||||
),
|
||||
user_api_key_dict=admin,
|
||||
)
|
||||
assert exc_info.value.code == "403"
|
||||
assert "changed heuristic scorer settings or tier models" in str(exc_info.value.message)
|
||||
assert "changed heuristic scoring rules" in str(exc_info.value.message)
|
||||
fake.tx_obj.litellm_proxymodeltable.create.assert_not_awaited()
|
||||
fake.litellm_proxymodeltable.create.assert_not_awaited()
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,196 @@
|
|||
from collections.abc import Mapping
|
||||
from typing import Final
|
||||
from types import SimpleNamespace
|
||||
|
||||
from litellm.proxy.common_utils.encrypt_decrypt_utils import encrypt_value_helper
|
||||
|
||||
import pytest
|
||||
|
||||
from litellm.proxy.management_helpers.auto_router_availability import (
|
||||
auto_router_availability,
|
||||
build_auto_router_catalog,
|
||||
)
|
||||
from litellm.router_utils.auto_router_tuning_baseline import snapshot_tuning_baselines
|
||||
|
||||
|
||||
def deployment(
|
||||
model_id: str,
|
||||
classifier: str,
|
||||
*,
|
||||
model: str = "solver",
|
||||
tuned: bool = False,
|
||||
config: Mapping[str, object] | None = None,
|
||||
) -> Mapping[str, object]:
|
||||
return {
|
||||
"model_name": model_id,
|
||||
"model_info": {"id": model_id, "db_model": True},
|
||||
"litellm_params": {
|
||||
"model": "auto_router/complexity_router",
|
||||
"complexity_router_config": {
|
||||
"classifier_type": classifier,
|
||||
"tiers": {"SIMPLE": [model]},
|
||||
**({"code_keywords": ["internal-api"]} if tuned else {}),
|
||||
**(config or {}),
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("classifier", ("heuristic_v2", "capability", "llm_v2"))
|
||||
def test_occupied_allowance_blocks_new_router_but_not_owner(classifier: str) -> None:
|
||||
existing: Final = deployment("existing", classifier)
|
||||
candidate: Final = deployment("new", classifier)
|
||||
new: Final = auto_router_availability(others=(existing,), existing=None, candidate=candidate, baselines={}, limit=1)
|
||||
edit: Final = auto_router_availability(others=(), existing=existing, candidate=existing, baselines={}, limit=1)
|
||||
new_slot: Final = next(slot for slot in new.allowances if slot.key == classifier)
|
||||
edit_slot: Final = next(slot for slot in edit.allowances if slot.key == classifier)
|
||||
assert (new_slot.remaining, new_slot.used_by_this_router, new.error is not None) == (0, False, True)
|
||||
assert (edit_slot.remaining, edit_slot.used_by_this_router, edit.error) == (1, True, None)
|
||||
|
||||
|
||||
def test_edit_does_not_exempt_another_classifier_allowance() -> None:
|
||||
existing: Final = deployment("existing", "capability")
|
||||
result: Final = auto_router_availability(
|
||||
others=(deployment("other", "llm_v2"),),
|
||||
existing=existing,
|
||||
candidate=deployment("existing", "llm_v2"),
|
||||
baselines={},
|
||||
limit=1,
|
||||
)
|
||||
assert result.error is not None
|
||||
assert next(slot for slot in result.allowances if slot.key == "llm_v2").remaining == 0
|
||||
|
||||
|
||||
def test_model_selection_does_not_claim_occupied_scoring_allowance() -> None:
|
||||
original: Final = deployment("legacy", "heuristic")
|
||||
changed: Final = deployment("other", "heuristic", tuned=True)
|
||||
baselines: Final = snapshot_tuning_baselines((original,))
|
||||
unchanged: Final = auto_router_availability(
|
||||
others=(changed,),
|
||||
existing=original,
|
||||
candidate=original,
|
||||
baselines=baselines,
|
||||
limit=1,
|
||||
)
|
||||
edited: Final = auto_router_availability(
|
||||
others=(changed,),
|
||||
existing=original,
|
||||
candidate=deployment("legacy", "heuristic", model="new"),
|
||||
baselines=baselines,
|
||||
limit=1,
|
||||
)
|
||||
assert unchanged.error is None
|
||||
assert next(slot for slot in unchanged.allowances if slot.key == "heuristic_tuning").remaining == 0
|
||||
assert edited.error is None
|
||||
tuned: Final = auto_router_availability(
|
||||
others=(changed,),
|
||||
existing=original,
|
||||
candidate=deployment("legacy", "heuristic", tuned=True),
|
||||
baselines=baselines,
|
||||
limit=1,
|
||||
)
|
||||
assert tuned.error is not None
|
||||
assert "weights, thresholds, keywords, and custom dimensions" in tuned.error
|
||||
|
||||
|
||||
def test_missing_baselines_are_reported_as_unknown() -> None:
|
||||
result: Final = auto_router_availability(
|
||||
others=(),
|
||||
existing=None,
|
||||
candidate=deployment("new", "heuristic"),
|
||||
baselines=None,
|
||||
limit=1,
|
||||
)
|
||||
slot: Final = next(slot for slot in result.allowances if slot.key == "heuristic_tuning")
|
||||
assert (slot.available, slot.remaining, slot.limit) == (False, None, 1)
|
||||
|
||||
|
||||
def test_unlimited_entitlement_does_not_report_exhausted_allowances() -> None:
|
||||
result: Final = auto_router_availability(
|
||||
others=(deployment("other", "heuristic_v2"),),
|
||||
existing=None,
|
||||
candidate=deployment("new", "heuristic_v2"),
|
||||
baselines=None,
|
||||
limit=None,
|
||||
)
|
||||
assert all(slot.available and slot.limit is None and slot.remaining is None for slot in result.allowances)
|
||||
assert result.error is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"customization",
|
||||
(
|
||||
{"tier_definitions": [{"name": "SIMPLE"}, {"name": "AUDIT", "description": "Review risks"}]},
|
||||
{"classification_prompt": "Use the simplest sufficient tier"},
|
||||
{"classification_examples": "Review this code -> COMPLEX"},
|
||||
{"classifier_llm_config": {"model": "judge", "system_prompt": "Route by urgency"}},
|
||||
),
|
||||
)
|
||||
def test_customization_owner_can_edit_models_and_restoring_defaults_clears_the_gate(
|
||||
customization: Mapping[str, object],
|
||||
) -> None:
|
||||
owner: Final = deployment("owner", "llm", config=customization)
|
||||
blocked: Final = auto_router_availability(
|
||||
others=(owner,), existing=None, candidate=deployment("new", "llm", config=customization), baselines={}, limit=1
|
||||
)
|
||||
assert blocked.error is not None
|
||||
assert "Custom tiers or classifier instructions" in blocked.error
|
||||
edited: Final = auto_router_availability(
|
||||
others=(),
|
||||
existing=owner,
|
||||
candidate=deployment("owner", "llm", model="new", config=customization),
|
||||
baselines={},
|
||||
limit=1,
|
||||
)
|
||||
assert edited.error is None
|
||||
assert next(slot for slot in edited.allowances if slot.key == "tier_or_classifier_prompt").used_by_this_router
|
||||
restored: Final = auto_router_availability(
|
||||
others=(owner,), existing=None, candidate=deployment("new", "llm"), baselines={}, limit=1
|
||||
)
|
||||
assert restored.error is None
|
||||
assert next(slot for slot in restored.allowances if slot.key == "tier_or_classifier_prompt").remaining == 0
|
||||
|
||||
|
||||
def test_restoring_tiers_does_not_exempt_a_retained_custom_prompt() -> None:
|
||||
prompt: Final = {"classification_prompt": "Use the simplest sufficient tier"}
|
||||
result: Final = auto_router_availability(
|
||||
others=(deployment("owner", "llm", config=prompt),),
|
||||
existing=None,
|
||||
candidate=deployment("new", "llm", config=prompt),
|
||||
baselines={},
|
||||
limit=1,
|
||||
)
|
||||
assert result.error is not None
|
||||
assert "Custom tiers or classifier instructions" in result.error
|
||||
|
||||
|
||||
@pytest.mark.parametrize("blocked", (False, True))
|
||||
def test_catalog_keeps_unloaded_routers_and_ownership_without_provider_credentials(blocked: bool, monkeypatch) -> None:
|
||||
monkeypatch.setenv("LITELLM_SALT_KEY", "catalog-test-key")
|
||||
source: Final = SimpleNamespace(
|
||||
model_id="saved",
|
||||
created_by="owner",
|
||||
model_info={"team_id": "team"},
|
||||
blocked=blocked,
|
||||
litellm_params={
|
||||
"model": encrypt_value_helper("auto_router/complexity_router"),
|
||||
"api_key": "private-key",
|
||||
"complexity_router_config": {"classifier_type": "heuristic_v2"},
|
||||
},
|
||||
)
|
||||
provider: Final = SimpleNamespace(model_id="provider", litellm_params={"model": "openai/model"})
|
||||
catalog: Final = build_auto_router_catalog((source, provider))
|
||||
assert catalog is not None and len(catalog) == 1
|
||||
assert (catalog[0].model_id, catalog[0].team_id, catalog[0].created_by) == ("saved", "team", "owner")
|
||||
assert catalog[0].deployment == {
|
||||
"model_info": {"id": "saved", "db_model": True},
|
||||
"litellm_params": {
|
||||
"model": "auto_router/complexity_router",
|
||||
"complexity_router_config": {"classifier_type": "heuristic_v2"},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def test_catalog_distinguishes_missing_data_from_an_empty_model_table() -> None:
|
||||
assert build_auto_router_catalog(()) == ()
|
||||
assert build_auto_router_catalog((SimpleNamespace(model_id="incomplete"),)) is None
|
||||
|
|
@ -920,7 +920,7 @@ def test_proxy_startup_event_warns_for_global_budget_without_database():
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_tuning_baseline_v2_is_created_alongside_the_legacy_row():
|
||||
async def test_tuning_baseline_v3_is_created_alongside_the_legacy_row():
|
||||
from litellm.router_utils.auto_router_tuning_baseline import DEFAULT_TUNING_FINGERPRINT
|
||||
|
||||
prisma_client = MagicMock()
|
||||
|
|
@ -935,11 +935,61 @@ async def test_tuning_baseline_v2_is_created_alongside_the_legacy_row():
|
|||
|
||||
assert result == {'yaml:["a",[]]': DEFAULT_TUNING_FINGERPRINT}
|
||||
assert prisma_client.db.litellm_config.create.await_args.kwargs["data"] == {
|
||||
"param_name": "auto_router_tuning_baseline_v2",
|
||||
"param_name": "auto_router_tuning_baseline_v3",
|
||||
"param_value": json.dumps(dict(result)),
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_scorer_baseline_upgrade_preserves_existing_routers_and_is_not_refreshed_on_restart():
|
||||
from litellm.router_utils.auto_router_tuning_baseline import mutable_tuned_identities, snapshot_tuning_baselines
|
||||
|
||||
deployments = [
|
||||
{
|
||||
"model_name": name,
|
||||
"litellm_params": {
|
||||
"model": "auto_router/complexity_router",
|
||||
"complexity_router_config": {"tiers": {"SIMPLE": name}, "code_keywords": [name]},
|
||||
},
|
||||
}
|
||||
for name in ("a", "b")
|
||||
]
|
||||
prisma_client = MagicMock()
|
||||
prisma_client.db.litellm_config.find_unique = AsyncMock(
|
||||
side_effect=lambda where: (
|
||||
MagicMock(param_value='{"legacy-router":"old-combined-hash"}')
|
||||
if where["param_name"] == "auto_router_tuning_baseline_v2"
|
||||
else None
|
||||
)
|
||||
)
|
||||
prisma_client.db.litellm_config.create = AsyncMock()
|
||||
|
||||
baseline = await ProxyStartupEvent._load_heuristic_v1_tuning_baselines(prisma_client, deployments)
|
||||
|
||||
assert baseline == snapshot_tuning_baselines(deployments)
|
||||
assert mutable_tuned_identities(deployments, baseline) == frozenset()
|
||||
prisma_client.db.litellm_config.create.assert_awaited_once_with(
|
||||
data={"param_name": "auto_router_tuning_baseline_v3", "param_value": json.dumps(dict(baseline))}
|
||||
)
|
||||
prisma_client.db.litellm_config.find_unique.side_effect = None
|
||||
prisma_client.db.litellm_config.find_unique.return_value = MagicMock(param_value=json.dumps(dict(baseline)))
|
||||
changed = [
|
||||
{
|
||||
"model_name": "a",
|
||||
"litellm_params": {
|
||||
"model": "auto_router/complexity_router",
|
||||
"complexity_router_config": {"tiers": {"SIMPLE": "different-model"}, "code_keywords": ["new-rule"]},
|
||||
},
|
||||
}
|
||||
]
|
||||
|
||||
reloaded = await ProxyStartupEvent._load_heuristic_v1_tuning_baselines(prisma_client, changed)
|
||||
|
||||
assert reloaded == baseline
|
||||
assert mutable_tuned_identities(changed, reloaded) == frozenset({'yaml:["a",[]]'})
|
||||
prisma_client.db.litellm_config.create.assert_awaited_once()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_tuning_baseline_waits_for_a_complete_db_model_census(monkeypatch):
|
||||
prisma_client = MagicMock()
|
||||
|
|
|
|||
|
|
@ -875,9 +875,7 @@ class _ConfigTable:
|
|||
await asyncio.sleep(0)
|
||||
return _ConfigRow(param_value=value) if value is not None else None
|
||||
|
||||
async def upsert(
|
||||
self, *, where: Mapping[str, str], data: Mapping[str, Mapping[str, str]]
|
||||
) -> _ConfigRow:
|
||||
async def upsert(self, *, where: Mapping[str, str], data: Mapping[str, Mapping[str, str]]) -> _ConfigRow:
|
||||
param_name: Final = where["param_name"]
|
||||
value: Final = _CONFIG_VALUE.validate_json(data["update"]["param_value"])
|
||||
self.rows[param_name] = value
|
||||
|
|
@ -926,7 +924,9 @@ class _ConfigPrisma:
|
|||
self.db.litellm_config.upserted_param_names.append(param_name)
|
||||
|
||||
|
||||
def _db_backed_proxy_config(monkeypatch, rows: Mapping[str, Mapping[str, JsonValue]]) -> tuple[ProxyConfig, _ConfigTable]:
|
||||
def _db_backed_proxy_config(
|
||||
monkeypatch, rows: Mapping[str, Mapping[str, JsonValue]]
|
||||
) -> tuple[ProxyConfig, _ConfigTable]:
|
||||
table: Final = _ConfigTable(rows)
|
||||
monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", _ConfigPrisma(db=_ConfigDb(litellm_config=table)))
|
||||
monkeypatch.setattr("litellm.proxy.proxy_server.store_model_in_db", True)
|
||||
|
|
@ -4750,9 +4750,7 @@ def test_validate_deployment_access_windows_rejects_malformed_time():
|
|||
"model_name": "gpt-4o-shared",
|
||||
"litellm_params": {"model": "gpt-4o"},
|
||||
"model_info": {
|
||||
"access_windows": [
|
||||
{"start": "25:00", "end": "06:00", "timezone": "America/New_York", "team_ids": ["t"]}
|
||||
]
|
||||
"access_windows": [{"start": "25:00", "end": "06:00", "timezone": "America/New_York", "team_ids": ["t"]}]
|
||||
},
|
||||
}
|
||||
|
||||
|
|
@ -4767,9 +4765,7 @@ def test_validate_deployment_access_windows_rejects_unknown_timezone():
|
|||
"model_name": "gpt-4o-shared",
|
||||
"litellm_params": {"model": "gpt-4o"},
|
||||
"model_info": {
|
||||
"access_windows": [
|
||||
{"start": "22:00", "end": "06:00", "timezone": "Mars/Olympus", "team_ids": ["t"]}
|
||||
]
|
||||
"access_windows": [{"start": "22:00", "end": "06:00", "timezone": "Mars/Olympus", "team_ids": ["t"]}]
|
||||
},
|
||||
}
|
||||
|
||||
|
|
@ -4799,3 +4795,28 @@ def test_validate_deployment_access_windows_accepts_valid_and_absent():
|
|||
)
|
||||
is None
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_model_refresh_updates_availability_catalog_and_retains_it_on_db_failure():
|
||||
pc = ProxyConfig()
|
||||
row = SimpleNamespace(
|
||||
model_id="gated",
|
||||
created_by="owner",
|
||||
model_info={},
|
||||
litellm_params={
|
||||
"model": "auto_router/complexity_router",
|
||||
"complexity_router_config": {"classifier_type": "heuristic_v2"},
|
||||
},
|
||||
)
|
||||
find_many = AsyncMock(side_effect=[[row], RuntimeError("database unavailable"), []])
|
||||
client = SimpleNamespace(db=SimpleNamespace(litellm_proxymodeltable=SimpleNamespace(find_many=find_many)))
|
||||
assert pc.auto_router_db_catalog is None
|
||||
assert await pc._get_models_from_db(client) == [row]
|
||||
loaded = pc.auto_router_db_catalog
|
||||
assert loaded is not None and loaded[0].model_id == "gated"
|
||||
assert await pc._get_models_from_db(client) is None
|
||||
assert pc.auto_router_db_catalog == loaded
|
||||
assert await pc._get_models_from_db(client) == []
|
||||
assert pc.auto_router_db_catalog == ()
|
||||
assert find_many.await_count == 3
|
||||
|
|
|
|||
|
|
@ -1218,13 +1218,13 @@ def test_get_autorouter_presets_local_mode_serves_bundled_catalog(
|
|||
assert "anthropic_family" in payload
|
||||
assert payload["1m_context"]["complexity_router_config"]["classifier_type"] == "heuristic_v2"
|
||||
assert payload["1m_context"]["complexity_router_config"]["tiers"] == {
|
||||
"SIMPLE": ["gpt-5.6-luna"],
|
||||
"SIMPLE": ["gpt-6-luna"],
|
||||
"MEDIUM": ["gpt-5.6-terra"],
|
||||
"COMPLEX": ["gpt-5.6-sol"],
|
||||
"REASONING": ["claude-opus-5"],
|
||||
"COMPLEX": ["gpt-6-sol"],
|
||||
"REASONING": ["claude-opus-5-5"],
|
||||
}
|
||||
assert payload["1m_context"]["complexity_router_config"]["tier_model_configs"] == {
|
||||
"REASONING": [{"model_name": "claude-opus-5", "litellm_params": {"reasoning_effort": "high"}}]
|
||||
"REASONING": [{"model_name": "claude-opus-5-5", "litellm_params": {"reasoning_effort": "high"}}]
|
||||
}
|
||||
for preset in payload.values():
|
||||
assert isinstance(preset["label"], str)
|
||||
|
|
|
|||
|
|
@ -33,21 +33,6 @@ _HISTORICAL_FINGERPRINTS: Final = (
|
|||
{"custom_dimensions": [{"name": "sqlDdl", "weight": 0.4, "patterns": [r"\bCREATE\s{1,4}TABLE\b"]}]},
|
||||
"814ce0017fc7f60a160b262f658d910e9bdf784e6139a4ba4f1e2657aa203950",
|
||||
),
|
||||
(
|
||||
{
|
||||
"tiers": _TIERS,
|
||||
"dimension_weights": {"codePresence": 0.3},
|
||||
"custom_dimensions": [
|
||||
{
|
||||
"name": "internalFrameworks",
|
||||
"weight": 0.2,
|
||||
"keywords": ["orbitmesh", "fluxgate"],
|
||||
"patterns": [r"\bALTER\s{1,4}TABLE\b"],
|
||||
}
|
||||
],
|
||||
},
|
||||
"38970dc9224e265ab38c89674563d8d0537822591f9239b45251db6f5ca6cc39",
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -78,11 +63,9 @@ class TestTuningFingerprint:
|
|||
{"tiers": {"SIMPLE": "x"}}
|
||||
)
|
||||
|
||||
@pytest.mark.parametrize("field", sorted(set(HEURISTIC_V1_TUNING_FIELDS) - {"tier_model_configs"}))
|
||||
@pytest.mark.parametrize("field", HEURISTIC_V1_TUNING_FIELDS)
|
||||
def test_every_tuning_field_changes_the_fingerprint(self, field: str) -> None:
|
||||
samples: dict[str, object] = {
|
||||
"tiers": _ALT_TIERS,
|
||||
"classifier_type": "heuristic_first",
|
||||
"tier_boundaries": {"simple_medium": 0.2, "medium_complex": 0.4, "complex_reasoning": 0.7},
|
||||
"reasoning_override_min_score": 0.05,
|
||||
"token_thresholds": {"simple": 20, "complex": 500},
|
||||
|
|
@ -97,9 +80,6 @@ class TestTuningFingerprint:
|
|||
"keyword_tier_rules": [{"keywords": ["urgent"], "tier": "COMPLEX"}],
|
||||
}
|
||||
config: dict[str, object] = {field: samples[field]}
|
||||
if field == "classifier_type":
|
||||
config["heuristic_first_max_tier"] = "MEDIUM"
|
||||
config["classifier_llm_config"] = {"model": "judge"}
|
||||
assert tuning_fingerprint(config) != DEFAULT_TUNING_FINGERPRINT
|
||||
|
||||
def test_explicit_empty_tier_model_configs_follow_omission(self) -> None:
|
||||
|
|
@ -126,12 +106,33 @@ class TestTuningFingerprint:
|
|||
!= historical
|
||||
)
|
||||
|
||||
def test_tier_model_overrides_change_the_fingerprint(self) -> None:
|
||||
def test_tier_model_overrides_do_not_change_the_fingerprint(self) -> None:
|
||||
plain = tuning_fingerprint({"tiers": {"SIMPLE": "x"}})
|
||||
with_override = tuning_fingerprint(
|
||||
{"tiers": {"SIMPLE": {"model_name": "x", "litellm_params": {"temperature": 0.1}}}}
|
||||
)
|
||||
assert plain != with_override
|
||||
assert plain == with_override == DEFAULT_TUNING_FINGERPRINT
|
||||
|
||||
@pytest.mark.parametrize("classifier_type", ("heuristic", "heuristic_first", "hybrid"))
|
||||
def test_model_selection_and_classifier_switching_do_not_claim_tuning(self, classifier_type: str) -> None:
|
||||
config: Final = {
|
||||
"classifier_type": classifier_type,
|
||||
**({"classifier_llm_config": {"model": "judge"}} if classifier_type != "heuristic" else {}),
|
||||
**({"heuristic_first_max_tier": "MEDIUM"} if classifier_type == "heuristic_first" else {}),
|
||||
**({"hybrid_boundary_margin": 0.1} if classifier_type == "hybrid" else {}),
|
||||
"tiers": _ALT_TIERS,
|
||||
"escalation_keywords": ["LITELLM ESCALATE"],
|
||||
"tier_model_configs": {"COMPLEX": [{"model_name": "other-strong", "litellm_params": {"temperature": 0.1}}]},
|
||||
}
|
||||
tuned: Final = _router("tuned", {"dimension_weights": {"codePresence": 0.9}})
|
||||
model_only: Final = _router("model-only", {"tiers": _TIERS})
|
||||
candidate: Final = _router("another", config)
|
||||
assert tuning_fingerprint(config) == DEFAULT_TUNING_FINGERPRINT
|
||||
assert tuning_quota_violation(candidate=candidate, others=(tuned, model_only), baselines={}, limit=1) is None
|
||||
|
||||
def test_disabling_or_replacing_escalation_is_still_a_custom_rule(self) -> None:
|
||||
assert tuning_fingerprint({"escalation_keywords": []}) != DEFAULT_TUNING_FINGERPRINT
|
||||
assert tuning_fingerprint({"escalation_keywords": ["USE A STRONGER MODEL"]}) != DEFAULT_TUNING_FINGERPRINT
|
||||
|
||||
def test_non_tuning_fields_do_not_change_the_fingerprint(self) -> None:
|
||||
assert (
|
||||
|
|
@ -230,17 +231,18 @@ class TestQuota:
|
|||
def test_router_added_after_snapshot_is_mutable_only_when_tuned(self) -> None:
|
||||
baselines = snapshot_tuning_baselines([_router("a", {"tiers": _TIERS})])
|
||||
assert mutable_tuned_identities([_router("new", {})], baselines) == frozenset()
|
||||
assert mutable_tuned_identities([_router("new", {"tiers": _TIERS})], baselines) == {
|
||||
router_identity(_router("new", {}))
|
||||
}
|
||||
assert mutable_tuned_identities([_router("new", {"tiers": _TIERS})], baselines) == frozenset()
|
||||
assert mutable_tuned_identities(
|
||||
[_router("new", {"tiers": _TIERS, "code_keywords": ["internal-api"]})], baselines
|
||||
) == {router_identity(_router("new", {}))}
|
||||
|
||||
def test_quota_matrix(self) -> None:
|
||||
legacy_a = _router("a", {"tiers": _TIERS})
|
||||
legacy_b = _router("b", {"tiers": _ALT_TIERS})
|
||||
baselines = snapshot_tuning_baselines([legacy_a, legacy_b])
|
||||
edited_a = _router("a", {"tiers": _TIERS, "dimension_weights": {"codePresence": 0.9}})
|
||||
edited_b = _router("b", {"tiers": _TIERS})
|
||||
new_c = _router("c", {"tiers": _TIERS})
|
||||
edited_b = _router("b", {"tiers": _TIERS, "code_keywords": ["internal-api"]})
|
||||
new_c = _router("c", {"tiers": _TIERS, "code_keywords": ["internal-api"]})
|
||||
|
||||
assert tuning_quota_violation(candidate=edited_a, others=[legacy_b], baselines=baselines, limit=1) is None
|
||||
assert (
|
||||
|
|
@ -260,7 +262,7 @@ class TestQuota:
|
|||
legacy_a = _router("a", {"tiers": _TIERS})
|
||||
legacy_b = _router("b", {"tiers": _ALT_TIERS})
|
||||
baselines = snapshot_tuning_baselines([legacy_a, legacy_b])
|
||||
edited_b = _router("b", {"tiers": _TIERS})
|
||||
edited_b = _router("b", {"tiers": _TIERS, "code_keywords": ["internal-api"]})
|
||||
assert tuning_quota_violation(candidate=edited_b, others=[legacy_a], baselines=baselines, limit=1) is None
|
||||
assert (
|
||||
tuning_quota_violation(candidate=edited_b, others=[legacy_a, edited_b], baselines=baselines, limit=1)
|
||||
|
|
@ -304,5 +306,6 @@ class TestQuota:
|
|||
assert message is not None
|
||||
assert "At most 1 auto-router(s)" in message
|
||||
assert "revert the other changed router to its baseline" in message
|
||||
assert "Selecting models does not use this allowance" in message
|
||||
assert tuning_limit_violation(held=1, limit=1) is None
|
||||
assert tuning_limit_violation(held=5, limit=None) is None
|
||||
|
|
|
|||
|
|
@ -0,0 +1,165 @@
|
|||
import { createContext, useContext, useEffect, useState } from "react";
|
||||
import { useQuery, type UseQueryOptions } from "@tanstack/react-query";
|
||||
import { apiClient } from "@/components/networking";
|
||||
import { Popover, PopoverContent, PopoverTitle, PopoverTrigger } from "@/components/ui/popover";
|
||||
import type { components } from "@/lib/http/schema";
|
||||
|
||||
type Availability = components["schemas"]["AutoRouterAvailabilityResponse"];
|
||||
type Request = components["schemas"]["AutoRouterAvailabilityRequest"];
|
||||
export type Allowance = components["schemas"]["AutoRouterAllowance"];
|
||||
|
||||
type AvailabilityState = {
|
||||
data?: Availability;
|
||||
isPending: boolean;
|
||||
isError: boolean;
|
||||
isChecking?: boolean;
|
||||
refetch?: () => unknown;
|
||||
};
|
||||
|
||||
export const AutoRouterAvailabilityContext = createContext<AvailabilityState>({ isPending: true, isError: false });
|
||||
|
||||
export const useAutoRouterAvailability = (accessToken: string, body: Request, enabled = true) => {
|
||||
const serialized = JSON.stringify(body.complexity_router_config ?? null);
|
||||
const [debounced, setDebounced] = useState(serialized);
|
||||
useEffect(() => {
|
||||
const timeout = setTimeout(() => setDebounced(serialized), 300);
|
||||
return () => clearTimeout(timeout);
|
||||
}, [serialized]);
|
||||
const options: UseQueryOptions<Availability> = {
|
||||
queryKey: ["autoRouterAvailability", accessToken, body.team_id, body.saved_model_id, debounced],
|
||||
queryFn: ({ signal }) =>
|
||||
apiClient.post<Availability>("/auto_router/availability", {
|
||||
accessToken,
|
||||
body: { ...body, complexity_router_config: JSON.parse(debounced) },
|
||||
signal,
|
||||
}),
|
||||
enabled: enabled && Boolean(accessToken),
|
||||
placeholderData: (previous, previousQuery) => {
|
||||
const key = previousQuery?.queryKey;
|
||||
return key?.[1] === accessToken && key[2] === body.team_id && key[3] === body.saved_model_id
|
||||
? previous
|
||||
: undefined;
|
||||
},
|
||||
refetchOnMount: "always",
|
||||
staleTime: 0,
|
||||
retry: false,
|
||||
};
|
||||
const query = useQuery(options);
|
||||
const isChecking = query.isFetching || query.isPlaceholderData || serialized !== debounced;
|
||||
const saveBlockedReason = () => {
|
||||
if (!enabled) return null;
|
||||
if (query.isPending || isChecking) return "Checking availability";
|
||||
if (query.isError || !query.data) return "Could not check availability. Retry before saving.";
|
||||
return query.data.error ?? null;
|
||||
};
|
||||
return {
|
||||
...query,
|
||||
isPending: query.isPending || (query.isFetching && !query.isFetchedAfterMount),
|
||||
isChecking,
|
||||
saveBlockedReason: saveBlockedReason(),
|
||||
};
|
||||
};
|
||||
|
||||
export const allowanceLabel = (allowance?: Allowance): string | null => {
|
||||
if (!allowance?.available) return "Availability unavailable";
|
||||
if (allowance.limit == null) return null;
|
||||
if (allowance.used_by_this_router) return "Used by this router";
|
||||
return `${allowance.remaining} of ${allowance.limit} available`;
|
||||
};
|
||||
|
||||
const availabilityLabel = (state: AvailabilityState, key: string) => {
|
||||
if (state.isPending || state.isChecking) return "Checking availability";
|
||||
if (state.isError) return "Availability unavailable";
|
||||
return allowanceLabel(state.data?.allowances.find((entry) => entry.key === key));
|
||||
};
|
||||
|
||||
export const useAllowanceLabel = (key: string) => availabilityLabel(useContext(AutoRouterAvailabilityContext), key);
|
||||
|
||||
export const isAllowanceExhausted = (allowance?: Allowance) =>
|
||||
Boolean(allowance?.available && allowance.limit != null && allowance.remaining === 0) &&
|
||||
!allowance?.used_by_this_router;
|
||||
|
||||
export const AUTO_ROUTER_CONTACT_URL = "https://calendly.com/tin-berri/litellm-auto-router-pricing-discussion";
|
||||
|
||||
export const AutoRouterContactLink = ({ features, message }: { features?: string[]; message?: string }) => {
|
||||
const state = useContext(AutoRouterAvailabilityContext);
|
||||
if (state.isPending || state.isError || state.isChecking) return null;
|
||||
const exhausted = state.data?.allowances.some(
|
||||
(entry) => (!features || features.includes(entry.key)) && isAllowanceExhausted(entry),
|
||||
);
|
||||
if (!exhausted) return null;
|
||||
return (
|
||||
<span className="inline-flex flex-wrap items-baseline gap-x-1 text-xs leading-5 text-muted-foreground">
|
||||
{message}
|
||||
<a
|
||||
href={AUTO_ROUTER_CONTACT_URL}
|
||||
target="_blank"
|
||||
rel="noopener noreferrer"
|
||||
className="font-medium text-blue-600 hover:underline dark:text-blue-400"
|
||||
>
|
||||
Talk to our team
|
||||
</a>
|
||||
</span>
|
||||
);
|
||||
};
|
||||
|
||||
export const AutoRouterAllowanceLabel = ({ feature }: { feature: string }) => {
|
||||
const label = useAllowanceLabel(feature);
|
||||
return label ? (
|
||||
<span className="shrink-0 whitespace-nowrap text-xs leading-5 tabular-nums text-muted-foreground">{label}</span>
|
||||
) : null;
|
||||
};
|
||||
|
||||
export const AutoRouterAllowanceNote = ({ feature, label }: { feature: string; label: string }) => {
|
||||
const availability = useAllowanceLabel(feature);
|
||||
return availability ? (
|
||||
<p className="text-xs leading-5 text-muted-foreground">
|
||||
{label}: {availability} <AutoRouterContactLink features={[feature]} />
|
||||
</p>
|
||||
) : null;
|
||||
};
|
||||
|
||||
export const AutoRouterLimits = () => {
|
||||
const state = useContext(AutoRouterAvailabilityContext);
|
||||
const limits = [
|
||||
["heuristic_v2", "Heuristic v2 routers"],
|
||||
["capability", "Capability routers"],
|
||||
["llm_v2", "Fuse v2 routers"],
|
||||
["tier_or_classifier_prompt", "Custom tiers or prompts"],
|
||||
["heuristic_tuning", "Rule-based tuning"],
|
||||
];
|
||||
return (
|
||||
<Popover>
|
||||
<PopoverTrigger className="shrink-0 whitespace-nowrap text-xs font-normal text-muted-foreground underline underline-offset-4 hover:text-foreground">
|
||||
View limits
|
||||
</PopoverTrigger>
|
||||
<PopoverContent align="end" className="w-96 max-w-[calc(100vw-2rem)] gap-3">
|
||||
<PopoverTitle>Routing and customization limits</PopoverTitle>
|
||||
<p className="text-xs leading-5 text-muted-foreground">
|
||||
Rule-based, Complexity, and Jev are unlimited with built-in settings. Choose or change tier models freely.
|
||||
Customization allowances are shared across this proxy.
|
||||
</p>
|
||||
<dl className="space-y-2 text-xs">
|
||||
{limits.map(([key, label]) => (
|
||||
<div key={key} className="flex items-center justify-between gap-3">
|
||||
<dt>{label}</dt>
|
||||
<dd className="shrink-0 tabular-nums text-muted-foreground">
|
||||
{availabilityLabel(state, key) ?? "Unlimited"}
|
||||
</dd>
|
||||
</div>
|
||||
))}
|
||||
</dl>
|
||||
<p className="text-xs leading-5 text-muted-foreground">
|
||||
Custom tier definitions and written classifier instructions share one allowance. Built-in prompts and
|
||||
display-name changes do not use it.
|
||||
</p>
|
||||
<p className="text-xs leading-5 text-muted-foreground">
|
||||
Changing scoring rules, such as weights, thresholds, keywords, or custom dimensions, uses the Rule-based
|
||||
tuning allowance. It also applies to Heuristic first and Hybrid. Recorded settings on existing routers are
|
||||
preserved; new routers start from built-in rules.
|
||||
</p>
|
||||
<AutoRouterContactLink message="Need a higher limit?" />
|
||||
</PopoverContent>
|
||||
</Popover>
|
||||
);
|
||||
};
|
||||
|
|
@ -1,7 +1,9 @@
|
|||
import React, { useState } from "react";
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { fireEvent, renderWithProviders, screen } from "../../../tests/test-utils";
|
||||
import { fireEvent, renderWithProviders, screen, waitFor, within } from "../../../tests/test-utils";
|
||||
import { selectAutoRouterOption } from "../../../tests/autoRouterSetup";
|
||||
import AutoRouterClassifierTabs from "./AutoRouterClassifierTabs";
|
||||
import { AutoRouterAllowanceNote, AutoRouterAvailabilityContext } from "./AutoRouterAvailability";
|
||||
import type { ComplexityRouterConfigValue } from "./ComplexityRouterConfig";
|
||||
|
||||
const initial: ComplexityRouterConfigValue = {
|
||||
|
|
@ -9,28 +11,67 @@ const initial: ComplexityRouterConfigValue = {
|
|||
tiers: { SIMPLE: ["efficient"], MEDIUM: [], COMPLEX: [], REASONING: ["capable"] },
|
||||
};
|
||||
|
||||
function Form({ initialValue = initial }: { initialValue?: ComplexityRouterConfigValue }) {
|
||||
function Form({
|
||||
initialValue = initial,
|
||||
remaining = 1,
|
||||
limit = 1,
|
||||
ownedFeature,
|
||||
availabilityState,
|
||||
}: {
|
||||
initialValue?: ComplexityRouterConfigValue;
|
||||
remaining?: number;
|
||||
limit?: number | null;
|
||||
ownedFeature?: string;
|
||||
availabilityState?: Partial<React.ContextType<typeof AutoRouterAvailabilityContext>>;
|
||||
}) {
|
||||
const [value, setValue] = useState(initialValue);
|
||||
return (
|
||||
<AutoRouterClassifierTabs value={value} onChange={setValue}>
|
||||
<output aria-label="Classifier type">{value.classifier_type}</output>
|
||||
</AutoRouterClassifierTabs>
|
||||
<AutoRouterAvailabilityContext.Provider
|
||||
value={{
|
||||
isPending: false,
|
||||
isError: false,
|
||||
data: {
|
||||
allowances: ["heuristic_v2", "capability", "llm_v2", "tier_or_classifier_prompt", "heuristic_tuning"].map(
|
||||
(key) => ({
|
||||
key,
|
||||
limit,
|
||||
remaining,
|
||||
available: true,
|
||||
used_by_this_router: key === ownedFeature,
|
||||
}),
|
||||
),
|
||||
error: null,
|
||||
},
|
||||
...availabilityState,
|
||||
}}
|
||||
>
|
||||
<AutoRouterClassifierTabs value={value} onChange={setValue}>
|
||||
<output aria-label="Classifier type">{value.classifier_type}</output>
|
||||
</AutoRouterClassifierTabs>
|
||||
</AutoRouterAvailabilityContext.Provider>
|
||||
);
|
||||
}
|
||||
|
||||
describe("AutoRouterClassifierTabs", () => {
|
||||
it.each(["heuristic", "heuristic_v2", "llm", "heuristic_first", "hybrid"] as const)(
|
||||
"groups %s under Complexity without resetting its configuration",
|
||||
(classifier_type) => {
|
||||
describe("Auto-router classifier selection", () => {
|
||||
it.each(["heuristic", "heuristic_v2", "llm", "heuristic_first", "hybrid", "jev"] as const)(
|
||||
"shows saved %s without changing its configuration",
|
||||
async (classifier_type) => {
|
||||
const onChange = vi.fn();
|
||||
renderWithProviders(
|
||||
<AutoRouterClassifierTabs value={{ ...initial, classifier_type }} onChange={onChange}>
|
||||
Existing classifier settings
|
||||
Existing settings
|
||||
</AutoRouterClassifierTabs>,
|
||||
);
|
||||
expect(screen.getByRole("tab", { name: "Complexity" })).toHaveAttribute("aria-selected", "true");
|
||||
expect(screen.getByRole("tabpanel", { name: "Complexity" })).toHaveTextContent("Existing classifier settings");
|
||||
fireEvent.click(screen.getByRole("tab", { name: "Complexity" }));
|
||||
const family = {
|
||||
heuristic: "Heuristics",
|
||||
heuristic_v2: "Heuristics",
|
||||
llm: "LLM",
|
||||
heuristic_first: "LLM",
|
||||
hybrid: "LLM",
|
||||
jev: "Jev",
|
||||
}[classifier_type];
|
||||
expect(screen.getByRole("radio", { name: new RegExp(`^${family}$`) })).toBeChecked();
|
||||
fireEvent.click(screen.getByRole("radio", { name: new RegExp(`^${family}$`) }));
|
||||
expect(onChange).not.toHaveBeenCalled();
|
||||
},
|
||||
);
|
||||
|
|
@ -38,38 +79,186 @@ describe("AutoRouterClassifierTabs", () => {
|
|||
it.each([
|
||||
["capability", "Capability"],
|
||||
["llm_v2", "Fuse v2"],
|
||||
] as const)("opens saved %s settings and switches back to local Complexity", (classifier_type, label) => {
|
||||
renderWithProviders(<Form initialValue={{ ...initial, classifier_type }} />);
|
||||
expect(screen.getByRole("tab", { name: label })).toHaveAttribute("aria-selected", "true");
|
||||
fireEvent.click(screen.getByRole("tab", { name: "Complexity" }));
|
||||
expect(screen.getByRole("tab", { name: "Complexity" })).toHaveAttribute("aria-selected", "true");
|
||||
expect(screen.getByRole("status", { name: "Classifier type" })).toHaveTextContent("heuristic");
|
||||
] as const)(
|
||||
"opens saved %s and retains the LLM family when switching to Complexity",
|
||||
async (classifier_type, label) => {
|
||||
renderWithProviders(<Form initialValue={{ ...initial, classifier_type }} />);
|
||||
expect(screen.getByRole("button", { name: "Routing approach" })).toHaveTextContent(label);
|
||||
await selectAutoRouterOption("Routing approach", "Complexity");
|
||||
expect(screen.getByRole("status", { name: "Classifier type" })).toHaveTextContent("llm");
|
||||
},
|
||||
);
|
||||
|
||||
it.each([
|
||||
[1, "heuristic"],
|
||||
[0, "heuristic"],
|
||||
])("defaults to Rule-based when %s v2 slots remain", async (remaining, classifier) => {
|
||||
renderWithProviders(<Form remaining={Number(remaining)} />);
|
||||
fireEvent.click(screen.getByRole("radio", { name: /^Heuristics$/ }));
|
||||
expect(screen.getByRole("status", { name: "Classifier type" })).toHaveTextContent(String(classifier));
|
||||
fireEvent.click(screen.getByRole("button", { name: "Heuristic" }));
|
||||
expect(screen.getByRole("menuitemradio", { name: /^Heuristic v2/ })).toHaveTextContent(
|
||||
`${remaining} of 1 available`,
|
||||
);
|
||||
});
|
||||
|
||||
it("keeps custom tiers editable under Complexity and explains why forecast tabs are disabled", () => {
|
||||
it.each([
|
||||
{ data: undefined },
|
||||
{ isPending: true },
|
||||
{ isError: true },
|
||||
{ isChecking: true },
|
||||
{ data: { allowances: [], error: null } },
|
||||
{ data: { allowances: [{ key: "heuristic_v2", limit: 1, remaining: null, available: false }], error: null } },
|
||||
])("uses Rule-based when v2 availability is unverified: %j", async (availabilityState) => {
|
||||
renderWithProviders(<Form availabilityState={availabilityState} />);
|
||||
fireEvent.click(screen.getByRole("radio", { name: "Heuristics" }));
|
||||
expect(screen.getByRole("status", { name: "Classifier type" })).toHaveTextContent(/^heuristic$/);
|
||||
});
|
||||
|
||||
it("does not present Rule-based as having a classifier quota", () => {
|
||||
renderWithProviders(<Form initialValue={{ ...initial, classifier_type: "heuristic" }} remaining={0} />);
|
||||
expect(screen.getByRole("button", { name: "Heuristic" })).toHaveTextContent(/^Rule-based/);
|
||||
fireEvent.click(screen.getByRole("button", { name: "Heuristic" }));
|
||||
expect(screen.getAllByRole("menuitemradio")[0]).toHaveTextContent(/^Rule-based/);
|
||||
expect(screen.getByRole("menuitemradio", { name: /^Rule-based/ })).not.toHaveTextContent("of 1 available");
|
||||
expect(screen.getByRole("menuitemradio", { name: /^Heuristic v2/ })).toHaveTextContent("0 of 1 available");
|
||||
});
|
||||
|
||||
it("omits allowance labels with an unlimited entitlement", async () => {
|
||||
renderWithProviders(<Form initialValue={{ ...initial, classifier_type: "heuristic_v2" }} limit={null} />);
|
||||
fireEvent.click(screen.getByRole("button", { name: "Heuristic" }));
|
||||
expect(screen.getByRole("menuitemradio", { name: /^Heuristic v2/ })).not.toHaveTextContent("available");
|
||||
});
|
||||
|
||||
it.each([
|
||||
["heuristic", "Heuristic", "Heuristic v2"],
|
||||
["llm", "Routing approach", "Capability"],
|
||||
["llm", "Routing approach", "Fuse v2"],
|
||||
] as const)("blocks exhausted %s options: %s / %s", (classifier_type, field, option) => {
|
||||
renderWithProviders(<Form initialValue={{ ...initial, classifier_type }} remaining={0} />);
|
||||
fireEvent.click(screen.getByRole("button", { name: field }));
|
||||
const unavailable = screen.getByRole("menuitemradio", { name: new RegExp(`^${option}`) });
|
||||
expect(unavailable).toHaveAttribute("aria-disabled", "true");
|
||||
fireEvent.click(unavailable);
|
||||
expect(screen.getByRole("status", { name: "Classifier type" })).toHaveTextContent(classifier_type);
|
||||
});
|
||||
|
||||
it.each([
|
||||
["heuristic_v2", "heuristic", "Heuristic", "Heuristic v2"],
|
||||
["capability", "llm", "Routing approach", "Capability"],
|
||||
["llm_v2", "llm", "Routing approach", "Fuse v2"],
|
||||
] as const)("lets a saved router reselect its own %s allowance", async (feature, classifier_type, field, option) => {
|
||||
renderWithProviders(<Form initialValue={{ ...initial, classifier_type }} remaining={0} ownedFeature={feature} />);
|
||||
await selectAutoRouterOption(field, option);
|
||||
expect(screen.getByRole("status", { name: "Classifier type" })).toHaveTextContent(feature);
|
||||
expect(screen.getByRole("button", { name: field })).toHaveTextContent("Used by this router");
|
||||
});
|
||||
|
||||
it("shows Jev's single Complexity approach without changing saved configuration", () => {
|
||||
const onChange = vi.fn();
|
||||
renderWithProviders(
|
||||
<AutoRouterClassifierTabs
|
||||
value={{
|
||||
<AutoRouterClassifierTabs value={{ ...initial, classifier_type: "jev" }} onChange={onChange}>
|
||||
Existing settings
|
||||
</AutoRouterClassifierTabs>,
|
||||
);
|
||||
expect(screen.getByRole("button", { name: "Routing approach" })).toHaveTextContent("ComplexityUnlimited");
|
||||
fireEvent.click(screen.getByRole("button", { name: "Routing approach" }));
|
||||
expect(screen.getAllByRole("menuitemradio")).toHaveLength(1);
|
||||
fireEvent.click(screen.getByRole("menuitemradio", { name: /^Complexity/ }));
|
||||
expect(onChange).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("keeps custom tiers editable and disables incompatible choices", async () => {
|
||||
renderWithProviders(
|
||||
<Form
|
||||
initialValue={{
|
||||
...initial,
|
||||
custom_tier_set: {
|
||||
tiers: [{ id: "review", name: "REVIEW", definition: "Code reviews", models: ["capable"] }],
|
||||
fallback_tier_id: "review",
|
||||
},
|
||||
}}
|
||||
onChange={onChange}
|
||||
>
|
||||
Custom tiers
|
||||
</AutoRouterClassifierTabs>,
|
||||
/>,
|
||||
);
|
||||
expect(screen.getByRole("tabpanel", { name: "Complexity" })).toHaveTextContent("Custom tiers");
|
||||
expect(screen.getByRole("radio", { name: /^Heuristics$/ })).toHaveAttribute("aria-disabled", "true");
|
||||
fireEvent.click(screen.getByRole("button", { name: "Routing approach" }));
|
||||
for (const name of ["Capability", "Fuse v2"]) {
|
||||
const tab = screen.getByRole("tab", { name });
|
||||
expect(tab).toHaveAttribute("aria-disabled", "true");
|
||||
expect(tab).toHaveAccessibleDescription("Restore standard tiers to use Capability or Fuse v2.");
|
||||
fireEvent.click(tab);
|
||||
expect(screen.getByRole("menuitemradio", { name: new RegExp(`^${name}`) })).toHaveAttribute(
|
||||
"aria-disabled",
|
||||
"true",
|
||||
);
|
||||
}
|
||||
expect(onChange).not.toHaveBeenCalled();
|
||||
expect(screen.getByText("Restore standard tiers to use Capability or Fuse v2.")).toBeVisible();
|
||||
expect(screen.getByRole("status", { name: "Classifier type" })).toHaveTextContent("llm");
|
||||
});
|
||||
});
|
||||
|
||||
describe("Gated routing contact action", () => {
|
||||
it("offers a pricing discussion in View limits", async () => {
|
||||
renderWithProviders(<Form remaining={0} />);
|
||||
expect(screen.queryByRole("link", { name: "Talk to our team" })).not.toBeInTheDocument();
|
||||
fireEvent.click(screen.getByRole("button", { name: "View limits" }));
|
||||
const link = within(screen.getByRole("dialog")).getByRole("link", { name: "Talk to our team" });
|
||||
await waitFor(() => expect(link).toBeVisible());
|
||||
expect(link).toHaveAttribute("href", "https://calendly.com/tin-berri/litellm-auto-router-pricing-discussion");
|
||||
expect(link).toHaveAttribute("target", "_blank");
|
||||
expect(link).toHaveAttribute("rel", "noopener noreferrer");
|
||||
});
|
||||
|
||||
it.each([
|
||||
["heuristic", "Heuristic", "Heuristic v2"],
|
||||
["llm", "Routing approach", "Capability"],
|
||||
] as const)(
|
||||
"keeps the contact action available beside the disabled %s choice",
|
||||
async (classifier_type, field, option) => {
|
||||
renderWithProviders(<Form initialValue={{ ...initial, classifier_type }} remaining={0} />);
|
||||
expect(screen.queryByText(/Need more/)).not.toBeInTheDocument();
|
||||
fireEvent.click(screen.getByRole("button", { name: field }));
|
||||
const disabled = screen.getByRole("menuitemradio", { name: new RegExp(`^${option}`) });
|
||||
expect(disabled).toHaveAttribute("aria-disabled", "true");
|
||||
const link = screen.getByRole("menuitem", { name: `Talk to our team about ${option}` });
|
||||
await waitFor(() => expect(link).toBeVisible());
|
||||
expect(link).toHaveAttribute("href", "https://calendly.com/tin-berri/litellm-auto-router-pricing-discussion");
|
||||
expect(link).toHaveAttribute("target", "_blank");
|
||||
fireEvent.click(link);
|
||||
expect(screen.getByRole("status", { name: "Classifier type" })).toHaveTextContent(classifier_type);
|
||||
},
|
||||
);
|
||||
|
||||
it.each([
|
||||
{ remaining: 1 },
|
||||
{ remaining: 0, limit: null },
|
||||
{ remaining: 0, availabilityState: { isPending: true } },
|
||||
{ remaining: 0, availabilityState: { isError: true } },
|
||||
{ remaining: 0, availabilityState: { isChecking: true } },
|
||||
])("does not pitch an upgrade for a free or unverified option: %j", (props) => {
|
||||
renderWithProviders(<Form {...props} />);
|
||||
fireEvent.click(screen.getByRole("button", { name: "Routing approach" }));
|
||||
expect(screen.queryByRole("menuitem", { name: /Talk to our team/ })).not.toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("does not pitch an upgrade for the saved heuristic's own slot", () => {
|
||||
renderWithProviders(
|
||||
<Form initialValue={{ ...initial, classifier_type: "heuristic_v2" }} remaining={0} ownedFeature="heuristic_v2" />,
|
||||
);
|
||||
fireEvent.click(screen.getByRole("button", { name: "Heuristic" }));
|
||||
expect(screen.queryByRole("menuitem", { name: /Talk to our team/ })).not.toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("includes the sales action beside customization limits and blocked changes", () => {
|
||||
const allowance = { key: "tier_or_classifier_prompt", limit: 1, remaining: 0, available: true };
|
||||
const state = {
|
||||
isPending: false,
|
||||
isError: false,
|
||||
data: { allowances: [allowance], error: "Custom tiers have no available allowance" },
|
||||
};
|
||||
renderWithProviders(
|
||||
<AutoRouterAvailabilityContext.Provider value={state}>
|
||||
<AutoRouterClassifierTabs value={initial} onChange={vi.fn()}>
|
||||
<AutoRouterAllowanceNote feature="tier_or_classifier_prompt" label="Custom tiers" />
|
||||
</AutoRouterClassifierTabs>
|
||||
</AutoRouterAvailabilityContext.Provider>,
|
||||
);
|
||||
expect(screen.getByText(/Custom tiers: 0 of 1 available/)).toHaveTextContent("Talk to our team");
|
||||
expect(within(screen.getByRole("alert")).getByRole("link", { name: "Talk to our team" })).toBeVisible();
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -1,8 +1,119 @@
|
|||
import React, { useId } from "react";
|
||||
import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs";
|
||||
import { effectiveClassifierType, type ComplexityRouterConfigValue } from "./ComplexityRouterConfig";
|
||||
import React, { useContext, useId } from "react";
|
||||
import { Label } from "@/components/ui/label";
|
||||
import { RadioGroup, RadioGroupItem } from "@/components/ui/radio-group";
|
||||
import { ChevronDownIcon } from "lucide-react";
|
||||
import { Button } from "@/components/ui/button";
|
||||
import {
|
||||
DropdownMenu,
|
||||
DropdownMenuContent,
|
||||
DropdownMenuItem,
|
||||
DropdownMenuRadioGroup,
|
||||
DropdownMenuRadioItem,
|
||||
DropdownMenuTrigger,
|
||||
} from "@/components/ui/dropdown-menu";
|
||||
import {
|
||||
effectiveClassifierType,
|
||||
type ClassifierType,
|
||||
type ComplexityRouterConfigValue,
|
||||
} from "./ComplexityRouterConfig";
|
||||
import { transitionClassifierType } from "./classifier_type_transition";
|
||||
import { isForecastClassifier } from "./forecast_classifier_config";
|
||||
import {
|
||||
AutoRouterAllowanceLabel,
|
||||
AutoRouterAvailabilityContext,
|
||||
AutoRouterLimits,
|
||||
AutoRouterContactLink,
|
||||
isAllowanceExhausted,
|
||||
AUTO_ROUTER_CONTACT_URL,
|
||||
} from "./AutoRouterAvailability";
|
||||
|
||||
function ClassifierOption({
|
||||
value,
|
||||
label,
|
||||
description,
|
||||
feature,
|
||||
disabled,
|
||||
unlimited = true,
|
||||
}: {
|
||||
value: string;
|
||||
label: string;
|
||||
description: string;
|
||||
feature?: string;
|
||||
disabled?: boolean;
|
||||
unlimited?: boolean;
|
||||
}) {
|
||||
const state = useContext(AutoRouterAvailabilityContext);
|
||||
const allowance = state.data?.allowances.find((entry) => entry.key === feature);
|
||||
const fresh = !state.isPending && !state.isError && !state.isChecking;
|
||||
const exhausted = isAllowanceExhausted(allowance);
|
||||
return (
|
||||
<div className="relative">
|
||||
<DropdownMenuRadioItem value={value} disabled={disabled || exhausted} closeOnClick className="py-3">
|
||||
<span className="grid w-full min-w-0 gap-1 whitespace-normal">
|
||||
<span className="flex items-center justify-between gap-3">
|
||||
<span className="font-medium">{label}</span>
|
||||
{feature ? (
|
||||
<AutoRouterAllowanceLabel feature={feature} />
|
||||
) : (
|
||||
unlimited && <span className="shrink-0 text-xs leading-5 text-muted-foreground">Unlimited</span>
|
||||
)}
|
||||
</span>
|
||||
<span className={`text-xs leading-5 text-muted-foreground ${exhausted ? "pr-28" : ""}`}>{description}</span>
|
||||
</span>
|
||||
</DropdownMenuRadioItem>
|
||||
{fresh && exhausted && (
|
||||
<DropdownMenuItem
|
||||
render={<a href={AUTO_ROUTER_CONTACT_URL} target="_blank" rel="noopener noreferrer" />}
|
||||
aria-label={`Talk to our team about ${label}`}
|
||||
className="absolute top-9 right-8 cursor-pointer px-0 py-0 text-xs leading-5 font-medium text-blue-600 focus:text-blue-600 hover:underline dark:text-blue-400 dark:focus:text-blue-400"
|
||||
>
|
||||
Talk to our team
|
||||
</DropdownMenuItem>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
function ClassifierMenu({
|
||||
id,
|
||||
label,
|
||||
value,
|
||||
selectedLabel,
|
||||
feature,
|
||||
onValueChange,
|
||||
children,
|
||||
}: {
|
||||
id: string;
|
||||
label: string;
|
||||
value: string;
|
||||
selectedLabel: string;
|
||||
feature?: string;
|
||||
onValueChange: (value: string) => void;
|
||||
children: React.ReactNode;
|
||||
}) {
|
||||
return (
|
||||
<DropdownMenu>
|
||||
<DropdownMenuTrigger
|
||||
id={id}
|
||||
aria-label={label}
|
||||
render={<Button variant="outline" className="w-full justify-between font-normal" />}
|
||||
>
|
||||
<span className="flex-1 text-left">{selectedLabel}</span>
|
||||
{feature ? (
|
||||
<AutoRouterAllowanceLabel feature={feature} />
|
||||
) : (
|
||||
<span className="text-xs text-muted-foreground">Unlimited</span>
|
||||
)}
|
||||
<ChevronDownIcon className="size-4 text-muted-foreground" />
|
||||
</DropdownMenuTrigger>
|
||||
<DropdownMenuContent>
|
||||
<DropdownMenuRadioGroup value={value} onValueChange={onValueChange}>
|
||||
{children}
|
||||
</DropdownMenuRadioGroup>
|
||||
</DropdownMenuContent>
|
||||
</DropdownMenu>
|
||||
);
|
||||
}
|
||||
|
||||
interface AutoRouterClassifierTabsProps {
|
||||
value: ComplexityRouterConfigValue;
|
||||
|
|
@ -11,47 +122,174 @@ interface AutoRouterClassifierTabsProps {
|
|||
}
|
||||
|
||||
const AutoRouterClassifierTabs: React.FC<AutoRouterClassifierTabsProps> = ({ value, onChange, children }) => {
|
||||
const restrictionId = useId();
|
||||
const id = useId();
|
||||
const availability = useContext(AutoRouterAvailabilityContext);
|
||||
const classifierType = effectiveClassifierType(value);
|
||||
const selected = isForecastClassifier(classifierType) ? classifierType : "complexity";
|
||||
const familyByType: Record<ClassifierType, string> = {
|
||||
heuristic: "heuristics",
|
||||
heuristic_v2: "heuristics",
|
||||
llm: "llm",
|
||||
heuristic_first: "llm",
|
||||
hybrid: "llm",
|
||||
capability: "llm",
|
||||
llm_v2: "llm",
|
||||
jev: "jev",
|
||||
custom: "custom",
|
||||
};
|
||||
const family = familyByType[classifierType];
|
||||
const hasCustomTiers = Boolean(value.custom_tier_set);
|
||||
|
||||
const handleChange = (tab: unknown) => {
|
||||
if (tab === selected) return;
|
||||
if (tab === "complexity") {
|
||||
onChange(transitionClassifierType(value, isForecastClassifier(classifierType) ? "heuristic" : classifierType));
|
||||
} else if (!hasCustomTiers && (tab === "capability" || tab === "llm_v2")) {
|
||||
onChange(transitionClassifierType(value, tab));
|
||||
}
|
||||
const changeType = (next: ClassifierType) => {
|
||||
if (next !== classifierType) onChange(transitionClassifierType(value, next));
|
||||
};
|
||||
|
||||
const changeFamily = (next: unknown) => {
|
||||
if (next === family) return;
|
||||
if (next === "heuristics") changeType("heuristic");
|
||||
if (next === "llm") changeType("llm");
|
||||
if (next === "jev") changeType("jev");
|
||||
};
|
||||
const approachLabels: Partial<Record<ClassifierType, string>> = { capability: "Capability", llm_v2: "Fuse v2" };
|
||||
const approachDescription: Partial<Record<ClassifierType, string>> = {
|
||||
capability: "Use the efficient model when it is likely to succeed",
|
||||
llm_v2: "Use the efficient model when its predicted quality is close enough to the capable model",
|
||||
};
|
||||
return (
|
||||
<Tabs value={selected} onValueChange={handleChange}>
|
||||
<p className="text-sm font-medium">Classifier type</p>
|
||||
<TabsList aria-label="Classifier type" className="w-full">
|
||||
<TabsTrigger value="complexity">Complexity</TabsTrigger>
|
||||
<TabsTrigger
|
||||
value="capability"
|
||||
disabled={hasCustomTiers}
|
||||
aria-describedby={hasCustomTiers ? restrictionId : undefined}
|
||||
>
|
||||
Capability
|
||||
</TabsTrigger>
|
||||
<TabsTrigger
|
||||
value="llm_v2"
|
||||
disabled={hasCustomTiers}
|
||||
aria-describedby={hasCustomTiers ? restrictionId : undefined}
|
||||
>
|
||||
Fuse v2
|
||||
</TabsTrigger>
|
||||
</TabsList>
|
||||
<div className="flex flex-col gap-6">
|
||||
<fieldset>
|
||||
<legend className="mb-3 flex w-full items-center justify-between gap-3 text-sm font-medium">
|
||||
What classifies your requests?
|
||||
<AutoRouterLimits />
|
||||
</legend>
|
||||
<RadioGroup value={family} onValueChange={changeFamily} className="grid gap-3 sm:grid-cols-3">
|
||||
{[
|
||||
{ value: "heuristics", label: "Heuristics", description: "Classify locally, with no API call" },
|
||||
{ value: "llm", label: "LLM", description: "Use a judge model to choose a solver" },
|
||||
{ value: "jev", label: "Jev", description: "Use TypeSafe System One Choice to choose a tier" },
|
||||
].map((option) => (
|
||||
<Label
|
||||
key={option.value}
|
||||
className="cursor-pointer items-start rounded-lg border p-4 transition-colors hover:bg-muted/50 has-data-checked:border-primary has-data-checked:bg-primary/5"
|
||||
>
|
||||
<RadioGroupItem
|
||||
aria-describedby={`${id}-${option.value}-description`}
|
||||
value={option.value}
|
||||
disabled={option.value === "heuristics" && hasCustomTiers}
|
||||
/>
|
||||
<span className="space-y-1">
|
||||
<span className="block font-medium">{option.label}</span>
|
||||
<span
|
||||
id={`${id}-${option.value}-description`}
|
||||
aria-hidden="true"
|
||||
className="block text-xs font-normal text-muted-foreground"
|
||||
>
|
||||
{option.description}
|
||||
</span>
|
||||
</span>
|
||||
</Label>
|
||||
))}
|
||||
</RadioGroup>
|
||||
</fieldset>
|
||||
{family === "custom" && (
|
||||
<p className="text-sm text-muted-foreground">This router uses a custom classifier plugin</p>
|
||||
)}
|
||||
{family === "heuristics" && (
|
||||
<div className="space-y-2">
|
||||
<Label htmlFor={`${id}-heuristic`}>Heuristic</Label>
|
||||
<ClassifierMenu
|
||||
id={`${id}-heuristic`}
|
||||
label="Heuristic"
|
||||
selectedLabel={classifierType === "heuristic_v2" ? "Heuristic v2" : "Rule-based"}
|
||||
feature={classifierType === "heuristic_v2" ? "heuristic_v2" : undefined}
|
||||
value={classifierType}
|
||||
onValueChange={(next) => {
|
||||
if (next === "heuristic" || next === "heuristic_v2") changeType(next);
|
||||
}}
|
||||
>
|
||||
<ClassifierOption
|
||||
value="heuristic"
|
||||
label="Rule-based"
|
||||
description="Score requests with local rules to choose a tier, with no API call"
|
||||
/>
|
||||
<ClassifierOption
|
||||
value="heuristic_v2"
|
||||
label="Heuristic v2"
|
||||
description="Use calibrated probabilities to match requests to a tier, with no API call"
|
||||
feature="heuristic_v2"
|
||||
/>
|
||||
</ClassifierMenu>
|
||||
<p className="text-xs text-muted-foreground">
|
||||
{classifierType === "heuristic_v2"
|
||||
? "Use calibrated probabilities to match requests to a tier"
|
||||
: "Match requests using scoring rules. Choose or change tier models freely"}
|
||||
</p>
|
||||
</div>
|
||||
)}
|
||||
{(family === "llm" || family === "jev") && (
|
||||
<div className="space-y-2">
|
||||
<Label htmlFor={`${id}-approach`}>Routing approach</Label>
|
||||
<ClassifierMenu
|
||||
id={`${id}-approach`}
|
||||
label="Routing approach"
|
||||
selectedLabel={approachLabels[classifierType] ?? "Complexity"}
|
||||
feature={isForecastClassifier(classifierType) ? classifierType : undefined}
|
||||
value={isForecastClassifier(classifierType) ? classifierType : "llm"}
|
||||
onValueChange={(next) => {
|
||||
if (next === "llm" || next === "capability" || next === "llm_v2") {
|
||||
if (next === "llm" && !isForecastClassifier(classifierType)) return;
|
||||
changeType(next);
|
||||
}
|
||||
}}
|
||||
>
|
||||
<ClassifierOption
|
||||
value="llm"
|
||||
label="Complexity"
|
||||
description="Match task difficulty to a tier, then use one of that tier's models"
|
||||
/>
|
||||
{family === "llm" && (
|
||||
<>
|
||||
<ClassifierOption
|
||||
value="capability"
|
||||
label="Capability"
|
||||
description="Use the efficient model when it is likely to succeed; otherwise use the capable model"
|
||||
feature="capability"
|
||||
disabled={hasCustomTiers}
|
||||
/>
|
||||
<ClassifierOption
|
||||
value="llm_v2"
|
||||
label="Fuse v2"
|
||||
description="Use the efficient model when its predicted quality is close enough to the capable model"
|
||||
feature="llm_v2"
|
||||
disabled={hasCustomTiers}
|
||||
/>
|
||||
</>
|
||||
)}
|
||||
</ClassifierMenu>
|
||||
<p className="text-xs text-muted-foreground">
|
||||
{approachDescription[classifierType] ?? "Match task difficulty to a tier"}
|
||||
</p>
|
||||
</div>
|
||||
)}
|
||||
{hasCustomTiers && (
|
||||
<p id={restrictionId} className="text-sm text-muted-foreground">
|
||||
Restore standard tiers to use Capability or Fuse v2.
|
||||
<p className="text-xs text-muted-foreground">
|
||||
Restore standard tiers to use Heuristics, Capability, or Fuse v2
|
||||
</p>
|
||||
)}
|
||||
<TabsContent value={selected}>{children}</TabsContent>
|
||||
</Tabs>
|
||||
{availability.data?.error && !availability.isChecking && (
|
||||
<div role="alert" className="space-y-1">
|
||||
<p className="text-sm text-destructive">{availability.data.error}</p>
|
||||
<AutoRouterContactLink />
|
||||
</div>
|
||||
)}
|
||||
{availability.isError && (
|
||||
<p className="text-xs text-muted-foreground">
|
||||
Could not check availability.{" "}
|
||||
<button type="button" className="underline" onClick={() => availability.refetch?.()}>
|
||||
Retry
|
||||
</button>
|
||||
</p>
|
||||
)}
|
||||
{children}
|
||||
</div>
|
||||
);
|
||||
};
|
||||
|
||||
|
|
|
|||
|
|
@ -1,9 +1,10 @@
|
|||
import ClassifierPrimarySettings from "./ClassifierPrimarySettings";
|
||||
import { AutoRouterAllowanceNote } from "./AutoRouterAvailability";
|
||||
import { transitionClassifierType } from "./classifier_type_transition";
|
||||
import JevClassifierConfig from "./JevClassifierConfig";
|
||||
import { Info } from "lucide-react";
|
||||
import { SimpleTooltip } from "@/components/ui/tooltip";
|
||||
import { MultiSelect } from "@/components/shared/MultiSelect";
|
||||
import { SearchSelect } from "@/components/shared/SearchSelect";
|
||||
import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select";
|
||||
import { Card, CardContent } from "@/components/ui/card";
|
||||
import { Button } from "@/components/ui/button";
|
||||
|
|
@ -25,13 +26,10 @@ import ClassifierTypeRadios from "./ClassifierTypeRadios";
|
|||
import type { ReasoningEffort } from "./complexity_router_tiers";
|
||||
import { useComplexityScorerDefaults } from "@/app/(dashboard)/hooks/autoRouter/useComplexityScorerDefaults";
|
||||
import {
|
||||
ClassificationFrequency,
|
||||
ClassifierFallback,
|
||||
ClassifierLLMConfig,
|
||||
ClassifierType,
|
||||
ComplexityRouterConfigValue,
|
||||
classificationFrequency,
|
||||
withClassificationFrequency,
|
||||
DEFAULT_CLASSIFIER_CONTEXT_BUDGET_CHARS,
|
||||
MIN_QUOTED_CONTEXT_TURN_CHARS,
|
||||
DEFAULT_CLASSIFIER_CONTEXT_WINDOW_SIZE,
|
||||
|
|
@ -171,6 +169,7 @@ interface ClassificationMethodConfigProps {
|
|||
showValidationErrors?: boolean;
|
||||
/** The resolved default model - see resolveComplexityDefaultModel. Names and gates the radio. */
|
||||
defaultModel?: string;
|
||||
advancedOnly?: boolean;
|
||||
}
|
||||
|
||||
export const InactiveHeuristicV2Threshold: React.FC<Pick<ClassificationMethodConfigProps, "value" | "onChange">> = ({
|
||||
|
|
@ -215,13 +214,11 @@ const ClassificationMethodConfig: React.FC<ClassificationMethodConfigProps> = ({
|
|||
onCustomTechnicalKeywordsChange,
|
||||
showValidationErrors = false,
|
||||
defaultModel,
|
||||
advancedOnly = false,
|
||||
}) => {
|
||||
const [draft, setDraft] = React.useState<{ id: string; raw: string } | null>(null);
|
||||
const hasDefaultModel = Boolean(defaultModel);
|
||||
const classifierType = effectiveClassifierType(value);
|
||||
const sessionFrequencyRestriction = restrictedBy(value, "sessionAffinity");
|
||||
const classifierModelMissing =
|
||||
showValidationErrors && usesLlmClassifier(classifierType) && !value.classifier_llm_config?.model;
|
||||
const usesCustomPrompt = Boolean(value.classifier_llm_config?.system_prompt?.trim());
|
||||
const contextBudget = value.classifier_context_budget_chars ?? DEFAULT_CLASSIFIER_CONTEXT_BUDGET_CHARS;
|
||||
const contextBudgetQuotesNothing = contextBudget > 0 && contextBudget < MIN_QUOTED_CONTEXT_TURN_CHARS;
|
||||
|
|
@ -282,23 +279,6 @@ const ClassificationMethodConfig: React.FC<ClassificationMethodConfigProps> = ({
|
|||
onChange(nextValue);
|
||||
};
|
||||
|
||||
const handleClassifierModelChange = (model: string | null) => {
|
||||
if (model === null) return;
|
||||
if (model === value.classifier_llm_config?.model) return;
|
||||
const { reasoning_effort: _reasoningEffort, ...classifierLlmConfig } = value.classifier_llm_config ?? {
|
||||
model: "",
|
||||
timeout_ms: DEFAULT_CLASSIFIER_TIMEOUT_MS,
|
||||
};
|
||||
onChange({
|
||||
...value,
|
||||
classifier_llm_config: {
|
||||
...classifierLlmConfig,
|
||||
model,
|
||||
timeout_ms: classifierLlmConfig.timeout_ms,
|
||||
},
|
||||
});
|
||||
};
|
||||
|
||||
const handleClassifierReasoningEffortChange = (reasoningEffort: ReasoningEffort | undefined) => {
|
||||
if (!value.classifier_llm_config) return;
|
||||
const { reasoning_effort: _reasoningEffort, ...classifierLlmConfig } = value.classifier_llm_config;
|
||||
|
|
@ -350,10 +330,6 @@ const ClassificationMethodConfig: React.FC<ClassificationMethodConfigProps> = ({
|
|||
onChange({ ...value, classifier_fallback: fallback });
|
||||
};
|
||||
|
||||
const handleClassificationFrequencyChange = (frequency: ClassificationFrequency) => {
|
||||
onChange(withClassificationFrequency(value, frequency));
|
||||
};
|
||||
|
||||
const handleClassifierContextWindowSizeChange = (windowSize: number) => {
|
||||
onChange({
|
||||
...value,
|
||||
|
|
@ -389,7 +365,50 @@ const ClassificationMethodConfig: React.FC<ClassificationMethodConfigProps> = ({
|
|||
|
||||
return (
|
||||
<>
|
||||
<ClassifierTypeRadios value={value} classifierType={classifierType} onTypeChange={handleClassifierTypeChange} />
|
||||
{!advancedOnly && (
|
||||
<>
|
||||
<ClassifierTypeRadios
|
||||
value={value}
|
||||
classifierType={classifierType}
|
||||
onTypeChange={handleClassifierTypeChange}
|
||||
/>
|
||||
<ClassifierPrimarySettings
|
||||
value={value}
|
||||
onChange={onChange}
|
||||
modelOptions={modelOptions}
|
||||
showValidationErrors={showValidationErrors}
|
||||
/>
|
||||
</>
|
||||
)}
|
||||
{advancedOnly && ["llm", "heuristic_first", "hybrid"].includes(classifierType) && (
|
||||
<div className="space-y-2">
|
||||
<Label htmlFor="auto-router-local-checks">Local checks before the judge</Label>
|
||||
<Select
|
||||
items={[
|
||||
{ value: "llm", label: "Always use the judge" },
|
||||
{ value: "heuristic_first", label: "Heuristic first" },
|
||||
{ value: "hybrid", label: "Hybrid" },
|
||||
]}
|
||||
value={classifierType}
|
||||
onValueChange={(next) => {
|
||||
if (next === "llm" || next === "heuristic_first" || next === "hybrid") handleClassifierTypeChange(next);
|
||||
}}
|
||||
>
|
||||
<SelectTrigger id="auto-router-local-checks" className="w-full">
|
||||
<SelectValue />
|
||||
</SelectTrigger>
|
||||
<SelectContent>
|
||||
<SelectItem value="llm">Always use the judge</SelectItem>
|
||||
<SelectItem value="heuristic_first" disabled={Boolean(value.custom_tier_set)}>
|
||||
Heuristic first
|
||||
</SelectItem>
|
||||
<SelectItem value="hybrid" disabled={Boolean(value.custom_tier_set)}>
|
||||
Hybrid
|
||||
</SelectItem>
|
||||
</SelectContent>
|
||||
</Select>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{classifierType === "custom" && (
|
||||
<ClassifierPluginTimeoutField value={value} onChange={onChange} showValidationErrors={showValidationErrors} />
|
||||
|
|
@ -474,66 +493,9 @@ const ClassificationMethodConfig: React.FC<ClassificationMethodConfigProps> = ({
|
|||
</div>
|
||||
)}
|
||||
|
||||
<div className="mt-4 space-y-2">
|
||||
<strong className="block font-semibold">How often to classify</strong>
|
||||
<RadioGroup
|
||||
value={classificationFrequency(value)}
|
||||
onValueChange={(frequency: unknown) =>
|
||||
handleClassificationFrequencyChange(frequency as ClassificationFrequency)
|
||||
}
|
||||
>
|
||||
<div className="inline-flex flex-col gap-2">
|
||||
<Label className="items-start font-normal leading-normal">
|
||||
<RadioGroupItem value="every_request" className="mt-0.5" />
|
||||
<span>
|
||||
<span>Every request</span>{" "}
|
||||
<span className="text-muted-foreground">: score every turn, tool-result continuations included</span>
|
||||
</span>
|
||||
</Label>
|
||||
<Label className="items-start font-normal leading-normal">
|
||||
<RadioGroupItem value="user_turn" className="mt-0.5" />
|
||||
<span>
|
||||
<span>Every new user message</span>{" "}
|
||||
<span className="text-muted-foreground">
|
||||
: score each new human ask, then hold that tier for the tool calls that follow it
|
||||
</span>
|
||||
</span>
|
||||
</Label>
|
||||
<Label className="items-start font-normal leading-normal">
|
||||
<RadioGroupItem value="session" className="mt-0.5" disabled={Boolean(sessionFrequencyRestriction)} />
|
||||
<span>
|
||||
<span>Once per session</span>{" "}
|
||||
<span className="text-muted-foreground">
|
||||
{sessionFrequencyRestriction?.reason ??
|
||||
": score the first turn only, then hold that tier and its deployment for the whole session"}
|
||||
</span>
|
||||
</span>
|
||||
</Label>
|
||||
</div>
|
||||
</RadioGroup>
|
||||
<p className="text-sm text-muted-foreground">
|
||||
Holding the tier keeps an agent on one model for a whole tool loop and cuts scoring cost. A turn the router
|
||||
cannot match to a held decision, such as one with no session id or an expired one, is scored again
|
||||
</p>
|
||||
</div>
|
||||
|
||||
{classifierType === "jev" && <JevClassifierConfig value={value} onChange={onChange} />}
|
||||
{usesLlmClassifier(classifierType) && (
|
||||
<div className="mt-4 space-y-3">
|
||||
<div>
|
||||
<strong className="block mb-1 font-semibold">Classifier Model</strong>
|
||||
<SearchSelect
|
||||
options={modelOptions}
|
||||
value={value.classifier_llm_config?.model ?? ""}
|
||||
onValueChange={handleClassifierModelChange}
|
||||
placeholder="Select the model that will classify request complexity"
|
||||
emptyText="No models found"
|
||||
allowClear={false}
|
||||
className={classifierModelMissing ? "border-destructive" : undefined}
|
||||
aria-label="Classifier Model"
|
||||
/>
|
||||
{classifierModelMissing && <span className="text-xs text-destructive">A classifier model is required</span>}
|
||||
</div>
|
||||
<ClassifierReasoningEffortSelect
|
||||
model={classifierModel}
|
||||
value={classifierReasoningEffort}
|
||||
|
|
@ -583,6 +545,10 @@ const ClassificationMethodConfig: React.FC<ClassificationMethodConfigProps> = ({
|
|||
<Info className="size-4 text-muted-foreground" />
|
||||
</SimpleTooltip>
|
||||
</div>
|
||||
<AutoRouterAllowanceNote
|
||||
feature="tier_or_classifier_prompt"
|
||||
label="Custom instructions and examples share the custom-tier allowance"
|
||||
/>
|
||||
{!value.custom_tier_set && usesCustomPrompt ? (
|
||||
<ClassifierPromptEditor
|
||||
systemPrompt={value.classifier_llm_config?.system_prompt}
|
||||
|
|
@ -676,7 +642,7 @@ const ClassificationMethodConfig: React.FC<ClassificationMethodConfigProps> = ({
|
|||
/>
|
||||
<span className="text-xs text-muted-foreground">
|
||||
Number of prior user turns sent to the classifier provider, excluding tool output and harness reminders.
|
||||
LLM and JEV default to 3 turns; JEV sends them to the configured TypeSafe endpoint. Set to 0 to omit
|
||||
LLM and Jev default to 3 turns; Jev sends them to the configured TypeSafe endpoint. Set to 0 to omit
|
||||
conversation history. The current message and selected system text are still sent.
|
||||
</span>
|
||||
</div>
|
||||
|
|
@ -769,6 +735,9 @@ const ClassificationMethodConfig: React.FC<ClassificationMethodConfigProps> = ({
|
|||
</div>
|
||||
)}
|
||||
|
||||
{["heuristic", "heuristic_first", "hybrid"].includes(classifierType) && (
|
||||
<AutoRouterAllowanceNote feature="heuristic_tuning" label="Custom scoring rules" />
|
||||
)}
|
||||
<HeuristicScoringConfig value={value} onChange={onChange} />
|
||||
|
||||
<HowClassificationWorks value={value} />
|
||||
|
|
|
|||
|
|
@ -0,0 +1,98 @@
|
|||
import React from "react";
|
||||
import { Label } from "@/components/ui/label";
|
||||
import { SearchSelect } from "@/components/shared/SearchSelect";
|
||||
import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select";
|
||||
import {
|
||||
classificationFrequency,
|
||||
withClassificationFrequency,
|
||||
effectiveClassifierType,
|
||||
usesLlmClassifier,
|
||||
DEFAULT_CLASSIFIER_TIMEOUT_MS,
|
||||
type ComplexityRouterConfigValue,
|
||||
type ClassificationFrequency,
|
||||
} from "./ComplexityRouterConfig";
|
||||
import { restrictedBy } from "./TierRestrictions";
|
||||
|
||||
export default function ClassifierPrimarySettings({
|
||||
value,
|
||||
onChange,
|
||||
modelOptions,
|
||||
showValidationErrors = false,
|
||||
}: {
|
||||
value: ComplexityRouterConfigValue;
|
||||
onChange: (value: ComplexityRouterConfigValue) => void;
|
||||
modelOptions: { value: string; label: string }[];
|
||||
showValidationErrors?: boolean;
|
||||
}) {
|
||||
const id = React.useId();
|
||||
const restriction = restrictedBy(value, "sessionAffinity");
|
||||
const frequency = classificationFrequency(value);
|
||||
const frequencyDescription = {
|
||||
every_request: "Choose a model again for every request",
|
||||
user_turn: "Reclassify when the user sends a new message",
|
||||
session: "Keep the same tier for the session. Requires a client session ID",
|
||||
}[frequency];
|
||||
const usesJudge = usesLlmClassifier(effectiveClassifierType(value));
|
||||
const missingJudge = showValidationErrors && usesJudge && !value.classifier_llm_config?.model;
|
||||
return (
|
||||
<div className="mb-6 grid gap-4 sm:grid-cols-2">
|
||||
<div className="space-y-2">
|
||||
<Label htmlFor={`${id}-frequency`}>How often to classify</Label>
|
||||
<Select
|
||||
items={[
|
||||
{ value: "every_request", label: "Every request" },
|
||||
{ value: "user_turn", label: "Every new user message" },
|
||||
{ value: "session", label: "Once per session" },
|
||||
]}
|
||||
value={frequency}
|
||||
onValueChange={(frequency) => {
|
||||
if (frequency) onChange(withClassificationFrequency(value, frequency as ClassificationFrequency));
|
||||
}}
|
||||
>
|
||||
<SelectTrigger id={`${id}-frequency`} className="w-full">
|
||||
<SelectValue />
|
||||
</SelectTrigger>
|
||||
<SelectContent>
|
||||
<SelectItem value="every_request">Every request</SelectItem>
|
||||
<SelectItem value="user_turn">Every new user message</SelectItem>
|
||||
<SelectItem value="session" disabled={Boolean(restriction)}>
|
||||
Once per session
|
||||
</SelectItem>
|
||||
</SelectContent>
|
||||
</Select>
|
||||
<p className="text-xs text-muted-foreground">{restriction?.reason ?? frequencyDescription}</p>
|
||||
</div>
|
||||
{usesJudge && (
|
||||
<div className="space-y-2">
|
||||
<Label htmlFor={`${id}-judge`}>Judge model</Label>
|
||||
<SearchSelect
|
||||
inputId={`${id}-judge`}
|
||||
aria-label="Judge model"
|
||||
options={modelOptions}
|
||||
value={value.classifier_llm_config?.model ?? ""}
|
||||
placeholder="Select the judge model"
|
||||
allowClear={false}
|
||||
className={missingJudge ? "border-destructive" : undefined}
|
||||
onValueChange={(model) => {
|
||||
if (!model || model === value.classifier_llm_config?.model) return;
|
||||
onChange({
|
||||
...value,
|
||||
classifier_llm_config: {
|
||||
...value.classifier_llm_config,
|
||||
model,
|
||||
timeout_ms: value.classifier_llm_config?.timeout_ms ?? DEFAULT_CLASSIFIER_TIMEOUT_MS,
|
||||
reasoning_effort: undefined,
|
||||
},
|
||||
});
|
||||
}}
|
||||
/>
|
||||
{missingJudge && (
|
||||
<p role="alert" className="text-xs text-destructive">
|
||||
A judge model is required
|
||||
</p>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
|
@ -54,7 +54,7 @@ const ClassifierTypeRadios: React.FC<ClassifierTypeRadiosProps> = ({ value, clas
|
|||
<Label className="items-start font-normal leading-normal">
|
||||
<RadioGroupItem value="jev" className="mt-0.5" />
|
||||
<span>
|
||||
<strong className="font-semibold">JEV Classifier</strong>{" "}
|
||||
<strong className="font-semibold">Jev Classifier</strong>{" "}
|
||||
<span className="text-muted-foreground">uses TypeSafe System One Choice to decide the tier</span>
|
||||
</span>
|
||||
</Label>
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ import type { ModelGroup } from "@/components/llm_calls/fetch_models";
|
|||
import type { ComplexityRouterConfigValue } from "./ComplexityRouterConfig";
|
||||
import AdaptiveRoutingConfig from "./AdaptiveRoutingConfig";
|
||||
import ClassificationMethodConfig from "./ClassificationMethodConfig";
|
||||
import ForecastClassifierConfig from "./ForecastClassifierConfig";
|
||||
import ContextWindowEscalationConfig from "./ContextWindowEscalationConfig";
|
||||
import ResponseFormatControls from "./ResponseFormatControls";
|
||||
import StallEscalationConfig from "./StallEscalationConfig";
|
||||
|
|
@ -79,13 +80,31 @@ const ComplexityRouterAdvancedSections: React.FC<ComplexityRouterAdvancedSection
|
|||
customTierSet,
|
||||
}) => {
|
||||
const sections = [
|
||||
...(forecast
|
||||
? [
|
||||
{
|
||||
key: "classifier",
|
||||
label: <strong className="text-foreground font-semibold">Classifier tuning</strong>,
|
||||
children: (
|
||||
<ForecastClassifierConfig
|
||||
section="advanced"
|
||||
value={value}
|
||||
onChange={onChange}
|
||||
modelOptions={modelOptions}
|
||||
effortOptionsByModel={classifierEffortOptionsByModel}
|
||||
/>
|
||||
),
|
||||
},
|
||||
]
|
||||
: []),
|
||||
...(!forecast
|
||||
? [
|
||||
{
|
||||
key: "classifier",
|
||||
label: <strong className="text-foreground font-semibold">Advanced: Classification Method</strong>,
|
||||
label: <strong className="text-foreground font-semibold">Classification Method</strong>,
|
||||
children: (
|
||||
<ClassificationMethodConfig
|
||||
advancedOnly
|
||||
value={value}
|
||||
onChange={onChange}
|
||||
modelOptions={modelOptions}
|
||||
|
|
@ -103,14 +122,14 @@ const ComplexityRouterAdvancedSections: React.FC<ComplexityRouterAdvancedSection
|
|||
? [
|
||||
{
|
||||
key: "keyword-overrides",
|
||||
label: <strong className="text-foreground font-semibold">Advanced: Heuristic Keyword Overrides</strong>,
|
||||
label: <strong className="text-foreground font-semibold">Heuristic Keyword Overrides</strong>,
|
||||
children: <HeuristicKeywordOverrides value={value} onChange={onChange} />,
|
||||
},
|
||||
]
|
||||
: []),
|
||||
{
|
||||
key: "adaptive",
|
||||
label: <strong className="text-foreground font-semibold">Advanced: Adaptive Routing</strong>,
|
||||
label: <strong className="text-foreground font-semibold">Adaptive Routing</strong>,
|
||||
children: (
|
||||
<Restricted by={restrictedBy(value, "adaptive")}>
|
||||
<AdaptiveRoutingConfig value={value} onChange={onChange} />
|
||||
|
|
@ -119,39 +138,39 @@ const ComplexityRouterAdvancedSections: React.FC<ComplexityRouterAdvancedSection
|
|||
},
|
||||
{
|
||||
key: "affinity",
|
||||
label: <strong className="text-foreground font-semibold">Advanced: Affinity</strong>,
|
||||
label: <strong className="text-foreground font-semibold">Affinity</strong>,
|
||||
children: <AffinityControls value={value} onChange={onChange} />,
|
||||
},
|
||||
{
|
||||
key: "modality",
|
||||
label: <strong className="text-foreground font-semibold">Advanced: Modality Routing</strong>,
|
||||
label: <strong className="text-foreground font-semibold">Modality Routing</strong>,
|
||||
children: <ModalityRoutingControls value={value} onChange={onChange} />,
|
||||
},
|
||||
{
|
||||
key: "plan-mode",
|
||||
label: <strong className="text-foreground font-semibold">Advanced: Plan-Mode Override</strong>,
|
||||
label: <strong className="text-foreground font-semibold">Plan-Mode Override</strong>,
|
||||
children: (
|
||||
<PlanModeOverrideControls value={value} onChange={onChange} planModeTierOptions={planModeTierOptions} />
|
||||
),
|
||||
},
|
||||
{
|
||||
key: "housekeeping",
|
||||
label: <strong className="text-foreground font-semibold">Advanced: Housekeeping Routing</strong>,
|
||||
label: <strong className="text-foreground font-semibold">Housekeeping Routing</strong>,
|
||||
children: <HousekeepingRoutingControls value={value} onChange={onChange} />,
|
||||
},
|
||||
{
|
||||
key: "reminder-markers",
|
||||
label: <strong className="text-foreground font-semibold">Advanced: Ignore Custom Tags</strong>,
|
||||
label: <strong className="text-foreground font-semibold">Ignore Custom Tags</strong>,
|
||||
children: <ReminderMarkers value={value} onChange={onChange} showValidationErrors={showValidationErrors} />,
|
||||
},
|
||||
{
|
||||
key: "context-window",
|
||||
label: <strong className="text-foreground font-semibold">Advanced: Context Window Escalation</strong>,
|
||||
label: <strong className="text-foreground font-semibold">Context Window Escalation</strong>,
|
||||
children: <ContextWindowEscalationConfig value={value} onChange={onChange} />,
|
||||
},
|
||||
{
|
||||
key: "stall-escalation",
|
||||
label: <strong className="text-foreground font-semibold">Advanced: Stalled Task Escalation</strong>,
|
||||
label: <strong className="text-foreground font-semibold">Stalled Task Escalation</strong>,
|
||||
children: (
|
||||
<Restricted by={restrictedBy(value, "stallEscalation")}>
|
||||
<StallEscalationConfig value={value} onChange={onChange} />
|
||||
|
|
@ -160,14 +179,14 @@ const ComplexityRouterAdvancedSections: React.FC<ComplexityRouterAdvancedSection
|
|||
},
|
||||
{
|
||||
key: "response",
|
||||
label: <strong className="text-foreground font-semibold">Advanced: Response Format</strong>,
|
||||
label: <strong className="text-foreground font-semibold">Response Format</strong>,
|
||||
children: <ResponseFormatControls value={value} onChange={onChange} />,
|
||||
},
|
||||
...(onEscalationKeywordsChange
|
||||
? [
|
||||
{
|
||||
key: "escalation",
|
||||
label: <strong className="text-foreground font-semibold">Advanced: Escalation Keywords</strong>,
|
||||
label: <strong className="text-foreground font-semibold">Escalation Keywords</strong>,
|
||||
children: (
|
||||
<Restricted by={restrictedBy(value, "escalation")}>
|
||||
<EscalationKeywords keywords={escalationKeywords} onChange={onEscalationKeywordsChange} />
|
||||
|
|
@ -180,7 +199,7 @@ const ComplexityRouterAdvancedSections: React.FC<ComplexityRouterAdvancedSection
|
|||
? [
|
||||
{
|
||||
key: "compression",
|
||||
label: <strong className="text-foreground font-semibold">Advanced: Compression</strong>,
|
||||
label: <strong className="text-foreground font-semibold">Compression</strong>,
|
||||
children: <CompressionControls value={autoRouterCompression} onChange={onAutoRouterCompressionChange} />,
|
||||
},
|
||||
]
|
||||
|
|
@ -189,7 +208,7 @@ const ComplexityRouterAdvancedSections: React.FC<ComplexityRouterAdvancedSection
|
|||
? [
|
||||
{
|
||||
key: "keyword-semantic",
|
||||
label: <strong className="text-foreground font-semibold">Advanced: Keyword/Semantic Matching</strong>,
|
||||
label: <strong className="text-foreground font-semibold">Keyword/Semantic Matching</strong>,
|
||||
children: (
|
||||
<>
|
||||
{onKeywordTierRulesChange && (
|
||||
|
|
@ -220,20 +239,65 @@ const ComplexityRouterAdvancedSections: React.FC<ComplexityRouterAdvancedSection
|
|||
: []),
|
||||
];
|
||||
|
||||
const groups = [
|
||||
{ label: "Classifier tuning", keys: ["classifier", "keyword-overrides", "reminder-markers"] },
|
||||
{
|
||||
label: "Routing rules and recovery",
|
||||
keys: [
|
||||
"modality",
|
||||
"plan-mode",
|
||||
"housekeeping",
|
||||
"context-window",
|
||||
"stall-escalation",
|
||||
"escalation",
|
||||
"keyword-semantic",
|
||||
],
|
||||
},
|
||||
{ label: "Sessions and efficiency", keys: ["affinity", "adaptive", "compression"] },
|
||||
{ label: "Compatibility", keys: ["response"] },
|
||||
];
|
||||
const [openGroups, setOpenGroups] = React.useState<string[]>(() =>
|
||||
showValidationErrors ? groups.map((group) => group.label) : [],
|
||||
);
|
||||
const [previousValidation, setPreviousValidation] = React.useState(showValidationErrors);
|
||||
if (previousValidation !== showValidationErrors) {
|
||||
setPreviousValidation(showValidationErrors);
|
||||
if (showValidationErrors) setOpenGroups(groups.map((group) => group.label));
|
||||
}
|
||||
return (
|
||||
<>
|
||||
{sections
|
||||
.filter(({ key }) => !forecast || !["adaptive", "context-window", "escalation"].includes(key))
|
||||
.map(({ key, label, children }) => (
|
||||
<Collapsible key={key} className="border-b border-border last:border-b-0">
|
||||
<CollapsibleTrigger className="group flex w-full items-center gap-2 px-4 py-3 text-left">
|
||||
<ChevronRight className="size-4 shrink-0 text-muted-foreground transition-transform group-data-panel-open:rotate-90" />
|
||||
{label}
|
||||
</CollapsibleTrigger>
|
||||
<CollapsibleContent className="px-4 pb-4">{children}</CollapsibleContent>
|
||||
</Collapsible>
|
||||
))}
|
||||
</>
|
||||
<div>
|
||||
{groups.map((group) => (
|
||||
<Collapsible
|
||||
key={group.label}
|
||||
open={openGroups.includes(group.label)}
|
||||
onOpenChange={(open) =>
|
||||
setOpenGroups((current) =>
|
||||
open ? [...current, group.label] : current.filter((label) => label !== group.label),
|
||||
)
|
||||
}
|
||||
className="border-b border-border last:border-b-0"
|
||||
>
|
||||
<CollapsibleTrigger className="group flex w-full items-center gap-2 px-4 py-3 text-left font-medium">
|
||||
<ChevronRight className="size-4 shrink-0 text-muted-foreground transition-transform group-data-panel-open:rotate-90" />
|
||||
{group.label}
|
||||
</CollapsibleTrigger>
|
||||
<CollapsibleContent className="space-y-6 px-4 pb-4">
|
||||
{sections
|
||||
.filter(
|
||||
({ key }) =>
|
||||
group.keys.includes(key) &&
|
||||
(!forecast || !["adaptive", "context-window", "escalation"].includes(key)),
|
||||
)
|
||||
.map(({ key, label, children }) => (
|
||||
<section key={key} className="space-y-3">
|
||||
{key !== "classifier" && <h4>{label}</h4>}
|
||||
{children}
|
||||
</section>
|
||||
))}
|
||||
</CollapsibleContent>
|
||||
</Collapsible>
|
||||
))}
|
||||
</div>
|
||||
);
|
||||
};
|
||||
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
|
|
@ -1,4 +1,6 @@
|
|||
import RoutingOptions from "./RoutingOptions";
|
||||
import ClassifierPrimarySettings from "./ClassifierPrimarySettings";
|
||||
import { AutoRouterAllowanceNote } from "./AutoRouterAvailability";
|
||||
import type { JevClassifierConfig } from "./jev_classifier_config";
|
||||
import { type ClassifierType } from "./classifier_types";
|
||||
export { type ClassifierType, usesLlmClassifier, usesClassifierContext } from "./classifier_types";
|
||||
|
|
@ -6,11 +8,11 @@ import ForecastClassifierConfig, { ForecastSolverModels } from "./ForecastClassi
|
|||
import { isForecastClassifier, type CapabilitySettings, type FuseSettings } from "./forecast_classifier_config";
|
||||
import { SimpleTooltip } from "@/components/ui/tooltip";
|
||||
import { MultiSelect } from "@/components/shared/MultiSelect";
|
||||
import TierConfigIntro from "./TierConfigIntro";
|
||||
import DefaultModelField from "./DefaultModelField";
|
||||
import { Info, Plus, Trash2, X } from "lucide-react";
|
||||
|
||||
import NonReasoningTierToggle from "./NonReasoningTierToggle";
|
||||
import TierConfigIntro from "./TierConfigIntro";
|
||||
import TierRowSelect from "./TierRowSelect";
|
||||
import { Card, CardContent } from "@/components/ui/card";
|
||||
import { InputGroup, InputGroupAddon, InputGroupButton, InputGroupInput } from "@/components/ui/input-group";
|
||||
|
|
@ -226,10 +228,16 @@ const TierSetToolbar: React.FC<{
|
|||
)
|
||||
)}
|
||||
</div>
|
||||
{editing && (
|
||||
<AutoRouterAllowanceNote
|
||||
feature="tier_or_classifier_prompt"
|
||||
label="Custom tiers and written prompts share this allowance"
|
||||
/>
|
||||
)}
|
||||
{editing && (
|
||||
<span className="block mt-1 text-xs text-muted-foreground">
|
||||
Add or remove tiers to define your own set. Every custom tier needs a definition the classifier routes on, and
|
||||
an edited set requires the LLM or JEV classification method
|
||||
an edited set requires the LLM or Jev classification method
|
||||
</span>
|
||||
)}
|
||||
{editing && keywordRulesError && (
|
||||
|
|
@ -596,10 +604,14 @@ const ComplexityRouterConfig: React.FC<ComplexityRouterConfigProps> = ({
|
|||
|
||||
return (
|
||||
<div className="w-full max-w-none">
|
||||
<ClassifierPrimarySettings
|
||||
value={value}
|
||||
onChange={onChange}
|
||||
modelOptions={modelOptions}
|
||||
showValidationErrors={showValidationErrors}
|
||||
/>
|
||||
<div className="inline-flex items-center gap-2 mb-4">
|
||||
<h4 className="m-0 text-xl font-semibold text-foreground">
|
||||
{forecast ? "Solver models" : "Complexity Tier Configuration"}
|
||||
</h4>
|
||||
<h4 className="m-0 text-xl font-semibold text-foreground">{forecast ? "Solver models" : "Models by tier"}</h4>
|
||||
{!forecast && (
|
||||
<SimpleTooltip content="Map each complexity tier to one or more models. Simple queries use cheaper/faster models, complex queries use more capable models.">
|
||||
<Info className="size-4 text-muted-foreground" />
|
||||
|
|
@ -619,6 +631,7 @@ const ComplexityRouterConfig: React.FC<ComplexityRouterConfigProps> = ({
|
|||
fastModeByModel={fastModeByModel}
|
||||
/>
|
||||
<ForecastClassifierConfig
|
||||
section="required"
|
||||
value={value}
|
||||
onChange={onChange}
|
||||
modelOptions={modelOptions}
|
||||
|
|
@ -628,7 +641,6 @@ const ComplexityRouterConfig: React.FC<ComplexityRouterConfigProps> = ({
|
|||
) : (
|
||||
<>
|
||||
<TierConfigIntro value={value} />
|
||||
|
||||
<Card>
|
||||
<CardContent>
|
||||
{!customTierSet && (
|
||||
|
|
@ -750,13 +762,19 @@ const ComplexityRouterConfig: React.FC<ComplexityRouterConfigProps> = ({
|
|||
</Card>
|
||||
</>
|
||||
)}
|
||||
{!forecast && <DefaultModelField value={value} onChange={onChange} modelOptions={modelOptions} />}
|
||||
<DefaultModelField value={value} onChange={onChange} modelOptions={modelOptions} />
|
||||
<Separator className="my-6" />
|
||||
|
||||
<RoutingOptions forecast={forecast}>
|
||||
<RoutingOptions
|
||||
showValidationErrors={showValidationErrors}
|
||||
summary={
|
||||
{ hybrid: "Hybrid local checks enabled", heuristic_first: "Heuristic first enabled" }[
|
||||
value.classifier_type as "hybrid" | "heuristic_first"
|
||||
]
|
||||
}
|
||||
>
|
||||
{forecast && (
|
||||
<>
|
||||
<DefaultModelField value={value} onChange={onChange} modelOptions={modelOptions} />
|
||||
<ForecastSolverModels
|
||||
additionalPoolsOnly
|
||||
value={value}
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
import { openAutoRouterAdvanced } from "../../../tests/autoRouterSetup";
|
||||
import userEvent from "@testing-library/user-event";
|
||||
import React from "react";
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
|
|
@ -215,7 +216,7 @@ it.each(["MEDIUM", "REASONING"])("clears a legacy Capability pool while reconcil
|
|||
const value = hydrateComplexityRouterConfig(stored, undefined);
|
||||
const onChange = vi.fn<(value: ComplexityRouterConfigValue) => void>();
|
||||
renderWithProviders(<ComplexityRouterConfig modelInfo={modelInfo} value={value} onChange={onChange} />);
|
||||
await user.click(screen.getByRole("button", { name: "Advanced routing options" }));
|
||||
openAutoRouterAdvanced("Keyword/Semantic Matching");
|
||||
expect(screen.getByRole("switch", { name: "Fast mode for secondary in the Medium routing pool tier" })).toBeChecked();
|
||||
expect(onChange).not.toHaveBeenCalled();
|
||||
await user.click(screen.getByRole("combobox", { name: "Select medium routing pool models" }));
|
||||
|
|
@ -245,7 +246,7 @@ it.each(["capability", "llm_v2"] as const)(
|
|||
<ComplexityRouterConfig modelInfo={modelInfo} value={value} onChange={onChange} />
|
||||
);
|
||||
const view = renderWithProviders(editor(hydrateComplexityRouterConfig(stored, undefined)));
|
||||
await user.click(screen.getByRole("button", { name: "Advanced routing options" }));
|
||||
openAutoRouterAdvanced("Keyword/Semantic Matching");
|
||||
const select = () => screen.getByRole("combobox", { name: "Default model" });
|
||||
expect(select()).toHaveValue("legacy-default");
|
||||
expect(onChange).not.toHaveBeenCalled();
|
||||
|
|
@ -284,8 +285,8 @@ it.each(["capability", "llm_v2"] as const)("offers only populated keyword target
|
|||
/>
|
||||
);
|
||||
const view = renderWithProviders(editor([]));
|
||||
await user.click(screen.getByRole("button", { name: "Advanced routing options" }));
|
||||
await user.click(screen.getByText("Advanced: Keyword/Semantic Matching"));
|
||||
openAutoRouterAdvanced("Keyword/Semantic Matching");
|
||||
openAutoRouterAdvanced("Keyword/Semantic Matching");
|
||||
await user.click(screen.getByRole("button", { name: "Add keyword rule" }));
|
||||
const rules = onRulesChange.mock.lastCall![0];
|
||||
expect(rules[0].tier).toBe("SIMPLE");
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
import { selectAutoRouterApproach } from "../../../tests/autoRouterSetup";
|
||||
import React, { useState } from "react";
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import userEvent from "@testing-library/user-event";
|
||||
|
|
@ -306,7 +307,7 @@ describe("forecast classifier form", () => {
|
|||
);
|
||||
});
|
||||
|
||||
it("switches a populated standard router to Capability without saving hidden pools or their overrides", () => {
|
||||
it("switches a populated standard router to Capability without saving hidden pools or their overrides", async () => {
|
||||
renderWithProviders(
|
||||
<Form
|
||||
initialValue={{
|
||||
|
|
@ -329,7 +330,7 @@ describe("forecast classifier form", () => {
|
|||
}}
|
||||
/>,
|
||||
);
|
||||
fireEvent.click(screen.getByRole("tab", { name: "Capability" }));
|
||||
await selectAutoRouterApproach("Capability");
|
||||
fireEvent.change(screen.getByLabelText("Solve probability threshold"), { target: { value: "0.7" } });
|
||||
expect(screen.getByRole("button", { name: "Save configuration" })).toBeEnabled();
|
||||
fireEvent.click(screen.getByRole("button", { name: "Save configuration" }));
|
||||
|
|
@ -347,7 +348,7 @@ describe("forecast classifier form", () => {
|
|||
|
||||
it.each(["capability", "llm_v2"] as const)(
|
||||
"carries non-default solver assignments when switching away from %s",
|
||||
(source) => {
|
||||
async (source) => {
|
||||
const pair = { efficient_tier: "MEDIUM", capable_tier: "COMPLEX" };
|
||||
const previous: ComplexityRouterConfigValue = {
|
||||
...(source === "capability" ? initial : fuseInitial),
|
||||
|
|
@ -359,7 +360,7 @@ describe("forecast classifier form", () => {
|
|||
tier_model_params: { MEDIUM: { efficient: { max_tokens: 128 } }, COMPLEX: { capable: { speed: "fast" } } },
|
||||
};
|
||||
renderWithProviders(<Form initialValue={previous} />);
|
||||
fireEvent.click(screen.getByRole("tab", { name: source === "capability" ? "Fuse v2" : "Capability" }));
|
||||
await selectAutoRouterApproach(source === "capability" ? "Fuse v2" : "Capability");
|
||||
if (source === "capability") {
|
||||
fireEvent.change(screen.getByLabelText("Efficient solver profile"), { target: { value: "Small solver" } });
|
||||
fireEvent.change(screen.getByLabelText("Capable solver profile"), { target: { value: "Large solver" } });
|
||||
|
|
@ -402,20 +403,22 @@ describe("forecast classifier form", () => {
|
|||
] as const)("restores the current rubric when switching %s through Complexity to %s", async (source, target) => {
|
||||
const user = userEvent.setup();
|
||||
renderWithProviders(<Form initialValue={source === "capability" ? initial : fuseInitial} />);
|
||||
fireEvent.click(screen.getByRole("tab", { name: "Complexity" }));
|
||||
await selectAutoRouterApproach("Complexity");
|
||||
fireEvent.click(screen.getByRole("radio", { name: new RegExp(`^${target}`) }));
|
||||
await user.click(screen.getByRole("combobox", { name: "Classifier Model" }));
|
||||
await user.click(screen.getByRole("combobox", { name: "Judge model" }));
|
||||
await user.click(screen.getByRole("option", { name: "judge" }));
|
||||
fireEvent.click(screen.getByRole("button", { name: "Save configuration" }));
|
||||
const output = screen.getByRole("status", { name: "Saved configuration" });
|
||||
expect(output).toHaveTextContent('"classification_rubric":"agentic"');
|
||||
expect(output).toHaveTextContent('"model":"judge"');
|
||||
expect(output).toHaveTextContent('"timeout_ms":3000');
|
||||
expect(output).toHaveTextContent(
|
||||
`"timeout_ms":${(source === "capability" ? initial : fuseInitial).classifier_llm_config?.timeout_ms}`,
|
||||
);
|
||||
expect(output).not.toHaveTextContent('"capability_classifier_config"');
|
||||
expect(output).not.toHaveTextContent('"llm_v2_config"');
|
||||
});
|
||||
|
||||
it("saves capability threshold edits together with fitted calibration", () => {
|
||||
it("saves capability threshold edits together with fitted calibration", async () => {
|
||||
renderWithProviders(<Form />);
|
||||
fireEvent.change(screen.getByLabelText("Solve probability threshold"), { target: { value: "0.6" } });
|
||||
fireEvent.click(screen.getByRole("button", { name: "Classifier options" }));
|
||||
|
|
@ -432,9 +435,9 @@ describe("forecast classifier form", () => {
|
|||
expect(screen.getByRole("button", { name: "Save configuration" })).toBeDisabled();
|
||||
});
|
||||
|
||||
it("switches to Fuse, requires solver context, and saves the filled fields", () => {
|
||||
it("switches to Fuse, requires solver context, and saves the filled fields", async () => {
|
||||
renderWithProviders(<Form />);
|
||||
fireEvent.click(screen.getByRole("tab", { name: "Fuse v2" }));
|
||||
await selectAutoRouterApproach("Fuse v2");
|
||||
expect(screen.queryByLabelText("Solve probability threshold")).not.toBeInTheDocument();
|
||||
expect(screen.getByRole("button", { name: "Save configuration" })).toBeDisabled();
|
||||
fireEvent.change(screen.getByLabelText("Efficient solver profile"), {
|
||||
|
|
|
|||
|
|
@ -36,6 +36,7 @@ interface Props {
|
|||
onChange: (value: ComplexityRouterConfigValue) => void;
|
||||
modelOptions: { value: string; label: string }[];
|
||||
effortOptionsByModel: Record<string, string[] | null | undefined>;
|
||||
section?: "all" | "required" | "advanced";
|
||||
}
|
||||
|
||||
const NumberField = ({
|
||||
|
|
@ -188,7 +189,7 @@ const CalibrationFields = ({
|
|||
|
||||
const emptyCoefficients = () => ({ slope: Number.NaN, intercept: Number.NaN });
|
||||
|
||||
const ForecastClassifierConfig = ({ value, onChange, modelOptions, effortOptionsByModel }: Props) => {
|
||||
const ForecastClassifierConfig = ({ value, onChange, modelOptions, effortOptionsByModel, section = "all" }: Props) => {
|
||||
const id = React.useId();
|
||||
const isCapability = value.classifier_type === "capability";
|
||||
const capability = value.capability_classifier_config ?? newCapabilitySettings();
|
||||
|
|
@ -212,72 +213,195 @@ const ForecastClassifierConfig = ({ value, onChange, modelOptions, effortOptions
|
|||
? "Forecasts whether the efficient solver can complete the task using the bundled capability card"
|
||||
: "Forecasts success for both solvers and selects efficient when the estimated quality gap is within your allowance"}
|
||||
</p>
|
||||
<div className="space-y-1">
|
||||
<Label htmlFor={`${id}-judge`}>Judge model</Label>
|
||||
<SearchSelect
|
||||
inputId={`${id}-judge`}
|
||||
aria-label="Judge model"
|
||||
options={modelOptions}
|
||||
value={llm.model}
|
||||
placeholder="Select the judge model"
|
||||
onValueChange={(model) => {
|
||||
if (model === llm.model) return;
|
||||
onChange({ ...value, classifier_llm_config: { ...llm, model: model ?? "", reasoning_effort: undefined } });
|
||||
}}
|
||||
/>
|
||||
</div>
|
||||
{isCapability ? (
|
||||
{section === "all" && (
|
||||
<>
|
||||
<NumberField
|
||||
label="Solve probability threshold"
|
||||
value={capability.base_threshold}
|
||||
min={0}
|
||||
max={1}
|
||||
help="Minimum estimated chance of whole-task success required to use the efficient solver"
|
||||
onChange={(base_threshold) => updateCapability({ ...capability, base_threshold })}
|
||||
/>
|
||||
</>
|
||||
) : (
|
||||
<>
|
||||
<FuseProfilePresets value={fuse} onChange={updateFuse} />
|
||||
<NumberField
|
||||
label="Maximum quality gap"
|
||||
value={fuse.max_quality_gap}
|
||||
min={0}
|
||||
max={1}
|
||||
help="Allowed difference between capable and efficient success probabilities, from 0 to 1. Tune on held-out tasks from your workload; this estimate is not a measured quality guarantee. A gap of 0 still selects efficient on tied or higher forecasts. Route directly to one model to avoid judging when you do not want model selection"
|
||||
onChange={(max_quality_gap) => updateFuse({ ...fuse, max_quality_gap })}
|
||||
/>
|
||||
<div className="space-y-1">
|
||||
<Label htmlFor={`${id}-judge`}>Judge model</Label>
|
||||
<SearchSelect
|
||||
inputId={`${id}-judge`}
|
||||
aria-label="Judge model"
|
||||
options={modelOptions}
|
||||
value={llm.model}
|
||||
placeholder="Select the judge model"
|
||||
onValueChange={(model) => {
|
||||
if (model === llm.model) return;
|
||||
onChange({
|
||||
...value,
|
||||
classifier_llm_config: { ...llm, model: model ?? "", reasoning_effort: undefined },
|
||||
});
|
||||
}}
|
||||
/>
|
||||
</div>
|
||||
</>
|
||||
)}
|
||||
<Collapsible className="rounded-lg border">
|
||||
<CollapsibleTrigger className="group flex w-full items-center gap-2 px-4 py-3 text-left font-medium">
|
||||
<ChevronRight className="size-4 transition-transform group-data-panel-open:rotate-90" />
|
||||
Classifier options
|
||||
</CollapsibleTrigger>
|
||||
<CollapsibleContent className="space-y-4 px-4 pb-4">
|
||||
<ClassifierReasoningEffortSelect
|
||||
model={llm.model}
|
||||
value={llm.reasoning_effort}
|
||||
explicitlySupported={effortOptionsByModel[llm.model]}
|
||||
onChange={(reasoning_effort) => onChange({ ...value, classifier_llm_config: { ...llm, reasoning_effort } })}
|
||||
/>
|
||||
<NumberField
|
||||
label="Timeout (ms)"
|
||||
min={1}
|
||||
step={1}
|
||||
value={llm.timeout_ms}
|
||||
help="Allow enough time for the judge to produce its forecast"
|
||||
onChange={(timeout_ms) => onChange({ ...value, classifier_llm_config: { ...llm, timeout_ms } })}
|
||||
/>
|
||||
<ClassifierCircuitBreakerConfig
|
||||
value={llm}
|
||||
onChange={(classifier_llm_config) => onChange({ ...value, classifier_llm_config })}
|
||||
/>
|
||||
<ClassifierVisionConfig
|
||||
value={llm}
|
||||
onChange={(classifier_llm_config) => onChange({ ...value, classifier_llm_config })}
|
||||
/>
|
||||
{section !== "advanced" && (
|
||||
<>
|
||||
{isCapability ? (
|
||||
<>
|
||||
<NumberField
|
||||
label="Solve probability threshold"
|
||||
value={capability.base_threshold}
|
||||
min={0}
|
||||
max={1}
|
||||
help="Minimum estimated chance of whole-task success required to use the efficient solver"
|
||||
onChange={(base_threshold) => updateCapability({ ...capability, base_threshold })}
|
||||
/>
|
||||
</>
|
||||
) : (
|
||||
<>
|
||||
<FuseProfilePresets value={fuse} onChange={updateFuse} />
|
||||
<NumberField
|
||||
label="Maximum quality gap"
|
||||
value={fuse.max_quality_gap}
|
||||
min={0}
|
||||
max={1}
|
||||
help="Allowed difference between capable and efficient success probabilities, from 0 to 1. Tune on held-out tasks from your workload; this estimate is not a measured quality guarantee. A gap of 0 still selects efficient on tied or higher forecasts. Route directly to one model to avoid judging when you do not want model selection"
|
||||
onChange={(max_quality_gap) => updateFuse({ ...fuse, max_quality_gap })}
|
||||
/>
|
||||
</>
|
||||
)}
|
||||
</>
|
||||
)}
|
||||
{section !== "required" && (
|
||||
<Collapsible
|
||||
open={section === "advanced" ? true : undefined}
|
||||
className={section === "advanced" ? undefined : "rounded-lg border"}
|
||||
>
|
||||
{section === "all" && (
|
||||
<CollapsibleTrigger className="group flex w-full items-center gap-2 px-4 py-3 text-left font-medium">
|
||||
<ChevronRight className="size-4 transition-transform group-data-panel-open:rotate-90" />
|
||||
Classifier options
|
||||
</CollapsibleTrigger>
|
||||
)}
|
||||
<CollapsibleContent className="space-y-4 px-4 pb-4">
|
||||
<ClassifierReasoningEffortSelect
|
||||
model={llm.model}
|
||||
value={llm.reasoning_effort}
|
||||
explicitlySupported={effortOptionsByModel[llm.model]}
|
||||
onChange={(reasoning_effort) =>
|
||||
onChange({ ...value, classifier_llm_config: { ...llm, reasoning_effort } })
|
||||
}
|
||||
/>
|
||||
<NumberField
|
||||
label="Timeout (ms)"
|
||||
min={1}
|
||||
step={1}
|
||||
value={llm.timeout_ms}
|
||||
help="Allow enough time for the judge to produce its forecast"
|
||||
onChange={(timeout_ms) => onChange({ ...value, classifier_llm_config: { ...llm, timeout_ms } })}
|
||||
/>
|
||||
<ClassifierCircuitBreakerConfig
|
||||
value={llm}
|
||||
onChange={(classifier_llm_config) => onChange({ ...value, classifier_llm_config })}
|
||||
/>
|
||||
<ClassifierVisionConfig
|
||||
value={llm}
|
||||
onChange={(classifier_llm_config) => onChange({ ...value, classifier_llm_config })}
|
||||
/>
|
||||
{isCapability && (
|
||||
<NumberField
|
||||
label="Capability boundary step"
|
||||
value={capability.threshold_step ?? 0}
|
||||
min={0}
|
||||
max={0.5}
|
||||
help="Added once for uncertain or unmatched tasks and twice for unsupported tasks; the final threshold cannot exceed 1"
|
||||
onChange={(threshold_step) => updateCapability({ ...capability, threshold_step })}
|
||||
/>
|
||||
)}
|
||||
<NumberField
|
||||
label="Classifier output token limit"
|
||||
min={1}
|
||||
step={1}
|
||||
value={config.max_output_tokens ?? (isCapability ? 4096 : 1024)}
|
||||
onChange={(max_output_tokens) => updateTransport({ max_output_tokens })}
|
||||
/>
|
||||
<div className="space-y-1">
|
||||
<Label htmlFor={`${id}-format`}>Forecast response format</Label>
|
||||
<SearchSelect
|
||||
inputId={`${id}-format`}
|
||||
aria-label="Forecast response format"
|
||||
value={config.response_format ?? "json_schema"}
|
||||
allowClear={false}
|
||||
options={[
|
||||
{ value: "json_schema", label: "Strict JSON schema" },
|
||||
{ value: "json_object", label: "JSON object (for judges without strict schema support)" },
|
||||
]}
|
||||
onValueChange={(response_format) => {
|
||||
if (response_format === "json_schema" || response_format === "json_object")
|
||||
updateTransport({ response_format });
|
||||
}}
|
||||
/>
|
||||
</div>
|
||||
<div className="space-y-3 rounded-md border p-3">
|
||||
<Label>
|
||||
<Switch
|
||||
checked={Boolean(config.calibration)}
|
||||
onCheckedChange={(enabled) =>
|
||||
isCapability
|
||||
? updateCapability({
|
||||
...capability,
|
||||
calibration: enabled ? { version: "", ...emptyCoefficients() } : undefined,
|
||||
})
|
||||
: updateFuse({
|
||||
...fuse,
|
||||
calibration: enabled
|
||||
? {
|
||||
version: "",
|
||||
prompt_version: "llm-v2-1",
|
||||
efficient: emptyCoefficients(),
|
||||
capable: emptyCoefficients(),
|
||||
}
|
||||
: undefined,
|
||||
})
|
||||
}
|
||||
/>
|
||||
Use fitted calibration
|
||||
</Label>
|
||||
<p className="text-xs text-muted-foreground">
|
||||
Optional coefficients fitted for your judge, solvers, and harness. Leave off to use raw forecasts
|
||||
</p>
|
||||
{config.calibration && (
|
||||
<div className="space-y-1">
|
||||
<Label htmlFor={`${id}-version`}>Calibration version</Label>
|
||||
<Input
|
||||
id={`${id}-version`}
|
||||
value={config.calibration.version}
|
||||
maxLength={isCapability ? 128 : 512}
|
||||
onChange={(event) => setCalibrationVersion(event.target.value)}
|
||||
/>
|
||||
</div>
|
||||
)}
|
||||
{isCapability && capability.calibration && (
|
||||
<CalibrationFields
|
||||
label="Efficient"
|
||||
bounded
|
||||
value={capability.calibration}
|
||||
onChange={(next) =>
|
||||
updateCapability({
|
||||
...capability,
|
||||
calibration: { version: capability.calibration?.version ?? "", ...next },
|
||||
})
|
||||
}
|
||||
/>
|
||||
)}
|
||||
{!isCapability &&
|
||||
fuse.calibration &&
|
||||
(["efficient", "capable"] as const).map((role) => (
|
||||
<CalibrationFields
|
||||
key={role}
|
||||
label={role === "efficient" ? "Efficient" : "Capable"}
|
||||
value={fuse.calibration![role]}
|
||||
onChange={(next) => {
|
||||
if (fuse.calibration) updateFuse({ ...fuse, calibration: { ...fuse.calibration, [role]: next } });
|
||||
}}
|
||||
/>
|
||||
))}
|
||||
</div>
|
||||
</CollapsibleContent>
|
||||
</Collapsible>
|
||||
)}
|
||||
{section === "all" && (
|
||||
<>
|
||||
<div className="space-y-1">
|
||||
<Label htmlFor={`${id}-frequency`}>How often to classify</Label>
|
||||
<SearchSelect
|
||||
|
|
@ -295,111 +419,12 @@ const ForecastClassifierConfig = ({ value, onChange, modelOptions, effortOptions
|
|||
}}
|
||||
/>
|
||||
</div>
|
||||
{isCapability && (
|
||||
<NumberField
|
||||
label="Capability boundary step"
|
||||
value={capability.threshold_step ?? 0}
|
||||
min={0}
|
||||
max={0.5}
|
||||
help="Added once for uncertain or unmatched tasks and twice for unsupported tasks; the final threshold cannot exceed 1"
|
||||
onChange={(threshold_step) => updateCapability({ ...capability, threshold_step })}
|
||||
/>
|
||||
)}
|
||||
<NumberField
|
||||
label="Classifier output token limit"
|
||||
min={1}
|
||||
step={1}
|
||||
value={config.max_output_tokens ?? (isCapability ? 4096 : 1024)}
|
||||
onChange={(max_output_tokens) => updateTransport({ max_output_tokens })}
|
||||
/>
|
||||
<div className="space-y-1">
|
||||
<Label htmlFor={`${id}-format`}>Forecast response format</Label>
|
||||
<SearchSelect
|
||||
inputId={`${id}-format`}
|
||||
aria-label="Forecast response format"
|
||||
value={config.response_format ?? "json_schema"}
|
||||
allowClear={false}
|
||||
options={[
|
||||
{ value: "json_schema", label: "Strict JSON schema" },
|
||||
{ value: "json_object", label: "JSON object (for judges without strict schema support)" },
|
||||
]}
|
||||
onValueChange={(response_format) => {
|
||||
if (response_format === "json_schema" || response_format === "json_object")
|
||||
updateTransport({ response_format });
|
||||
}}
|
||||
/>
|
||||
</div>
|
||||
<div className="space-y-3 rounded-md border p-3">
|
||||
<Label>
|
||||
<Switch
|
||||
checked={Boolean(config.calibration)}
|
||||
onCheckedChange={(enabled) =>
|
||||
isCapability
|
||||
? updateCapability({
|
||||
...capability,
|
||||
calibration: enabled ? { version: "", ...emptyCoefficients() } : undefined,
|
||||
})
|
||||
: updateFuse({
|
||||
...fuse,
|
||||
calibration: enabled
|
||||
? {
|
||||
version: "",
|
||||
prompt_version: "llm-v2-1",
|
||||
efficient: emptyCoefficients(),
|
||||
capable: emptyCoefficients(),
|
||||
}
|
||||
: undefined,
|
||||
})
|
||||
}
|
||||
/>
|
||||
Use fitted calibration
|
||||
</Label>
|
||||
<p className="text-xs text-muted-foreground">
|
||||
Optional coefficients fitted for your judge, solvers, and harness. Leave off to use raw forecasts
|
||||
</p>
|
||||
{config.calibration && (
|
||||
<div className="space-y-1">
|
||||
<Label htmlFor={`${id}-version`}>Calibration version</Label>
|
||||
<Input
|
||||
id={`${id}-version`}
|
||||
value={config.calibration.version}
|
||||
maxLength={isCapability ? 128 : 512}
|
||||
onChange={(event) => setCalibrationVersion(event.target.value)}
|
||||
/>
|
||||
</div>
|
||||
)}
|
||||
{isCapability && capability.calibration && (
|
||||
<CalibrationFields
|
||||
label="Efficient"
|
||||
bounded
|
||||
value={capability.calibration}
|
||||
onChange={(next) =>
|
||||
updateCapability({
|
||||
...capability,
|
||||
calibration: { version: capability.calibration?.version ?? "", ...next },
|
||||
})
|
||||
}
|
||||
/>
|
||||
)}
|
||||
{!isCapability &&
|
||||
fuse.calibration &&
|
||||
(["efficient", "capable"] as const).map((role) => (
|
||||
<CalibrationFields
|
||||
key={role}
|
||||
label={role === "efficient" ? "Efficient" : "Capable"}
|
||||
value={fuse.calibration![role]}
|
||||
onChange={(next) => {
|
||||
if (fuse.calibration) updateFuse({ ...fuse, calibration: { ...fuse.calibration, [role]: next } });
|
||||
}}
|
||||
/>
|
||||
))}
|
||||
</div>
|
||||
</CollapsibleContent>
|
||||
</Collapsible>
|
||||
</>
|
||||
)}
|
||||
<p className="text-xs text-muted-foreground">
|
||||
The classifier uses its bundled prompt and always falls back to the capable solver
|
||||
</p>
|
||||
{error && (
|
||||
{section !== "advanced" && error && (
|
||||
<p role="alert" className="text-sm text-destructive">
|
||||
{error}
|
||||
</p>
|
||||
|
|
|
|||
|
|
@ -98,28 +98,28 @@ describe("JEV classifier editor", () => {
|
|||
afterEach(() => vi.mocked(useAuthorized).mockReset());
|
||||
it("uses built-in JEV without a license and preserves custom tiers and context through reload", () => {
|
||||
renderWithProviders(<Form />);
|
||||
expect(screen.getByLabelText("Classifier Model")).toBeInTheDocument();
|
||||
expect(screen.getByLabelText("Judge model")).toBeInTheDocument();
|
||||
expect(screen.getByText("Reasoning Effort")).toBeInTheDocument();
|
||||
expect(screen.getByText("Classifier Prompt")).toBeInTheDocument();
|
||||
expect(screen.getByRole("switch", { name: "Use images for classification" })).toBeInTheDocument();
|
||||
fireEvent.click(screen.getByRole("radio", { name: /JEV Classifier/ }));
|
||||
expect(screen.getByRole("tab", { name: "Complexity" })).toHaveAttribute("aria-selected", "true");
|
||||
expect(screen.getByLabelText("JEV Model")).toHaveValue("jev-latest");
|
||||
expect(screen.getByLabelText("JEV Instructions")).toBeDisabled();
|
||||
expect(screen.queryByLabelText("Classifier Model")).not.toBeInTheDocument();
|
||||
fireEvent.click(screen.getByRole("radio", { name: /Jev Classifier/ }));
|
||||
expect(screen.getByRole("radio", { name: /^Jev Classifier/ })).toBeChecked();
|
||||
expect(screen.getByLabelText("Jev Model")).toHaveValue("jev-latest");
|
||||
expect(screen.getByLabelText("Jev Instructions")).toBeEnabled();
|
||||
expect(screen.queryByLabelText("Judge model")).not.toBeInTheDocument();
|
||||
expect(screen.queryByText("Reasoning Effort")).not.toBeInTheDocument();
|
||||
expect(screen.queryByText("Classifier Prompt")).not.toBeInTheDocument();
|
||||
expect(screen.queryByRole("switch", { name: "Use images for classification" })).not.toBeInTheDocument();
|
||||
fireEvent.change(screen.getByLabelText("JEV Model"), { target: { value: "jev-test" } });
|
||||
fireEvent.change(screen.getByLabelText("JEV Timeout (ms)"), { target: { value: "4200" } });
|
||||
fireEvent.change(screen.getByLabelText("Jev Model"), { target: { value: "jev-test" } });
|
||||
fireEvent.change(screen.getByLabelText("Jev Timeout (ms)"), { target: { value: "4200" } });
|
||||
fireEvent.change(screen.getByLabelText("Context Window Size"), { target: { value: "6" } });
|
||||
fireEvent.change(screen.getByLabelText("Circuit breaker cooldown (seconds)"), { target: { value: "50" } });
|
||||
fireEvent.click(screen.getByRole("switch", { name: "Classifier circuit breaker" }));
|
||||
fireEvent.click(screen.getByRole("button", { name: "Customize tiers" }));
|
||||
fireEvent.click(screen.getByRole("button", { name: "Save and reload" }));
|
||||
expect(screen.getByRole("radio", { name: /JEV Classifier/ })).toBeChecked();
|
||||
expect(screen.getByLabelText("JEV Model")).toHaveValue("jev-test");
|
||||
expect(screen.getByLabelText("JEV Timeout (ms)")).toHaveValue(4200);
|
||||
expect(screen.getByRole("radio", { name: /Jev Classifier/ })).toBeChecked();
|
||||
expect(screen.getByLabelText("Jev Model")).toHaveValue("jev-test");
|
||||
expect(screen.getByLabelText("Jev Timeout (ms)")).toHaveValue(4200);
|
||||
expect(screen.getByLabelText("Context Window Size")).toHaveValue("6");
|
||||
expect(screen.getByRole("switch", { name: "Classifier circuit breaker" })).not.toBeChecked();
|
||||
fireEvent.click(screen.getByRole("button", { name: "Probe current config" }));
|
||||
|
|
@ -152,10 +152,10 @@ describe("JEV classifier editor", () => {
|
|||
return <JevEditor value={value} onChange={setValue} />;
|
||||
};
|
||||
renderWithProviders(<LicensedForm />);
|
||||
expect(screen.getByLabelText("JEV Instructions")).toBeEnabled();
|
||||
fireEvent.change(screen.getByLabelText("JEV Instructions"), { target: { value: "New instructions" } });
|
||||
expect(screen.getByLabelText("JEV Instructions")).toHaveValue("New instructions");
|
||||
fireEvent.click(screen.getByRole("button", { name: "Restore built-in JEV instructions" }));
|
||||
expect(screen.getByLabelText("JEV Instructions")).toHaveValue("");
|
||||
expect(screen.getByLabelText("Jev Instructions")).toBeEnabled();
|
||||
fireEvent.change(screen.getByLabelText("Jev Instructions"), { target: { value: "New instructions" } });
|
||||
expect(screen.getByLabelText("Jev Instructions")).toHaveValue("New instructions");
|
||||
fireEvent.click(screen.getByRole("button", { name: "Restore built-in Jev instructions" }));
|
||||
expect(screen.getByLabelText("Jev Instructions")).toHaveValue("");
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -1,10 +1,9 @@
|
|||
import React, { useId } from "react";
|
||||
import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized";
|
||||
import { AutoRouterAllowanceNote } from "./AutoRouterAvailability";
|
||||
import { Button } from "@/components/ui/button";
|
||||
import { Input } from "@/components/ui/input";
|
||||
import { Label } from "@/components/ui/label";
|
||||
import { Textarea } from "@/components/ui/textarea";
|
||||
import { SimpleTooltip } from "@/components/ui/tooltip";
|
||||
import ClassifierCircuitBreakerConfig from "./ClassifierCircuitBreakerConfig";
|
||||
import type { ComplexityRouterConfigValue } from "./ComplexityRouterConfig";
|
||||
import { defaultJevClassifierConfig } from "./jev_classifier_config";
|
||||
|
|
@ -17,7 +16,6 @@ export default function JevClassifierConfig({
|
|||
onChange: (value: ComplexityRouterConfigValue) => void;
|
||||
}) {
|
||||
const id = useId();
|
||||
const { premiumUser } = useAuthorized();
|
||||
const config = value.jev_classifier_config ?? defaultJevClassifierConfig();
|
||||
const update = (patch: Partial<typeof config>) =>
|
||||
onChange({ ...value, jev_classifier_config: { ...config, ...patch } });
|
||||
|
|
@ -28,11 +26,11 @@ export default function JevClassifierConfig({
|
|||
Uses TypeSafe System One Choice evaluation with your configured tiers
|
||||
</p>
|
||||
<div>
|
||||
<Label htmlFor={`${id}-model`}>JEV Model</Label>
|
||||
<Label htmlFor={`${id}-model`}>Jev Model</Label>
|
||||
<Input id={`${id}-model`} value={config.model} onChange={(event) => update({ model: event.target.value })} />
|
||||
</div>
|
||||
<div>
|
||||
<Label htmlFor={`${id}-timeout`}>JEV Timeout (ms)</Label>
|
||||
<Label htmlFor={`${id}-timeout`}>Jev Timeout (ms)</Label>
|
||||
<Input
|
||||
id={`${id}-timeout`}
|
||||
type="number"
|
||||
|
|
@ -52,35 +50,24 @@ export default function JevClassifierConfig({
|
|||
}
|
||||
/>
|
||||
<div>
|
||||
<Label htmlFor={`${id}-instructions`}>JEV Instructions</Label>
|
||||
<SimpleTooltip
|
||||
content={!premiumUser ? "Custom JEV instructions require a LiteLLM Enterprise license" : undefined}
|
||||
>
|
||||
<div>
|
||||
<Textarea
|
||||
id={`${id}-instructions`}
|
||||
value={config.instructions ?? ""}
|
||||
disabled={!premiumUser}
|
||||
placeholder="Leave blank to use the built-in instructions"
|
||||
onChange={(event) => update({ instructions: event.target.value || undefined })}
|
||||
/>
|
||||
</div>
|
||||
</SimpleTooltip>
|
||||
<Label htmlFor={`${id}-instructions`}>Jev Instructions</Label>
|
||||
<AutoRouterAllowanceNote
|
||||
feature="tier_or_classifier_prompt"
|
||||
label="Custom instructions share the custom-tier allowance"
|
||||
/>
|
||||
<Textarea
|
||||
id={`${id}-instructions`}
|
||||
value={config.instructions ?? ""}
|
||||
placeholder="Leave blank to use the built-in instructions"
|
||||
onChange={(event) => update({ instructions: event.target.value || undefined })}
|
||||
/>
|
||||
{config.instructions && (
|
||||
<Button variant="outline" type="button" onClick={() => update({ instructions: undefined })}>
|
||||
Restore built-in JEV instructions
|
||||
Restore built-in Jev instructions
|
||||
</Button>
|
||||
)}
|
||||
<p className="text-xs text-muted-foreground">
|
||||
Built-in JEV is available without a license and uses the shipped tier criteria
|
||||
{!premiumUser && (
|
||||
<>
|
||||
. Custom instructions require LiteLLM Enterprise. Get a trial key{" "}
|
||||
<a href="https://www.litellm.ai/#pricing" target="_blank" rel="noopener noreferrer" className="underline">
|
||||
here
|
||||
</a>
|
||||
</>
|
||||
)}
|
||||
Built-in Jev is available without a license and uses the shipped tier criteria
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
|
|
|
|||
|
|
@ -107,10 +107,10 @@ describe("JEV network probes", () => {
|
|||
expect(JSON.parse(String(routingCall?.[1]?.body))).toEqual(expectedRequest);
|
||||
expect(fetchMock).toHaveBeenCalledTimes(5);
|
||||
expect(screen.getAllByTestId("test-status-success")).toHaveLength(4);
|
||||
expect(screen.getByRole("status", { name: "JEV connection" })).toHaveTextContent(
|
||||
expect(screen.getByRole("status", { name: "Jev connection" })).toHaveTextContent(
|
||||
cause === "jev_classifier"
|
||||
? "JEV classification succeeded"
|
||||
: `JEV was not reached successfully (routing cause: ${cause})`,
|
||||
? "Jev classification succeeded"
|
||||
: `Jev was not reached successfully (routing cause: ${cause})`,
|
||||
);
|
||||
},
|
||||
);
|
||||
|
|
|
|||
|
|
@ -39,7 +39,7 @@ const NonReasoningTierToggle: React.FC<{
|
|||
<span className="block text-xs text-muted-foreground">
|
||||
Adds NON_REASONING below Simple, for operational agent traffic that relays or reformats information rather than
|
||||
reasoning about it. Escalation still moves up out of it when a request needs more.
|
||||
{!available && " Requires the LLM or JEV classification method"}
|
||||
{!available && " Requires the LLM or Jev classification method"}
|
||||
</span>
|
||||
<Separator className="my-4" />
|
||||
</>
|
||||
|
|
|
|||
|
|
@ -3,21 +3,28 @@ import { ChevronRight } from "lucide-react";
|
|||
import { Collapsible, CollapsibleContent, CollapsibleTrigger } from "@/components/ui/collapsible";
|
||||
|
||||
interface RoutingOptionsProps {
|
||||
forecast: boolean;
|
||||
showValidationErrors?: boolean;
|
||||
summary?: string;
|
||||
children: React.ReactNode;
|
||||
}
|
||||
|
||||
const RoutingOptions = ({ forecast, children }: RoutingOptionsProps) =>
|
||||
forecast ? (
|
||||
<Collapsible className="rounded-lg border">
|
||||
<CollapsibleTrigger className="group flex w-full items-center gap-2 px-4 py-3 text-left font-medium">
|
||||
const RoutingOptions = ({ showValidationErrors = false, summary, children }: RoutingOptionsProps) => {
|
||||
const [open, setOpen] = React.useState(false);
|
||||
const [previousValidation, setPreviousValidation] = React.useState(showValidationErrors);
|
||||
if (previousValidation !== showValidationErrors) {
|
||||
setPreviousValidation(showValidationErrors);
|
||||
if (showValidationErrors) setOpen(true);
|
||||
}
|
||||
return (
|
||||
<Collapsible open={open} onOpenChange={setOpen} className="rounded-lg border">
|
||||
<CollapsibleTrigger className="group flex w-full flex-wrap items-center gap-2 px-4 py-3 text-left font-medium">
|
||||
<ChevronRight className="size-4 transition-transform group-data-panel-open:rotate-90" />
|
||||
Advanced routing options
|
||||
Advanced settings
|
||||
{summary && <span className="text-xs font-normal text-muted-foreground">{summary}</span>}
|
||||
</CollapsibleTrigger>
|
||||
<CollapsibleContent className="space-y-4 px-4 pb-4">{children}</CollapsibleContent>
|
||||
<CollapsibleContent className="space-y-4">{children}</CollapsibleContent>
|
||||
</Collapsible>
|
||||
) : (
|
||||
<>{children}</>
|
||||
);
|
||||
};
|
||||
|
||||
export default RoutingOptions;
|
||||
|
|
|
|||
|
|
@ -1,33 +1,18 @@
|
|||
import React from "react";
|
||||
|
||||
import { type ComplexityRouterConfigValue, heuristicScoringRole, usesLlmClassifier } from "./ComplexityRouterConfig";
|
||||
import { type ComplexityRouterConfigValue, usesLlmClassifier } from "./ComplexityRouterConfig";
|
||||
import { restrictedBy } from "./TierRestrictions";
|
||||
|
||||
const tierConfigIntroText = (value: ComplexityRouterConfigValue): string => {
|
||||
if (value.classifier_type === "jev") {
|
||||
return "JEV classifies each request with TypeSafe System One Choice evaluation and routes it to a tier. Configure which models handle each tier";
|
||||
}
|
||||
if (value.classifier_type === "heuristic_v2") {
|
||||
return "The complexity router classifies each request with a calibrated local four-tier model (no API calls). Configure which model(s) handle each tier.";
|
||||
}
|
||||
if (heuristicScoringRole(value) === "never") {
|
||||
return "The complexity router classifies each request with your classifier model and routes it to that tier. Configure which model(s) handle each tier.";
|
||||
}
|
||||
return "The complexity router automatically classifies requests by complexity using rule-based scoring (no API calls, <1ms latency). Configure which model(s) handle each tier.";
|
||||
};
|
||||
|
||||
const TierConfigIntro: React.FC<{ value: ComplexityRouterConfigValue }> = ({ value }) => (
|
||||
<>
|
||||
<span className="block mb-6 text-muted-foreground">{tierConfigIntroText(value)}</span>
|
||||
|
||||
<span className="block mb-4 text-xs text-muted-foreground">
|
||||
{restrictedBy(value, "displayNames")?.reason ??
|
||||
"Rename a tier to use your own vocabulary in the dashboard and your spend logs. Renaming doesn't change how requests are classified, and callers never see these names."}
|
||||
<div className="mb-4 space-y-2 text-sm text-muted-foreground">
|
||||
<p>Choose one or more models for each tier. Requests use the models in their assigned tier</p>
|
||||
<p className="text-xs">
|
||||
{restrictedBy(value, "displayNames")?.reason ?? "Display names appear in the dashboard and spend logs"}
|
||||
{!value.custom_tier_set &&
|
||||
usesLlmClassifier(value.classifier_type) &&
|
||||
" Your classifier model reads these names, so clearer ones can sharpen its choices."}
|
||||
</span>
|
||||
</>
|
||||
". Your judge model also uses these names to classify requests"}
|
||||
</p>
|
||||
</div>
|
||||
);
|
||||
|
||||
export default TierConfigIntro;
|
||||
|
|
|
|||
|
|
@ -1,3 +1,8 @@
|
|||
import {
|
||||
openAutoRouterAdvanced,
|
||||
selectAutoRouterOption,
|
||||
selectAutoRouterApproach,
|
||||
} from "../../../tests/autoRouterSetup";
|
||||
import {
|
||||
renderWithProviders,
|
||||
screen,
|
||||
|
|
@ -5,6 +10,7 @@ import {
|
|||
within,
|
||||
fireEvent,
|
||||
testQueryClient,
|
||||
act,
|
||||
chooseSelectOption,
|
||||
} from "../../../tests/test-utils";
|
||||
import userEvent from "@testing-library/user-event";
|
||||
|
|
@ -15,7 +21,7 @@ import { handleAddAutoRouterSubmit } from "./handle_add_auto_router_submit";
|
|||
import { getMissingTiersError } from "./build_complexity_router_config";
|
||||
import { getSubmitBlockedReason } from "./add_auto_router_tab";
|
||||
import { buildModelAvailability } from "@/lib/autorouter_presets";
|
||||
import { modelCreateCall, testAutoRouterRouting } from "../networking";
|
||||
import { apiClient, modelCreateCall, testAutoRouterRouting } from "../networking";
|
||||
import { ModelGroup } from "@/components/llm_calls/fetch_models";
|
||||
import { AutoRouterPreset, getRequiredModelsInPreset } from "@/lib/autorouter_presets";
|
||||
import { BUNDLED_PRESETS, LOADED_PRESETS_QUERY, useAutoRouterPresets } from "../../../tests/mocks/autoRouterPresets";
|
||||
|
|
@ -47,10 +53,8 @@ const openTemplateDropdown = (): void => {
|
|||
fireEvent.click(screen.getByTestId("template-selector"));
|
||||
};
|
||||
|
||||
// Detailed Configuration is collapsed by default, so any test reaching into it (a tier select, an
|
||||
// "Advanced: ..." sub-section) has to open it first.
|
||||
const expandDetailedConfiguration = (): void => {
|
||||
fireEvent.click(screen.getByTestId("detailed-configuration-toggle"));
|
||||
expect(screen.getByText("Models by tier")).toBeVisible();
|
||||
};
|
||||
|
||||
const visibleOptions = (): HTMLElement[] => screen.queryAllByRole("option");
|
||||
|
|
@ -104,6 +108,12 @@ const { validateAutoRouterConfig } = vi.hoisted(() => ({
|
|||
}));
|
||||
|
||||
vi.mock("../networking", () => ({
|
||||
apiClient: {
|
||||
post: vi.fn().mockResolvedValue({
|
||||
allowances: [{ key: "heuristic_v2", limit: 1, remaining: 1, available: true }],
|
||||
error: null,
|
||||
}),
|
||||
},
|
||||
modelCreateCall: vi.fn().mockResolvedValue({}),
|
||||
modelAvailableCall: vi.fn().mockResolvedValue({ data: [] }),
|
||||
testAutoRouterRouting: vi.fn(),
|
||||
|
|
@ -159,6 +169,10 @@ const Harness = () => <AddAutoRouterTab handleOk={vi.fn()} accessToken="token" u
|
|||
describe("AddAutoRouterTab", () => {
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
vi.mocked(apiClient.post).mockResolvedValue({
|
||||
allowances: [{ key: "heuristic_v2", limit: 1, remaining: 1, available: true }],
|
||||
error: null,
|
||||
});
|
||||
// testQueryClient is a shared singleton with staleTime: Infinity, so cached model lists would
|
||||
// otherwise bleed across tests (a later test reusing accessToken="token" would read an earlier
|
||||
// test's data instead of its own mock).
|
||||
|
|
@ -167,6 +181,194 @@ describe("AddAutoRouterTab", () => {
|
|||
mockFetchAllModelDeployments.mockResolvedValue([]);
|
||||
});
|
||||
|
||||
it.each([1, 0])("defaults to Rule-based with %s v2 slots remaining", async (remaining) => {
|
||||
vi.mocked(apiClient.post).mockResolvedValueOnce({
|
||||
allowances: [{ key: "heuristic_v2", limit: 1, remaining, available: true }],
|
||||
error: null,
|
||||
});
|
||||
renderWithProviders(<Harness />);
|
||||
await waitFor(() => expect(screen.getByRole("button", { name: "Heuristic" })).toHaveTextContent("Rule-based"));
|
||||
});
|
||||
|
||||
it("does not show a tuning rejection for an incomplete Rule-based draft", async () => {
|
||||
vi.mocked(apiClient.post).mockImplementationOnce(async (_url, options) => ({
|
||||
allowances: [{ key: "heuristic_v2", limit: 1, remaining: 0, available: true }],
|
||||
error: (options?.body as { complexity_router_config?: unknown })?.complexity_router_config
|
||||
? "This change needs an available rule-based tuning allowance"
|
||||
: null,
|
||||
}));
|
||||
renderWithProviders(<Harness />);
|
||||
await waitFor(() => expect(screen.getByRole("button", { name: "Heuristic" })).toHaveTextContent("Rule-based"));
|
||||
expect(screen.queryByRole("alert")).not.toBeInTheDocument();
|
||||
expect(screen.getByRole("button", { name: "Add Auto Router" })).toBeDisabled();
|
||||
});
|
||||
|
||||
it("keeps the Rule-based default when the form reopens with a cached allowance", async () => {
|
||||
const first = renderWithProviders(<Harness />);
|
||||
await waitFor(() => expect(apiClient.post).toHaveBeenCalled());
|
||||
first.unmount();
|
||||
vi.mocked(apiClient.post).mockResolvedValueOnce({
|
||||
allowances: [{ key: "heuristic_v2", limit: 1, remaining: 0, available: true }],
|
||||
error: null,
|
||||
});
|
||||
renderWithProviders(<Harness />);
|
||||
await waitFor(() => expect(screen.getByRole("button", { name: "Heuristic" })).toHaveTextContent("Rule-based"));
|
||||
});
|
||||
|
||||
it.each(["exhausted", "available", "failed"])(
|
||||
"keeps an exhausted option disabled through a background refresh that becomes %s",
|
||||
async (result) => {
|
||||
vi.mocked(apiClient.post).mockResolvedValue({
|
||||
allowances: [{ key: "heuristic_v2", limit: 1, remaining: 0, available: true }],
|
||||
error: null,
|
||||
});
|
||||
renderWithProviders(<Harness />);
|
||||
fireEvent.click(screen.getByRole("button", { name: "Heuristic" }));
|
||||
const option = screen.getByRole("menuitemradio", { name: /^Heuristic v2/ });
|
||||
await waitFor(() => expect(option).toHaveTextContent("0 of 1 available"));
|
||||
expect(option).toHaveAttribute("aria-disabled", "true");
|
||||
let complete: (() => void) | undefined;
|
||||
vi.mocked(apiClient.post).mockImplementationOnce(
|
||||
() =>
|
||||
new Promise((resolve, reject) => {
|
||||
complete = () =>
|
||||
result === "failed"
|
||||
? reject(new Error("availability unavailable"))
|
||||
: resolve({
|
||||
allowances: [
|
||||
{ key: "heuristic_v2", limit: 1, remaining: result === "available" ? 1 : 0, available: true },
|
||||
],
|
||||
error: null,
|
||||
});
|
||||
}),
|
||||
);
|
||||
await act(async () => {
|
||||
void testQueryClient.refetchQueries({ queryKey: ["autoRouterAvailability"] });
|
||||
});
|
||||
await waitFor(() => expect(option).toHaveTextContent("Checking availability"));
|
||||
expect(option).toHaveAttribute("aria-disabled", "true");
|
||||
fireEvent.click(option);
|
||||
expect(screen.getByRole("button", { name: "Heuristic" })).toHaveTextContent("Rule-based");
|
||||
await act(async () => complete?.());
|
||||
await waitFor(() => expect(option).not.toHaveTextContent("Checking availability"));
|
||||
if (result === "available") {
|
||||
expect(option).not.toHaveAttribute("aria-disabled", "true");
|
||||
fireEvent.click(option);
|
||||
expect(screen.getByRole("button", { name: "Heuristic" })).toHaveTextContent("Heuristic v2");
|
||||
} else {
|
||||
expect(option).toHaveAttribute("aria-disabled", "true");
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
it("clears a customization rejection after restoring tiers and keeps the selected models", async () => {
|
||||
const user = userEvent.setup();
|
||||
const blocked = "Custom tiers or classifier instructions has no available allowance";
|
||||
vi.mocked(apiClient.post).mockImplementation(async (_url, options) => {
|
||||
const body = options?.body as { complexity_router_config?: { tier_definitions?: unknown } };
|
||||
return {
|
||||
allowances: [{ key: "tier_or_classifier_prompt", limit: 1, remaining: 0, available: true }],
|
||||
error: body?.complexity_router_config?.tier_definitions ? blocked : null,
|
||||
};
|
||||
});
|
||||
mockFetchAvailableModels.mockResolvedValue(ALL_FAMILY_MODELS);
|
||||
renderWithProviders(<Harness />);
|
||||
await user.click(await screen.findByRole("button", { name: "Choose models for me" }));
|
||||
await user.click(screen.getByRole("radio", { name: "Jev" }));
|
||||
await waitFor(() =>
|
||||
expect(apiClient.post).toHaveBeenLastCalledWith(
|
||||
"/auto_router/availability",
|
||||
expect.objectContaining({
|
||||
body: expect.objectContaining({
|
||||
complexity_router_config: expect.objectContaining({ classifier_type: "jev" }),
|
||||
}),
|
||||
}),
|
||||
),
|
||||
);
|
||||
const initialRequest = vi.mocked(apiClient.post).mock.calls.at(-1)?.[1]?.body as {
|
||||
complexity_router_config: { tiers: Record<string, string[]> };
|
||||
};
|
||||
fireEvent.change(screen.getByLabelText("Auto Router Name"), { target: { value: "restored-router" } });
|
||||
await user.click(screen.getByRole("button", { name: "Edit tiers" }));
|
||||
await user.click(screen.getByRole("button", { name: "Add tier" }));
|
||||
expect(screen.getByLabelText("Name for tier 5")).toBeInTheDocument();
|
||||
expect(screen.getByRole("button", { name: "Add Auto Router" })).toBeDisabled();
|
||||
await user.click(screen.getByRole("button", { name: "Restore defaults" }));
|
||||
expect(screen.queryByLabelText("Name for tier 5")).not.toBeInTheDocument();
|
||||
fireEvent.change(screen.getByLabelText("Definition for tier 1"), { target: { value: "Only short requests" } });
|
||||
expect(await screen.findByRole("alert")).toHaveTextContent(blocked);
|
||||
expect(screen.getByRole("button", { name: "Add Auto Router" })).toBeDisabled();
|
||||
expect(within(screen.getByRole("alert")).getByRole("link", { name: "Talk to our team" })).toBeVisible();
|
||||
await user.click(screen.getByRole("button", { name: "Restore defaults" }));
|
||||
await waitFor(() => expect(screen.queryByRole("alert")).not.toBeInTheDocument());
|
||||
expect(screen.getByRole("radio", { name: "Jev" })).toBeChecked();
|
||||
await waitFor(() => expect(screen.getByRole("button", { name: "Add Auto Router" })).toBeEnabled());
|
||||
await user.click(screen.getByRole("button", { name: "Add Auto Router" }));
|
||||
await waitFor(() => expect(handleAddAutoRouterSubmit).toHaveBeenCalled());
|
||||
const saved = vi.mocked(handleAddAutoRouterSubmit).mock.calls.at(-1)?.[0].complexity_router_config;
|
||||
expect(saved).not.toHaveProperty("tier_definitions");
|
||||
expect(saved?.classifier_type).toBe("jev");
|
||||
expect(Object.keys(saved?.tiers ?? {})).toEqual(["SIMPLE", "MEDIUM", "COMPLEX", "REASONING"]);
|
||||
expect(saved?.tiers).toEqual(initialRequest.complexity_router_config.tiers);
|
||||
});
|
||||
|
||||
it("blocks button and Enter submissions until the edited draft is checked", async () => {
|
||||
const user = userEvent.setup();
|
||||
mockFetchAvailableModels.mockResolvedValue(ALL_FAMILY_MODELS);
|
||||
renderWithProviders(<Harness />);
|
||||
await user.click(await screen.findByRole("button", { name: "Choose models for me" }));
|
||||
await user.click(screen.getByRole("radio", { name: "Jev" }));
|
||||
fireEvent.change(screen.getByLabelText("Auto Router Name"), { target: { value: "checked-router" } });
|
||||
await waitFor(() => expect(screen.getByRole("button", { name: "Add Auto Router" })).toBeEnabled());
|
||||
let complete: ((result: unknown) => void) | undefined;
|
||||
vi.mocked(apiClient.post).mockImplementationOnce(
|
||||
() =>
|
||||
new Promise((resolve) => {
|
||||
complete = resolve;
|
||||
}),
|
||||
);
|
||||
await user.click(screen.getByRole("button", { name: "Edit tiers" }));
|
||||
fireEvent.change(screen.getByLabelText("Definition for tier 1"), { target: { value: "Custom definition" } });
|
||||
expect(screen.getByRole("button", { name: "Add Auto Router" })).toBeDisabled();
|
||||
fireEvent.submit(screen.getByLabelText("Auto Router Name").closest("form")!);
|
||||
await waitFor(() => expect(complete).toBeDefined());
|
||||
expect(handleAddAutoRouterSubmit).not.toHaveBeenCalled();
|
||||
expect(screen.getByRole("button", { name: "Add Auto Router" })).toBeDisabled();
|
||||
await act(async () => complete?.({ allowances: [], error: "Custom tiers have no available allowance" }));
|
||||
expect(await screen.findByRole("alert")).toHaveTextContent("Custom tiers have no available allowance");
|
||||
expect(screen.getByRole("button", { name: "Add Auto Router" })).toBeDisabled();
|
||||
await user.click(screen.getByRole("button", { name: "Restore defaults" }));
|
||||
await waitFor(() => expect(screen.getByRole("button", { name: "Add Auto Router" })).toBeEnabled());
|
||||
await user.click(screen.getByRole("button", { name: "Add Auto Router" }));
|
||||
await waitFor(() => expect(handleAddAutoRouterSubmit).toHaveBeenCalledOnce());
|
||||
});
|
||||
|
||||
it("does not overwrite a classifier chosen while availability is loading", async () => {
|
||||
let complete: ((result: unknown) => void) | undefined;
|
||||
vi.mocked(apiClient.post).mockReturnValueOnce(
|
||||
new Promise((resolve) => {
|
||||
complete = resolve;
|
||||
}),
|
||||
);
|
||||
renderWithProviders(<Harness />);
|
||||
await userEvent.click(screen.getByRole("radio", { name: "LLM" }));
|
||||
complete?.({ allowances: [{ key: "heuristic_v2", limit: 1, remaining: 0, available: true }], error: null });
|
||||
await waitFor(() => expect(screen.getByRole("radio", { name: "LLM" })).toBeChecked());
|
||||
expect(screen.getByRole("button", { name: "Routing approach" })).toHaveTextContent("Complexity");
|
||||
});
|
||||
|
||||
it.each(["LLM", "Jev"])("keeps %s and the frequency when choosing models automatically", async (family) => {
|
||||
mockFetchAvailableModels.mockResolvedValue(ALL_FAMILY_MODELS);
|
||||
renderWithProviders(<Harness />);
|
||||
const automatic = await screen.findByRole("button", { name: "Choose models for me" });
|
||||
await userEvent.click(screen.getByRole("radio", { name: family }));
|
||||
await selectAutoRouterOption("How often to classify", "Every new user message");
|
||||
await userEvent.click(automatic);
|
||||
expect(screen.getByRole("radio", { name: family })).toBeChecked();
|
||||
expect(screen.getByRole("combobox", { name: "How often to classify" })).toHaveTextContent("Every new user message");
|
||||
expect(screen.getByRole("button", { name: "Advanced settings" })).toHaveAttribute("aria-expanded", "false");
|
||||
});
|
||||
|
||||
it.each(["Capability", "Fuse v2"])(
|
||||
"creates %s from its dedicated tab without complexity templates",
|
||||
async (label) => {
|
||||
|
|
@ -178,14 +380,14 @@ describe("AddAutoRouterTab", () => {
|
|||
]);
|
||||
renderWithProviders(<Harness />);
|
||||
await user.type(screen.getByLabelText("Auto Router Name"), "forecast-router");
|
||||
await user.click(screen.getByRole("tab", { name: label, exact: true }));
|
||||
await selectAutoRouterApproach(label);
|
||||
expect(screen.getByLabelText("Auto Router Name")).toHaveValue("forecast-router");
|
||||
expect(screen.queryByTestId("template-selector")).not.toBeInTheDocument();
|
||||
expect(screen.queryByTestId("configure-automatically-button")).not.toBeInTheDocument();
|
||||
expect(screen.queryByTestId("detailed-configuration-toggle")).not.toBeInTheDocument();
|
||||
expect(screen.queryByText("Complexity Tier Configuration")).not.toBeInTheDocument();
|
||||
expect(screen.queryByText("Advanced: Classification Method")).not.toBeInTheDocument();
|
||||
expect(screen.queryByText("Advanced: Adaptive Routing")).not.toBeInTheDocument();
|
||||
expect(screen.getByRole("button", { name: "Advanced settings" })).toHaveAttribute("aria-expanded", "false");
|
||||
expect(screen.queryByText("Classification Method")).not.toBeInTheDocument();
|
||||
expect(screen.queryByText("Adaptive Routing")).not.toBeInTheDocument();
|
||||
const capability = label === "Capability";
|
||||
for (const [role, model] of [
|
||||
["Efficient", "efficient"],
|
||||
|
|
@ -211,15 +413,21 @@ describe("AddAutoRouterTab", () => {
|
|||
fireEvent.change(screen.getByLabelText("Maximum quality gap"), { target: { value: "0.05" } });
|
||||
}
|
||||
expect(screen.queryByRole("alert")).not.toBeInTheDocument();
|
||||
expect(screen.getByRole("button", { name: "Add Auto Router" })).toBeEnabled();
|
||||
await user.click(screen.getByRole("button", { name: "Advanced routing options" }));
|
||||
await waitFor(() => expect(screen.getByRole("button", { name: "Add Auto Router" })).toBeEnabled());
|
||||
openAutoRouterAdvanced("Housekeeping Routing");
|
||||
openAutoRouterAdvanced("Affinity");
|
||||
openAutoRouterAdvanced("Response Format");
|
||||
for (const label of ["Adaptive Routing", "Context Window Escalation", "Escalation Keywords"]) {
|
||||
expect(screen.queryByText(`Advanced: ${label}`)).not.toBeInTheDocument();
|
||||
expect(screen.queryByText(`${label}`)).not.toBeInTheDocument();
|
||||
}
|
||||
expect(screen.getByText("Advanced: Stalled Task Escalation")).toBeInTheDocument();
|
||||
expect(screen.getByText("Advanced: Response Format")).toBeInTheDocument();
|
||||
expect(screen.queryByText("Advanced: Classification Method")).not.toBeInTheDocument();
|
||||
expect(screen.getByText("Advanced: Affinity")).toBeInTheDocument();
|
||||
openAutoRouterAdvanced("Stalled Task Escalation");
|
||||
expect(screen.getByText("Stalled Task Escalation")).toBeInTheDocument();
|
||||
openAutoRouterAdvanced("Response Format");
|
||||
expect(screen.getByText("Response Format")).toBeInTheDocument();
|
||||
expect(screen.queryByText("Classification Method")).not.toBeInTheDocument();
|
||||
openAutoRouterAdvanced("Affinity");
|
||||
expect(screen.getByText("Affinity")).toBeInTheDocument();
|
||||
await waitFor(() => expect(screen.getByRole("button", { name: "Add Auto Router" })).toBeEnabled());
|
||||
await user.click(screen.getByRole("button", { name: "Add Auto Router" }));
|
||||
await waitFor(() => expect(handleAddAutoRouterSubmit).toHaveBeenCalledTimes(1));
|
||||
const expected = {
|
||||
|
|
@ -239,20 +447,21 @@ describe("AddAutoRouterTab", () => {
|
|||
mockFetchAvailableModels.mockResolvedValue(ALL_FAMILY_MODELS);
|
||||
renderWithProviders(<Harness />);
|
||||
await screen.findByTestId("configure-automatically-button");
|
||||
await user.click(screen.getByRole("tab", { name: "Capability", exact: true }));
|
||||
await selectAutoRouterApproach("Capability");
|
||||
expect(screen.queryByTestId("configure-automatically-button")).not.toBeInTheDocument();
|
||||
await user.click(screen.getByRole("tab", { name: "Complexity", exact: true }));
|
||||
await selectAutoRouterApproach("Complexity");
|
||||
expect(screen.getByTestId("configure-automatically-button")).toBeInTheDocument();
|
||||
expect(screen.getByTestId("template-selector")).toBeInTheDocument();
|
||||
expandDetailedConfiguration();
|
||||
expect(screen.getByText("Complexity Tier Configuration")).toBeInTheDocument();
|
||||
expect(screen.getByText("Models by tier")).toBeInTheDocument();
|
||||
for (const label of ["Adaptive Routing", "Context Window Escalation", "Escalation Keywords"]) {
|
||||
expect(screen.getByText(`Advanced: ${label}`)).toBeInTheDocument();
|
||||
openAutoRouterAdvanced(label);
|
||||
expect(screen.getAllByText(label)).not.toHaveLength(0);
|
||||
}
|
||||
await user.click(screen.getByText("Advanced: Classification Method"));
|
||||
openAutoRouterAdvanced("Classification Method");
|
||||
expect(screen.queryByRole("radio", { name: /^Capability/ })).not.toBeInTheDocument();
|
||||
expect(screen.queryByRole("radio", { name: /^Fuse v2/ })).not.toBeInTheDocument();
|
||||
expect(screen.getByRole("radio", { name: /^Heuristic \(default/ })).toBeChecked();
|
||||
expect(screen.getByRole("button", { name: "Routing approach" })).toHaveTextContent("Complexity");
|
||||
});
|
||||
|
||||
it.each(["Capability", "Fuse v2"])(
|
||||
|
|
@ -265,7 +474,7 @@ describe("AddAutoRouterTab", () => {
|
|||
{ model_group: "judge", mode: "chat" },
|
||||
]);
|
||||
renderWithProviders(<Harness />);
|
||||
await user.click(screen.getByRole("tab", { name: label, exact: true }));
|
||||
await selectAutoRouterApproach(label);
|
||||
await user.type(screen.getByLabelText("Auto Router Name"), "forecast-retry");
|
||||
const capability = label === "Capability";
|
||||
const policyField = capability ? "Solve probability threshold" : "Efficient solver profile";
|
||||
|
|
@ -276,7 +485,7 @@ describe("AddAutoRouterTab", () => {
|
|||
expect(screen.queryByTestId("template-selector")).not.toBeInTheDocument();
|
||||
await user.click(screen.getByRole("button", { name: "Retry", exact: true }));
|
||||
await waitFor(() => expect(screen.queryByText("Could not load available models.")).not.toBeInTheDocument());
|
||||
expect(screen.getByRole("tab", { name: label, exact: true })).toHaveAttribute("aria-selected", "true");
|
||||
expect(screen.getByRole("button", { name: "Routing approach" })).toHaveTextContent(label);
|
||||
expect(screen.getByLabelText("Auto Router Name")).toHaveValue("forecast-retry");
|
||||
expect(screen.getByLabelText(policyField)).toHaveValue(capability ? 0.7 : "Small solver");
|
||||
await user.click(screen.getByRole("combobox", { name: "Judge model" }));
|
||||
|
|
@ -287,14 +496,14 @@ describe("AddAutoRouterTab", () => {
|
|||
|
||||
// Detailed Configuration starts collapsed so the modal opens onto just Name + Template; a caller
|
||||
// opts into the full tier/classifier form rather than always seeing it up front.
|
||||
it("keeps Detailed Configuration collapsed until a caller opens it", () => {
|
||||
it("shows models immediately and keeps advanced settings collapsed", async () => {
|
||||
renderWithProviders(<Harness />);
|
||||
|
||||
expect(screen.queryByText("Complexity Tier Configuration")).not.toBeInTheDocument();
|
||||
expect(screen.getByRole("button", { name: "Advanced settings" })).toHaveAttribute("aria-expanded", "false");
|
||||
|
||||
fireEvent.click(screen.getByTestId("detailed-configuration-toggle"));
|
||||
expect(screen.getByText("Models by tier")).toBeVisible();
|
||||
|
||||
expect(screen.getByText("Complexity Tier Configuration")).toBeInTheDocument();
|
||||
expect(screen.getByText("Models by tier")).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("hides automatic setup when no available model is recommended", async () => {
|
||||
|
|
@ -325,8 +534,8 @@ describe("AddAutoRouterTab", () => {
|
|||
|
||||
expectTierModel("Simple", "gpt-5.6-luna");
|
||||
expectTierModel("Medium", "claude-sonnet-5");
|
||||
expectTierModel("Complex", "claude-opus-5");
|
||||
expectTierModel("Reasoning", "claude-opus-5");
|
||||
expectTierModel("Complex", "claude-opus-5-5");
|
||||
expectTierModel("Reasoning", "claude-opus-5-5");
|
||||
expect(toast.success).not.toHaveBeenCalledWith(expect.stringContaining("Configured with"));
|
||||
});
|
||||
|
||||
|
|
@ -354,11 +563,11 @@ describe("AddAutoRouterTab", () => {
|
|||
mockFetchAvailableModels.mockResolvedValue([...ALL_FAMILY_MODELS, { model_group: simpleModel, mode: "chat" }]);
|
||||
renderWithProviders(<Harness />);
|
||||
|
||||
expect(screen.queryByText("Complexity Tier Configuration")).not.toBeInTheDocument();
|
||||
expect(screen.getByRole("button", { name: "Advanced settings" })).toHaveAttribute("aria-expanded", "false");
|
||||
|
||||
await userEvent.click(await screen.findByTestId("configure-automatically-button"));
|
||||
|
||||
expect(screen.getByText("Complexity Tier Configuration")).toBeInTheDocument();
|
||||
expect(screen.getByText("Models by tier")).toBeInTheDocument();
|
||||
expectTierModel("Simple", simpleModel);
|
||||
});
|
||||
|
||||
|
|
@ -375,13 +584,14 @@ describe("AddAutoRouterTab", () => {
|
|||
vi.mocked(getMissingTiersError).mockReturnValue(null);
|
||||
renderWithProviders(<Harness />);
|
||||
|
||||
await waitFor(() => expect(screen.getByRole("button", { name: /add auto router/i })).toBeEnabled());
|
||||
await user.click(screen.getByRole("button", { name: /add auto router/i }));
|
||||
|
||||
expect(await screen.findByText("Auto router name is required")).toBeInTheDocument();
|
||||
expect(toast.fromError).toHaveBeenCalledWith("Please enter an Auto Router Name");
|
||||
});
|
||||
|
||||
it("offers no team selector to a proxy admin, who may create an unscoped router", () => {
|
||||
it("offers no team selector to a proxy admin, who may create an unscoped router", async () => {
|
||||
renderWithProviders(<Harness />);
|
||||
|
||||
expect(screen.queryByTestId("team-dropdown")).not.toBeInTheDocument();
|
||||
|
|
@ -409,6 +619,7 @@ describe("AddAutoRouterTab", () => {
|
|||
|
||||
await user.type(screen.getByPlaceholderText(/smart_router/i), "team-scoped-router");
|
||||
await user.selectOptions(screen.getByTestId("team-dropdown"), "team-1");
|
||||
await waitFor(() => expect(screen.getByRole("button", { name: /add auto router/i })).toBeEnabled());
|
||||
await user.click(screen.getByRole("button", { name: /add auto router/i }));
|
||||
|
||||
await waitFor(() => expect(handleAddAutoRouterSubmit).toHaveBeenCalled());
|
||||
|
|
@ -454,8 +665,9 @@ describe("AddAutoRouterTab", () => {
|
|||
expect(screen.getByPlaceholderText(/smart_router/i)).toHaveValue("my-router");
|
||||
await user.selectOptions(screen.getByTestId("team-dropdown"), "team-1");
|
||||
expandDetailedConfiguration();
|
||||
expect(screen.queryByText("Advanced: Compression")).not.toBeInTheDocument();
|
||||
expect(screen.queryByText("Compression")).not.toBeInTheDocument();
|
||||
expect(screen.queryByText("Model Access Groups")).not.toBeInTheDocument();
|
||||
await waitFor(() => expect(screen.getByRole("button", { name: /add auto router/i })).toBeEnabled());
|
||||
await user.click(screen.getByRole("button", { name: /add auto router/i }));
|
||||
|
||||
await waitFor(() => expect(modelCreateCall).toHaveBeenCalled());
|
||||
|
|
@ -481,6 +693,7 @@ describe("AddAutoRouterTab", () => {
|
|||
|
||||
renderWithProviders(<Harness />);
|
||||
await user.type(screen.getByPlaceholderText(/smart_router/i), "rejected-router");
|
||||
await waitFor(() => expect(screen.getByRole("button", { name: /add auto router/i })).toBeEnabled());
|
||||
await user.click(screen.getByRole("button", { name: /add auto router/i }));
|
||||
|
||||
await waitFor(() => expect(validateAutoRouterConfig).toHaveBeenCalled());
|
||||
|
|
@ -500,8 +713,10 @@ describe("AddAutoRouterTab", () => {
|
|||
const { container } = renderWithProviders(<Harness />);
|
||||
fireEvent.change(screen.getByPlaceholderText(/smart_router/i), { target: { value: "double-submit-router" } });
|
||||
|
||||
await waitFor(() => expect(screen.getByRole("button", { name: /add auto router/i })).toBeEnabled());
|
||||
fireEvent.submit(container.querySelector("form")!);
|
||||
await waitFor(() => expect(screen.getByRole("button", { name: /add auto router/i })).toBeDisabled());
|
||||
await waitFor(() => expect(validateAutoRouterConfig).toHaveBeenCalledOnce());
|
||||
expect(screen.getByRole("button", { name: /add auto router/i })).toBeDisabled();
|
||||
fireEvent.submit(container.querySelector("form")!);
|
||||
|
||||
resolveVerdict({ valid: true });
|
||||
|
|
@ -517,6 +732,7 @@ describe("AddAutoRouterTab", () => {
|
|||
|
||||
renderWithProviders(<Harness />);
|
||||
await user.type(screen.getByPlaceholderText(/smart_router/i), "accepted-router");
|
||||
await waitFor(() => expect(screen.getByRole("button", { name: /add auto router/i })).toBeEnabled());
|
||||
await user.click(screen.getByRole("button", { name: /add auto router/i }));
|
||||
|
||||
await waitFor(() => expect(handleAddAutoRouterSubmit).toHaveBeenCalled());
|
||||
|
|
@ -533,7 +749,7 @@ describe("AddAutoRouterTab", () => {
|
|||
|
||||
await user.type(screen.getByPlaceholderText(/smart_router/i), "keyword-router");
|
||||
expandDetailedConfiguration();
|
||||
await user.click(screen.getByText("Advanced: Keyword/Semantic Matching"));
|
||||
openAutoRouterAdvanced("Keyword/Semantic Matching");
|
||||
await user.click(screen.getByRole("button", { name: /add keyword rule/i }));
|
||||
|
||||
expect(screen.getByRole("button", { name: /add auto router/i })).toBeDisabled();
|
||||
|
|
@ -550,13 +766,13 @@ describe("AddAutoRouterTab", () => {
|
|||
|
||||
await user.type(screen.getByPlaceholderText(/smart_router/i), "keyword-router");
|
||||
expandDetailedConfiguration();
|
||||
await user.click(screen.getByText("Advanced: Keyword/Semantic Matching"));
|
||||
openAutoRouterAdvanced("Keyword/Semantic Matching");
|
||||
await user.click(screen.getByRole("button", { name: /add keyword rule/i }));
|
||||
expect(screen.getByRole("button", { name: /add auto router/i })).toBeDisabled();
|
||||
|
||||
await addKeyword(user, screen.getByText("Keywords 1").closest("div") as HTMLElement, "invoice");
|
||||
|
||||
expect(screen.getByRole("button", { name: /add auto router/i })).toBeEnabled();
|
||||
await waitFor(() => expect(screen.getByRole("button", { name: /add auto router/i })).toBeEnabled());
|
||||
expect(screen.queryByText("At least one keyword is required")).not.toBeInTheDocument();
|
||||
});
|
||||
|
||||
|
|
@ -568,7 +784,7 @@ describe("AddAutoRouterTab", () => {
|
|||
|
||||
await user.type(screen.getByPlaceholderText(/smart_router/i), "orphan-rule-router");
|
||||
expandDetailedConfiguration();
|
||||
await user.click(screen.getByText("Advanced: Keyword/Semantic Matching"));
|
||||
openAutoRouterAdvanced("Keyword/Semantic Matching");
|
||||
await user.click(screen.getByRole("button", { name: /add keyword rule/i }));
|
||||
await addKeyword(user, screen.getByText("Keywords 1").closest("div") as HTMLElement, "invoice");
|
||||
|
||||
|
|
@ -587,7 +803,7 @@ describe("AddAutoRouterTab", () => {
|
|||
|
||||
await user.type(screen.getByPlaceholderText(/smart_router/i), "keyword-router");
|
||||
expandDetailedConfiguration();
|
||||
await user.click(screen.getByText("Advanced: Keyword/Semantic Matching"));
|
||||
openAutoRouterAdvanced("Keyword/Semantic Matching");
|
||||
await user.click(screen.getByRole("button", { name: /add keyword rule/i }));
|
||||
await addKeyword(user, screen.getByText("Keywords 1").closest("div") as HTMLElement, "invoice");
|
||||
await user.click(screen.getByRole("button", { name: /add keyword rule/i }));
|
||||
|
|
@ -604,10 +820,11 @@ describe("AddAutoRouterTab", () => {
|
|||
|
||||
await user.type(screen.getByPlaceholderText(/smart_router/i), "keyword-router");
|
||||
expandDetailedConfiguration();
|
||||
await user.click(screen.getByText("Advanced: Keyword/Semantic Matching"));
|
||||
openAutoRouterAdvanced("Keyword/Semantic Matching");
|
||||
await user.click(screen.getByRole("button", { name: /add keyword rule/i }));
|
||||
const keywordsField = screen.getByText("Keywords 1").closest("div") as HTMLElement;
|
||||
await addKeyword(user, keywordsField, "invoice");
|
||||
await waitFor(() => expect(screen.getByRole("button", { name: /add auto router/i })).toBeEnabled());
|
||||
await user.click(screen.getByRole("button", { name: /add auto router/i }));
|
||||
|
||||
await waitFor(() => expect(handleAddAutoRouterSubmit).toHaveBeenCalled());
|
||||
|
|
@ -625,6 +842,7 @@ describe("AddAutoRouterTab", () => {
|
|||
);
|
||||
|
||||
await user.type(screen.getByPlaceholderText(/smart_router/i), "team-scoped-router");
|
||||
await waitFor(() => expect(screen.getByRole("button", { name: /add auto router/i })).toBeEnabled());
|
||||
await user.click(screen.getByRole("button", { name: /add auto router/i }));
|
||||
|
||||
expect(await screen.findByText("Please select a team to continue")).toBeInTheDocument();
|
||||
|
|
@ -644,6 +862,7 @@ describe("AddAutoRouterTab", () => {
|
|||
await user.type(screen.getByPlaceholderText(/smart_router/i), "team-scoped-router");
|
||||
await user.selectOptions(screen.getByTestId("team-dropdown"), "team-1");
|
||||
await user.click(screen.getByTestId("team-dropdown-clear"));
|
||||
await waitFor(() => expect(screen.getByRole("button", { name: /add auto router/i })).toBeEnabled());
|
||||
await user.click(screen.getByRole("button", { name: /add auto router/i }));
|
||||
|
||||
expect(await screen.findByText("Please select a team to continue")).toBeInTheDocument();
|
||||
|
|
@ -658,9 +877,12 @@ describe("AddAutoRouterTab", () => {
|
|||
|
||||
await user.type(screen.getByPlaceholderText(/smart_router/i), "affinity-router");
|
||||
expandDetailedConfiguration();
|
||||
await user.click(screen.getByText("Advanced: Classification Method"));
|
||||
expect(await screen.findByRole("radio", { name: /Once per session/ })).not.toBeChecked();
|
||||
openAutoRouterAdvanced("Classification Method");
|
||||
expect(await screen.findByRole("combobox", { name: "How often to classify" })).not.toHaveTextContent(
|
||||
"Once per session",
|
||||
);
|
||||
|
||||
await waitFor(() => expect(screen.getByRole("button", { name: /add auto router/i })).toBeEnabled());
|
||||
await user.click(screen.getByRole("button", { name: /add auto router/i }));
|
||||
|
||||
await waitFor(() => expect(handleAddAutoRouterSubmit).toHaveBeenCalled());
|
||||
|
|
@ -675,8 +897,8 @@ describe("AddAutoRouterTab", () => {
|
|||
renderWithProviders(<Harness />);
|
||||
fireEvent.change(screen.getByLabelText("Auto Router Name"), { target: { value: "threshold-router" } });
|
||||
expandDetailedConfiguration();
|
||||
await user.click(screen.getByText("Advanced: Classification Method"));
|
||||
await user.click(screen.getByRole("radio", { name: /^Heuristic v2/ }));
|
||||
openAutoRouterAdvanced("Classification Method");
|
||||
await selectAutoRouterOption("Heuristic", "Heuristic v2");
|
||||
|
||||
const threshold = screen.getByRole("textbox", { name: "Success threshold" });
|
||||
expect(threshold).toHaveValue("");
|
||||
|
|
@ -687,12 +909,13 @@ describe("AddAutoRouterTab", () => {
|
|||
expect(screen.getByRole("button", { name: "Add Auto Router" })).toBeDisabled();
|
||||
expect(screen.getByTestId("auto-router-test-routing-btn")).toBeDisabled();
|
||||
|
||||
await user.click(screen.getByRole("radio", { name: /^Heuristic \(default\)/ }));
|
||||
await selectAutoRouterOption("Heuristic", "Rule-based");
|
||||
expect(screen.getByRole("button", { name: "Add Auto Router" })).toBeDisabled();
|
||||
await user.click(screen.getByRole("radio", { name: /^Heuristic v2/ }));
|
||||
await selectAutoRouterOption("Heuristic", "Heuristic v2");
|
||||
fireEvent.change(screen.getByRole("textbox", { name: "Success threshold" }), { target: { value: "1.01" } });
|
||||
expect(screen.getByRole("button", { name: "Add Auto Router" })).toBeDisabled();
|
||||
fireEvent.change(screen.getByRole("textbox", { name: "Success threshold" }), { target: { value: "0" } });
|
||||
await waitFor(() => expect(screen.getByRole("button", { name: "Add Auto Router" })).toBeEnabled());
|
||||
await user.click(screen.getByRole("button", { name: "Add Auto Router" }));
|
||||
|
||||
await waitFor(() => expect(handleAddAutoRouterSubmit).toHaveBeenCalledOnce());
|
||||
|
|
@ -702,21 +925,24 @@ describe("AddAutoRouterTab", () => {
|
|||
});
|
||||
});
|
||||
|
||||
it("clears an invalid threshold draft when automatic setup replaces the configuration", async () => {
|
||||
it("preserves classifier tuning when choosing models automatically", async () => {
|
||||
const user = userEvent.setup();
|
||||
mockFetchAvailableModels.mockResolvedValue(ALL_FAMILY_MODELS);
|
||||
renderWithProviders(<Harness />);
|
||||
const automaticSetup = await screen.findByRole("button", { name: "Configure automatically" });
|
||||
const automaticSetup = await screen.findByRole("button", { name: "Choose models for me" });
|
||||
await waitFor(() => expect(automaticSetup).toBeEnabled());
|
||||
await user.click(automaticSetup);
|
||||
fireEvent.change(screen.getByLabelText("Auto Router Name"), { target: { value: "reset-threshold-router" } });
|
||||
await user.click(screen.getByText("Advanced: Classification Method"));
|
||||
openAutoRouterAdvanced("Classification Method");
|
||||
await selectAutoRouterOption("Heuristic", "Heuristic v2");
|
||||
fireEvent.change(screen.getByRole("textbox", { name: "Success threshold" }), { target: { value: "1.1" } });
|
||||
expect(screen.getByRole("button", { name: "Add Auto Router" })).toBeDisabled();
|
||||
|
||||
await user.click(automaticSetup);
|
||||
expect(screen.getByRole("textbox", { name: "Success threshold" })).toHaveValue("");
|
||||
expect(screen.getByRole("textbox", { name: "Success threshold" })).toHaveAttribute("aria-invalid", "false");
|
||||
expect(screen.getByRole("textbox", { name: "Success threshold" })).toHaveValue("1.1");
|
||||
expect(screen.getByRole("button", { name: "Add Auto Router" })).toBeDisabled();
|
||||
fireEvent.change(screen.getByRole("textbox", { name: "Success threshold" }), { target: { value: "" } });
|
||||
await waitFor(() => expect(screen.getByRole("button", { name: "Add Auto Router" })).toBeEnabled());
|
||||
await user.click(screen.getByRole("button", { name: "Add Auto Router" }));
|
||||
await waitFor(() => expect(handleAddAutoRouterSubmit).toHaveBeenCalledOnce());
|
||||
expect(vi.mocked(handleAddAutoRouterSubmit).mock.calls.at(-1)?.[0].complexity_router_config).not.toHaveProperty(
|
||||
|
|
@ -730,13 +956,14 @@ describe("AddAutoRouterTab", () => {
|
|||
renderWithProviders(<Harness />);
|
||||
fireEvent.change(screen.getByLabelText("Auto Router Name"), { target: { value: "clear-threshold-router" } });
|
||||
expandDetailedConfiguration();
|
||||
await user.click(screen.getByText("Advanced: Classification Method"));
|
||||
await user.click(screen.getByRole("radio", { name: /^Heuristic v2/ }));
|
||||
openAutoRouterAdvanced("Classification Method");
|
||||
await selectAutoRouterOption("Heuristic", "Heuristic v2");
|
||||
fireEvent.change(screen.getByRole("textbox", { name: "Success threshold" }), { target: { value: "invalid" } });
|
||||
await user.click(screen.getByRole("radio", { name: /^Heuristic \(default\)/ }));
|
||||
await selectAutoRouterOption("Heuristic", "Rule-based");
|
||||
expect(screen.getByRole("button", { name: "Add Auto Router" })).toBeDisabled();
|
||||
await user.click(screen.getByRole("button", { name: "Clear Heuristic v2 threshold" }));
|
||||
expect(screen.queryByRole("region", { name: "Inactive Heuristic v2 threshold" })).not.toBeInTheDocument();
|
||||
await waitFor(() => expect(screen.getByRole("button", { name: "Add Auto Router" })).toBeEnabled());
|
||||
await user.click(screen.getByRole("button", { name: "Add Auto Router" }));
|
||||
await waitFor(() => expect(handleAddAutoRouterSubmit).toHaveBeenCalledOnce());
|
||||
expect(vi.mocked(handleAddAutoRouterSubmit).mock.calls.at(-1)?.[0].complexity_router_config).not.toHaveProperty(
|
||||
|
|
@ -752,12 +979,13 @@ describe("AddAutoRouterTab", () => {
|
|||
|
||||
await user.type(screen.getByPlaceholderText(/smart_router/i), "ctx-window-router");
|
||||
expandDetailedConfiguration();
|
||||
await user.click(screen.getByText("Advanced: Context Window Escalation"));
|
||||
openAutoRouterAdvanced("Context Window Escalation");
|
||||
const toggle = await screen.findByRole("switch", { name: "Escalate oversized prompts to a tier that fits" });
|
||||
expect(toggle).not.toBeChecked();
|
||||
expect(screen.queryByLabelText("Window fit buffer")).not.toBeInTheDocument();
|
||||
await user.click(toggle);
|
||||
|
||||
await waitFor(() => expect(screen.getByRole("button", { name: /add auto router/i })).toBeEnabled());
|
||||
await user.click(screen.getByRole("button", { name: /add auto router/i }));
|
||||
|
||||
await waitFor(() => expect(handleAddAutoRouterSubmit).toHaveBeenCalled());
|
||||
|
|
@ -774,12 +1002,13 @@ describe("AddAutoRouterTab", () => {
|
|||
|
||||
await user.type(screen.getByPlaceholderText(/smart_router/i), "ctx-buffer-router");
|
||||
expandDetailedConfiguration();
|
||||
await user.click(screen.getByText("Advanced: Context Window Escalation"));
|
||||
openAutoRouterAdvanced("Context Window Escalation");
|
||||
await user.click(screen.getByRole("switch", { name: "Escalate oversized prompts to a tier that fits" }));
|
||||
const buffer = await screen.findByLabelText("Window fit buffer");
|
||||
fireEvent.change(buffer, { target: { value: "1.5" } });
|
||||
fireEvent.blur(buffer, { target: { value: "1.5" } });
|
||||
|
||||
await waitFor(() => expect(screen.getByRole("button", { name: /add auto router/i })).toBeEnabled());
|
||||
await user.click(screen.getByRole("button", { name: /add auto router/i }));
|
||||
|
||||
await waitFor(() => expect(handleAddAutoRouterSubmit).toHaveBeenCalled());
|
||||
|
|
@ -796,7 +1025,7 @@ describe("AddAutoRouterTab", () => {
|
|||
|
||||
await user.type(screen.getByPlaceholderText(/smart_router/i), "ctx-clear-router");
|
||||
expandDetailedConfiguration();
|
||||
await user.click(screen.getByText("Advanced: Context Window Escalation"));
|
||||
openAutoRouterAdvanced("Context Window Escalation");
|
||||
await user.click(screen.getByRole("switch", { name: "Escalate oversized prompts to a tier that fits" }));
|
||||
const buffer = await screen.findByLabelText("Window fit buffer");
|
||||
fireEvent.change(buffer, { target: { value: "0.8" } });
|
||||
|
|
@ -804,6 +1033,7 @@ describe("AddAutoRouterTab", () => {
|
|||
fireEvent.change(buffer, { target: { value: "" } });
|
||||
fireEvent.blur(buffer, { target: { value: "" } });
|
||||
|
||||
await waitFor(() => expect(screen.getByRole("button", { name: /add auto router/i })).toBeEnabled());
|
||||
await user.click(screen.getByRole("button", { name: /add auto router/i }));
|
||||
|
||||
await waitFor(() => expect(handleAddAutoRouterSubmit).toHaveBeenCalled());
|
||||
|
|
@ -820,6 +1050,7 @@ describe("AddAutoRouterTab", () => {
|
|||
renderWithProviders(<Harness />);
|
||||
|
||||
await user.type(screen.getByPlaceholderText(/smart_router/i), "no-compression-router");
|
||||
await waitFor(() => expect(screen.getByRole("button", { name: /add auto router/i })).toBeEnabled());
|
||||
await user.click(screen.getByRole("button", { name: /add auto router/i }));
|
||||
|
||||
await waitFor(() => expect(handleAddAutoRouterSubmit).toHaveBeenCalled());
|
||||
|
|
@ -836,13 +1067,14 @@ describe("AddAutoRouterTab", () => {
|
|||
|
||||
await user.type(screen.getByPlaceholderText(/smart_router/i), "no-compression-explicit-router");
|
||||
expandDetailedConfiguration();
|
||||
await user.click(screen.getByText("Advanced: Compression"));
|
||||
openAutoRouterAdvanced("Compression");
|
||||
await chooseSelectOption(
|
||||
user,
|
||||
screen.getByRole("combobox", { name: "Routing decision compression" }),
|
||||
"None (no compression)",
|
||||
);
|
||||
|
||||
await waitFor(() => expect(screen.getByRole("button", { name: /add auto router/i })).toBeEnabled());
|
||||
await user.click(screen.getByRole("button", { name: /add auto router/i }));
|
||||
|
||||
await waitFor(() => expect(handleAddAutoRouterSubmit).toHaveBeenCalled());
|
||||
|
|
@ -859,7 +1091,7 @@ describe("AddAutoRouterTab", () => {
|
|||
|
||||
await user.type(screen.getByPlaceholderText(/smart_router/i), "different-compression-router");
|
||||
expandDetailedConfiguration();
|
||||
await user.click(screen.getByText("Advanced: Compression"));
|
||||
openAutoRouterAdvanced("Compression");
|
||||
await chooseSelectOption(
|
||||
user,
|
||||
screen.getByRole("combobox", { name: "Routing decision compression" }),
|
||||
|
|
@ -868,6 +1100,7 @@ describe("AddAutoRouterTab", () => {
|
|||
await user.click(screen.getByText("Use a different compression"));
|
||||
expect(screen.getByRole("combobox", { name: "Model call compression" })).toBeInTheDocument();
|
||||
|
||||
await waitFor(() => expect(screen.getByRole("button", { name: /add auto router/i })).toBeEnabled());
|
||||
await user.click(screen.getByRole("button", { name: /add auto router/i }));
|
||||
|
||||
await waitFor(() => expect(handleAddAutoRouterSubmit).toHaveBeenCalled());
|
||||
|
|
@ -887,10 +1120,12 @@ describe("AddAutoRouterTab", () => {
|
|||
|
||||
await user.type(screen.getByPlaceholderText(/smart_router/i), "override-floor-router");
|
||||
expandDetailedConfiguration();
|
||||
await user.click(screen.getByText("Advanced: Classification Method"));
|
||||
openAutoRouterAdvanced("Classification Method");
|
||||
await selectAutoRouterOption("Heuristic", "Rule-based");
|
||||
await user.click(await screen.findByText("Advanced scoring"));
|
||||
fireEvent.change(await screen.findByLabelText("Minimum score"), { target: { value: "0" } });
|
||||
|
||||
await waitFor(() => expect(screen.getByRole("button", { name: /add auto router/i })).toBeEnabled());
|
||||
await user.click(screen.getByRole("button", { name: /add auto router/i }));
|
||||
|
||||
await waitFor(() => expect(handleAddAutoRouterSubmit).toHaveBeenCalled());
|
||||
|
|
@ -907,13 +1142,14 @@ describe("AddAutoRouterTab", () => {
|
|||
|
||||
await user.type(screen.getByPlaceholderText(/smart_router/i), "affinity-router");
|
||||
expandDetailedConfiguration();
|
||||
await user.click(screen.getByText("Advanced: Classification Method"));
|
||||
await user.click(await screen.findByRole("radio", { name: /Once per session/ }));
|
||||
await user.click(screen.getByText("Advanced: Affinity"));
|
||||
openAutoRouterAdvanced("Classification Method");
|
||||
await selectAutoRouterOption("How often to classify", "Once per session");
|
||||
openAutoRouterAdvanced("Affinity");
|
||||
const ttl = await screen.findByLabelText("How long a pin survives idle (seconds)");
|
||||
fireEvent.change(ttl, { target: { value: "300" } });
|
||||
fireEvent.blur(ttl);
|
||||
|
||||
await waitFor(() => expect(screen.getByRole("button", { name: /add auto router/i })).toBeEnabled());
|
||||
await user.click(screen.getByRole("button", { name: /add auto router/i }));
|
||||
|
||||
await waitFor(() => expect(handleAddAutoRouterSubmit).toHaveBeenCalled());
|
||||
|
|
@ -931,9 +1167,10 @@ describe("AddAutoRouterTab", () => {
|
|||
|
||||
await user.type(screen.getByPlaceholderText(/smart_router/i), "user-turn-router");
|
||||
expandDetailedConfiguration();
|
||||
await user.click(screen.getByText("Advanced: Classification Method"));
|
||||
await user.click(await screen.findByRole("radio", { name: /Every new user message/ }));
|
||||
openAutoRouterAdvanced("Classification Method");
|
||||
await selectAutoRouterOption("How often to classify", "Every new user message");
|
||||
|
||||
await waitFor(() => expect(screen.getByRole("button", { name: /add auto router/i })).toBeEnabled());
|
||||
await user.click(screen.getByRole("button", { name: /add auto router/i }));
|
||||
|
||||
await waitFor(() => expect(handleAddAutoRouterSubmit).toHaveBeenCalled());
|
||||
|
|
@ -950,9 +1187,10 @@ describe("AddAutoRouterTab", () => {
|
|||
|
||||
await user.type(screen.getByPlaceholderText(/smart_router/i), "default-timing-router");
|
||||
expandDetailedConfiguration();
|
||||
await user.click(screen.getByText("Advanced: Classification Method"));
|
||||
expect(await screen.findByRole("radio", { name: /Every request/ })).toBeChecked();
|
||||
openAutoRouterAdvanced("Classification Method");
|
||||
expect(await screen.findByRole("combobox", { name: "How often to classify" })).toHaveTextContent("Every request");
|
||||
|
||||
await waitFor(() => expect(screen.getByRole("button", { name: /add auto router/i })).toBeEnabled());
|
||||
await user.click(screen.getByRole("button", { name: /add auto router/i }));
|
||||
|
||||
await waitFor(() => expect(handleAddAutoRouterSubmit).toHaveBeenCalled());
|
||||
|
|
@ -969,9 +1207,10 @@ describe("AddAutoRouterTab", () => {
|
|||
|
||||
await user.type(screen.getByPlaceholderText(/smart_router/i), "affinity-router");
|
||||
expandDetailedConfiguration();
|
||||
await user.click(screen.getByText("Advanced: Affinity"));
|
||||
openAutoRouterAdvanced("Affinity");
|
||||
expect(await screen.findByRole("switch", { name: "Pin one model deployment per tier" })).toBeChecked();
|
||||
|
||||
await waitFor(() => expect(screen.getByRole("button", { name: /add auto router/i })).toBeEnabled());
|
||||
await user.click(screen.getByRole("button", { name: /add auto router/i }));
|
||||
|
||||
await waitFor(() => expect(handleAddAutoRouterSubmit).toHaveBeenCalled());
|
||||
|
|
@ -988,9 +1227,10 @@ describe("AddAutoRouterTab", () => {
|
|||
|
||||
await user.type(screen.getByPlaceholderText(/smart_router/i), "affinity-router");
|
||||
expandDetailedConfiguration();
|
||||
await user.click(screen.getByText("Advanced: Affinity"));
|
||||
openAutoRouterAdvanced("Affinity");
|
||||
await user.click(await screen.findByRole("switch", { name: "Pin one model deployment per tier" }));
|
||||
|
||||
await waitFor(() => expect(screen.getByRole("button", { name: /add auto router/i })).toBeEnabled());
|
||||
await user.click(screen.getByRole("button", { name: /add auto router/i }));
|
||||
|
||||
await waitFor(() => expect(handleAddAutoRouterSubmit).toHaveBeenCalled());
|
||||
|
|
@ -1007,6 +1247,7 @@ describe("AddAutoRouterTab", () => {
|
|||
|
||||
fireEvent.change(screen.getByPlaceholderText(/smart_router/i), { target: { value: "modality-router" } });
|
||||
|
||||
await waitFor(() => expect(screen.getByRole("button", { name: /add auto router/i })).toBeEnabled());
|
||||
await user.click(screen.getByRole("button", { name: /add auto router/i }));
|
||||
|
||||
await waitFor(() => expect(handleAddAutoRouterSubmit).toHaveBeenCalled());
|
||||
|
|
@ -1024,10 +1265,11 @@ describe("AddAutoRouterTab", () => {
|
|||
|
||||
fireEvent.change(screen.getByPlaceholderText(/smart_router/i), { target: { value: "modality-router" } });
|
||||
expandDetailedConfiguration();
|
||||
await user.click(screen.getByText("Advanced: Modality Routing"));
|
||||
openAutoRouterAdvanced("Modality Routing");
|
||||
await user.click(await screen.findByRole("switch", { name: "Route image requests to vision-capable models" }));
|
||||
await user.click(await screen.findByRole("switch", { name: "Override session pin for image requests" }));
|
||||
|
||||
await waitFor(() => expect(screen.getByRole("button", { name: /add auto router/i })).toBeEnabled());
|
||||
await user.click(screen.getByRole("button", { name: /add auto router/i }));
|
||||
|
||||
await waitFor(() => expect(handleAddAutoRouterSubmit).toHaveBeenCalled());
|
||||
|
|
@ -1039,7 +1281,7 @@ describe("AddAutoRouterTab", () => {
|
|||
|
||||
// Custom is the escape hatch, not the headline choice, so it's listed after every bundled preset
|
||||
// rather than first.
|
||||
it("lists Custom Configuration after the bundled presets", () => {
|
||||
it("lists Custom Configuration after the bundled presets", async () => {
|
||||
renderWithProviders(<Harness />);
|
||||
openTemplateDropdown();
|
||||
|
||||
|
|
@ -1082,7 +1324,7 @@ describe("AddAutoRouterTab", () => {
|
|||
renderWithProviders(<Harness />);
|
||||
await user.type(screen.getByPlaceholderText(/smart_router/i), "keyword-router");
|
||||
expandDetailedConfiguration();
|
||||
await user.click(screen.getByText("Advanced: Keyword/Semantic Matching"));
|
||||
openAutoRouterAdvanced("Keyword/Semantic Matching");
|
||||
await user.click(screen.getByRole("button", { name: /add keyword rule/i }));
|
||||
const keywordsField = screen.getByText("Keywords 1").closest("div") as HTMLElement;
|
||||
await addKeyword(user, keywordsField, "invoice");
|
||||
|
|
@ -1199,20 +1441,14 @@ describe("AddAutoRouterTab", () => {
|
|||
await waitForPresetEnabled("OpenAI Family");
|
||||
});
|
||||
|
||||
it("collapses detailed configuration and shows a tier summary once a preset is applied", async () => {
|
||||
it("shows the template's model choices while advanced settings stay collapsed", async () => {
|
||||
mockFetchAvailableModels.mockResolvedValue(ALL_FAMILY_MODELS);
|
||||
renderWithProviders(<Harness />);
|
||||
await waitForPresetEnabled("Anthropic Family");
|
||||
|
||||
await selectTemplate("Anthropic Family");
|
||||
|
||||
expect(screen.queryByText("Advanced: Keyword/Semantic Matching")).not.toBeInTheDocument();
|
||||
expect(
|
||||
screen.getByText(
|
||||
`Simple: ${ANTHROPIC_TIERS.SIMPLE.join(", ")} · Medium: ${ANTHROPIC_TIERS.MEDIUM.join(", ")} · ` +
|
||||
`Complex: ${ANTHROPIC_TIERS.COMPLEX.join(", ")} · Reasoning: ${ANTHROPIC_TIERS.REASONING.join(", ")}`,
|
||||
),
|
||||
).toBeInTheDocument();
|
||||
expect(screen.getByRole("button", { name: "Advanced settings" })).toHaveAttribute("aria-expanded", "false");
|
||||
expectTierModel("simple", ANTHROPIC_TIERS.SIMPLE[0]);
|
||||
expectTierModel("reasoning", ANTHROPIC_TIERS.REASONING[0]);
|
||||
});
|
||||
|
||||
it("expands detailed configuration when Custom Configuration is chosen", async () => {
|
||||
|
|
@ -1221,7 +1457,9 @@ describe("AddAutoRouterTab", () => {
|
|||
|
||||
await selectTemplate("Custom Configuration");
|
||||
|
||||
expect(screen.getByText("Advanced: Keyword/Semantic Matching")).toBeInTheDocument();
|
||||
openAutoRouterAdvanced("Keyword/Semantic Matching");
|
||||
|
||||
expect(screen.getByText("Keyword/Semantic Matching")).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("lets a caller manually re-expand a detailed configuration a preset just collapsed", async () => {
|
||||
|
|
@ -1229,11 +1467,13 @@ describe("AddAutoRouterTab", () => {
|
|||
renderWithProviders(<Harness />);
|
||||
await waitForPresetEnabled("Anthropic Family");
|
||||
await selectTemplate("Anthropic Family");
|
||||
expect(screen.queryByText("Advanced: Keyword/Semantic Matching")).not.toBeInTheDocument();
|
||||
expect(screen.queryByText("Keyword/Semantic Matching")).not.toBeInTheDocument();
|
||||
|
||||
fireEvent.click(screen.getByTestId("detailed-configuration-toggle"));
|
||||
expect(screen.getByText("Models by tier")).toBeVisible();
|
||||
|
||||
expect(screen.getByText("Advanced: Keyword/Semantic Matching")).toBeInTheDocument();
|
||||
openAutoRouterAdvanced("Keyword/Semantic Matching");
|
||||
|
||||
expect(screen.getByText("Keyword/Semantic Matching")).toBeInTheDocument();
|
||||
});
|
||||
|
||||
// This is the regression test for the whole feature: if handlePresetChange stopped prefilling
|
||||
|
|
@ -1248,6 +1488,7 @@ describe("AddAutoRouterTab", () => {
|
|||
await selectTemplate("Anthropic Family");
|
||||
|
||||
await user.type(screen.getByPlaceholderText(/smart_router/i), "anthropic-router");
|
||||
await waitFor(() => expect(screen.getByRole("button", { name: /add auto router/i })).toBeEnabled());
|
||||
await user.click(screen.getByRole("button", { name: /add auto router/i }));
|
||||
|
||||
await waitFor(() => expect(handleAddAutoRouterSubmit).toHaveBeenCalled());
|
||||
|
|
@ -1269,7 +1510,7 @@ describe("AddAutoRouterTab", () => {
|
|||
await waitForPresetEnabled("Anthropic Family");
|
||||
await selectTemplate("Anthropic Family");
|
||||
fireEvent.change(screen.getByPlaceholderText(/smart_router/i), { target: { value: "stale-model-router" } });
|
||||
expect(screen.getByRole("button", { name: /add auto router/i })).toBeEnabled();
|
||||
await waitFor(() => expect(screen.getByRole("button", { name: /add auto router/i })).toBeEnabled());
|
||||
|
||||
// The model list changed after the tiers were filled in (e.g. a deployment removed
|
||||
// elsewhere) - update the query cache directly rather than a real refetch, since that's the
|
||||
|
|
@ -1294,6 +1535,7 @@ describe("AddAutoRouterTab", () => {
|
|||
await selectTemplate("Anthropic Family");
|
||||
|
||||
await user.type(screen.getByPlaceholderText(/smart_router/i), "anthropic-router");
|
||||
await waitFor(() => expect(screen.getByRole("button", { name: /add auto router/i })).toBeEnabled());
|
||||
await user.click(screen.getByRole("button", { name: /add auto router/i }));
|
||||
|
||||
await waitFor(() => expect(handleAddAutoRouterSubmit).toHaveBeenCalled());
|
||||
|
|
@ -1330,6 +1572,7 @@ describe("AddAutoRouterTab", () => {
|
|||
|
||||
await applyPresetAndPin(user);
|
||||
await user.type(screen.getByPlaceholderText(/smart_router/i), "pinned-router");
|
||||
await waitFor(() => expect(screen.getByRole("button", { name: /add auto router/i })).toBeEnabled());
|
||||
await user.click(screen.getByRole("button", { name: /add auto router/i }));
|
||||
|
||||
await waitFor(() => expect(handleAddAutoRouterSubmit).toHaveBeenCalled());
|
||||
|
|
@ -1349,7 +1592,7 @@ describe("AddAutoRouterTab", () => {
|
|||
|
||||
await applyPresetAndPin(user);
|
||||
fireEvent.change(screen.getByPlaceholderText(/smart_router/i), { target: { value: "stale-pin-router" } });
|
||||
expect(screen.getByRole("button", { name: /add auto router/i })).toBeEnabled();
|
||||
await waitFor(() => expect(screen.getByRole("button", { name: /add auto router/i })).toBeEnabled());
|
||||
|
||||
// Only the pinned model disappears - the tier models all survive, so nothing but the pin can
|
||||
// be what blocks the submit.
|
||||
|
|
@ -1375,6 +1618,7 @@ describe("AddAutoRouterTab", () => {
|
|||
await waitForPresetEnabled("Anthropic Family");
|
||||
await selectTemplate("Anthropic Family");
|
||||
await user.type(screen.getByPlaceholderText(/smart_router/i), "no-plan-router");
|
||||
await waitFor(() => expect(screen.getByRole("button", { name: /add auto router/i })).toBeEnabled());
|
||||
await user.click(screen.getByRole("button", { name: /add auto router/i }));
|
||||
|
||||
await waitFor(() => expect(handleAddAutoRouterSubmit).toHaveBeenCalled());
|
||||
|
|
@ -1390,10 +1634,11 @@ describe("AddAutoRouterTab", () => {
|
|||
await waitForPresetEnabled("Anthropic Family");
|
||||
await selectTemplate("Anthropic Family");
|
||||
expandDetailedConfiguration();
|
||||
await user.click(screen.getByText("Advanced: Plan-Mode Override"));
|
||||
openAutoRouterAdvanced("Plan-Mode Override");
|
||||
await user.click(await screen.findByRole("switch", { name: "Route plan-mode requests to a minimum tier" }));
|
||||
|
||||
await user.type(screen.getByPlaceholderText(/smart_router/i), "plan-router");
|
||||
await waitFor(() => expect(screen.getByRole("button", { name: /add auto router/i })).toBeEnabled());
|
||||
await user.click(screen.getByRole("button", { name: /add auto router/i }));
|
||||
|
||||
await waitFor(() => expect(handleAddAutoRouterSubmit).toHaveBeenCalled());
|
||||
|
|
@ -1444,9 +1689,12 @@ describe("AddAutoRouterTab", () => {
|
|||
});
|
||||
await selectTemplate("Anthropic Family");
|
||||
|
||||
expect(screen.getByText("Advanced: Keyword/Semantic Matching")).toBeInTheDocument();
|
||||
openAutoRouterAdvanced("Keyword/Semantic Matching");
|
||||
|
||||
expect(screen.getByText("Keyword/Semantic Matching")).toBeInTheDocument();
|
||||
|
||||
await user.type(screen.getByPlaceholderText(/smart_router/i), "renamed-router");
|
||||
await waitFor(() => expect(screen.getByRole("button", { name: /add auto router/i })).toBeEnabled());
|
||||
await user.click(screen.getByRole("button", { name: /add auto router/i }));
|
||||
|
||||
await waitFor(() => expect(handleAddAutoRouterSubmit).toHaveBeenCalled());
|
||||
|
|
@ -1541,6 +1789,7 @@ describe("AddAutoRouterTab", () => {
|
|||
await selectTemplate("Anthropic Family");
|
||||
|
||||
await user.type(screen.getByPlaceholderText(/smart_router/i), "wildcard-router");
|
||||
await waitFor(() => expect(screen.getByRole("button", { name: /add auto router/i })).toBeEnabled());
|
||||
await user.click(screen.getByRole("button", { name: /add auto router/i }));
|
||||
|
||||
await waitFor(() => expect(handleAddAutoRouterSubmit).toHaveBeenCalled());
|
||||
|
|
@ -1575,17 +1824,17 @@ describe("getSubmitBlockedReason", () => {
|
|||
defaultModel: undefined,
|
||||
};
|
||||
|
||||
it("lets a complete heuristic router through", () => {
|
||||
it("lets a complete heuristic router through", async () => {
|
||||
expect(getSubmitBlockedReason({ tiers, classifier_type: "heuristic" }, [], referenced, availability)).toBeNull();
|
||||
});
|
||||
|
||||
it("blocks an LLM classifier with no model, which the button previously left enabled", () => {
|
||||
it("blocks an LLM classifier with no model, which the button previously left enabled", async () => {
|
||||
expect(getSubmitBlockedReason({ tiers, classifier_type: "llm" }, [], referenced, availability)).toContain(
|
||||
"Please select a classifier model",
|
||||
);
|
||||
});
|
||||
|
||||
it("blocks an edited tier set with no classifier model, since the set forces the LLM classifier", () => {
|
||||
it("blocks an edited tier set with no classifier model, since the set forces the LLM classifier", async () => {
|
||||
const config = {
|
||||
tiers,
|
||||
classifier_type: "heuristic" as const,
|
||||
|
|
@ -1602,7 +1851,7 @@ describe("getSubmitBlockedReason", () => {
|
|||
);
|
||||
});
|
||||
|
||||
it("blocks a keyword rule aimed at a tier this router does not have", () => {
|
||||
it("blocks a keyword rule aimed at a tier this router does not have", async () => {
|
||||
const rules = [{ id: "r1", keywords: ["audit"], tier: "AUDIT" }];
|
||||
expect(getSubmitBlockedReason({ tiers, classifier_type: "heuristic" }, rules, referenced, availability)).toContain(
|
||||
"no longer has",
|
||||
|
|
@ -1638,6 +1887,7 @@ describe("preset catalog fetch states", () => {
|
|||
await waitForPresetEnabled("Bounded JEV");
|
||||
await selectTemplate("Bounded JEV");
|
||||
fireEvent.change(screen.getByLabelText("Auto Router Name"), { target: { value: "bounded-router" } });
|
||||
await waitFor(() => expect(screen.getByRole("button", { name: "Add Auto Router" })).toBeEnabled());
|
||||
fireEvent.click(screen.getByRole("button", { name: "Add Auto Router" }));
|
||||
|
||||
await waitFor(() => expect(handleAddAutoRouterSubmit).toHaveBeenCalledOnce());
|
||||
|
|
@ -1647,7 +1897,7 @@ describe("preset catalog fetch states", () => {
|
|||
});
|
||||
});
|
||||
|
||||
it("keeps showing cached presets without the error banner when only a refetch fails", () => {
|
||||
it("keeps showing cached presets without the error banner when only a refetch fails", async () => {
|
||||
vi.mocked(useAutoRouterPresets).mockReturnValue({
|
||||
...LOADED_PRESETS_QUERY,
|
||||
isError: true,
|
||||
|
|
@ -1660,7 +1910,7 @@ describe("preset catalog fetch states", () => {
|
|||
expect(screen.queryAllByRole("option").length).toBeGreaterThan(1);
|
||||
});
|
||||
|
||||
it("shows a loading hint while the catalog fetch is pending", () => {
|
||||
it("shows a loading hint while the catalog fetch is pending", async () => {
|
||||
vi.mocked(useAutoRouterPresets).mockReturnValue({
|
||||
...LOADED_PRESETS_QUERY,
|
||||
data: undefined,
|
||||
|
|
@ -1,9 +1,9 @@
|
|||
import { AutoRouterAvailabilityContext, useAutoRouterAvailability } from "./AutoRouterAvailability";
|
||||
import AutoRouterClassifierTabs from "./AutoRouterClassifierTabs";
|
||||
import { getForecastConfigError, isForecastClassifier } from "../add_model/forecast_classifier_config";
|
||||
import React, { useEffect, useState } from "react";
|
||||
import { useQuery } from "@tanstack/react-query";
|
||||
import { useWatch } from "react-hook-form";
|
||||
import { ChevronDown, ChevronRight } from "lucide-react";
|
||||
import { z } from "zod/v4";
|
||||
import { FieldGroup } from "@/components/ui/field";
|
||||
import { FormField } from "@/components/shared/form/FormField";
|
||||
|
|
@ -233,7 +233,6 @@ const AddAutoRouterTab: React.FC<AddAutoRouterTabProps> = ({
|
|||
const [isSubmitting, setIsSubmitting] = useState(false);
|
||||
|
||||
const [selectedPreset, setSelectedPreset] = useState<string | undefined>(undefined);
|
||||
const [detailsExpanded, setDetailsExpanded] = useState<boolean>(false);
|
||||
|
||||
const [isRoutingTestVisible, setIsRoutingTestVisible] = useState<boolean>(false);
|
||||
const [isTestModalVisible, setIsTestModalVisible] = useState<boolean>(false);
|
||||
|
|
@ -357,16 +356,20 @@ const AddAutoRouterTab: React.FC<AddAutoRouterTabProps> = ({
|
|||
const handleAutomaticSetup = () => {
|
||||
if (automaticRouterConfig === null) return;
|
||||
setSelectedPreset(undefined);
|
||||
applyPrefill({ ...buildEmptyPrefill(), complexityRouterConfig: automaticRouterConfig });
|
||||
setDetailsExpanded(true);
|
||||
toast.success("Automatic setup created", { description: tierConfigSummary(automaticRouterConfig) });
|
||||
setComplexityRouterConfig({
|
||||
...complexityRouterConfig,
|
||||
tiers: { ...complexityRouterConfig.tiers, ...automaticRouterConfig.tiers },
|
||||
tier_model_params: { ...complexityRouterConfig.tier_model_params, ...automaticRouterConfig.tier_model_params },
|
||||
});
|
||||
|
||||
toast.success("Models selected", { description: tierConfigSummary(automaticRouterConfig) });
|
||||
};
|
||||
|
||||
const handlePresetChange = (presetKey: string | undefined) => {
|
||||
if (!presetKey || presetKey === "custom") {
|
||||
setSelectedPreset(presetKey);
|
||||
applyPrefill(buildEmptyPrefill());
|
||||
setDetailsExpanded(true);
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
|
|
@ -379,7 +382,6 @@ const AddAutoRouterTab: React.FC<AddAutoRouterTabProps> = ({
|
|||
|
||||
setSelectedPreset(presetKey);
|
||||
applyPrefill(buildPresetPrefill(preset.complexity_router_config, availability));
|
||||
setDetailsExpanded(presetState.viaDeployments);
|
||||
};
|
||||
|
||||
const referencedModelsParams = {
|
||||
|
|
@ -408,6 +410,20 @@ const AddAutoRouterTab: React.FC<AddAutoRouterTabProps> = ({
|
|||
matchThreshold,
|
||||
escalationKeywords,
|
||||
};
|
||||
const routerAvailability = useAutoRouterAvailability(
|
||||
accessToken,
|
||||
{
|
||||
team_id: requiresTeamScope ? watchedTeamId : undefined,
|
||||
complexity_router_config:
|
||||
submitBlockedReason === null ? { ...buildComplexityRouterConfig(complexityRouterConfigParams) } : null,
|
||||
},
|
||||
!requiresTeamScope || Boolean(watchedTeamId),
|
||||
);
|
||||
const saveBlockedReason = submitBlockedReason ?? routerAvailability.saveBlockedReason;
|
||||
const handleClassifierChange = (config: ComplexityRouterConfigValue) => {
|
||||
setSelectedPreset(undefined);
|
||||
setComplexityRouterConfig(config);
|
||||
};
|
||||
const jevRequestParams =
|
||||
effectiveClassifierType(complexityRouterConfig) === "jev"
|
||||
? {
|
||||
|
|
@ -425,13 +441,15 @@ const AddAutoRouterTab: React.FC<AddAutoRouterTabProps> = ({
|
|||
// disagree about why. The handler needs it in its own right: the form fires this on Enter
|
||||
// regardless of the button's disabled state.
|
||||
const blockedReason =
|
||||
saveBlockedReason ??
|
||||
getSubmitBlockedReason(
|
||||
complexityRouterConfig,
|
||||
keywordTierRules,
|
||||
referencedModelsParams,
|
||||
groupsOnlyAvailability,
|
||||
modelInfo,
|
||||
) ?? getSemanticConfigError({ semanticMatchingEnabled, embeddingModel, keywordTierRules });
|
||||
) ??
|
||||
getSemanticConfigError({ semanticMatchingEnabled, embeddingModel, keywordTierRules });
|
||||
if (blockedReason) {
|
||||
setShowValidationErrors(true);
|
||||
toast.fromError(blockedReason);
|
||||
|
|
@ -556,281 +574,261 @@ const AddAutoRouterTab: React.FC<AddAutoRouterTabProps> = ({
|
|||
const forecast = isForecastClassifier(complexityRouterConfig.classifier_type);
|
||||
|
||||
return (
|
||||
<TooltipProvider>
|
||||
<Card>
|
||||
<CardContent>
|
||||
<form onSubmit={form.handleSubmit(() => handleAutoRouterSubmit())} noValidate>
|
||||
<div className="mb-6">
|
||||
<FormField
|
||||
control={form.control}
|
||||
name="auto_router_name"
|
||||
label={labelWithHint("Auto Router Name", "Unique name for this auto router configuration")}
|
||||
>
|
||||
{({ ref, ...field }) => <Input {...field} ref={ref} placeholder="e.g., smart_router, auto_router_1" />}
|
||||
</FormField>
|
||||
</div>
|
||||
<AutoRouterClassifierTabs
|
||||
value={complexityRouterConfig}
|
||||
onChange={(config) => {
|
||||
setSelectedPreset(undefined);
|
||||
setComplexityRouterConfig(config);
|
||||
}}
|
||||
>
|
||||
<FieldGroup>
|
||||
<div>
|
||||
{!forecast && (
|
||||
<>
|
||||
{!automaticSetupLoading && automaticRouterConfig && (
|
||||
<div className="mt-5 flex flex-wrap items-center justify-between gap-3 rounded-lg border border-border bg-muted px-4 py-3">
|
||||
<div>
|
||||
<p className="text-sm font-medium text-foreground">Not sure where to start?</p>
|
||||
<p className="text-sm text-muted-foreground">
|
||||
Let us pick models for each complexity tier.
|
||||
</p>
|
||||
</div>
|
||||
<Button
|
||||
type="button"
|
||||
data-testid="configure-automatically-button"
|
||||
onClick={handleAutomaticSetup}
|
||||
>
|
||||
Configure automatically
|
||||
</Button>
|
||||
</div>
|
||||
)}
|
||||
|
||||
<div className="mt-5">
|
||||
<label className="block text-sm font-medium text-foreground mb-2">Template</label>
|
||||
<Select
|
||||
items={templateItems}
|
||||
value={selectedPreset ?? null}
|
||||
onValueChange={(presetKey: string | null) => handlePresetChange(presetKey ?? undefined)}
|
||||
>
|
||||
<SelectTrigger data-testid="template-selector" className="w-full">
|
||||
<SelectValue placeholder="Choose a template or select Custom to define your own" />
|
||||
</SelectTrigger>
|
||||
<SelectContent>
|
||||
{sortedPresetOptions.map(({ preset, availability: presetState }) => {
|
||||
const disabledHint = presetDisabledHint(presetState);
|
||||
const hintClass = isPresetHintAlarming(presetState)
|
||||
? "text-destructive"
|
||||
: "text-muted-foreground";
|
||||
const matchedHint =
|
||||
presetState.kind === "available" && presetState.viaDeployments
|
||||
? "Matches your deployments"
|
||||
: null;
|
||||
|
||||
return (
|
||||
<SelectItem
|
||||
key={preset.key}
|
||||
value={preset.key}
|
||||
label={preset.label}
|
||||
disabled={disabledHint !== null}
|
||||
title={disabledHint ?? preset.description}
|
||||
>
|
||||
<div>
|
||||
<div className="font-medium">{preset.label}</div>
|
||||
<div className="text-xs text-muted-foreground">{preset.description}</div>
|
||||
{disabledHint && <div className={`text-xs mt-1 ${hintClass}`}>{disabledHint}</div>}
|
||||
{matchedHint && <div className="text-xs mt-1 text-success">{matchedHint}</div>}
|
||||
</div>
|
||||
</SelectItem>
|
||||
);
|
||||
})}
|
||||
<SelectItem value="custom" label="Custom Configuration">
|
||||
<div>
|
||||
<div className="font-medium">Custom Configuration</div>
|
||||
<div className="text-xs text-muted-foreground">
|
||||
Define your auto router from scratch
|
||||
</div>
|
||||
</div>
|
||||
</SelectItem>
|
||||
</SelectContent>
|
||||
</Select>
|
||||
{presetsPending && (
|
||||
<div className="text-xs mt-1 text-muted-foreground">Loading templates...</div>
|
||||
)}
|
||||
{presetsUnavailable && (
|
||||
<div className="text-xs mt-1 text-destructive">
|
||||
Could not load templates, so only Custom Configuration is shown.{" "}
|
||||
<button type="button" className="underline" onClick={() => void refetchPresets()}>
|
||||
Retry
|
||||
</button>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</>
|
||||
<AutoRouterAvailabilityContext.Provider value={routerAvailability}>
|
||||
<TooltipProvider>
|
||||
<Card>
|
||||
<CardContent>
|
||||
<form onSubmit={form.handleSubmit(() => handleAutoRouterSubmit())} noValidate>
|
||||
<div className="mb-6">
|
||||
<FormField
|
||||
control={form.control}
|
||||
name="auto_router_name"
|
||||
label={labelWithHint("Auto Router Name", "Unique name for this auto router configuration")}
|
||||
>
|
||||
{({ ref, ...field }) => (
|
||||
<Input {...field} ref={ref} placeholder="e.g., smart_router, auto_router_1" />
|
||||
)}
|
||||
{modelsUnverifiable && (
|
||||
<div className="text-xs mt-1 text-destructive">
|
||||
Could not load available models.{" "}
|
||||
<button type="button" className="underline" onClick={() => refetchModels()}>
|
||||
Retry
|
||||
</button>
|
||||
</div>
|
||||
</FormField>
|
||||
</div>
|
||||
{requiresTeamScope && (
|
||||
<FormField
|
||||
control={form.control}
|
||||
name="team_id"
|
||||
label={labelWithHint(
|
||||
"Select Team",
|
||||
"Select the team this auto router belongs to. Only keys for this team will be able to call it.",
|
||||
)}
|
||||
</div>
|
||||
|
||||
{requiresTeamScope && (
|
||||
<FormField
|
||||
control={form.control}
|
||||
name="team_id"
|
||||
label={labelWithHint(
|
||||
"Select Team",
|
||||
"Select the team this auto router belongs to. Only keys for this team will be able to call it.",
|
||||
)}
|
||||
>
|
||||
{({ id, value, onChange }) => (
|
||||
<TeamDropdown
|
||||
id={id}
|
||||
value={value}
|
||||
onChange={onChange}
|
||||
filterTeam={(team) => canCreateAutoRouterForTeam(actor, team)}
|
||||
/>
|
||||
)}
|
||||
</FormField>
|
||||
)}
|
||||
|
||||
{forecast ? (
|
||||
configurationForm
|
||||
) : (
|
||||
<div className="border border-border rounded-lg">
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => setDetailsExpanded((expanded) => !expanded)}
|
||||
className="w-full flex flex-col gap-1 px-4 py-3 text-left hover:bg-muted"
|
||||
data-testid="detailed-configuration-toggle"
|
||||
>
|
||||
<span className="flex items-center gap-2 font-medium text-foreground">
|
||||
{detailsExpanded ? (
|
||||
<ChevronDown className="size-3 text-muted-foreground" />
|
||||
) : (
|
||||
<ChevronRight className="size-3 text-muted-foreground" />
|
||||
)}
|
||||
Detailed Configuration
|
||||
</span>
|
||||
{!detailsExpanded && (
|
||||
<span className="text-xs text-muted-foreground line-clamp-2">
|
||||
{tierConfigSummary(complexityRouterConfig)}
|
||||
</span>
|
||||
)}
|
||||
</button>
|
||||
{detailsExpanded && <div className="px-4 pb-4">{configurationForm}</div>}
|
||||
</div>
|
||||
)}
|
||||
|
||||
{isAdmin && (
|
||||
<FormField
|
||||
control={form.control}
|
||||
name="model_access_group"
|
||||
label={labelWithHint(
|
||||
"Model Access Group",
|
||||
"Use model access groups to control who can access this auto router",
|
||||
)}
|
||||
>
|
||||
{({ id, value, onChange, "aria-invalid": ariaInvalid, "aria-describedby": ariaDescribedBy }) => (
|
||||
<AccessGroupTagsCombobox
|
||||
id={id}
|
||||
value={value}
|
||||
onChange={onChange}
|
||||
options={modelAccessGroups}
|
||||
ariaInvalid={ariaInvalid}
|
||||
ariaDescribedBy={ariaDescribedBy}
|
||||
/>
|
||||
)}
|
||||
</FormField>
|
||||
)}
|
||||
|
||||
<div className="flex justify-between items-center">
|
||||
<Tooltip>
|
||||
<TooltipTrigger
|
||||
render={
|
||||
<a
|
||||
href="https://github.com/BerriAI/litellm/issues"
|
||||
className="text-sm text-primary underline-offset-4 hover:underline"
|
||||
>
|
||||
Need Help?
|
||||
</a>
|
||||
}
|
||||
>
|
||||
{({ id, value, onChange }) => (
|
||||
<TeamDropdown
|
||||
id={id}
|
||||
value={value}
|
||||
onChange={onChange}
|
||||
filterTeam={(team) => canCreateAutoRouterForTeam(actor, team)}
|
||||
/>
|
||||
<TooltipContent>Get help on our github</TooltipContent>
|
||||
</Tooltip>
|
||||
<div className="flex gap-2">
|
||||
<BlockedReasonTooltip reason={submitBlockedReason}>
|
||||
)}
|
||||
</FormField>
|
||||
)}
|
||||
|
||||
<AutoRouterClassifierTabs value={complexityRouterConfig} onChange={handleClassifierChange}>
|
||||
<FieldGroup>
|
||||
<div className="space-y-4 empty:hidden">
|
||||
{!forecast && (
|
||||
<>
|
||||
{!automaticSetupLoading && automaticRouterConfig && !complexityRouterConfig.custom_tier_set && (
|
||||
<div className="flex flex-wrap items-center justify-between gap-3 rounded-lg border border-border bg-muted px-4 py-3">
|
||||
<div>
|
||||
<p className="text-sm font-medium text-foreground">Choose models quickly</p>
|
||||
<p className="text-sm text-muted-foreground">
|
||||
Fill the tiers while keeping your classifier and settings.
|
||||
</p>
|
||||
</div>
|
||||
<Button
|
||||
type="button"
|
||||
data-testid="configure-automatically-button"
|
||||
onClick={handleAutomaticSetup}
|
||||
>
|
||||
Choose models for me
|
||||
</Button>
|
||||
</div>
|
||||
)}
|
||||
|
||||
<div className="space-y-2">
|
||||
<label className="block text-sm font-medium text-foreground mb-2">Template</label>
|
||||
<Select
|
||||
items={templateItems}
|
||||
value={selectedPreset ?? null}
|
||||
onValueChange={(presetKey: string | null) => handlePresetChange(presetKey ?? undefined)}
|
||||
>
|
||||
<SelectTrigger data-testid="template-selector" className="w-full">
|
||||
<SelectValue placeholder="Choose a template or select Custom to define your own" />
|
||||
</SelectTrigger>
|
||||
<SelectContent>
|
||||
{sortedPresetOptions.map(({ preset, availability: presetState }) => {
|
||||
const disabledHint = presetDisabledHint(presetState);
|
||||
const hintClass = isPresetHintAlarming(presetState)
|
||||
? "text-destructive"
|
||||
: "text-muted-foreground";
|
||||
const matchedHint =
|
||||
presetState.kind === "available" && presetState.viaDeployments
|
||||
? "Matches your deployments"
|
||||
: null;
|
||||
|
||||
return (
|
||||
<SelectItem
|
||||
key={preset.key}
|
||||
value={preset.key}
|
||||
label={preset.label}
|
||||
disabled={disabledHint !== null}
|
||||
title={disabledHint ?? preset.description}
|
||||
>
|
||||
<div>
|
||||
<div className="font-medium">{preset.label}</div>
|
||||
<div className="text-xs text-muted-foreground">{preset.description}</div>
|
||||
<div className="text-xs text-muted-foreground">
|
||||
Replaces classifier and model choices
|
||||
</div>
|
||||
{disabledHint && (
|
||||
<div className={`text-xs mt-1 ${hintClass}`}>{disabledHint}</div>
|
||||
)}
|
||||
{matchedHint && <div className="text-xs mt-1 text-success">{matchedHint}</div>}
|
||||
</div>
|
||||
</SelectItem>
|
||||
);
|
||||
})}
|
||||
<SelectItem value="custom" label="Custom Configuration">
|
||||
<div>
|
||||
<div className="font-medium">Custom Configuration</div>
|
||||
<div className="text-xs text-muted-foreground">
|
||||
Define your auto router from scratch
|
||||
</div>
|
||||
</div>
|
||||
</SelectItem>
|
||||
</SelectContent>
|
||||
</Select>
|
||||
{presetsPending && (
|
||||
<div className="text-xs mt-1 text-muted-foreground">Loading templates...</div>
|
||||
)}
|
||||
{presetsUnavailable && (
|
||||
<div className="text-xs mt-1 text-destructive">
|
||||
Could not load templates, so only Custom Configuration is shown.{" "}
|
||||
<button type="button" className="underline" onClick={() => void refetchPresets()}>
|
||||
Retry
|
||||
</button>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</>
|
||||
)}
|
||||
{modelsUnverifiable && (
|
||||
<div className="text-xs mt-1 text-destructive">
|
||||
Could not load available models.{" "}
|
||||
<button type="button" className="underline" onClick={() => refetchModels()}>
|
||||
Retry
|
||||
</button>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
|
||||
{configurationForm}
|
||||
|
||||
{isAdmin && (
|
||||
<FormField
|
||||
control={form.control}
|
||||
name="model_access_group"
|
||||
label={labelWithHint(
|
||||
"Model Access Group",
|
||||
"Use model access groups to control who can access this auto router",
|
||||
)}
|
||||
>
|
||||
{({ id, value, onChange, "aria-invalid": ariaInvalid, "aria-describedby": ariaDescribedBy }) => (
|
||||
<AccessGroupTagsCombobox
|
||||
id={id}
|
||||
value={value}
|
||||
onChange={onChange}
|
||||
options={modelAccessGroups}
|
||||
ariaInvalid={ariaInvalid}
|
||||
ariaDescribedBy={ariaDescribedBy}
|
||||
/>
|
||||
)}
|
||||
</FormField>
|
||||
)}
|
||||
|
||||
<div className="flex flex-wrap items-center justify-between gap-3">
|
||||
<Tooltip>
|
||||
<TooltipTrigger
|
||||
render={
|
||||
<a
|
||||
href="https://github.com/BerriAI/litellm/issues"
|
||||
className="text-sm text-primary underline-offset-4 hover:underline"
|
||||
>
|
||||
Need Help?
|
||||
</a>
|
||||
}
|
||||
/>
|
||||
<TooltipContent>Get help on our github</TooltipContent>
|
||||
</Tooltip>
|
||||
<div className="flex flex-wrap gap-2">
|
||||
<BlockedReasonTooltip reason={submitBlockedReason}>
|
||||
<Button
|
||||
type="button"
|
||||
variant="outline"
|
||||
data-testid="auto-router-test-routing-btn"
|
||||
disabled={submitBlockedReason !== null || isSubmitting}
|
||||
onClick={() => setIsRoutingTestVisible(true)}
|
||||
>
|
||||
Test Routing
|
||||
</Button>
|
||||
</BlockedReasonTooltip>
|
||||
<Button
|
||||
type="button"
|
||||
variant="outline"
|
||||
data-testid="auto-router-test-routing-btn"
|
||||
disabled={submitBlockedReason !== null || isSubmitting}
|
||||
onClick={() => setIsRoutingTestVisible(true)}
|
||||
data-testid="auto-router-test-connect-btn"
|
||||
onClick={handleTestConnection}
|
||||
disabled={isTestingConnection}
|
||||
>
|
||||
Test Routing
|
||||
{isTestingConnection && <UiLoadingSpinner className="size-4" />}
|
||||
Test Connection
|
||||
</Button>
|
||||
</BlockedReasonTooltip>
|
||||
<Button
|
||||
type="button"
|
||||
variant="outline"
|
||||
data-testid="auto-router-test-connect-btn"
|
||||
onClick={handleTestConnection}
|
||||
disabled={isTestingConnection}
|
||||
>
|
||||
{isTestingConnection && <UiLoadingSpinner className="size-4" />}
|
||||
Test Connection
|
||||
</Button>
|
||||
<BlockedReasonTooltip reason={submitBlockedReason}>
|
||||
<Button
|
||||
type="button"
|
||||
disabled={submitBlockedReason !== null || isSubmitting}
|
||||
onClick={() => {
|
||||
void handleAutoRouterSubmit();
|
||||
}}
|
||||
>
|
||||
Add Auto Router
|
||||
</Button>
|
||||
</BlockedReasonTooltip>
|
||||
<BlockedReasonTooltip reason={saveBlockedReason}>
|
||||
<Button
|
||||
type="button"
|
||||
disabled={saveBlockedReason !== null || isSubmitting}
|
||||
onClick={() => {
|
||||
void handleAutoRouterSubmit();
|
||||
}}
|
||||
>
|
||||
Add Auto Router
|
||||
</Button>
|
||||
</BlockedReasonTooltip>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</FieldGroup>
|
||||
</AutoRouterClassifierTabs>
|
||||
</form>
|
||||
</CardContent>
|
||||
</Card>
|
||||
</FieldGroup>
|
||||
</AutoRouterClassifierTabs>
|
||||
</form>
|
||||
</CardContent>
|
||||
</Card>
|
||||
|
||||
<Dialog open={isRoutingTestVisible} onOpenChange={(open) => !open && setIsRoutingTestVisible(false)}>
|
||||
<DialogContent className="max-h-[calc(100dvh-2rem)] overflow-y-auto sm:max-w-[760px]">
|
||||
<DialogHeader>
|
||||
<DialogTitle>Test Routing</DialogTitle>
|
||||
</DialogHeader>
|
||||
{isRoutingTestVisible && (
|
||||
<AutoRouterRoutingTest
|
||||
accessToken={accessToken}
|
||||
config={buildComplexityRouterConfig(complexityRouterConfigParams)}
|
||||
defaultModel={resolveComplexityDefaultModel(complexityRouterConfig, complexityRouterConfig.default_model)}
|
||||
routerName={watchedName}
|
||||
teamId={requiresTeamScope ? watchedTeamId ?? undefined : undefined}
|
||||
/>
|
||||
)}
|
||||
<DialogFooter>
|
||||
{" "}
|
||||
<Button variant="outline" onClick={() => setIsRoutingTestVisible(false)}>
|
||||
Close
|
||||
</Button>
|
||||
</DialogFooter>
|
||||
</DialogContent>
|
||||
</Dialog>
|
||||
<Dialog open={isRoutingTestVisible} onOpenChange={(open) => !open && setIsRoutingTestVisible(false)}>
|
||||
<DialogContent className="max-h-[calc(100dvh-2rem)] overflow-y-auto sm:max-w-[760px]">
|
||||
<DialogHeader>
|
||||
<DialogTitle>Test Routing</DialogTitle>
|
||||
</DialogHeader>
|
||||
{isRoutingTestVisible && (
|
||||
<AutoRouterRoutingTest
|
||||
accessToken={accessToken}
|
||||
config={buildComplexityRouterConfig(complexityRouterConfigParams)}
|
||||
defaultModel={resolveComplexityDefaultModel(
|
||||
complexityRouterConfig,
|
||||
complexityRouterConfig.default_model,
|
||||
)}
|
||||
routerName={watchedName}
|
||||
teamId={requiresTeamScope ? watchedTeamId ?? undefined : undefined}
|
||||
/>
|
||||
)}
|
||||
<DialogFooter>
|
||||
{" "}
|
||||
<Button variant="outline" onClick={() => setIsRoutingTestVisible(false)}>
|
||||
Close
|
||||
</Button>
|
||||
</DialogFooter>
|
||||
</DialogContent>
|
||||
</Dialog>
|
||||
|
||||
<AutoRouterConnectionTestDialog
|
||||
open={isTestModalVisible}
|
||||
onClose={() => {
|
||||
setIsTestModalVisible(false);
|
||||
setIsTestingConnection(false);
|
||||
}}
|
||||
testId={connectionTestId}
|
||||
accessToken={accessToken}
|
||||
targets={testTargets}
|
||||
jevRequest={jevRequest}
|
||||
onTestComplete={() => setIsTestingConnection(false)}
|
||||
/>
|
||||
</TooltipProvider>
|
||||
<AutoRouterConnectionTestDialog
|
||||
open={isTestModalVisible}
|
||||
onClose={() => {
|
||||
setIsTestModalVisible(false);
|
||||
setIsTestingConnection(false);
|
||||
}}
|
||||
testId={connectionTestId}
|
||||
accessToken={accessToken}
|
||||
targets={testTargets}
|
||||
jevRequest={jevRequest}
|
||||
onTestComplete={() => setIsTestingConnection(false)}
|
||||
/>
|
||||
</TooltipProvider>
|
||||
</AutoRouterAvailabilityContext.Provider>
|
||||
);
|
||||
};
|
||||
|
||||
|
|
|
|||
|
|
@ -50,7 +50,7 @@ const AutoRouterConnectionTest: React.FC<AutoRouterConnectionTestProps> = ({
|
|||
? { status: "success" }
|
||||
: {
|
||||
status: "error",
|
||||
error: `JEV was not reached successfully (routing cause: ${decision.cause ?? "unknown"})`,
|
||||
error: `Jev was not reached successfully (routing cause: ${decision.cause ?? "unknown"})`,
|
||||
},
|
||||
);
|
||||
};
|
||||
|
|
@ -91,11 +91,11 @@ const AutoRouterConnectionTest: React.FC<AutoRouterConnectionTestProps> = ({
|
|||
classifier probe includes its reasoning effort override.
|
||||
</p>
|
||||
{jevRequest && (
|
||||
<div role="status" aria-label="JEV connection" className="rounded-lg border p-3 text-sm">
|
||||
<strong>JEV Classifier</strong>
|
||||
<div role="status" aria-label="Jev connection" className="rounded-lg border p-3 text-sm">
|
||||
<strong>Jev Classifier</strong>
|
||||
<p>
|
||||
{jevResult.status === "pending" && "Testing JEV classification"}
|
||||
{jevResult.status === "success" && "JEV classification succeeded"}
|
||||
{jevResult.status === "pending" && "Testing Jev classification"}
|
||||
{jevResult.status === "success" && "Jev classification succeeded"}
|
||||
{jevResult.status === "error" && jevResult.error}
|
||||
</p>
|
||||
</div>
|
||||
|
|
|
|||
|
|
@ -1,10 +1,12 @@
|
|||
import { openAutoRouterAdvanced, selectAutoRouterOption } from "../../../tests/autoRouterSetup";
|
||||
import userEvent from "@testing-library/user-event";
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
|
||||
import { fireEvent, renderWithProviders, screen, waitFor, within } from "@/../tests/test-utils";
|
||||
import { act, fireEvent, renderWithProviders, screen, waitFor, within, testQueryClient } from "@/../tests/test-utils";
|
||||
|
||||
import { toast } from "@/lib/toast";
|
||||
import EditAutoRouterModal from "./edit_auto_router_modal";
|
||||
import { apiClient } from "../networking";
|
||||
vi.mock(
|
||||
"@/app/(dashboard)/hooks/autoRouter/useComplexityScorerDefaults",
|
||||
async () => await import("../../../tests/mocks/complexityScorerDefaults"),
|
||||
|
|
@ -25,6 +27,7 @@ const {
|
|||
}));
|
||||
|
||||
vi.mock("../networking", () => ({
|
||||
apiClient: { post: vi.fn().mockResolvedValue({ allowances: [], error: null }) },
|
||||
modelPatchUpdateCall,
|
||||
modelAvailableCall,
|
||||
getAutoRouterClassifierDefaultPromptCall,
|
||||
|
|
@ -102,10 +105,11 @@ describe("EditAutoRouterModal keyword matching", () => {
|
|||
});
|
||||
|
||||
expect(await screen.findByRole("textbox", { name: "Auto Router Name" })).toHaveAttribute("readonly");
|
||||
expect(screen.queryByText("Advanced: Compression")).not.toBeInTheDocument();
|
||||
expect(screen.queryByText("Compression")).not.toBeInTheDocument();
|
||||
expect(screen.queryByText("Model Access Groups")).not.toBeInTheDocument();
|
||||
await user.click(screen.getByText("Advanced: Affinity"));
|
||||
openAutoRouterAdvanced("Affinity");
|
||||
await user.click(await screen.findByRole("switch", { name: "Pin one model deployment per tier" }));
|
||||
await waitFor(() => expect(screen.getByRole("button", { name: /save changes/i })).toBeEnabled());
|
||||
await user.click(screen.getByRole("button", { name: /save changes/i }));
|
||||
await waitFor(() => expect(modelPatchUpdateCall).toHaveBeenCalled());
|
||||
expect(modelPatchUpdateCall).toHaveBeenLastCalledWith(
|
||||
|
|
@ -128,7 +132,9 @@ describe("EditAutoRouterModal keyword matching", () => {
|
|||
it("renders the advanced sections the create form offers", async () => {
|
||||
renderModal();
|
||||
|
||||
expect(await screen.findByText(/Escalation Keywords/i)).toBeInTheDocument();
|
||||
await screen.findByRole("combobox", { name: "How often to classify" });
|
||||
openAutoRouterAdvanced("Escalation Keywords");
|
||||
expect(screen.getAllByText(/Escalation Keywords/i)).not.toHaveLength(0);
|
||||
expect(await screen.findByText(/Keyword\/Semantic Matching/i)).toBeInTheDocument();
|
||||
});
|
||||
|
||||
|
|
@ -147,10 +153,11 @@ describe("EditAutoRouterModal keyword matching", () => {
|
|||
},
|
||||
},
|
||||
});
|
||||
await user.click(await screen.findByText("Advanced: Classification Method"));
|
||||
openAutoRouterAdvanced("Classification Method");
|
||||
const threshold = screen.getByRole("textbox", { name: "Success threshold" });
|
||||
expect(threshold).toHaveValue("0.91");
|
||||
fireEvent.change(threshold, { target: { value: raw } });
|
||||
await waitFor(() => expect(screen.getByRole("button", { name: "Save Changes" })).toBeEnabled());
|
||||
await user.click(screen.getByRole("button", { name: "Save Changes" }));
|
||||
await waitFor(() => expect(modelPatchUpdateCall).toHaveBeenCalledOnce());
|
||||
if (raw === "") expect(savedConfig()).not.toHaveProperty("heuristic_v2_success_threshold");
|
||||
|
|
@ -172,14 +179,15 @@ describe("EditAutoRouterModal keyword matching", () => {
|
|||
},
|
||||
},
|
||||
});
|
||||
await user.click(await screen.findByText("Advanced: Classification Method"));
|
||||
openAutoRouterAdvanced("Classification Method");
|
||||
fireEvent.change(screen.getByRole("textbox", { name: "Success threshold" }), { target: { value: "-0.1" } });
|
||||
expect(screen.getByRole("button", { name: "Save Changes" })).toBeDisabled();
|
||||
expect(modelPatchUpdateCall).not.toHaveBeenCalled();
|
||||
|
||||
fireEvent.change(screen.getByRole("textbox", { name: "Success threshold" }), { target: { value: "0.88" } });
|
||||
await user.click(screen.getByRole("radio", { name: /^Heuristic \(default\)/ }));
|
||||
await selectAutoRouterOption("Heuristic", "Rule-based");
|
||||
expect(screen.queryByRole("textbox", { name: "Success threshold" })).not.toBeInTheDocument();
|
||||
await waitFor(() => expect(screen.getByRole("button", { name: "Save Changes" })).toBeEnabled());
|
||||
await user.click(screen.getByRole("button", { name: "Save Changes" }));
|
||||
await waitFor(() => expect(modelPatchUpdateCall).toHaveBeenCalledOnce());
|
||||
expect(savedConfig()).toMatchObject({ classifier_type: "heuristic", heuristic_v2_success_threshold: 0.88 });
|
||||
|
|
@ -188,13 +196,14 @@ describe("EditAutoRouterModal keyword matching", () => {
|
|||
it("clears an invalid inactive threshold before saving the router", async () => {
|
||||
const user = userEvent.setup();
|
||||
renderModal();
|
||||
await user.click(await screen.findByText("Advanced: Classification Method"));
|
||||
await user.click(screen.getByRole("radio", { name: /^Heuristic v2/ }));
|
||||
openAutoRouterAdvanced("Classification Method");
|
||||
await selectAutoRouterOption("Heuristic", "Heuristic v2");
|
||||
fireEvent.change(screen.getByRole("textbox", { name: "Success threshold" }), { target: { value: "1.1" } });
|
||||
await user.click(screen.getByRole("radio", { name: /^Heuristic \(default\)/ }));
|
||||
await selectAutoRouterOption("Heuristic", "Rule-based");
|
||||
expect(screen.getByRole("button", { name: "Save Changes" })).toBeDisabled();
|
||||
await user.click(screen.getByRole("button", { name: "Clear Heuristic v2 threshold" }));
|
||||
expect(screen.queryByRole("region", { name: "Inactive Heuristic v2 threshold" })).not.toBeInTheDocument();
|
||||
await waitFor(() => expect(screen.getByRole("button", { name: "Save Changes" })).toBeEnabled());
|
||||
await user.click(screen.getByRole("button", { name: "Save Changes" }));
|
||||
await waitFor(() => expect(modelPatchUpdateCall).toHaveBeenCalledOnce());
|
||||
expect(savedConfig()).not.toHaveProperty("heuristic_v2_success_threshold");
|
||||
|
|
@ -207,7 +216,8 @@ describe("EditAutoRouterModal keyword matching", () => {
|
|||
const user = userEvent.setup();
|
||||
renderModal();
|
||||
|
||||
await screen.findByText(/Escalation Keywords/i);
|
||||
await screen.findByRole("combobox", { name: "How often to classify" });
|
||||
await waitFor(() => expect(screen.getByRole("button", { name: /save changes/i })).toBeEnabled());
|
||||
await user.click(screen.getByRole("button", { name: /save changes/i }));
|
||||
|
||||
await waitFor(() => expect(modelPatchUpdateCall).toHaveBeenCalled());
|
||||
|
|
@ -230,7 +240,8 @@ describe("EditAutoRouterModal keyword matching", () => {
|
|||
});
|
||||
|
||||
renderModal();
|
||||
await screen.findByText(/Escalation Keywords/i);
|
||||
await screen.findByRole("combobox", { name: "How often to classify" });
|
||||
await waitFor(() => expect(screen.getByRole("button", { name: /save changes/i })).toBeEnabled());
|
||||
await user.click(screen.getByRole("button", { name: /save changes/i }));
|
||||
|
||||
await waitFor(() => expect(validateAutoRouterConfig).toHaveBeenCalled());
|
||||
|
|
@ -264,7 +275,8 @@ describe("EditAutoRouterModal keyword matching", () => {
|
|||
/>,
|
||||
);
|
||||
|
||||
await screen.findByText(/Escalation Keywords/i);
|
||||
await screen.findByRole("combobox", { name: "How often to classify" });
|
||||
await waitFor(() => expect(screen.getByRole("button", { name: /save changes/i })).toBeEnabled());
|
||||
await user.click(screen.getByRole("button", { name: /save changes/i }));
|
||||
|
||||
await waitFor(() => expect(toast.fromError).toHaveBeenCalled());
|
||||
|
|
@ -297,8 +309,8 @@ describe("EditAutoRouterModal keyword matching", () => {
|
|||
/>,
|
||||
);
|
||||
|
||||
await screen.findByText(/Escalation Keywords/i);
|
||||
fireEvent.click(screen.getByText("Advanced: Keyword/Semantic Matching"));
|
||||
await screen.findByRole("combobox", { name: "How often to classify" });
|
||||
openAutoRouterAdvanced("Keyword/Semantic Matching");
|
||||
await user.click(screen.getByRole("button", { name: /add keyword rule/i }));
|
||||
|
||||
// The modal renders the same controls as the create form, so it owes the same treatment:
|
||||
|
|
@ -331,8 +343,8 @@ describe("EditAutoRouterModal keyword matching", () => {
|
|||
/>,
|
||||
);
|
||||
|
||||
await screen.findByText(/Escalation Keywords/i);
|
||||
fireEvent.click(screen.getByText("Advanced: Keyword/Semantic Matching"));
|
||||
await screen.findByRole("combobox", { name: "How often to classify" });
|
||||
openAutoRouterAdvanced("Keyword/Semantic Matching");
|
||||
await user.click(screen.getByRole("button", { name: /add keyword rule/i }));
|
||||
expect(screen.getByRole("button", { name: /save changes/i })).toBeDisabled();
|
||||
|
||||
|
|
@ -342,7 +354,7 @@ describe("EditAutoRouterModal keyword matching", () => {
|
|||
);
|
||||
await user.click(await screen.findByText('Create "chargeback"'));
|
||||
|
||||
expect(screen.getByRole("button", { name: /save changes/i })).toBeEnabled();
|
||||
await waitFor(() => expect(screen.getByRole("button", { name: /save changes/i })).toBeEnabled());
|
||||
expect(screen.queryByText("At least one keyword is required")).not.toBeInTheDocument();
|
||||
});
|
||||
});
|
||||
|
|
@ -373,17 +385,18 @@ describe("EditAutoRouterModal advanced field round trips", () => {
|
|||
const user = userEvent.setup();
|
||||
renderAdvancedModal();
|
||||
|
||||
await user.click(await screen.findByText("Advanced: Housekeeping Routing"));
|
||||
openAutoRouterAdvanced("Housekeeping Routing");
|
||||
expect(screen.getByRole("switch", { name: "Route housekeeping calls to the cheapest tier" })).not.toBeChecked();
|
||||
expect(screen.getByRole("combobox", { name: "e.g., conversation title" })).toHaveValue("");
|
||||
|
||||
await user.click(screen.getByText("Advanced: Ignore Custom Tags"));
|
||||
openAutoRouterAdvanced("Ignore Custom Tags");
|
||||
expect(screen.getByLabelText("Opening tag")).toHaveValue("<a>");
|
||||
expect(screen.getByLabelText("Closing tag")).toHaveValue("</a>");
|
||||
|
||||
await user.click(screen.getByText("Advanced: Response Format"));
|
||||
openAutoRouterAdvanced("Response Format");
|
||||
const maxTokensSwitch = screen.getByRole("switch", { name: "Cap max_tokens at the tier model's output ceiling" });
|
||||
await user.click(maxTokensSwitch);
|
||||
await waitFor(() => expect(screen.getByRole("button", { name: /save changes/i })).toBeEnabled());
|
||||
await user.click(screen.getByRole("button", { name: /save changes/i }));
|
||||
await waitFor(() => expect(modelPatchUpdateCall).toHaveBeenCalledOnce());
|
||||
|
||||
|
|
@ -407,6 +420,7 @@ describe("EditAutoRouterModal advanced field round trips", () => {
|
|||
it("preserves all stored advanced fields through an untouched save", async () => {
|
||||
const user = userEvent.setup();
|
||||
renderAdvancedModal();
|
||||
await waitFor(() => expect(screen.getByRole("button", { name: /save changes/i })).toBeEnabled());
|
||||
await user.click(screen.getByRole("button", { name: /save changes/i }));
|
||||
await waitFor(() => expect(modelPatchUpdateCall).toHaveBeenCalledOnce());
|
||||
expect(savedConfig()).toMatchObject({
|
||||
|
|
@ -454,11 +468,12 @@ describe("EditAutoRouterModal classifier context window", () => {
|
|||
const user = userEvent.setup();
|
||||
renderLlmModal();
|
||||
|
||||
await user.click(await screen.findByText("Advanced: Classification Method"));
|
||||
openAutoRouterAdvanced("Classification Method");
|
||||
await screen.findByText("Context Window Size");
|
||||
expect(screen.getByDisplayValue("5")).toBeInTheDocument();
|
||||
expect(screen.queryByText("Context Per-Turn Character Limit")).not.toBeInTheDocument();
|
||||
|
||||
await waitFor(() => expect(screen.getByRole("button", { name: /save changes/i })).toBeEnabled());
|
||||
await user.click(screen.getByRole("button", { name: /save changes/i }));
|
||||
|
||||
await waitFor(() => expect(modelPatchUpdateCall).toHaveBeenCalled());
|
||||
|
|
@ -475,7 +490,7 @@ describe("EditAutoRouterModal classifier context window", () => {
|
|||
const user = userEvent.setup();
|
||||
const { baseElement } = renderLlmModal();
|
||||
|
||||
await user.click(await screen.findByText("Advanced: Classification Method"));
|
||||
openAutoRouterAdvanced("Classification Method");
|
||||
await user.click(await screen.findByRole("button", { name: /prompt/i }));
|
||||
|
||||
expect(await screen.findByLabelText("Classification instructions")).toBeInTheDocument();
|
||||
|
|
@ -486,10 +501,11 @@ describe("EditAutoRouterModal classifier context window", () => {
|
|||
const user = userEvent.setup();
|
||||
renderLlmModal();
|
||||
|
||||
await user.click(await screen.findByText("Advanced: Classification Method"));
|
||||
openAutoRouterAdvanced("Classification Method");
|
||||
const input = await screen.findByLabelText("Context Window Size");
|
||||
fireEvent.change(input, { target: { value: "8" } });
|
||||
|
||||
await waitFor(() => expect(screen.getByRole("button", { name: /save changes/i })).toBeEnabled());
|
||||
await user.click(screen.getByRole("button", { name: /save changes/i }));
|
||||
|
||||
await waitFor(() => expect(modelPatchUpdateCall).toHaveBeenCalled());
|
||||
|
|
@ -531,10 +547,11 @@ describe("EditAutoRouterModal assistant turns", () => {
|
|||
const user = userEvent.setup();
|
||||
renderModal();
|
||||
|
||||
await user.click(await screen.findByText("Advanced: Classification Method"));
|
||||
openAutoRouterAdvanced("Classification Method");
|
||||
await screen.findByText("Include Assistant Turns");
|
||||
expect(screen.getByRole("switch", { name: "Include Assistant Turns" })).toBeChecked();
|
||||
|
||||
await waitFor(() => expect(screen.getByRole("button", { name: /save changes/i })).toBeEnabled());
|
||||
await user.click(screen.getByRole("button", { name: /save changes/i }));
|
||||
|
||||
await waitFor(() => expect(modelPatchUpdateCall).toHaveBeenCalled());
|
||||
|
|
@ -545,10 +562,11 @@ describe("EditAutoRouterModal assistant turns", () => {
|
|||
const user = userEvent.setup();
|
||||
renderModal();
|
||||
|
||||
await user.click(await screen.findByText("Advanced: Classification Method"));
|
||||
openAutoRouterAdvanced("Classification Method");
|
||||
await screen.findByText("Include Assistant Turns");
|
||||
await user.click(screen.getByRole("switch", { name: "Include Assistant Turns" }));
|
||||
|
||||
await waitFor(() => expect(screen.getByRole("button", { name: /save changes/i })).toBeEnabled());
|
||||
await user.click(screen.getByRole("button", { name: /save changes/i }));
|
||||
|
||||
await waitFor(() => expect(modelPatchUpdateCall).toHaveBeenCalled());
|
||||
|
|
@ -580,9 +598,10 @@ describe("EditAutoRouterModal classification frequency", () => {
|
|||
const user = userEvent.setup();
|
||||
renderWithStoredConfig(STORED_CONFIG);
|
||||
|
||||
await user.click(await screen.findByText("Advanced: Classification Method"));
|
||||
expect(await screen.findByRole("radio", { name: /Every request/ })).toBeChecked();
|
||||
openAutoRouterAdvanced("Classification Method");
|
||||
expect(await screen.findByRole("combobox", { name: "How often to classify" })).toHaveTextContent("Every request");
|
||||
|
||||
await waitFor(() => expect(screen.getByRole("button", { name: /save changes/i })).toBeEnabled());
|
||||
await user.click(screen.getByRole("button", { name: /save changes/i }));
|
||||
|
||||
await waitFor(() => expect(modelPatchUpdateCall).toHaveBeenCalled());
|
||||
|
|
@ -593,9 +612,12 @@ describe("EditAutoRouterModal classification frequency", () => {
|
|||
const user = userEvent.setup();
|
||||
renderWithStoredConfig({ ...STORED_CONFIG, session_affinity: true });
|
||||
|
||||
await user.click(await screen.findByText("Advanced: Classification Method"));
|
||||
expect(await screen.findByRole("radio", { name: /Once per session/ })).toBeChecked();
|
||||
openAutoRouterAdvanced("Classification Method");
|
||||
expect(await screen.findByRole("combobox", { name: "How often to classify" })).toHaveTextContent(
|
||||
"Once per session",
|
||||
);
|
||||
|
||||
await waitFor(() => expect(screen.getByRole("button", { name: /save changes/i })).toBeEnabled());
|
||||
await user.click(screen.getByRole("button", { name: /save changes/i }));
|
||||
|
||||
await waitFor(() => expect(modelPatchUpdateCall).toHaveBeenCalled());
|
||||
|
|
@ -606,9 +628,10 @@ describe("EditAutoRouterModal classification frequency", () => {
|
|||
const user = userEvent.setup();
|
||||
renderWithStoredConfig(STORED_CONFIG);
|
||||
|
||||
await user.click(await screen.findByText("Advanced: Classification Method"));
|
||||
await user.click(await screen.findByRole("radio", { name: /Once per session/ }));
|
||||
openAutoRouterAdvanced("Classification Method");
|
||||
await selectAutoRouterOption("How often to classify", "Once per session");
|
||||
|
||||
await waitFor(() => expect(screen.getByRole("button", { name: /save changes/i })).toBeEnabled());
|
||||
await user.click(screen.getByRole("button", { name: /save changes/i }));
|
||||
|
||||
await waitFor(() => expect(modelPatchUpdateCall).toHaveBeenCalled());
|
||||
|
|
@ -619,9 +642,10 @@ describe("EditAutoRouterModal classification frequency", () => {
|
|||
const user = userEvent.setup();
|
||||
renderWithStoredConfig({ ...STORED_CONFIG, session_affinity: true });
|
||||
|
||||
await user.click(await screen.findByText("Advanced: Classification Method"));
|
||||
await user.click(await screen.findByRole("radio", { name: /Every request/ }));
|
||||
openAutoRouterAdvanced("Classification Method");
|
||||
await selectAutoRouterOption("How often to classify", "Every request");
|
||||
|
||||
await waitFor(() => expect(screen.getByRole("button", { name: /save changes/i })).toBeEnabled());
|
||||
await user.click(screen.getByRole("button", { name: /save changes/i }));
|
||||
|
||||
await waitFor(() => expect(modelPatchUpdateCall).toHaveBeenCalled());
|
||||
|
|
@ -632,9 +656,10 @@ describe("EditAutoRouterModal classification frequency", () => {
|
|||
const user = userEvent.setup();
|
||||
renderWithStoredConfig({ ...STORED_CONFIG, session_affinity: true });
|
||||
|
||||
await user.click(await screen.findByText("Advanced: Classification Method"));
|
||||
await user.click(await screen.findByRole("radio", { name: /Every new user message/ }));
|
||||
openAutoRouterAdvanced("Classification Method");
|
||||
await selectAutoRouterOption("How often to classify", "Every new user message");
|
||||
|
||||
await waitFor(() => expect(screen.getByRole("button", { name: /save changes/i })).toBeEnabled());
|
||||
await user.click(screen.getByRole("button", { name: /save changes/i }));
|
||||
|
||||
await waitFor(() => expect(modelPatchUpdateCall).toHaveBeenCalled());
|
||||
|
|
@ -646,9 +671,12 @@ describe("EditAutoRouterModal classification frequency", () => {
|
|||
const user = userEvent.setup();
|
||||
renderWithStoredConfig({ ...STORED_CONFIG, classification_mode: "user_turn" });
|
||||
|
||||
await user.click(await screen.findByText("Advanced: Classification Method"));
|
||||
expect(await screen.findByRole("radio", { name: /Every new user message/ })).toBeChecked();
|
||||
openAutoRouterAdvanced("Classification Method");
|
||||
expect(await screen.findByRole("combobox", { name: "How often to classify" })).toHaveTextContent(
|
||||
"Every new user message",
|
||||
);
|
||||
|
||||
await waitFor(() => expect(screen.getByRole("button", { name: /save changes/i })).toBeEnabled());
|
||||
await user.click(screen.getByRole("button", { name: /save changes/i }));
|
||||
|
||||
await waitFor(() => expect(modelPatchUpdateCall).toHaveBeenCalled());
|
||||
|
|
@ -659,9 +687,10 @@ describe("EditAutoRouterModal classification frequency", () => {
|
|||
const user = userEvent.setup();
|
||||
renderWithStoredConfig(STORED_CONFIG);
|
||||
|
||||
await user.click(await screen.findByText("Advanced: Classification Method"));
|
||||
await user.click(await screen.findByRole("radio", { name: /Every new user message/ }));
|
||||
openAutoRouterAdvanced("Classification Method");
|
||||
await selectAutoRouterOption("How often to classify", "Every new user message");
|
||||
|
||||
await waitFor(() => expect(screen.getByRole("button", { name: /save changes/i })).toBeEnabled());
|
||||
await user.click(screen.getByRole("button", { name: /save changes/i }));
|
||||
|
||||
await waitFor(() => expect(modelPatchUpdateCall).toHaveBeenCalled());
|
||||
|
|
@ -672,9 +701,10 @@ describe("EditAutoRouterModal classification frequency", () => {
|
|||
const user = userEvent.setup();
|
||||
renderWithStoredConfig({ ...STORED_CONFIG, classification_mode: "user_turn" });
|
||||
|
||||
await user.click(await screen.findByText("Advanced: Classification Method"));
|
||||
await user.click(await screen.findByRole("radio", { name: /Every request/ }));
|
||||
openAutoRouterAdvanced("Classification Method");
|
||||
await selectAutoRouterOption("How often to classify", "Every request");
|
||||
|
||||
await waitFor(() => expect(screen.getByRole("button", { name: /save changes/i })).toBeEnabled());
|
||||
await user.click(screen.getByRole("button", { name: /save changes/i }));
|
||||
|
||||
await waitFor(() => expect(modelPatchUpdateCall).toHaveBeenCalled());
|
||||
|
|
@ -703,9 +733,10 @@ describe("EditAutoRouterModal deployment affinity", () => {
|
|||
const user = userEvent.setup();
|
||||
renderWithStoredConfig(STORED_CONFIG);
|
||||
|
||||
await user.click(await screen.findByText("Advanced: Affinity"));
|
||||
openAutoRouterAdvanced("Affinity");
|
||||
expect(await screen.findByRole("switch", { name: "Pin one model deployment per tier" })).toBeChecked();
|
||||
|
||||
await waitFor(() => expect(screen.getByRole("button", { name: /save changes/i })).toBeEnabled());
|
||||
await user.click(screen.getByRole("button", { name: /save changes/i }));
|
||||
|
||||
await waitFor(() => expect(modelPatchUpdateCall).toHaveBeenCalled());
|
||||
|
|
@ -716,9 +747,10 @@ describe("EditAutoRouterModal deployment affinity", () => {
|
|||
const user = userEvent.setup();
|
||||
renderWithStoredConfig({ ...STORED_CONFIG, deployment_affinity: false });
|
||||
|
||||
await user.click(await screen.findByText("Advanced: Affinity"));
|
||||
openAutoRouterAdvanced("Affinity");
|
||||
expect(await screen.findByRole("switch", { name: "Pin one model deployment per tier" })).not.toBeChecked();
|
||||
|
||||
await waitFor(() => expect(screen.getByRole("button", { name: /save changes/i })).toBeEnabled());
|
||||
await user.click(screen.getByRole("button", { name: /save changes/i }));
|
||||
|
||||
await waitFor(() => expect(modelPatchUpdateCall).toHaveBeenCalled());
|
||||
|
|
@ -729,9 +761,10 @@ describe("EditAutoRouterModal deployment affinity", () => {
|
|||
const user = userEvent.setup();
|
||||
renderWithStoredConfig(STORED_CONFIG);
|
||||
|
||||
await user.click(await screen.findByText("Advanced: Affinity"));
|
||||
openAutoRouterAdvanced("Affinity");
|
||||
await user.click(await screen.findByRole("switch", { name: "Pin one model deployment per tier" }));
|
||||
|
||||
await waitFor(() => expect(screen.getByRole("button", { name: /save changes/i })).toBeEnabled());
|
||||
await user.click(screen.getByRole("button", { name: /save changes/i }));
|
||||
|
||||
await waitFor(() => expect(modelPatchUpdateCall).toHaveBeenCalled());
|
||||
|
|
@ -742,9 +775,10 @@ describe("EditAutoRouterModal deployment affinity", () => {
|
|||
const user = userEvent.setup();
|
||||
renderWithStoredConfig({ ...STORED_CONFIG, session_affinity_ttl_seconds: 300 });
|
||||
|
||||
await user.click(await screen.findByText("Advanced: Affinity"));
|
||||
openAutoRouterAdvanced("Affinity");
|
||||
expect(await screen.findByLabelText("How long a pin survives idle (seconds)")).toHaveValue("300");
|
||||
|
||||
await waitFor(() => expect(screen.getByRole("button", { name: /save changes/i })).toBeEnabled());
|
||||
await user.click(screen.getByRole("button", { name: /save changes/i }));
|
||||
|
||||
await waitFor(() => expect(modelPatchUpdateCall).toHaveBeenCalled());
|
||||
|
|
@ -755,11 +789,12 @@ describe("EditAutoRouterModal deployment affinity", () => {
|
|||
const user = userEvent.setup();
|
||||
renderWithStoredConfig(STORED_CONFIG);
|
||||
|
||||
await user.click(await screen.findByText("Advanced: Affinity"));
|
||||
openAutoRouterAdvanced("Affinity");
|
||||
const ttl = await screen.findByLabelText("How long a pin survives idle (seconds)");
|
||||
fireEvent.change(ttl, { target: { value: "300" } });
|
||||
fireEvent.blur(ttl);
|
||||
|
||||
await waitFor(() => expect(screen.getByRole("button", { name: /save changes/i })).toBeEnabled());
|
||||
await user.click(screen.getByRole("button", { name: /save changes/i }));
|
||||
|
||||
await waitFor(() => expect(modelPatchUpdateCall).toHaveBeenCalled());
|
||||
|
|
@ -770,11 +805,12 @@ describe("EditAutoRouterModal deployment affinity", () => {
|
|||
const user = userEvent.setup();
|
||||
renderWithStoredConfig({ ...STORED_CONFIG, session_affinity_ttl_seconds: 300 });
|
||||
|
||||
await user.click(await screen.findByText("Advanced: Affinity"));
|
||||
openAutoRouterAdvanced("Affinity");
|
||||
const ttl = await screen.findByLabelText("How long a pin survives idle (seconds)");
|
||||
fireEvent.change(ttl, { target: { value: "" } });
|
||||
fireEvent.blur(ttl);
|
||||
|
||||
await waitFor(() => expect(screen.getByRole("button", { name: /save changes/i })).toBeEnabled());
|
||||
await user.click(screen.getByRole("button", { name: /save changes/i }));
|
||||
|
||||
await waitFor(() => expect(modelPatchUpdateCall).toHaveBeenCalled());
|
||||
|
|
@ -787,9 +823,10 @@ describe("EditAutoRouterModal deployment affinity", () => {
|
|||
const user = userEvent.setup();
|
||||
renderWithStoredConfig({ ...STORED_CONFIG, modality_routing: true, modality_pin_override: true });
|
||||
|
||||
await user.click(await screen.findByText("Advanced: Modality Routing"));
|
||||
openAutoRouterAdvanced("Modality Routing");
|
||||
expect(await screen.findByRole("switch", { name: "Override session pin for image requests" })).toBeChecked();
|
||||
|
||||
await waitFor(() => expect(screen.getByRole("button", { name: /save changes/i })).toBeEnabled());
|
||||
await user.click(screen.getByRole("button", { name: /save changes/i }));
|
||||
|
||||
await waitFor(() => expect(modelPatchUpdateCall).toHaveBeenCalled());
|
||||
|
|
@ -800,9 +837,10 @@ describe("EditAutoRouterModal deployment affinity", () => {
|
|||
const user = userEvent.setup();
|
||||
renderWithStoredConfig({ ...STORED_CONFIG, modality_routing: true });
|
||||
|
||||
await user.click(await screen.findByText("Advanced: Modality Routing"));
|
||||
openAutoRouterAdvanced("Modality Routing");
|
||||
await user.click(await screen.findByRole("switch", { name: "Override session pin for image requests" }));
|
||||
|
||||
await waitFor(() => expect(screen.getByRole("button", { name: /save changes/i })).toBeEnabled());
|
||||
await user.click(screen.getByRole("button", { name: /save changes/i }));
|
||||
|
||||
await waitFor(() => expect(modelPatchUpdateCall).toHaveBeenCalled());
|
||||
|
|
@ -813,12 +851,13 @@ describe("EditAutoRouterModal deployment affinity", () => {
|
|||
const user = userEvent.setup();
|
||||
renderWithStoredConfig(STORED_CONFIG);
|
||||
|
||||
await user.click(await screen.findByText("Advanced: Modality Routing"));
|
||||
openAutoRouterAdvanced("Modality Routing");
|
||||
expect(await screen.findByRole("switch", { name: "Override session pin for image requests" })).toHaveAttribute(
|
||||
"aria-disabled",
|
||||
"true",
|
||||
);
|
||||
|
||||
await waitFor(() => expect(screen.getByRole("button", { name: /save changes/i })).toBeEnabled());
|
||||
await user.click(screen.getByRole("button", { name: /save changes/i }));
|
||||
|
||||
await waitFor(() => expect(modelPatchUpdateCall).toHaveBeenCalled());
|
||||
|
|
@ -863,10 +902,11 @@ describe("EditAutoRouterModal custom classifier prompt and fallback", () => {
|
|||
const user = userEvent.setup();
|
||||
renderCustomModal();
|
||||
|
||||
await user.click(await screen.findByText("Advanced: Classification Method"));
|
||||
openAutoRouterAdvanced("Classification Method");
|
||||
expect(await screen.findByRole("button", { name: "Edit custom prompt" })).toBeInTheDocument();
|
||||
expect(screen.getByRole("radio", { name: /Route to the default model/ })).toBeChecked();
|
||||
|
||||
await waitFor(() => expect(screen.getByRole("button", { name: /save changes/i })).toBeEnabled());
|
||||
await user.click(screen.getByRole("button", { name: /save changes/i }));
|
||||
|
||||
await waitFor(() => expect(modelPatchUpdateCall).toHaveBeenCalled());
|
||||
|
|
@ -879,8 +919,9 @@ describe("EditAutoRouterModal custom classifier prompt and fallback", () => {
|
|||
const user = userEvent.setup();
|
||||
renderCustomModal();
|
||||
|
||||
await user.click(await screen.findByText("Advanced: Classification Method"));
|
||||
openAutoRouterAdvanced("Classification Method");
|
||||
await user.click(await screen.findByRole("radio", { name: /Score with the heuristic/ }));
|
||||
await waitFor(() => expect(screen.getByRole("button", { name: /save changes/i })).toBeEnabled());
|
||||
await user.click(screen.getByRole("button", { name: /save changes/i }));
|
||||
|
||||
await waitFor(() => expect(modelPatchUpdateCall).toHaveBeenCalled());
|
||||
|
|
@ -891,8 +932,9 @@ describe("EditAutoRouterModal custom classifier prompt and fallback", () => {
|
|||
const user = userEvent.setup();
|
||||
renderCustomModal();
|
||||
|
||||
await user.click(await screen.findByText("Advanced: Classification Method"));
|
||||
openAutoRouterAdvanced("Classification Method");
|
||||
await user.click(await screen.findByRole("button", { name: "Reset to default" }));
|
||||
await waitFor(() => expect(screen.getByRole("button", { name: /save changes/i })).toBeEnabled());
|
||||
await user.click(screen.getByRole("button", { name: /save changes/i }));
|
||||
|
||||
await waitFor(() => expect(modelPatchUpdateCall).toHaveBeenCalled());
|
||||
|
|
@ -953,6 +995,7 @@ describe("EditAutoRouterModal default model", () => {
|
|||
const user = userEvent.setup();
|
||||
renderWithStoredPin("out-of-band-default");
|
||||
|
||||
await waitFor(() => expect(screen.getByRole("button", { name: /save changes/i })).toBeEnabled());
|
||||
await user.click(await screen.findByRole("button", { name: /save changes/i }));
|
||||
|
||||
await waitFor(() => expect(modelPatchUpdateCall).toHaveBeenCalled());
|
||||
|
|
@ -976,6 +1019,7 @@ describe("EditAutoRouterModal default model", () => {
|
|||
const select = await screen.findByRole("combobox", { name: "Default model" });
|
||||
expect(select).toHaveValue(STORED_CONFIG.tiers.MEDIUM[0]);
|
||||
|
||||
await waitFor(() => expect(screen.getByRole("button", { name: /save changes/i })).toBeEnabled());
|
||||
await user.click(screen.getByRole("button", { name: /save changes/i }));
|
||||
await waitFor(() => expect(modelPatchUpdateCall).toHaveBeenCalled());
|
||||
expect(savedConfig()).toMatchObject({ default_model: STORED_CONFIG.tiers.MEDIUM[0] });
|
||||
|
|
@ -992,6 +1036,7 @@ describe("EditAutoRouterModal default model", () => {
|
|||
const select = await screen.findByRole("combobox", { name: "Default model" });
|
||||
expect(select).toHaveValue("");
|
||||
|
||||
await waitFor(() => expect(screen.getByRole("button", { name: /save changes/i })).toBeEnabled());
|
||||
await user.click(screen.getByRole("button", { name: /save changes/i }));
|
||||
await waitFor(() => expect(modelPatchUpdateCall).toHaveBeenCalled());
|
||||
expect(savedConfig()).not.toHaveProperty("default_model");
|
||||
|
|
@ -1009,6 +1054,7 @@ describe("EditAutoRouterModal default model", () => {
|
|||
const select = await screen.findByRole("combobox", { name: "Default model" });
|
||||
expect(select).toHaveValue("claude-sonnet-4");
|
||||
|
||||
await waitFor(() => expect(screen.getByRole("button", { name: /save changes/i })).toBeEnabled());
|
||||
await user.click(screen.getByRole("button", { name: /save changes/i }));
|
||||
await waitFor(() => expect(modelPatchUpdateCall).toHaveBeenCalled());
|
||||
expect(savedConfig()).toMatchObject({ default_model: "claude-sonnet-4" });
|
||||
|
|
@ -1041,6 +1087,7 @@ describe("EditAutoRouterModal default model", () => {
|
|||
const select = await screen.findByRole("combobox", { name: "Default model" });
|
||||
expect(select).toHaveValue("blob-pin");
|
||||
|
||||
await waitFor(() => expect(screen.getByRole("button", { name: /save changes/i })).toBeEnabled());
|
||||
await user.click(screen.getByRole("button", { name: /save changes/i }));
|
||||
await waitFor(() => expect(modelPatchUpdateCall).toHaveBeenCalled());
|
||||
expect(savedConfig()).toMatchObject({ default_model: "blob-pin" });
|
||||
|
|
@ -1070,6 +1117,7 @@ describe("EditAutoRouterModal default model", () => {
|
|||
/>,
|
||||
);
|
||||
|
||||
await waitFor(() => expect(screen.getByRole("button", { name: /save changes/i })).toBeEnabled());
|
||||
await user.click(await screen.findByRole("button", { name: /save changes/i }));
|
||||
|
||||
await waitFor(() => expect(toast.fromError).toHaveBeenCalledWith(expect.stringContaining("Simple or Medium tier")));
|
||||
|
|
@ -1083,6 +1131,7 @@ describe("EditAutoRouterModal default model", () => {
|
|||
const select = await screen.findByRole("combobox", { name: "Default model" });
|
||||
expect(select).toHaveValue("");
|
||||
|
||||
await waitFor(() => expect(screen.getByRole("button", { name: /save changes/i })).toBeEnabled());
|
||||
await user.click(screen.getByRole("button", { name: /save changes/i }));
|
||||
await waitFor(() => expect(modelPatchUpdateCall).toHaveBeenCalled());
|
||||
expect(savedDefaultModel()).toBe(STORED_CONFIG.tiers.MEDIUM[0]);
|
||||
|
|
@ -1114,7 +1163,7 @@ describe("EditAutoRouterModal plan-mode minimum tier", () => {
|
|||
);
|
||||
|
||||
const openPlanModePanel = async (user: ReturnType<typeof userEvent.setup>) => {
|
||||
await user.click(await screen.findByText("Advanced: Plan-Mode Override"));
|
||||
openAutoRouterAdvanced("Plan-Mode Override");
|
||||
};
|
||||
|
||||
it("shows a stored tier as an enabled override, so the saved value is not a hidden one", async () => {
|
||||
|
|
@ -1128,6 +1177,7 @@ describe("EditAutoRouterModal plan-mode minimum tier", () => {
|
|||
const user = userEvent.setup();
|
||||
renderWithStoredTier("MEDIUM");
|
||||
|
||||
await waitFor(() => expect(screen.getByRole("button", { name: /save changes/i })).toBeEnabled());
|
||||
await user.click(await screen.findByRole("button", { name: /save changes/i }));
|
||||
|
||||
await waitFor(() => expect(modelPatchUpdateCall).toHaveBeenCalled());
|
||||
|
|
@ -1140,6 +1190,7 @@ describe("EditAutoRouterModal plan-mode minimum tier", () => {
|
|||
await openPlanModePanel(user);
|
||||
await user.click(await screen.findByRole("switch", { name: "Route plan-mode requests to a minimum tier" }));
|
||||
|
||||
await waitFor(() => expect(screen.getByRole("button", { name: /save changes/i })).toBeEnabled());
|
||||
await user.click(screen.getByRole("button", { name: /save changes/i }));
|
||||
|
||||
await waitFor(() => expect(modelPatchUpdateCall).toHaveBeenCalled());
|
||||
|
|
@ -1194,6 +1245,7 @@ describe("EditAutoRouterModal with a stored custom tier set", () => {
|
|||
renderCustomModal();
|
||||
|
||||
await screen.findByText("SECURITY_REVIEW Tier");
|
||||
await waitFor(() => expect(screen.getByRole("button", { name: /save changes/i })).toBeEnabled());
|
||||
await user.click(screen.getByRole("button", { name: /save changes/i }));
|
||||
await waitFor(() => expect(modelPatchUpdateCall).toHaveBeenCalled());
|
||||
|
||||
|
|
@ -1211,6 +1263,7 @@ describe("EditAutoRouterModal with a stored custom tier set", () => {
|
|||
renderCustomModal();
|
||||
|
||||
await screen.findByText("SECURITY_REVIEW Tier");
|
||||
await waitFor(() => expect(screen.getByRole("button", { name: /save changes/i })).toBeEnabled());
|
||||
await user.click(screen.getByRole("button", { name: /save changes/i }));
|
||||
await waitFor(() => expect(modelPatchUpdateCall).toHaveBeenCalled());
|
||||
|
||||
|
|
@ -1252,8 +1305,9 @@ describe("EditAutoRouterModal prompt compression", () => {
|
|||
auto_router_model_compression: "model-compressor",
|
||||
});
|
||||
|
||||
await user.click(await screen.findByText("Advanced: Compression"));
|
||||
openAutoRouterAdvanced("Compression");
|
||||
await user.click(screen.getAllByRole("button", { name: "Clear" })[0]);
|
||||
await waitFor(() => expect(screen.getByRole("button", { name: /save changes/i })).toBeEnabled());
|
||||
await user.click(screen.getByRole("button", { name: /save changes/i }));
|
||||
|
||||
await waitFor(() =>
|
||||
|
|
@ -1276,18 +1330,19 @@ describe("EditAutoRouterModal prompt compression", () => {
|
|||
const stored = { auto_router_routing_compression: "none", auto_router_model_compression: "model-compressor" };
|
||||
const view = renderWithStoredCompression(stored);
|
||||
|
||||
await user.click(await screen.findByText("Advanced: Compression"));
|
||||
openAutoRouterAdvanced("Compression");
|
||||
await user.click(screen.getAllByRole("button", { name: "Clear" })[0]);
|
||||
await user.click(screen.getByRole("button", { name: "Cancel" }));
|
||||
expect(modelPatchUpdateCall).not.toHaveBeenCalled();
|
||||
view.unmount();
|
||||
|
||||
renderWithStoredCompression(stored);
|
||||
await user.click(await screen.findByText("Advanced: Compression"));
|
||||
openAutoRouterAdvanced("Compression");
|
||||
expect(screen.getByRole("combobox", { name: "Routing decision compression" })).toHaveValue("None (no compression)");
|
||||
await user.click(screen.getAllByRole("button", { name: "Clear" })[0]);
|
||||
await user.click(screen.getByRole("combobox", { name: "Routing decision compression" }));
|
||||
await user.click(screen.getByRole("option", { name: "None (no compression)" }));
|
||||
await waitFor(() => expect(screen.getByRole("button", { name: /save changes/i })).toBeEnabled());
|
||||
await user.click(screen.getByRole("button", { name: /save changes/i }));
|
||||
|
||||
await waitFor(() => expect(modelPatchUpdateCall).toHaveBeenCalled());
|
||||
|
|
@ -1298,6 +1353,7 @@ describe("EditAutoRouterModal prompt compression", () => {
|
|||
const user = userEvent.setup();
|
||||
renderWithStoredCompression();
|
||||
|
||||
await waitFor(() => expect(screen.getByRole("button", { name: /save changes/i })).toBeEnabled());
|
||||
await user.click(await screen.findByRole("button", { name: /save changes/i }));
|
||||
|
||||
await waitFor(() => expect(modelPatchUpdateCall).toHaveBeenCalled());
|
||||
|
|
@ -1313,6 +1369,7 @@ describe("EditAutoRouterModal prompt compression", () => {
|
|||
const user = userEvent.setup();
|
||||
renderWithStoredCompression(stored);
|
||||
|
||||
await waitFor(() => expect(screen.getByRole("button", { name: /save changes/i })).toBeEnabled());
|
||||
await user.click(await screen.findByRole("button", { name: /save changes/i }));
|
||||
|
||||
await waitFor(() => expect(modelPatchUpdateCall).toHaveBeenCalled());
|
||||
|
|
@ -1332,7 +1389,7 @@ describe("EditAutoRouterModal prompt compression", () => {
|
|||
auto_router_model_compression: "none",
|
||||
});
|
||||
|
||||
await user.click(await screen.findByText("Advanced: Compression"));
|
||||
openAutoRouterAdvanced("Compression");
|
||||
|
||||
expect(await screen.findByRole("combobox", { name: "Routing decision compression" })).toHaveValue("headroom-a");
|
||||
expect(screen.getByRole("radio", { name: "Use a different compression" })).toBeChecked();
|
||||
|
|
@ -1346,6 +1403,7 @@ describe("EditAutoRouterModal prompt compression", () => {
|
|||
auto_router_model_compression: "none",
|
||||
});
|
||||
|
||||
await waitFor(() => expect(screen.getByRole("button", { name: /save changes/i })).toBeEnabled());
|
||||
await user.click(await screen.findByRole("button", { name: /save changes/i }));
|
||||
|
||||
await waitFor(() => expect(modelPatchUpdateCall).toHaveBeenCalled());
|
||||
|
|
@ -1388,10 +1446,11 @@ describe("EditAutoRouterModal classifier vision", () => {
|
|||
const user = userEvent.setup();
|
||||
renderModal();
|
||||
|
||||
await user.click(await screen.findByText("Advanced: Classification Method"));
|
||||
openAutoRouterAdvanced("Classification Method");
|
||||
expect(screen.getByRole("switch", { name: "Use images for classification" })).toBeChecked();
|
||||
expect(screen.getByLabelText("Maximum images per request")).toHaveValue("2");
|
||||
|
||||
await waitFor(() => expect(screen.getByRole("button", { name: /save changes/i })).toBeEnabled());
|
||||
await user.click(screen.getByRole("button", { name: /save changes/i }));
|
||||
|
||||
await waitFor(() => expect(modelPatchUpdateCall).toHaveBeenCalled());
|
||||
|
|
@ -1402,11 +1461,44 @@ describe("EditAutoRouterModal classifier vision", () => {
|
|||
const user = userEvent.setup();
|
||||
renderModal();
|
||||
|
||||
await user.click(await screen.findByText("Advanced: Classification Method"));
|
||||
openAutoRouterAdvanced("Classification Method");
|
||||
await user.click(screen.getByRole("switch", { name: "Use images for classification" }));
|
||||
await waitFor(() => expect(screen.getByRole("button", { name: /save changes/i })).toBeEnabled());
|
||||
await user.click(screen.getByRole("button", { name: /save changes/i }));
|
||||
|
||||
await waitFor(() => expect(modelPatchUpdateCall).toHaveBeenCalled());
|
||||
expect(savedConfig().classifier_llm_config).not.toHaveProperty("vision");
|
||||
});
|
||||
});
|
||||
|
||||
describe("EditAutoRouterModal availability checks", () => {
|
||||
it("blocks a rapid save while checking a changed definition and recovers after reset", async () => {
|
||||
testQueryClient.clear();
|
||||
modelPatchUpdateCall.mockClear();
|
||||
vi.mocked(apiClient.post).mockResolvedValue({ allowances: [], error: null });
|
||||
const user = userEvent.setup();
|
||||
renderModal();
|
||||
await waitFor(() => expect(screen.getByRole("button", { name: "Save Changes" })).toBeEnabled());
|
||||
let complete: ((result: unknown) => void) | undefined;
|
||||
vi.mocked(apiClient.post).mockImplementationOnce(
|
||||
() =>
|
||||
new Promise((resolve) => {
|
||||
complete = resolve;
|
||||
}),
|
||||
);
|
||||
await user.click(screen.getByRole("button", { name: "Edit tiers" }));
|
||||
fireEvent.change(screen.getByLabelText("Definition for tier 1"), { target: { value: "Custom definition" } });
|
||||
expect(screen.getByRole("button", { name: "Save Changes" })).toBeDisabled();
|
||||
await user.click(screen.getByRole("button", { name: "Save Changes" }));
|
||||
await waitFor(() => expect(complete).toBeDefined());
|
||||
expect(screen.getByRole("button", { name: "Save Changes" })).toBeDisabled();
|
||||
expect(modelPatchUpdateCall).not.toHaveBeenCalled();
|
||||
await act(async () => complete?.({ allowances: [], error: "Custom tiers have no available allowance" }));
|
||||
expect(await screen.findByRole("alert")).toHaveTextContent("Custom tiers have no available allowance");
|
||||
expect(screen.getByRole("button", { name: "Save Changes" })).toBeDisabled();
|
||||
await user.click(screen.getByRole("button", { name: "Restore defaults" }));
|
||||
await waitFor(() => expect(screen.getByRole("button", { name: "Save Changes" })).toBeEnabled());
|
||||
await user.click(screen.getByRole("button", { name: "Save Changes" }));
|
||||
await waitFor(() => expect(modelPatchUpdateCall).toHaveBeenCalledOnce());
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
import { AutoRouterAvailabilityContext, useAutoRouterAvailability } from "../add_model/AutoRouterAvailability";
|
||||
import AutoRouterClassifierTabs from "../add_model/AutoRouterClassifierTabs";
|
||||
import { usesClassifierContext } from "../add_model/classifier_types";
|
||||
export type { StoredComplexityRouterConfig } from "../add_model/build_complexity_router_config";
|
||||
|
|
@ -244,6 +245,20 @@ const EditAutoRouterModal: React.FC<EditAutoRouterModalProps> = ({
|
|||
});
|
||||
const isComplexityRouterModel = isComplexityRouter(modelData?.litellm_params);
|
||||
|
||||
const routerAvailability = useAutoRouterAvailability(
|
||||
accessToken,
|
||||
{
|
||||
saved_model_id: modelData?.model_info?.id,
|
||||
team_id: modelData?.model_info?.team_id,
|
||||
complexity_router_config: buildUpdatedComplexityRouterConfig(
|
||||
modelData?.litellm_params?.complexity_router_config,
|
||||
complexityRouterConfig,
|
||||
customTechnicalKeywords,
|
||||
{ keywordTierRules, escalationKeywords, semanticMatchingEnabled, embeddingModel, matchThreshold },
|
||||
),
|
||||
},
|
||||
isVisible && isComplexityRouterModel,
|
||||
);
|
||||
const schema = useMemo(
|
||||
() => (isComplexityRouterModel ? complexityRouterSchema : semanticRouterSchema),
|
||||
[isComplexityRouterModel],
|
||||
|
|
@ -253,7 +268,7 @@ const EditAutoRouterModal: React.FC<EditAutoRouterModalProps> = ({
|
|||
// Mirrors the create form: the button says why it is unavailable and disables on the same
|
||||
// answer. Tiers use this modal's own rule, which allows a partly filled router, so an edit that
|
||||
// is legal today stays legal.
|
||||
const submitBlockedReason = !isComplexityRouterModel
|
||||
const configBlockedReason = !isComplexityRouterModel
|
||||
? null
|
||||
: (complexityRouterConfig.custom_tier_set
|
||||
? getCustomTierRowsError(complexityRouterConfig.custom_tier_set) ??
|
||||
|
|
@ -270,6 +285,8 @@ const EditAutoRouterModal: React.FC<EditAutoRouterModalProps> = ({
|
|||
? customDimensionsError(complexityRouterConfig.custom_dimensions)
|
||||
: null);
|
||||
|
||||
const submitBlockedReason = configBlockedReason ?? routerAvailability.saveBlockedReason;
|
||||
|
||||
useEffect(() => {
|
||||
if (isVisible && modelData) {
|
||||
initializeForm();
|
||||
|
|
@ -384,6 +401,10 @@ const EditAutoRouterModal: React.FC<EditAutoRouterModalProps> = ({
|
|||
};
|
||||
|
||||
const saveValues = async (values: EditAutoRouterFormValues) => {
|
||||
if (routerAvailability.saveBlockedReason) {
|
||||
toast.fromError(routerAvailability.saveBlockedReason);
|
||||
return;
|
||||
}
|
||||
if (isComplexityRouterModel) {
|
||||
const { tiers, custom_tier_set, classifier_llm_config } = complexityRouterConfig;
|
||||
const rows = activeTierRows(complexityRouterConfig);
|
||||
|
|
@ -575,143 +596,145 @@ const EditAutoRouterModal: React.FC<EditAutoRouterModalProps> = ({
|
|||
);
|
||||
|
||||
return (
|
||||
<Dialog open={isVisible} onOpenChange={(open) => !open && onCancel()}>
|
||||
<DialogContent className="max-h-[90vh] overflow-y-auto sm:max-w-4xl">
|
||||
<TooltipProvider>
|
||||
<DialogHeader>
|
||||
<DialogTitle>Edit Auto Router Configuration</DialogTitle>
|
||||
<DialogDescription>
|
||||
Edit the auto router configuration including routing logic, default models, and access settings.
|
||||
</DialogDescription>
|
||||
</DialogHeader>
|
||||
<AutoRouterAvailabilityContext.Provider value={routerAvailability}>
|
||||
<Dialog open={isVisible} onOpenChange={(open) => !open && onCancel()}>
|
||||
<DialogContent className="max-h-[90vh] overflow-y-auto sm:max-w-4xl">
|
||||
<TooltipProvider>
|
||||
<DialogHeader>
|
||||
<DialogTitle>Edit Auto Router Configuration</DialogTitle>
|
||||
<DialogDescription>
|
||||
Edit the auto router configuration including routing logic, default models, and access settings.
|
||||
</DialogDescription>
|
||||
</DialogHeader>
|
||||
|
||||
<form onSubmit={(event) => event.preventDefault()} noValidate>
|
||||
<FieldGroup>
|
||||
{routerNameField}
|
||||
<form onSubmit={(event) => event.preventDefault()} noValidate>
|
||||
<FieldGroup>
|
||||
{routerNameField}
|
||||
|
||||
{isComplexityRouterModel ? (
|
||||
/* Complexity Router Configuration */
|
||||
<div className="w-full">
|
||||
<AutoRouterClassifierTabs value={complexityRouterConfig} onChange={setComplexityRouterConfig}>
|
||||
<ComplexityRouterConfig
|
||||
editingTiers={editingTiers}
|
||||
onEditingTiersChange={setEditingTiers}
|
||||
showValidationErrors={showValidationErrors}
|
||||
modelInfo={modelInfo}
|
||||
value={complexityRouterConfig}
|
||||
onChange={(config) => {
|
||||
setComplexityRouterConfig(config);
|
||||
}}
|
||||
customTechnicalKeywords={customTechnicalKeywords}
|
||||
onCustomTechnicalKeywordsChange={setCustomTechnicalKeywords}
|
||||
keywordTierRules={keywordTierRules}
|
||||
onKeywordTierRulesChange={setKeywordTierRules}
|
||||
keywordRulesError={getKeywordTierRulesError(
|
||||
keywordTierRules,
|
||||
activeTierRows(complexityRouterConfig),
|
||||
)}
|
||||
semanticMatchingEnabled={semanticMatchingEnabled}
|
||||
onSemanticMatchingEnabledChange={setSemanticMatchingEnabled}
|
||||
embeddingModel={embeddingModel}
|
||||
onEmbeddingModelChange={setEmbeddingModel}
|
||||
matchThreshold={matchThreshold}
|
||||
onMatchThresholdChange={setMatchThreshold}
|
||||
escalationKeywords={escalationKeywords}
|
||||
onEscalationKeywordsChange={setEscalationKeywords}
|
||||
autoRouterCompression={autoRouterCompression}
|
||||
onAutoRouterCompressionChange={isMemberManaged ? undefined : setAutoRouterCompression}
|
||||
/>
|
||||
</AutoRouterClassifierTabs>
|
||||
</div>
|
||||
) : (
|
||||
<>
|
||||
{/* Router Configuration Builder */}
|
||||
{isComplexityRouterModel ? (
|
||||
/* Complexity Router Configuration */
|
||||
<div className="w-full">
|
||||
<RouterConfigBuilder
|
||||
modelInfo={modelInfo}
|
||||
value={routerConfig}
|
||||
onChange={(config) => {
|
||||
setRouterConfig(config);
|
||||
}}
|
||||
/>
|
||||
<AutoRouterClassifierTabs value={complexityRouterConfig} onChange={setComplexityRouterConfig}>
|
||||
<ComplexityRouterConfig
|
||||
editingTiers={editingTiers}
|
||||
onEditingTiersChange={setEditingTiers}
|
||||
showValidationErrors={showValidationErrors}
|
||||
modelInfo={modelInfo}
|
||||
value={complexityRouterConfig}
|
||||
onChange={(config) => {
|
||||
setComplexityRouterConfig(config);
|
||||
}}
|
||||
customTechnicalKeywords={customTechnicalKeywords}
|
||||
onCustomTechnicalKeywordsChange={setCustomTechnicalKeywords}
|
||||
keywordTierRules={keywordTierRules}
|
||||
onKeywordTierRulesChange={setKeywordTierRules}
|
||||
keywordRulesError={getKeywordTierRulesError(
|
||||
keywordTierRules,
|
||||
activeTierRows(complexityRouterConfig),
|
||||
)}
|
||||
semanticMatchingEnabled={semanticMatchingEnabled}
|
||||
onSemanticMatchingEnabledChange={setSemanticMatchingEnabled}
|
||||
embeddingModel={embeddingModel}
|
||||
onEmbeddingModelChange={setEmbeddingModel}
|
||||
matchThreshold={matchThreshold}
|
||||
onMatchThresholdChange={setMatchThreshold}
|
||||
escalationKeywords={escalationKeywords}
|
||||
onEscalationKeywordsChange={setEscalationKeywords}
|
||||
autoRouterCompression={autoRouterCompression}
|
||||
onAutoRouterCompressionChange={isMemberManaged ? undefined : setAutoRouterCompression}
|
||||
/>
|
||||
</AutoRouterClassifierTabs>
|
||||
</div>
|
||||
) : (
|
||||
<>
|
||||
{/* Router Configuration Builder */}
|
||||
<div className="w-full">
|
||||
<RouterConfigBuilder
|
||||
modelInfo={modelInfo}
|
||||
value={routerConfig}
|
||||
onChange={(config) => {
|
||||
setRouterConfig(config);
|
||||
}}
|
||||
/>
|
||||
</div>
|
||||
|
||||
<FormField control={form.control} name="auto_router_default_model" label="Default Model">
|
||||
<FormField control={form.control} name="auto_router_default_model" label="Default Model">
|
||||
{({ id, value, onChange, "aria-invalid": ariaInvalid, "aria-describedby": ariaDescribedBy }) => (
|
||||
<ModelChoiceCombobox
|
||||
id={id}
|
||||
value={value}
|
||||
onChange={onChange}
|
||||
choices={modelChoices}
|
||||
placeholder="Select a default model"
|
||||
ariaInvalid={ariaInvalid}
|
||||
ariaDescribedBy={ariaDescribedBy}
|
||||
/>
|
||||
)}
|
||||
</FormField>
|
||||
|
||||
<FormField control={form.control} name="auto_router_embedding_model" label="Embedding Model">
|
||||
{({ id, value, onChange, "aria-invalid": ariaInvalid, "aria-describedby": ariaDescribedBy }) => (
|
||||
<ModelChoiceCombobox
|
||||
id={id}
|
||||
value={value}
|
||||
onChange={onChange}
|
||||
choices={modelChoices}
|
||||
placeholder="Select an embedding model"
|
||||
ariaInvalid={ariaInvalid}
|
||||
ariaDescribedBy={ariaDescribedBy}
|
||||
/>
|
||||
)}
|
||||
</FormField>
|
||||
</>
|
||||
)}
|
||||
|
||||
{userRole === "Admin" && !isMemberManaged && (
|
||||
<FormField
|
||||
control={form.control}
|
||||
name="model_access_group"
|
||||
label={labelWithHint("Model Access Groups", "Control who can access this auto router")}
|
||||
>
|
||||
{({ id, value, onChange, "aria-invalid": ariaInvalid, "aria-describedby": ariaDescribedBy }) => (
|
||||
<ModelChoiceCombobox
|
||||
<AccessGroupTagsCombobox
|
||||
id={id}
|
||||
value={value}
|
||||
onChange={onChange}
|
||||
choices={modelChoices}
|
||||
placeholder="Select a default model"
|
||||
options={modelAccessGroups}
|
||||
ariaInvalid={ariaInvalid}
|
||||
ariaDescribedBy={ariaDescribedBy}
|
||||
/>
|
||||
)}
|
||||
</FormField>
|
||||
)}
|
||||
</FieldGroup>
|
||||
</form>
|
||||
|
||||
<FormField control={form.control} name="auto_router_embedding_model" label="Embedding Model">
|
||||
{({ id, value, onChange, "aria-invalid": ariaInvalid, "aria-describedby": ariaDescribedBy }) => (
|
||||
<ModelChoiceCombobox
|
||||
id={id}
|
||||
value={value}
|
||||
onChange={onChange}
|
||||
choices={modelChoices}
|
||||
placeholder="Select an embedding model"
|
||||
ariaInvalid={ariaInvalid}
|
||||
ariaDescribedBy={ariaDescribedBy}
|
||||
/>
|
||||
)}
|
||||
</FormField>
|
||||
</>
|
||||
)}
|
||||
|
||||
{userRole === "Admin" && !isMemberManaged && (
|
||||
<FormField
|
||||
control={form.control}
|
||||
name="model_access_group"
|
||||
label={labelWithHint("Model Access Groups", "Control who can access this auto router")}
|
||||
>
|
||||
{({ id, value, onChange, "aria-invalid": ariaInvalid, "aria-describedby": ariaDescribedBy }) => (
|
||||
<AccessGroupTagsCombobox
|
||||
id={id}
|
||||
value={value}
|
||||
onChange={onChange}
|
||||
options={modelAccessGroups}
|
||||
ariaInvalid={ariaInvalid}
|
||||
ariaDescribedBy={ariaDescribedBy}
|
||||
/>
|
||||
)}
|
||||
</FormField>
|
||||
)}
|
||||
</FieldGroup>
|
||||
</form>
|
||||
|
||||
<DialogFooter>
|
||||
<Button variant="outline" onClick={onCancel}>
|
||||
Cancel
|
||||
</Button>
|
||||
{submitBlockedReason === null ? (
|
||||
<Button disabled={loading} onClick={handleSubmit}>
|
||||
{loading && <UiLoadingSpinner className="size-4" />}
|
||||
Save Changes
|
||||
<DialogFooter>
|
||||
<Button variant="outline" onClick={onCancel}>
|
||||
Cancel
|
||||
</Button>
|
||||
) : (
|
||||
<Tooltip>
|
||||
<TooltipTrigger
|
||||
render={
|
||||
<Button disabled onClick={handleSubmit}>
|
||||
Save Changes
|
||||
</Button>
|
||||
}
|
||||
/>
|
||||
<TooltipContent>{submitBlockedReason}</TooltipContent>
|
||||
</Tooltip>
|
||||
)}
|
||||
</DialogFooter>
|
||||
</TooltipProvider>
|
||||
</DialogContent>
|
||||
</Dialog>
|
||||
{submitBlockedReason === null ? (
|
||||
<Button disabled={loading} onClick={handleSubmit}>
|
||||
{loading && <UiLoadingSpinner className="size-4" />}
|
||||
Save Changes
|
||||
</Button>
|
||||
) : (
|
||||
<Tooltip>
|
||||
<TooltipTrigger
|
||||
render={
|
||||
<Button disabled onClick={handleSubmit}>
|
||||
Save Changes
|
||||
</Button>
|
||||
}
|
||||
/>
|
||||
<TooltipContent>{submitBlockedReason}</TooltipContent>
|
||||
</Tooltip>
|
||||
)}
|
||||
</DialogFooter>
|
||||
</TooltipProvider>
|
||||
</DialogContent>
|
||||
</Dialog>
|
||||
</AutoRouterAvailabilityContext.Provider>
|
||||
);
|
||||
};
|
||||
|
||||
|
|
|
|||
|
|
@ -146,13 +146,13 @@ describe("autorouter_presets", () => {
|
|||
expect(config.classifier_context_window_size).toBe(0);
|
||||
expect(config.classifier_context_per_turn_chars).toBeUndefined();
|
||||
expect(getRequiredModelsInPreset(lite)).toEqual(
|
||||
new Set(["deepseek-v4-flash", "muse-spark-1.2", "kimi-k3", "claude-opus-5"]),
|
||||
new Set(["deepseek-v4-flash", "muse-spark-1.3", "kimi-k3", "claude-opus-5-5"]),
|
||||
);
|
||||
});
|
||||
|
||||
it("pins the anthropic preset's reasoning tier to Fable 5.1 at high thinking", () => {
|
||||
const config = getPresetByKey("anthropic_family")!.complexity_router_config;
|
||||
expect(config.tiers.COMPLEX).toEqual(["claude-opus-5"]);
|
||||
expect(config.tiers.COMPLEX).toEqual(["claude-opus-5-5"]);
|
||||
expect(config.tiers.REASONING).toEqual(["claude-fable-5-1"]);
|
||||
expect(config.tier_model_configs).toEqual({
|
||||
REASONING: [{ model_name: "claude-fable-5-1", litellm_params: { reasoning_effort: "high" } }],
|
||||
|
|
@ -162,7 +162,7 @@ describe("autorouter_presets", () => {
|
|||
// Kimi K3 at max needs the map to declare max for kimi-k3, which is the commit below this one.
|
||||
it("pins the lite preset's per-tier reasoning efforts", () => {
|
||||
expect(getPresetByKey("lite")!.complexity_router_config.tier_model_configs).toEqual({
|
||||
MEDIUM: [{ model_name: "muse-spark-1.2", litellm_params: { reasoning_effort: "xhigh" } }],
|
||||
MEDIUM: [{ model_name: "muse-spark-1.3", litellm_params: { reasoning_effort: "xhigh" } }],
|
||||
COMPLEX: [{ model_name: "kimi-k3", litellm_params: { reasoning_effort: "max" } }],
|
||||
});
|
||||
});
|
||||
|
|
@ -222,7 +222,7 @@ describe("autorouter_presets", () => {
|
|||
const lite = getPresetByKey("lite")!;
|
||||
const prefill = buildPresetPrefill(lite.complexity_router_config, groupsOnly(getRequiredModelsInPreset(lite)));
|
||||
expect(prefill.complexityRouterConfig.tier_model_params).toEqual({
|
||||
MEDIUM: { "muse-spark-1.2": { reasoning_effort: "xhigh" } },
|
||||
MEDIUM: { "muse-spark-1.3": { reasoning_effort: "xhigh" } },
|
||||
COMPLEX: { "kimi-k3": { reasoning_effort: "max" } },
|
||||
});
|
||||
});
|
||||
|
|
@ -230,9 +230,9 @@ describe("autorouter_presets", () => {
|
|||
it("pins the OpenAI preset to the Luna, Terra, Sol, and Astra progression", () => {
|
||||
const preset = getPresetByKey("openai_family")!;
|
||||
const expectedTiers = {
|
||||
SIMPLE: ["gpt-5.6-luna"],
|
||||
SIMPLE: ["gpt-6-luna"],
|
||||
MEDIUM: ["gpt-5.6-terra"],
|
||||
COMPLEX: ["gpt-5.6-sol"],
|
||||
COMPLEX: ["gpt-6-sol"],
|
||||
REASONING: ["gpt-6-astra"],
|
||||
};
|
||||
expect(preset.complexity_router_config.tiers).toEqual(expectedTiers);
|
||||
|
|
@ -248,19 +248,19 @@ describe("autorouter_presets", () => {
|
|||
it("pins the 1M context preset to Luna, Terra, Sol, and Opus at high thinking", () => {
|
||||
const preset = getPresetByKey("1m_context")!;
|
||||
const expectedTiers = {
|
||||
SIMPLE: ["gpt-5.6-luna"],
|
||||
SIMPLE: ["gpt-6-luna"],
|
||||
MEDIUM: ["gpt-5.6-terra"],
|
||||
COMPLEX: ["gpt-5.6-sol"],
|
||||
REASONING: ["claude-opus-5"],
|
||||
COMPLEX: ["gpt-6-sol"],
|
||||
REASONING: ["claude-opus-5-5"],
|
||||
};
|
||||
expect(preset.complexity_router_config.classifier_type).toBe("heuristic_v2");
|
||||
expect(preset.complexity_router_config.tiers).toEqual(expectedTiers);
|
||||
expect(preset.complexity_router_config.tier_model_configs).toEqual({
|
||||
REASONING: [{ model_name: "claude-opus-5", litellm_params: { reasoning_effort: "high" } }],
|
||||
REASONING: [{ model_name: "claude-opus-5-5", litellm_params: { reasoning_effort: "high" } }],
|
||||
});
|
||||
const prefill = buildPresetPrefill(preset.complexity_router_config, groupsOnly(getRequiredModelsInPreset(preset)));
|
||||
expect(prefill.complexityRouterConfig.tier_model_params).toEqual({
|
||||
REASONING: { "claude-opus-5": { reasoning_effort: "high" } },
|
||||
REASONING: { "claude-opus-5-5": { reasoning_effort: "high" } },
|
||||
});
|
||||
});
|
||||
|
||||
|
|
@ -270,15 +270,15 @@ describe("autorouter_presets", () => {
|
|||
expect(config.classifier_type).toBe("heuristic");
|
||||
expect(config.classifier_llm_config).toBeUndefined();
|
||||
const expectedTiers = {
|
||||
SIMPLE: ["gemini-2.5-flash-lite"],
|
||||
MEDIUM: ["gemini-3.1-flash-lite"],
|
||||
COMPLEX: ["gemini-3.7-flash"],
|
||||
SIMPLE: ["gemini-3.5-flash-lite"],
|
||||
MEDIUM: ["gemini-3.8-flash"],
|
||||
COMPLEX: ["gemini-3.8-flash"],
|
||||
REASONING: ["gemini-3.1-pro-preview"],
|
||||
};
|
||||
expect(config.tiers).toEqual(expectedTiers);
|
||||
const required = getRequiredModelsInPreset(gemini);
|
||||
for (const model of required) expect(model).not.toMatch(/-latest$/);
|
||||
expect(required.size).toBe(4);
|
||||
expect(required.size).toBe(new Set(Object.values(expectedTiers).flat()).size);
|
||||
});
|
||||
|
||||
it("collects every tier model as a required model", () => {
|
||||
|
|
|
|||
87
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
87
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
|
|
@ -1234,6 +1234,23 @@ export interface paths {
|
|||
patch?: never;
|
||||
trace?: never;
|
||||
};
|
||||
"/auto_router/availability": {
|
||||
parameters: {
|
||||
query?: never;
|
||||
header?: never;
|
||||
path?: never;
|
||||
cookie?: never;
|
||||
};
|
||||
get?: never;
|
||||
put?: never;
|
||||
/** Get Auto Router Availability */
|
||||
post: operations["get_auto_router_availability_auto_router_availability_post"];
|
||||
delete?: never;
|
||||
options?: never;
|
||||
head?: never;
|
||||
patch?: never;
|
||||
trace?: never;
|
||||
};
|
||||
"/auto_router/benchmarks": {
|
||||
parameters: {
|
||||
query?: never;
|
||||
|
|
@ -24124,6 +24141,43 @@ export interface components {
|
|||
[key: string]: unknown;
|
||||
} | null;
|
||||
};
|
||||
/** AutoRouterAllowance */
|
||||
AutoRouterAllowance: {
|
||||
/**
|
||||
* Available
|
||||
* @default true
|
||||
*/
|
||||
available: boolean;
|
||||
/** Key */
|
||||
key: string;
|
||||
/** Limit */
|
||||
limit: number | null;
|
||||
/** Remaining */
|
||||
remaining: number | null;
|
||||
/**
|
||||
* Used By This Router
|
||||
* @default false
|
||||
*/
|
||||
used_by_this_router: boolean;
|
||||
};
|
||||
/** AutoRouterAvailabilityRequest */
|
||||
AutoRouterAvailabilityRequest: {
|
||||
/** Complexity Router Config */
|
||||
complexity_router_config?: {
|
||||
[key: string]: unknown;
|
||||
} | null;
|
||||
/** Saved Model Id */
|
||||
saved_model_id?: string | null;
|
||||
/** Team Id */
|
||||
team_id?: string | null;
|
||||
};
|
||||
/** AutoRouterAvailabilityResponse */
|
||||
AutoRouterAvailabilityResponse: {
|
||||
/** Allowances */
|
||||
allowances: components["schemas"]["AutoRouterAllowance"][];
|
||||
/** Error */
|
||||
error?: string | null;
|
||||
};
|
||||
/**
|
||||
* AutoRouterBenchmarkGroup
|
||||
* @description One auto-router's slice of the benchmarks.
|
||||
|
|
@ -44147,6 +44201,39 @@ export interface operations {
|
|||
};
|
||||
};
|
||||
};
|
||||
get_auto_router_availability_auto_router_availability_post: {
|
||||
parameters: {
|
||||
query?: never;
|
||||
header?: never;
|
||||
path?: never;
|
||||
cookie?: never;
|
||||
};
|
||||
requestBody: {
|
||||
content: {
|
||||
"application/json": components["schemas"]["AutoRouterAvailabilityRequest"];
|
||||
};
|
||||
};
|
||||
responses: {
|
||||
/** @description Successful Response */
|
||||
200: {
|
||||
headers: {
|
||||
[name: string]: unknown;
|
||||
};
|
||||
content: {
|
||||
"application/json": components["schemas"]["AutoRouterAvailabilityResponse"];
|
||||
};
|
||||
};
|
||||
/** @description Validation Error */
|
||||
422: {
|
||||
headers: {
|
||||
[name: string]: unknown;
|
||||
};
|
||||
content: {
|
||||
"application/json": components["schemas"]["HTTPValidationError"];
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
get_auto_router_benchmarks_auto_router_benchmarks_get: {
|
||||
parameters: {
|
||||
query?: {
|
||||
|
|
|
|||
34
ui/litellm-dashboard/tests/autoRouterSetup.ts
Normal file
34
ui/litellm-dashboard/tests/autoRouterSetup.ts
Normal file
|
|
@ -0,0 +1,34 @@
|
|||
import userEvent from "@testing-library/user-event";
|
||||
import { chooseSelectOption, fireEvent, screen } from "./test-utils";
|
||||
|
||||
const groups: Record<string, string> = {
|
||||
"Classification Method": "Classifier tuning",
|
||||
"Heuristic Keyword Overrides": "Classifier tuning",
|
||||
"Ignore Custom Tags": "Classifier tuning",
|
||||
Affinity: "Sessions and efficiency",
|
||||
"Adaptive Routing": "Sessions and efficiency",
|
||||
Compression: "Sessions and efficiency",
|
||||
"Response Format": "Compatibility",
|
||||
};
|
||||
|
||||
export const openAutoRouterAdvanced = (section: string) => {
|
||||
const advanced = screen.getByRole("button", { name: /^Advanced settings/ });
|
||||
if (advanced.getAttribute("aria-expanded") !== "true") fireEvent.click(advanced);
|
||||
const group = screen.getByRole("button", { name: groups[section] ?? "Routing rules and recovery" });
|
||||
if (group.getAttribute("aria-expanded") !== "true") fireEvent.click(group);
|
||||
};
|
||||
|
||||
export const selectAutoRouterOption = async (field: string, option: string) => {
|
||||
if (field === "Heuristic" || field === "Routing approach") {
|
||||
const user = userEvent.setup();
|
||||
fireEvent.click(screen.getByRole("button", { name: field }));
|
||||
await user.click(await screen.findByRole("menuitemradio", { name: new RegExp(`^${option}`) }));
|
||||
return;
|
||||
}
|
||||
await chooseSelectOption(userEvent.setup(), screen.getByRole("combobox", { name: field }), new RegExp(`^${option}`));
|
||||
};
|
||||
|
||||
export const selectAutoRouterApproach = async (option: string) => {
|
||||
await userEvent.click(screen.getByRole("radio", { name: "LLM" }));
|
||||
await selectAutoRouterOption("Routing approach", option);
|
||||
};
|
||||
Loading…
Add table
Reference in a new issue