From 2157351004d63e8a3db0f51c32bd8fe9c69ad6d5 Mon Sep 17 00:00:00 2001 From: moe-berri Date: Tue, 22 Sep 2026 18:03:32 -0700 Subject: [PATCH] feat(ui): simplify auto-router setup and clarify feature limits (#42625) * feat(ui): simplify auto-router setup and clarify feature limits * fix(ui): validate auto-router drafts before saving * fix: keep auto-router allowances consistent after deletes and refreshes --- litellm/proxy/_types.py | 1 + .../auto_router_endpoints.py | 50 ++ .../model_management_endpoints.py | 13 +- .../auto_router_availability.py | 148 +++++ litellm/proxy/proxy_server.py | 9 + .../public_endpoints/autorouter_presets.json | 36 +- .../complexity_router/fuse_presets.json | 23 +- .../auto_router_tuning_baseline.py | 16 +- .../auto_router_endpoints.py | 19 + .../test_auto_router_endpoints.py | 153 ++++- .../test_model_management_endpoints.py | 145 ++++- .../test_auto_router_availability.py | 196 +++++++ .../proxy/proxy_server/test_lifecycle.py | 54 +- .../proxy/proxy_server/test_proxy_config.py | 41 +- .../public_endpoints/test_public_endpoints.py | 8 +- .../test_auto_router_tuning_baseline.py | 61 +- .../add_model/AutoRouterAvailability.tsx | 165 ++++++ ...oRouterClassifierTabs.integration.test.tsx | 255 ++++++-- .../add_model/AutoRouterClassifierTabs.tsx | 310 ++++++++-- .../add_model/ClassificationMethodConfig.tsx | 143 ++--- .../add_model/ClassifierPrimarySettings.tsx | 98 ++++ .../add_model/ClassifierTypeRadios.tsx | 2 +- .../ComplexityRouterAdvancedSections.tsx | 118 +++- ...mplexityRouterConfig.integration.test.tsx} | 461 ++++++++------- .../add_model/ComplexityRouterConfig.tsx | 36 +- ...plexityRouterFastMode.integration.test.tsx | 9 +- ...ecastClassifierConfig.integration.test.tsx | 23 +- .../add_model/ForecastClassifierConfig.tsx | 357 ++++++------ .../JevClassifierConfig.integration.test.tsx | 32 +- .../add_model/JevClassifierConfig.tsx | 45 +- .../JevConnectionTest.integration.test.tsx | 6 +- .../add_model/NonReasoningTierToggle.tsx | 2 +- .../components/add_model/RoutingOptions.tsx | 25 +- .../components/add_model/TierConfigIntro.tsx | 31 +- ... add_auto_router_tab.integration.test.tsx} | 442 ++++++++++---- .../add_model/add_auto_router_tab.tsx | 542 +++++++++--------- .../add_model/auto_router_connection_test.tsx | 10 +- ...dit_auto_router_modal.integration.test.tsx | 214 +++++-- .../edit_auto_router_modal.tsx | 267 +++++---- .../src/lib/autorouter_presets.test.ts | 30 +- ui/litellm-dashboard/src/lib/http/schema.d.ts | 87 +++ ui/litellm-dashboard/tests/autoRouterSetup.ts | 34 ++ 42 files changed, 3364 insertions(+), 1353 deletions(-) create mode 100644 litellm/proxy/management_helpers/auto_router_availability.py create mode 100644 tests/test_litellm/proxy/management_helpers/test_auto_router_availability.py create mode 100644 ui/litellm-dashboard/src/components/add_model/AutoRouterAvailability.tsx create mode 100644 ui/litellm-dashboard/src/components/add_model/ClassifierPrimarySettings.tsx rename ui/litellm-dashboard/src/components/add_model/{ComplexityRouterConfig.test.tsx => ComplexityRouterConfig.integration.test.tsx} (85%) rename ui/litellm-dashboard/src/components/add_model/{add_auto_router_tab.test.tsx => add_auto_router_tab.integration.test.tsx} (77%) create mode 100644 ui/litellm-dashboard/tests/autoRouterSetup.ts diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 76df8790b92..c7273738fd0 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -937,6 +937,7 @@ class LiteLLMRoutes(enum.Enum): # proxy admin, or team admin naming their own team via team_id "/auto_router/test_routing", "/auto_router/validate_complexity_router_config", + "/auto_router/availability", # Per-session auto-router read - the endpoint scopes the row to the caller's own key hash "/auto_router/session", "/cost/predict-cache", diff --git a/litellm/proxy/management_endpoints/auto_router_endpoints.py b/litellm/proxy/management_endpoints/auto_router_endpoints.py index 03fc58622ce..2ae8639fe61 100644 --- a/litellm/proxy/management_endpoints/auto_router_endpoints.py +++ b/litellm/proxy/management_endpoints/auto_router_endpoints.py @@ -58,6 +58,8 @@ from litellm.router_utils.auto_router_model_naming import ( ) from litellm.types.management_endpoints.auto_router_endpoints import ( SHADOW_EVAL_TURN_VALVE, + AutoRouterAvailabilityRequest, + AutoRouterAvailabilityResponse, AutoRouterBenchmarkGroup, AutoRouterBenchmarksResponse, AutoRouterBenchmarkTotals, @@ -391,6 +393,54 @@ async def validate_complexity_router_config( return ComplexityRouterConfigValidationResponse(valid=error is None, error=error) +@router.post( + "/auto_router/availability", + tags=["model management"], # mutable-ok: FastAPI requires a list + response_model=AutoRouterAvailabilityResponse, +) +async def get_auto_router_availability( + data: AutoRouterAvailabilityRequest, + user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)], +) -> AutoRouterAvailabilityResponse: + from litellm.proxy.management_helpers.auto_router_availability import auto_router_availability + from litellm.proxy.proxy_server import ( + _license_check, # pyright: ignore[reportPrivateUsage] # same entitlement owner as the model write gate + heuristic_v1_tuning_baselines, + llm_router, + proxy_config, + ) + + member_team: Final = await _authorize_router_dry_run(user_api_key_dict, data.team_id) + rows: Final = proxy_config.auto_router_db_catalog + if rows is None or llm_router is None: + raise HTTPException(status_code=503, detail="Auto-router availability is unavailable") + saved: Final = next((row for row in rows if row.model_id == data.saved_model_id), None) + if data.saved_model_id is not None: + if saved is None: + raise HTTPException(status_code=404, detail="Saved auto router is unavailable") + if user_api_key_dict.user_role != LitellmUserRoles.PROXY_ADMIN and ( + saved.team_id != data.team_id or (member_team is not None and saved.created_by != user_api_key_dict.user_id) + ): + raise HTTPException(status_code=403, detail="Cannot check another user's auto router") + existing: Final = saved.deployment if saved is not None else None + others: Final = tuple(row.deployment for row in rows if row is not saved) + tuple(llm_router.config_deployments()) + candidate: Final = MappingProxyType( + { + "litellm_params": MappingProxyType( + {"model": "auto_router/complexity_router", "complexity_router_config": data.complexity_router_config} + ), + "model_info": MappingProxyType({"id": data.saved_model_id or "availability-new-router", "db_model": True}), + } + ) + return auto_router_availability( + others=others, + existing=existing, + candidate=candidate, + baselines=heuristic_v1_tuning_baselines, + limit=_license_check.auto_router_capability_limit(), + ) + + async def _resolve_saved_routing_test( data: AutoRouterRoutingTestRequest, user_api_key_dict: UserAPIKeyAuth, diff --git a/litellm/proxy/management_endpoints/model_management_endpoints.py b/litellm/proxy/management_endpoints/model_management_endpoints.py index fcadcfe2cae..615a528b552 100644 --- a/litellm/proxy/management_endpoints/model_management_endpoints.py +++ b/litellm/proxy/management_endpoints/model_management_endpoints.py @@ -1782,10 +1782,11 @@ async def delete_team_models( # Under MODEL_RECONCILE_LOCK, for the same reason as delete_model: the rows are # gone, but a reconcile holding a pre-delete snapshot would upsert these ids back # onto this pod. The lock orders the eviction after any in-flight reconcile. - if llm_router is not None: - from litellm.proxy.proxy_server import MODEL_RECONCILE_LOCK + from litellm.proxy.proxy_server import MODEL_RECONCILE_LOCK, proxy_config - async with MODEL_RECONCILE_LOCK: + async with MODEL_RECONCILE_LOCK: + proxy_config.remove_auto_router_catalog_entries(frozenset(deleted_model_ids)) + if llm_router is not None: for model_id in deleted_model_ids: llm_router.delete_deployment(id=model_id) @@ -2194,6 +2195,7 @@ async def delete_model( llm_router, premium_user, prisma_client, + proxy_config, proxy_logging_obj, store_model_in_db, user_api_key_cache, @@ -2245,8 +2247,9 @@ async def delete_model( # this pod serving a model the database no longer has, until the next # reconcile. Taking the lock orders this eviction after any such in-flight # reconcile's re-add, so the eviction is the last word. - if llm_router is not None: - async with MODEL_RECONCILE_LOCK: + async with MODEL_RECONCILE_LOCK: + proxy_config.remove_auto_router_catalog_entries(frozenset({model_info.id})) + if llm_router is not None: llm_router.delete_deployment(id=model_info.id) # Runs after the row delete so the sibling check sees post-delete state. diff --git a/litellm/proxy/management_helpers/auto_router_availability.py b/litellm/proxy/management_helpers/auto_router_availability.py new file mode 100644 index 00000000000..52cd9f0499b --- /dev/null +++ b/litellm/proxy/management_helpers/auto_router_availability.py @@ -0,0 +1,148 @@ +from collections.abc import Mapping, Sequence +from copy import deepcopy +from dataclasses import dataclass +from types import MappingProxyType +from typing import Final + +from pydantic import BaseModel, Json, TypeAdapter, ValidationError + +from litellm.proxy.common_utils.encrypt_decrypt_utils import decrypt_value_helper +from litellm.router_utils.auto_router_model_naming import ( + GATED_AUTO_ROUTER_CAPABILITIES, + capability_limit_violation, + classify_strategy_router_model, + count_capability_routers, + gated_capability_of, +) +from litellm.router_utils.auto_router_tuning_baseline import ( + is_mutable_tuned_candidate, + mutable_tuned_identities, + tuning_quota_violation, +) +from litellm.types.management_endpoints.auto_router_endpoints import ( + AutoRouterAllowance, + AutoRouterAvailabilityResponse, +) + + +class _CatalogModelInfo(BaseModel): + team_id: str | None = None + + +class _CatalogSource(BaseModel): + model_id: str + created_by: str | None = None + litellm_params: Json[dict[str, object]] | dict[str, object] + model_info: Json[_CatalogModelInfo] | _CatalogModelInfo | None = None + + +@dataclass(frozen=True, slots=True) +class AutoRouterCatalogEntry: + model_id: str + team_id: str | None + created_by: str | None + deployment: Mapping[str, object] + + +def _catalog_field(value: object, key: str) -> object: + if not isinstance(value, str): + return deepcopy(value) + return decrypt_value_helper(value, key=key, exception_type="debug", return_original_value=True) + + +def build_auto_router_catalog(rows: Sequence[object]) -> tuple[AutoRouterCatalogEntry, ...] | None: + try: + sources: Final = TypeAdapter(tuple[_CatalogSource, ...]).validate_python(rows, from_attributes=True) + except ValidationError: + return None + return tuple( + AutoRouterCatalogEntry( + model_id=row.model_id, + team_id=row.model_info.team_id if row.model_info is not None else None, + created_by=row.created_by, + deployment=MappingProxyType( + { + "litellm_params": MappingProxyType( + { + "model": model, + "complexity_router_config": _catalog_field( + row.litellm_params.get("complexity_router_config"), "complexity_router_config" + ), + } + ), + "model_info": MappingProxyType({"id": row.model_id, "db_model": True}), + } + ), + ) + for row in sources + if isinstance(model := _catalog_field(row.litellm_params.get("model"), "model"), str) + and classify_strategy_router_model(model) == "complexity" + ) + + +def auto_router_availability( + *, + others: Sequence[Mapping[str, object]], + existing: Mapping[str, object] | None, + candidate: Mapping[str, object], + baselines: Mapping[str, str] | None, + limit: int | None, +) -> AutoRouterAvailabilityResponse: + existing_params: Final = None if existing is None else existing.get("litellm_params") + candidate_params: Final = candidate.get("litellm_params") + owned: Final = gated_capability_of(existing_params) if isinstance(existing_params, Mapping) else None + claimed: Final = gated_capability_of(candidate_params) if isinstance(candidate_params, Mapping) else None + counts: Final = tuple( + (capability, count_capability_routers(others, capability=capability)) + for capability in GATED_AUTO_ROUTER_CAPABILITIES + ) + tuned_count: Final = len(mutable_tuned_identities(others, baselines)) if baselines is not None else 0 + allowances: Final = tuple( + AutoRouterAllowance( + key=capability.key, + limit=limit, + remaining=None if limit is None else max(0, limit - held), + used_by_this_router=owned is capability, + ) + for capability, held in counts + ) + capability_error: Final = next( + ( + capability_limit_violation(capability=capability, held=held + 1, limit=limit) + for capability, held in counts + if capability is claimed + ), + None, + ) + tuning_error: Final = ( + tuning_quota_violation(candidate=candidate, others=others, baselines=baselines, limit=limit) + if baselines is not None + else None + ) + capability_labels: Final = { + "heuristic_v2": "Heuristic v2", + "capability": "Capability", + "llm_v2": "Fuse v2", + "tier_or_classifier_prompt": "Custom tiers or classifier instructions", + } + return AutoRouterAvailabilityResponse( + allowances=( + *allowances, + AutoRouterAllowance( + key="heuristic_tuning", + limit=limit, + remaining=None if limit is None or baselines is None else max(0, limit - tuned_count), + available=limit is None or baselines is not None, + used_by_this_router=bool( + existing is not None and baselines is not None and is_mutable_tuned_candidate(existing, baselines) + ), + ), + ), + error=( + f"{capability_labels[claimed.key]} has no available allowance. Choose another option or free an existing allowance." + if capability_error is not None and claimed is not None + else "These scoring rules need an available Rule-based tuning allowance. Check the weights, thresholds, keywords, and custom dimensions in Advanced settings. Model choices do not use this allowance." + if tuning_error is not None + else None + ), + ) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index acb36fc4a59..dab7decd4dc 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -132,6 +132,7 @@ from litellm.proxy.common_utils.callback_utils import ( strip_callback_config, ) from litellm.proxy.common_utils.realtime_utils import _realtime_request_body +from litellm.proxy.management_helpers.auto_router_availability import AutoRouterCatalogEntry, build_auto_router_catalog from litellm.router_utils.access_windows import access_windows_config_error from litellm.router_utils.add_retry_fallback_headers import ( get_fallback_errors_from_headers, @@ -5068,6 +5069,7 @@ class ProxyConfig: def __init__(self) -> None: self.config: Mapping[str, object] = MappingProxyType({}) + self.auto_router_db_catalog: tuple[AutoRouterCatalogEntry, ...] | None = None self._last_semantic_filter_config: dict[str, object] | None = None self._last_websearch_interception_config: dict[str, object] | None = None self._last_hashicorp_vault_config: dict[str, object] | None = None @@ -7691,6 +7693,12 @@ class ProxyConfig: def _should_load_db_object(self, object_type: str | SupportedDBObjectType) -> bool: return should_load_db_object(object_type=object_type) + def remove_auto_router_catalog_entries(self, model_ids: frozenset[str]) -> None: + if self.auto_router_db_catalog is not None: + self.auto_router_db_catalog = tuple( + row for row in self.auto_router_db_catalog if row.model_id not in model_ids + ) + async def _get_models_from_db(self, prisma_client: PrismaClient) -> Sequence[_ProxyModelRow] | None: """ Fetch all model deployments from the DB. @@ -7711,6 +7719,7 @@ class ProxyConfig: new_models: Final[Sequence[_ProxyModelRow]] = await ModelRepository( WriterPinnedClient(prisma_client.db) ).table.find_many() + self.auto_router_db_catalog = build_auto_router_catalog(new_models) return new_models except Exception as e: verbose_proxy_logger.exception( diff --git a/litellm/proxy/public_endpoints/autorouter_presets.json b/litellm/proxy/public_endpoints/autorouter_presets.json index 7a251afc076..1c2f96350c6 100644 --- a/litellm/proxy/public_endpoints/autorouter_presets.json +++ b/litellm/proxy/public_endpoints/autorouter_presets.json @@ -1,18 +1,18 @@ { "1m_context": { "label": "1M Context", - "description": "Routes across models with 1M-token context windows: Luna for simple queries, Terra for medium, Sol for complex, Opus 5 at high thinking for reasoning.", + "description": "Routes across models with 1M-token context windows: GPT-6 Luna for simple queries, GPT-5.6 Terra for medium, GPT-6 Sol for complex, Opus 5.5 at high thinking for reasoning.", "complexity_router_config": { "tiers": { - "SIMPLE": ["gpt-5.6-luna"], + "SIMPLE": ["gpt-6-luna"], "MEDIUM": ["gpt-5.6-terra"], - "COMPLEX": ["gpt-5.6-sol"], - "REASONING": ["claude-opus-5"] + "COMPLEX": ["gpt-6-sol"], + "REASONING": ["claude-opus-5-5"] }, "tier_model_configs": { "REASONING": [ { - "model_name": "claude-opus-5", + "model_name": "claude-opus-5-5", "litellm_params": { "reasoning_effort": "high" } } ] @@ -28,12 +28,12 @@ }, "anthropic_family": { "label": "Anthropic Family", - "description": "Routes across the Claude model family: Haiku for simple queries, Sonnet for medium, Opus for complex, Fable 5.1 at high thinking for reasoning.", + "description": "Routes across the Claude model family: Haiku for simple queries, Sonnet for medium, Opus 5.5 for complex, Fable 5.1 at high thinking for reasoning.", "complexity_router_config": { "tiers": { "SIMPLE": ["claude-haiku-4-5"], "MEDIUM": ["claude-sonnet-5"], - "COMPLEX": ["claude-opus-5"], + "COMPLEX": ["claude-opus-5-5"], "REASONING": ["claude-fable-5-1"] }, "tier_model_configs": { @@ -55,12 +55,12 @@ }, "gemini_family": { "label": "Gemini Family", - "description": "Routes across the Gemini model family: Flash Lite 2.5 for simple queries, Flash Lite 3.1 for medium, Flash 3.7 for complex, Pro 3.1 for reasoning-heavy requests.", + "description": "Routes across the Gemini model family: Flash Lite 3.5 for simple queries, Flash 3.8 for medium and complex queries, Pro 3.1 for reasoning-heavy requests.", "complexity_router_config": { "tiers": { - "SIMPLE": ["gemini-2.5-flash-lite"], - "MEDIUM": ["gemini-3.1-flash-lite"], - "COMPLEX": ["gemini-3.7-flash"], + "SIMPLE": ["gemini-3.5-flash-lite"], + "MEDIUM": ["gemini-3.8-flash"], + "COMPLEX": ["gemini-3.8-flash"], "REASONING": ["gemini-3.1-pro-preview"] }, "classifier_type": "heuristic", @@ -74,18 +74,18 @@ }, "lite": { "label": "Lite", - "description": "Cost-optimized routing across providers: DeepSeek V4 Flash for simple queries, Muse Spark 1.2 at xhigh for medium, Kimi K3 at max for complex, Claude Opus 5 for reasoning. An LLM classifier with the agentic rubric assigns tiers.", + "description": "Cost-optimized routing across providers: DeepSeek V4 Flash for simple queries, Muse Spark 1.3 at xhigh for medium, Kimi K3 at max for complex, Claude Opus 5.5 for reasoning. An LLM classifier with the agentic rubric assigns tiers.", "complexity_router_config": { "tiers": { "SIMPLE": ["deepseek-v4-flash"], - "MEDIUM": ["muse-spark-1.2"], + "MEDIUM": ["muse-spark-1.3"], "COMPLEX": ["kimi-k3"], - "REASONING": ["claude-opus-5"] + "REASONING": ["claude-opus-5-5"] }, "tier_model_configs": { "MEDIUM": [ { - "model_name": "muse-spark-1.2", + "model_name": "muse-spark-1.3", "litellm_params": { "reasoning_effort": "xhigh" } } ], @@ -113,12 +113,12 @@ }, "openai_family": { "label": "OpenAI Family", - "description": "Routes across the GPT model family: Luna for simple queries, Terra for medium, Sol for complex, Astra at xhigh thinking for reasoning.", + "description": "Routes across the GPT model family: GPT-6 Luna for simple queries, GPT-5.6 Terra for medium, GPT-6 Sol for complex, GPT-6 Astra at xhigh thinking for reasoning.", "complexity_router_config": { "tiers": { - "SIMPLE": ["gpt-5.6-luna"], + "SIMPLE": ["gpt-6-luna"], "MEDIUM": ["gpt-5.6-terra"], - "COMPLEX": ["gpt-5.6-sol"], + "COMPLEX": ["gpt-6-sol"], "REASONING": ["gpt-6-astra"] }, "tier_model_configs": { diff --git a/litellm/router_strategy/complexity_router/fuse_presets.json b/litellm/router_strategy/complexity_router/fuse_presets.json index 4006366dc25..d3f9c48c0d4 100644 --- a/litellm/router_strategy/complexity_router/fuse_presets.json +++ b/litellm/router_strategy/complexity_router/fuse_presets.json @@ -1,5 +1,5 @@ { - "version": "2026-09-17-v1", + "version": "2026-09-22-v1", "models": [ { "id": "gpt-6-astra-v1", @@ -8,6 +8,20 @@ "text": "OpenAI model for demanding end-to-end work, including reasoning, coding, research, and document tasks", "sources": ["https://developers.openai.com/api/docs/models/gpt-6-astra"] }, + { + "id": "gpt-6-sol-v1", + "label": "GPT-6 Sol", + "model": "gpt-6-sol", + "text": "OpenAI model for complex coding and agentic workflows, supporting reasoning and tool calling through the Responses API", + "sources": ["https://developers.openai.com/api/docs/models/gpt-6-sol"] + }, + { + "id": "gpt-6-luna-v1", + "label": "GPT-6 Luna", + "model": "gpt-6-luna", + "text": "OpenAI model for efficient, high-volume workloads, supporting reasoning and tool calling through the Responses API", + "sources": ["https://developers.openai.com/api/docs/models/gpt-6-luna"] + }, { "id": "gpt-5.6-sol-v1", "label": "GPT-5.6 Sol", @@ -63,6 +77,13 @@ "model": "claude-fable-5-1", "text": "Anthropic model for demanding reasoning, long-running agentic coding, and multistep research, with always-on adaptive thinking", "sources": ["https://platform.claude.com/docs/en/models/fable-5-1/overview"] + }, + { + "id": "claude-opus-5-5-v1", + "label": "Claude Opus 5.5", + "model": "claude-opus-5-5", + "text": "Anthropic model for complex reasoning and agentic work, supporting adaptive thinking and tool use", + "sources": ["https://platform.claude.com/docs/en/models/opus-5-5/overview"] } ], "harnesses": [ diff --git a/litellm/router_utils/auto_router_tuning_baseline.py b/litellm/router_utils/auto_router_tuning_baseline.py index 9699ab886b9..e87548bf6de 100644 --- a/litellm/router_utils/auto_router_tuning_baseline.py +++ b/litellm/router_utils/auto_router_tuning_baseline.py @@ -10,12 +10,10 @@ from pydantic import ValidationError from litellm.router_strategy.complexity_router.config import ComplexityRouterConfig -TUNING_BASELINE_PARAM_NAME: Final = "auto_router_tuning_baseline_v2" +# v2 hashes combine models and scoring rules; a new snapshot is required to separate them. +TUNING_BASELINE_PARAM_NAME: Final = "auto_router_tuning_baseline_v3" HEURISTIC_V1_TUNING_FIELDS: Final = ( - "tiers", - "tier_model_configs", - "classifier_type", "tier_boundaries", "reasoning_override_min_score", "token_thresholds", @@ -49,8 +47,10 @@ def tuning_fingerprint(complexity_router_config: object) -> str | None: validated: Final = ComplexityRouterConfig.model_validate(raw) except ValidationError: return None - supplied: Final = ((_TUNING_FIELD_SET - frozenset(("tier_model_configs",))) & frozenset(raw)) | ( - frozenset(("tier_model_configs",)) if validated.tier_model_configs else frozenset() + # The UI always writes this built-in marker. Freeze its spelling so future defaults cannot change recorded hashes. + default_escalation: Final = validated.escalation_keywords in (None, ["LITELLM ESCALATE"]) + supplied: Final = (_TUNING_FIELD_SET & frozenset(raw)) - ( + frozenset(("escalation_keywords",)) if default_escalation else frozenset() ) payload: Final = validated.model_dump( mode="json", @@ -148,9 +148,9 @@ def tuning_limit_violation(*, held: int, limit: int | None) -> str | None: if limit is None or held <= limit: return None return ( - f"At most {limit} auto-router(s) with changed heuristic scorer settings or tier models can be modified " + f"At most {limit} auto-router(s) with changed heuristic scoring rules can be modified " "without an auto-router license. Keep this router on its recorded settings, or revert the other changed " - "router to its baseline, or remove one of them." + "router to its baseline, or remove one of them. Selecting models does not use this allowance." ) diff --git a/litellm/types/management_endpoints/auto_router_endpoints.py b/litellm/types/management_endpoints/auto_router_endpoints.py index 93ea925bd9e..334ca0dfb08 100644 --- a/litellm/types/management_endpoints/auto_router_endpoints.py +++ b/litellm/types/management_endpoints/auto_router_endpoints.py @@ -44,6 +44,25 @@ class ComplexityRouterConfigValidationResponse(BaseModel): error: str | None = None +class AutoRouterAvailabilityRequest(BaseModel): + team_id: str | None = None + saved_model_id: str | None = None + complexity_router_config: Mapping[str, object] | None = None + + +class AutoRouterAllowance(BaseModel): + key: str + limit: int | None + remaining: int | None + used_by_this_router: bool = False + available: bool = True + + +class AutoRouterAvailabilityResponse(BaseModel): + allowances: tuple[AutoRouterAllowance, ...] + error: str | None = None + + class AutoRouterRoutingTestRequest(BaseModel): """A single request to classify against a complexity-router config that need not be saved yet. diff --git a/tests/test_litellm/proxy/management_endpoints/test_auto_router_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_auto_router_endpoints.py index a282ae731dd..ff3d19e8637 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_auto_router_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_auto_router_endpoints.py @@ -723,11 +723,13 @@ class TestAutoRouterBenchmarks: def test_savings_compare_only_the_current_estimated_cohort(self, estimated_turns: int) -> None: from litellm.proxy.management_endpoints.auto_router_endpoints import _benchmark_totals - row: Final = self.ROW.model_copy(update={ - "savings_estimated_turns": estimated_turns, - "savings_estimated_actual_spend": 2.0 if estimated_turns else 0.0, - "savings_estimated_saved_spend": -0.5 if estimated_turns else 0.0, - }) + row: Final = self.ROW.model_copy( + update={ + "savings_estimated_turns": estimated_turns, + "savings_estimated_actual_spend": 2.0 if estimated_turns else 0.0, + "savings_estimated_saved_spend": -0.5 if estimated_turns else 0.0, + } + ) totals: Final = _benchmark_totals(row) assert totals.spend == 10.0 assert totals.savings_estimated_turns == estimated_turns @@ -755,10 +757,16 @@ class TestAutoRouterBenchmarks: _summed_agg_row, ) - other = self.ROW.model_copy(update={ - "router_name": "auto-2", "sessions": 1, "turns": 10, "spend": 0.0, - "savings_estimated_turns": 10, "savings_estimated_actual_spend": 0.0, - }) + other = self.ROW.model_copy( + update={ + "router_name": "auto-2", + "sessions": 1, + "turns": 10, + "spend": 0.0, + "savings_estimated_turns": 10, + "savings_estimated_actual_spend": 0.0, + } + ) summed = _summed_agg_row([self.ROW, other]) totals = _benchmark_totals(summed) assert summed.sessions == 5 @@ -1091,18 +1099,27 @@ class TestAutoRouterSession: return lookups @pytest.mark.asyncio - @pytest.mark.parametrize("turns, estimated", [(3, True), (10, True), (10, False)], ids=["full", "partial", "legacy"]) + @pytest.mark.parametrize( + "turns, estimated", [(3, True), (10, True), (10, False)], ids=["full", "partial", "legacy"] + ) async def test_a_key_reads_its_own_session_with_the_baseline_its_turns_were_priced_against( - self, monkeypatch: pytest.MonkeyPatch, turns: int, estimated: bool, + self, + monkeypatch: pytest.MonkeyPatch, + turns: int, + estimated: bool, ) -> None: from litellm.proxy.management_endpoints.auto_router_endpoints import get_auto_router_session caller = UserAPIKeyAuth(api_key="sk-caller") - row: Final = {key: value for key, value in self.ROW.items() if estimated or not key.startswith("savings_estimated_")} + row: Final = { + key: value for key, value in self.ROW.items() if estimated or not key.startswith("savings_estimated_") + } spend: Final = 0.14 if turns == 3 else 10.0 if estimated and turns != 3: row["savings_estimated_saved_spend"] = -0.04 - self._rig(monkeypatch, [{**row, "api_key": caller.api_key, "session_id": "sess-1", "turns": turns, "spend": spend}]) + self._rig( + monkeypatch, [{**row, "api_key": caller.api_key, "session_id": "sess-1", "turns": turns, "spend": spend}] + ) response = await get_auto_router_session(user_api_key_dict=caller, session_id="sess-1") assert response.model_dump() == { "session_id": "sess-1", @@ -1159,10 +1176,18 @@ class TestAutoRouterSession: from litellm.proxy.management_endpoints.auto_router_endpoints import get_auto_router_session priced = {"anthropic/claude-opus-5": 2, "anthropic/claude-sonnet-5": 1} - self._rig(monkeypatch, [{ - **self.ROW, "api_key": ADMIN.api_key, "session_id": "s", - "baseline_models": {"old-baseline": 100}, "savings_estimated_baseline_models": priced, - }]) + self._rig( + monkeypatch, + [ + { + **self.ROW, + "api_key": ADMIN.api_key, + "session_id": "s", + "baseline_models": {"old-baseline": 100}, + "savings_estimated_baseline_models": priced, + } + ], + ) response = await get_auto_router_session(user_api_key_dict=ADMIN, session_id="s") assert response.baseline_model == "anthropic/claude-opus-5" assert response.baseline_models == priced @@ -3562,3 +3587,97 @@ async def test_start_shadow_eval_seeds_a_zero_funnel_row_per_leg(monkeypatch: py if "group_id" in call.kwargs.get("where", {}) ] assert group_reads == [] + + +@pytest.mark.asyncio +async def test_availability_counts_db_and_yaml_without_disclosing_router_names(monkeypatch): + from litellm.models.model import LiteLLM_ProxyModelTable + from litellm.proxy.management_helpers.auto_router_availability import build_auto_router_catalog + from litellm.types.management_endpoints.auto_router_endpoints import AutoRouterAvailabilityRequest + + row = LiteLLM_ProxyModelTable( + model_id="db-router", + model_name="private-team-router", + created_by="someone-else", + litellm_params={ + "model": "auto_router/complexity_router", + "complexity_router_config": {"classifier_type": "heuristic_v2"}, + }, + ) + yaml_row = { + "model_name": "private-yaml-router", + "litellm_params": { + "model": "auto_router/complexity_router", + "complexity_router_config": {"classifier_type": "capability"}, + }, + } + find_many = AsyncMock(side_effect=AssertionError("Availability must not query the model table")) + monkeypatch.setattr( + proxy_server, + "prisma_client", + SimpleNamespace(db=SimpleNamespace(litellm_proxymodeltable=SimpleNamespace(find_many=find_many))), + ) + monkeypatch.setattr(proxy_server.proxy_config, "auto_router_db_catalog", build_auto_router_catalog((row,))) + monkeypatch.setattr(proxy_server, "llm_router", SimpleNamespace(config_deployments=lambda: (yaml_row,))) + monkeypatch.setattr(proxy_server, "_license_check", SimpleNamespace(auto_router_capability_limit=lambda: 1)) + monkeypatch.setattr(proxy_server, "heuristic_v1_tuning_baselines", {}) + result = await auto_router_endpoints.get_auto_router_availability(AutoRouterAvailabilityRequest(), ADMIN) + assert {slot.key: slot.remaining for slot in result.allowances} == { + "heuristic_v2": 0, + "capability": 0, + "llm_v2": 1, + "tier_or_classifier_prompt": 1, + "heuristic_tuning": 1, + } + assert "private" not in result.model_dump_json() + edit = await auto_router_endpoints.get_auto_router_availability( + AutoRouterAvailabilityRequest( + saved_model_id="db-router", complexity_router_config={"classifier_type": "heuristic_v2"} + ), + ADMIN, + ) + assert edit.allowances[0].used_by_this_router + assert edit.allowances[0].remaining == 1 + assert edit.error is None + find_many.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_availability_denies_another_teams_edit_exemption(monkeypatch): + from litellm.models.model import LiteLLM_ProxyModelTable + from litellm.proxy.management_helpers.auto_router_availability import build_auto_router_catalog + from litellm.types.management_endpoints.auto_router_endpoints import AutoRouterAvailabilityRequest + + row = LiteLLM_ProxyModelTable( + model_id="other-router", + model_name="other", + created_by="other", + model_info={"team_id": "other-team"}, + litellm_params={"model": "auto_router/complexity_router"}, + ) + find_many = AsyncMock(side_effect=AssertionError("Availability must not query the model table")) + monkeypatch.setattr( + proxy_server, + "prisma_client", + SimpleNamespace(db=SimpleNamespace(litellm_proxymodeltable=SimpleNamespace(find_many=find_many))), + ) + monkeypatch.setattr(proxy_server.proxy_config, "auto_router_db_catalog", build_auto_router_catalog((row,))) + monkeypatch.setattr(proxy_server, "llm_router", SimpleNamespace(config_deployments=lambda: ())) + monkeypatch.setattr(auto_router_endpoints, "_authorize_router_dry_run", AsyncMock(return_value=None)) + with pytest.raises(HTTPException) as error: + await auto_router_endpoints.get_auto_router_availability( + AutoRouterAvailabilityRequest(team_id="own-team", saved_model_id="other-router"), + UserAPIKeyAuth(user_role=LitellmUserRoles.INTERNAL_USER, user_id="owner"), + ) + assert error.value.status_code == 403 + + +@pytest.mark.asyncio +async def test_availability_waits_for_the_first_complete_catalog(monkeypatch): + from litellm.types.management_endpoints.auto_router_endpoints import AutoRouterAvailabilityRequest + + monkeypatch.setattr(proxy_server.proxy_config, "auto_router_db_catalog", None) + monkeypatch.setattr(proxy_server, "llm_router", SimpleNamespace(config_deployments=lambda: ())) + with pytest.raises(HTTPException) as error: + await auto_router_endpoints.get_auto_router_availability(AutoRouterAvailabilityRequest(), ADMIN) + assert error.value.status_code == 503 diff --git a/tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py index bd252169131..8d618f4699b 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py @@ -3,6 +3,7 @@ import asyncio import contextlib import json from collections.abc import Iterator, Mapping +from types import SimpleNamespace from typing import Dict, Final, Optional from unittest.mock import AsyncMock, MagicMock, patch @@ -1222,6 +1223,117 @@ class TestDeleteModelClearsRouterRegistry: assert mock_router.complexity_routers.get("shared-name") is config_router +@pytest.fixture +def deleted_auto_router_catalog(monkeypatch): + from litellm.proxy import proxy_server + from litellm.proxy.management_helpers.auto_router_availability import build_auto_router_catalog + + rows = tuple( + LiteLLM_ProxyModelTable( + model_id=model_id, + model_name=f"model_name_{team_id}_{model_id}", + litellm_params={ + "model": "auto_router/complexity_router", + "complexity_router_config": {"classifier_type": classifier}, + }, + model_info={"id": model_id, "team_id": team_id}, + created_by="admin", + updated_by="admin", + blocked=True, + ) + for model_id, team_id, classifier in ( + ("deleted-router", "deleted-team", "heuristic_v2"), + ("surviving-router", "surviving-team", "llm_v2"), + ) + ) + config = proxy_server.ProxyConfig() + config.auto_router_db_catalog = build_auto_router_catalog(rows) + monkeypatch.setattr(proxy_server, "proxy_config", config) + monkeypatch.setattr(proxy_server, "MODEL_RECONCILE_LOCK", asyncio.Lock()) + monkeypatch.setattr(proxy_server, "llm_router", Router(model_list=[])) + monkeypatch.setattr(proxy_server, "_license_check", SimpleNamespace(auto_router_capability_limit=lambda: 1)) + monkeypatch.setattr(proxy_server, "heuristic_v1_tuning_baselines", {}) + return config, rows + + +class TestDeletedAutoRouterAvailability: + @pytest.mark.asyncio + @pytest.mark.parametrize("delete_succeeds,has_router", ((True, True), (True, False), (False, True))) + async def test_single_delete_releases_allowance_only_after_success( + self, monkeypatch, deleted_auto_router_catalog, delete_succeeds, has_router + ): + from litellm.proxy import proxy_server + from litellm.proxy.management_endpoints.auto_router_endpoints import get_auto_router_availability + from litellm.proxy.management_endpoints.model_management_endpoints import ModelInfoDelete, delete_model + from litellm.types.management_endpoints.auto_router_endpoints import AutoRouterAvailabilityRequest + + config, rows = deleted_auto_router_catalog + original = config.auto_router_db_catalog + row = rows[0].model_copy(update={"model_info": {"id": rows[0].model_id}}) + table = SimpleNamespace( + find_unique=AsyncMock(return_value=row), + delete=AsyncMock(return_value=row, side_effect=None if delete_succeeds else RuntimeError("delete failed")), + ) + prisma = SimpleNamespace( + db=SimpleNamespace(litellm_proxymodeltable=table, query_raw=AsyncMock(return_value=[])) + ) + monkeypatch.setattr(proxy_server, "prisma_client", prisma) + monkeypatch.setattr(proxy_server, "store_model_in_db", True) + admin = UserAPIKeyAuth(user_id="admin", user_role=LitellmUserRoles.PROXY_ADMIN) + request = AutoRouterAvailabilityRequest(complexity_router_config={"classifier_type": "heuristic_v2"}) + before = await get_auto_router_availability(request, admin) + assert before.error is not None + if not has_router: + monkeypatch.setattr(proxy_server, "llm_router", None) + + if not delete_succeeds: + with pytest.raises(ProxyException, match="delete failed"): + await delete_model(ModelInfoDelete(id=row.model_id), admin) + assert config.auto_router_db_catalog == original + return + + await delete_model(ModelInfoDelete(id=row.model_id), admin) + monkeypatch.setattr(proxy_server, "llm_router", Router(model_list=[])) + after = await get_auto_router_availability(request, admin) + assert after.error is None + assert {slot.key: slot.remaining for slot in after.allowances} == { + "heuristic_v2": 1, + "capability": 1, + "llm_v2": 0, + "tier_or_classifier_prompt": 1, + "heuristic_tuning": 1, + } + + @pytest.mark.asyncio + @pytest.mark.parametrize("has_router", (True, False)) + async def test_team_delete_releases_only_its_routers_allowance( + self, monkeypatch, deleted_auto_router_catalog, has_router + ): + from litellm.proxy import proxy_server + from litellm.proxy.management_endpoints.auto_router_endpoints import get_auto_router_availability + from litellm.types.management_endpoints.auto_router_endpoints import AutoRouterAvailabilityRequest + + _, rows = deleted_auto_router_catalog + prisma = _TxPrismaClient(rows) + deleted = await delete_team_models( + team_ids=["deleted-team"], prisma_client=prisma, llm_router=proxy_server.llm_router if has_router else None + ) + + assert deleted == ["deleted-router"] + after = await get_auto_router_availability( + AutoRouterAvailabilityRequest(complexity_router_config={"classifier_type": "heuristic_v2"}), + UserAPIKeyAuth(user_id="admin", user_role=LitellmUserRoles.PROXY_ADMIN), + ) + assert after.error is None + assert {slot.key: slot.remaining for slot in after.allowances} == { + "heuristic_v2": 1, + "capability": 1, + "llm_v2": 0, + "tier_or_classifier_prompt": 1, + "heuristic_tuning": 1, + } + + class TestUpdateModel: """ Tests for the update_model (POST /model/update) handler. @@ -5274,7 +5386,7 @@ class TestDeleteEvictionsHoldTheReconcileLock: """ @staticmethod - async def _assert_evicts_under_lock(monkeypatch, call_endpoint, model_id: str) -> None: + async def _assert_evicts_under_lock(monkeypatch, call_endpoint, model_id: str, config) -> None: """Run ``call_endpoint`` with the lock already held and assert it blocks. Holding MODEL_RECONCILE_LOCK stands in for a reconcile that is mid-flight. If @@ -5291,6 +5403,7 @@ class TestDeleteEvictionsHoldTheReconcileLock: """ lock = asyncio.Lock() monkeypatch.setattr("litellm.proxy.proxy_server.MODEL_RECONCILE_LOCK", lock) + stale_catalog = config.auto_router_db_catalog async with lock: task = asyncio.create_task(call_endpoint()) @@ -5301,16 +5414,19 @@ class TestDeleteEvictionsHoldTheReconcileLock: f"deleting {model_id} did not wait for MODEL_RECONCILE_LOCK -- an " f"in-flight reconcile can resurrect the deployment it just evicted" ) + config.auto_router_db_catalog = stale_catalog await asyncio.wait_for(task, timeout=5) + assert tuple(row.model_id for row in config.auto_router_db_catalog) == ("surviving-router",) @pytest.mark.asyncio - async def test_delete_model_waits_for_an_in_flight_reconcile(self, monkeypatch): + async def test_delete_model_waits_for_an_in_flight_reconcile(self, monkeypatch, deleted_auto_router_catalog): from litellm.proxy.management_endpoints.model_management_endpoints import ( ModelInfoDelete, delete_model, ) - model_id = "m-doomed" + config, rows = deleted_auto_router_catalog + model_id = rows[0].model_id row = MagicMock() row.model_dump.return_value = { "model_name": "gpt-4o", @@ -5347,16 +5463,17 @@ class TestDeleteEvictionsHoldTheReconcileLock: ), ) - await self._assert_evicts_under_lock(monkeypatch, call, model_id) + await self._assert_evicts_under_lock(monkeypatch, call, model_id, config) router.delete_deployment.assert_called_once_with(id=model_id) @pytest.mark.asyncio - async def test_delete_team_models_waits_for_an_in_flight_reconcile(self, monkeypatch): + async def test_delete_team_models_waits_for_an_in_flight_reconcile(self, monkeypatch, deleted_auto_router_catalog): from litellm.proxy.management_endpoints.model_management_endpoints import ( delete_team_models, ) - model_id = "m-team-doomed" + config, rows = deleted_auto_router_catalog + model_id = rows[0].model_id router = MagicMock() router.delete_deployment = MagicMock(return_value=True) @@ -5392,7 +5509,7 @@ class TestDeleteEvictionsHoldTheReconcileLock: team_ids=["team-1"], prisma_client=prisma, llm_router=router ) - await self._assert_evicts_under_lock(monkeypatch, call, model_id) + await self._assert_evicts_under_lock(monkeypatch, call, model_id, config) router.delete_deployment.assert_called_once_with(id=model_id) @@ -6092,7 +6209,8 @@ class TestStrategyRouterWriteValidation: _TUNED_A = {"classifier_type": "heuristic", "tiers": {"SIMPLE": "gpt-4o-mini", "MEDIUM": "gpt-4o"}} _TUNED_A_EDITED = {**_TUNED_A, "dimension_weights": {"codePresence": 0.9}} _TUNED_B = {"classifier_type": "heuristic", "tiers": {"SIMPLE": "gpt-4o-mini", "MEDIUM": "gpt-4.1"}} - _TUNED_B_EDITED = {**_TUNED_B, "tiers": {"SIMPLE": "gpt-4o", "MEDIUM": "gpt-4.1"}} + _TUNED_B_EDITED = {**_TUNED_B, "code_keywords": ["internal-api"]} + _MODELS_ONLY_B = {**_TUNED_B, "tiers": {"SIMPLE": "fast-model", "MEDIUM": "capable-model"}} @staticmethod def _db_router_row(model_id: str, config: Mapping[str, object]) -> dict[str, object]: @@ -6110,7 +6228,9 @@ class TestStrategyRouterWriteValidation: (1, ["a", "b"], {"a": "_TUNED_A", "b": "_TUNED_B"}, "a", "_TUNED_A_EDITED", "allowed"), (1, ["a", "b"], {"a": "_TUNED_A_EDITED", "b": "_TUNED_B"}, "a", "_TUNED_A_EDITED", "allowed"), (1, ["a", "b"], {"a": "_TUNED_A_EDITED", "b": "_TUNED_B"}, "b", "_TUNED_B_EDITED", "refused"), - (1, ["a", "b"], {"a": "_TUNED_A_EDITED", "b": "_TUNED_B"}, "c", "_TUNED_B", "refused"), + (1, ["a", "b"], {"a": "_TUNED_A_EDITED", "b": "_TUNED_B"}, "c", "_TUNED_B_EDITED", "refused"), + (1, ["a", "b"], {"a": "_TUNED_A_EDITED", "b": "_TUNED_B"}, "c", "_TUNED_B", "allowed"), + (1, ["a", "b"], {"a": "_TUNED_A_EDITED", "b": "_TUNED_B"}, "b", "_MODELS_ONLY_B", "allowed"), (1, ["a", "b"], {"a": "_TUNED_A_EDITED", "b": "_TUNED_B"}, "b", "_TUNED_B", "allowed"), (None, ["a", "b"], {"a": "_TUNED_A_EDITED", "b": "_TUNED_B"}, "b", "_TUNED_B_EDITED", "allowed"), (1, [], {}, "c", "_TUNED_B", "allowed"), @@ -6138,6 +6258,7 @@ class TestStrategyRouterWriteValidation: "_TUNED_A_EDITED": self._TUNED_A_EDITED, "_TUNED_B": self._TUNED_B, "_TUNED_B_EDITED": self._TUNED_B_EDITED, + "_MODELS_ONLY_B": self._MODELS_ONLY_B, } baselines = snapshot_tuning_baselines( [self._db_router_row(row_id, configs["_TUNED_A" if row_id == "a" else "_TUNED_B"]) for row_id in baseline_rows] @@ -6172,7 +6293,7 @@ class TestStrategyRouterWriteValidation: async with _auto_router_capability_slot(fake, effective_params=effective_params, model_id=candidate_id): pass assert exc_info.value.status_code == 403 - assert "changed heuristic scorer settings or tier models" in str(exc_info.value.detail) + assert "changed heuristic scoring rules" in str(exc_info.value.detail) assert "'auto_router' feature lifts the limit" in str(exc_info.value.detail) return async with _auto_router_capability_slot(fake, effective_params=effective_params, model_id=candidate_id) as table: @@ -6222,13 +6343,13 @@ class TestStrategyRouterWriteValidation: model_params=Deployment( model_name="second-tuned", litellm_params=LiteLLM_Params( - model="auto_router/complexity_router", complexity_router_config=self._TUNED_B + model="auto_router/complexity_router", complexity_router_config=self._TUNED_B_EDITED ), ), user_api_key_dict=admin, ) assert exc_info.value.code == "403" - assert "changed heuristic scorer settings or tier models" in str(exc_info.value.message) + assert "changed heuristic scoring rules" in str(exc_info.value.message) fake.tx_obj.litellm_proxymodeltable.create.assert_not_awaited() fake.litellm_proxymodeltable.create.assert_not_awaited() diff --git a/tests/test_litellm/proxy/management_helpers/test_auto_router_availability.py b/tests/test_litellm/proxy/management_helpers/test_auto_router_availability.py new file mode 100644 index 00000000000..027930d9ebb --- /dev/null +++ b/tests/test_litellm/proxy/management_helpers/test_auto_router_availability.py @@ -0,0 +1,196 @@ +from collections.abc import Mapping +from typing import Final +from types import SimpleNamespace + +from litellm.proxy.common_utils.encrypt_decrypt_utils import encrypt_value_helper + +import pytest + +from litellm.proxy.management_helpers.auto_router_availability import ( + auto_router_availability, + build_auto_router_catalog, +) +from litellm.router_utils.auto_router_tuning_baseline import snapshot_tuning_baselines + + +def deployment( + model_id: str, + classifier: str, + *, + model: str = "solver", + tuned: bool = False, + config: Mapping[str, object] | None = None, +) -> Mapping[str, object]: + return { + "model_name": model_id, + "model_info": {"id": model_id, "db_model": True}, + "litellm_params": { + "model": "auto_router/complexity_router", + "complexity_router_config": { + "classifier_type": classifier, + "tiers": {"SIMPLE": [model]}, + **({"code_keywords": ["internal-api"]} if tuned else {}), + **(config or {}), + }, + }, + } + + +@pytest.mark.parametrize("classifier", ("heuristic_v2", "capability", "llm_v2")) +def test_occupied_allowance_blocks_new_router_but_not_owner(classifier: str) -> None: + existing: Final = deployment("existing", classifier) + candidate: Final = deployment("new", classifier) + new: Final = auto_router_availability(others=(existing,), existing=None, candidate=candidate, baselines={}, limit=1) + edit: Final = auto_router_availability(others=(), existing=existing, candidate=existing, baselines={}, limit=1) + new_slot: Final = next(slot for slot in new.allowances if slot.key == classifier) + edit_slot: Final = next(slot for slot in edit.allowances if slot.key == classifier) + assert (new_slot.remaining, new_slot.used_by_this_router, new.error is not None) == (0, False, True) + assert (edit_slot.remaining, edit_slot.used_by_this_router, edit.error) == (1, True, None) + + +def test_edit_does_not_exempt_another_classifier_allowance() -> None: + existing: Final = deployment("existing", "capability") + result: Final = auto_router_availability( + others=(deployment("other", "llm_v2"),), + existing=existing, + candidate=deployment("existing", "llm_v2"), + baselines={}, + limit=1, + ) + assert result.error is not None + assert next(slot for slot in result.allowances if slot.key == "llm_v2").remaining == 0 + + +def test_model_selection_does_not_claim_occupied_scoring_allowance() -> None: + original: Final = deployment("legacy", "heuristic") + changed: Final = deployment("other", "heuristic", tuned=True) + baselines: Final = snapshot_tuning_baselines((original,)) + unchanged: Final = auto_router_availability( + others=(changed,), + existing=original, + candidate=original, + baselines=baselines, + limit=1, + ) + edited: Final = auto_router_availability( + others=(changed,), + existing=original, + candidate=deployment("legacy", "heuristic", model="new"), + baselines=baselines, + limit=1, + ) + assert unchanged.error is None + assert next(slot for slot in unchanged.allowances if slot.key == "heuristic_tuning").remaining == 0 + assert edited.error is None + tuned: Final = auto_router_availability( + others=(changed,), + existing=original, + candidate=deployment("legacy", "heuristic", tuned=True), + baselines=baselines, + limit=1, + ) + assert tuned.error is not None + assert "weights, thresholds, keywords, and custom dimensions" in tuned.error + + +def test_missing_baselines_are_reported_as_unknown() -> None: + result: Final = auto_router_availability( + others=(), + existing=None, + candidate=deployment("new", "heuristic"), + baselines=None, + limit=1, + ) + slot: Final = next(slot for slot in result.allowances if slot.key == "heuristic_tuning") + assert (slot.available, slot.remaining, slot.limit) == (False, None, 1) + + +def test_unlimited_entitlement_does_not_report_exhausted_allowances() -> None: + result: Final = auto_router_availability( + others=(deployment("other", "heuristic_v2"),), + existing=None, + candidate=deployment("new", "heuristic_v2"), + baselines=None, + limit=None, + ) + assert all(slot.available and slot.limit is None and slot.remaining is None for slot in result.allowances) + assert result.error is None + + +@pytest.mark.parametrize( + "customization", + ( + {"tier_definitions": [{"name": "SIMPLE"}, {"name": "AUDIT", "description": "Review risks"}]}, + {"classification_prompt": "Use the simplest sufficient tier"}, + {"classification_examples": "Review this code -> COMPLEX"}, + {"classifier_llm_config": {"model": "judge", "system_prompt": "Route by urgency"}}, + ), +) +def test_customization_owner_can_edit_models_and_restoring_defaults_clears_the_gate( + customization: Mapping[str, object], +) -> None: + owner: Final = deployment("owner", "llm", config=customization) + blocked: Final = auto_router_availability( + others=(owner,), existing=None, candidate=deployment("new", "llm", config=customization), baselines={}, limit=1 + ) + assert blocked.error is not None + assert "Custom tiers or classifier instructions" in blocked.error + edited: Final = auto_router_availability( + others=(), + existing=owner, + candidate=deployment("owner", "llm", model="new", config=customization), + baselines={}, + limit=1, + ) + assert edited.error is None + assert next(slot for slot in edited.allowances if slot.key == "tier_or_classifier_prompt").used_by_this_router + restored: Final = auto_router_availability( + others=(owner,), existing=None, candidate=deployment("new", "llm"), baselines={}, limit=1 + ) + assert restored.error is None + assert next(slot for slot in restored.allowances if slot.key == "tier_or_classifier_prompt").remaining == 0 + + +def test_restoring_tiers_does_not_exempt_a_retained_custom_prompt() -> None: + prompt: Final = {"classification_prompt": "Use the simplest sufficient tier"} + result: Final = auto_router_availability( + others=(deployment("owner", "llm", config=prompt),), + existing=None, + candidate=deployment("new", "llm", config=prompt), + baselines={}, + limit=1, + ) + assert result.error is not None + assert "Custom tiers or classifier instructions" in result.error + + +@pytest.mark.parametrize("blocked", (False, True)) +def test_catalog_keeps_unloaded_routers_and_ownership_without_provider_credentials(blocked: bool, monkeypatch) -> None: + monkeypatch.setenv("LITELLM_SALT_KEY", "catalog-test-key") + source: Final = SimpleNamespace( + model_id="saved", + created_by="owner", + model_info={"team_id": "team"}, + blocked=blocked, + litellm_params={ + "model": encrypt_value_helper("auto_router/complexity_router"), + "api_key": "private-key", + "complexity_router_config": {"classifier_type": "heuristic_v2"}, + }, + ) + provider: Final = SimpleNamespace(model_id="provider", litellm_params={"model": "openai/model"}) + catalog: Final = build_auto_router_catalog((source, provider)) + assert catalog is not None and len(catalog) == 1 + assert (catalog[0].model_id, catalog[0].team_id, catalog[0].created_by) == ("saved", "team", "owner") + assert catalog[0].deployment == { + "model_info": {"id": "saved", "db_model": True}, + "litellm_params": { + "model": "auto_router/complexity_router", + "complexity_router_config": {"classifier_type": "heuristic_v2"}, + }, + } + + +def test_catalog_distinguishes_missing_data_from_an_empty_model_table() -> None: + assert build_auto_router_catalog(()) == () + assert build_auto_router_catalog((SimpleNamespace(model_id="incomplete"),)) is None diff --git a/tests/test_litellm/proxy/proxy_server/test_lifecycle.py b/tests/test_litellm/proxy/proxy_server/test_lifecycle.py index 9954351fa2e..36d2e16d261 100644 --- a/tests/test_litellm/proxy/proxy_server/test_lifecycle.py +++ b/tests/test_litellm/proxy/proxy_server/test_lifecycle.py @@ -920,7 +920,7 @@ def test_proxy_startup_event_warns_for_global_budget_without_database(): @pytest.mark.asyncio -async def test_tuning_baseline_v2_is_created_alongside_the_legacy_row(): +async def test_tuning_baseline_v3_is_created_alongside_the_legacy_row(): from litellm.router_utils.auto_router_tuning_baseline import DEFAULT_TUNING_FINGERPRINT prisma_client = MagicMock() @@ -935,11 +935,61 @@ async def test_tuning_baseline_v2_is_created_alongside_the_legacy_row(): assert result == {'yaml:["a",[]]': DEFAULT_TUNING_FINGERPRINT} assert prisma_client.db.litellm_config.create.await_args.kwargs["data"] == { - "param_name": "auto_router_tuning_baseline_v2", + "param_name": "auto_router_tuning_baseline_v3", "param_value": json.dumps(dict(result)), } +@pytest.mark.asyncio +async def test_scorer_baseline_upgrade_preserves_existing_routers_and_is_not_refreshed_on_restart(): + from litellm.router_utils.auto_router_tuning_baseline import mutable_tuned_identities, snapshot_tuning_baselines + + deployments = [ + { + "model_name": name, + "litellm_params": { + "model": "auto_router/complexity_router", + "complexity_router_config": {"tiers": {"SIMPLE": name}, "code_keywords": [name]}, + }, + } + for name in ("a", "b") + ] + prisma_client = MagicMock() + prisma_client.db.litellm_config.find_unique = AsyncMock( + side_effect=lambda where: ( + MagicMock(param_value='{"legacy-router":"old-combined-hash"}') + if where["param_name"] == "auto_router_tuning_baseline_v2" + else None + ) + ) + prisma_client.db.litellm_config.create = AsyncMock() + + baseline = await ProxyStartupEvent._load_heuristic_v1_tuning_baselines(prisma_client, deployments) + + assert baseline == snapshot_tuning_baselines(deployments) + assert mutable_tuned_identities(deployments, baseline) == frozenset() + prisma_client.db.litellm_config.create.assert_awaited_once_with( + data={"param_name": "auto_router_tuning_baseline_v3", "param_value": json.dumps(dict(baseline))} + ) + prisma_client.db.litellm_config.find_unique.side_effect = None + prisma_client.db.litellm_config.find_unique.return_value = MagicMock(param_value=json.dumps(dict(baseline))) + changed = [ + { + "model_name": "a", + "litellm_params": { + "model": "auto_router/complexity_router", + "complexity_router_config": {"tiers": {"SIMPLE": "different-model"}, "code_keywords": ["new-rule"]}, + }, + } + ] + + reloaded = await ProxyStartupEvent._load_heuristic_v1_tuning_baselines(prisma_client, changed) + + assert reloaded == baseline + assert mutable_tuned_identities(changed, reloaded) == frozenset({'yaml:["a",[]]'}) + prisma_client.db.litellm_config.create.assert_awaited_once() + + @pytest.mark.asyncio async def test_tuning_baseline_waits_for_a_complete_db_model_census(monkeypatch): prisma_client = MagicMock() diff --git a/tests/test_litellm/proxy/proxy_server/test_proxy_config.py b/tests/test_litellm/proxy/proxy_server/test_proxy_config.py index b47cce43dcc..89cb8356289 100644 --- a/tests/test_litellm/proxy/proxy_server/test_proxy_config.py +++ b/tests/test_litellm/proxy/proxy_server/test_proxy_config.py @@ -875,9 +875,7 @@ class _ConfigTable: await asyncio.sleep(0) return _ConfigRow(param_value=value) if value is not None else None - async def upsert( - self, *, where: Mapping[str, str], data: Mapping[str, Mapping[str, str]] - ) -> _ConfigRow: + async def upsert(self, *, where: Mapping[str, str], data: Mapping[str, Mapping[str, str]]) -> _ConfigRow: param_name: Final = where["param_name"] value: Final = _CONFIG_VALUE.validate_json(data["update"]["param_value"]) self.rows[param_name] = value @@ -926,7 +924,9 @@ class _ConfigPrisma: self.db.litellm_config.upserted_param_names.append(param_name) -def _db_backed_proxy_config(monkeypatch, rows: Mapping[str, Mapping[str, JsonValue]]) -> tuple[ProxyConfig, _ConfigTable]: +def _db_backed_proxy_config( + monkeypatch, rows: Mapping[str, Mapping[str, JsonValue]] +) -> tuple[ProxyConfig, _ConfigTable]: table: Final = _ConfigTable(rows) monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", _ConfigPrisma(db=_ConfigDb(litellm_config=table))) monkeypatch.setattr("litellm.proxy.proxy_server.store_model_in_db", True) @@ -4750,9 +4750,7 @@ def test_validate_deployment_access_windows_rejects_malformed_time(): "model_name": "gpt-4o-shared", "litellm_params": {"model": "gpt-4o"}, "model_info": { - "access_windows": [ - {"start": "25:00", "end": "06:00", "timezone": "America/New_York", "team_ids": ["t"]} - ] + "access_windows": [{"start": "25:00", "end": "06:00", "timezone": "America/New_York", "team_ids": ["t"]}] }, } @@ -4767,9 +4765,7 @@ def test_validate_deployment_access_windows_rejects_unknown_timezone(): "model_name": "gpt-4o-shared", "litellm_params": {"model": "gpt-4o"}, "model_info": { - "access_windows": [ - {"start": "22:00", "end": "06:00", "timezone": "Mars/Olympus", "team_ids": ["t"]} - ] + "access_windows": [{"start": "22:00", "end": "06:00", "timezone": "Mars/Olympus", "team_ids": ["t"]}] }, } @@ -4799,3 +4795,28 @@ def test_validate_deployment_access_windows_accepts_valid_and_absent(): ) is None ) + + +@pytest.mark.asyncio +async def test_model_refresh_updates_availability_catalog_and_retains_it_on_db_failure(): + pc = ProxyConfig() + row = SimpleNamespace( + model_id="gated", + created_by="owner", + model_info={}, + litellm_params={ + "model": "auto_router/complexity_router", + "complexity_router_config": {"classifier_type": "heuristic_v2"}, + }, + ) + find_many = AsyncMock(side_effect=[[row], RuntimeError("database unavailable"), []]) + client = SimpleNamespace(db=SimpleNamespace(litellm_proxymodeltable=SimpleNamespace(find_many=find_many))) + assert pc.auto_router_db_catalog is None + assert await pc._get_models_from_db(client) == [row] + loaded = pc.auto_router_db_catalog + assert loaded is not None and loaded[0].model_id == "gated" + assert await pc._get_models_from_db(client) is None + assert pc.auto_router_db_catalog == loaded + assert await pc._get_models_from_db(client) == [] + assert pc.auto_router_db_catalog == () + assert find_many.await_count == 3 diff --git a/tests/test_litellm/proxy/public_endpoints/test_public_endpoints.py b/tests/test_litellm/proxy/public_endpoints/test_public_endpoints.py index 0f1ff3b024d..0dec44af402 100644 --- a/tests/test_litellm/proxy/public_endpoints/test_public_endpoints.py +++ b/tests/test_litellm/proxy/public_endpoints/test_public_endpoints.py @@ -1218,13 +1218,13 @@ def test_get_autorouter_presets_local_mode_serves_bundled_catalog( assert "anthropic_family" in payload assert payload["1m_context"]["complexity_router_config"]["classifier_type"] == "heuristic_v2" assert payload["1m_context"]["complexity_router_config"]["tiers"] == { - "SIMPLE": ["gpt-5.6-luna"], + "SIMPLE": ["gpt-6-luna"], "MEDIUM": ["gpt-5.6-terra"], - "COMPLEX": ["gpt-5.6-sol"], - "REASONING": ["claude-opus-5"], + "COMPLEX": ["gpt-6-sol"], + "REASONING": ["claude-opus-5-5"], } assert payload["1m_context"]["complexity_router_config"]["tier_model_configs"] == { - "REASONING": [{"model_name": "claude-opus-5", "litellm_params": {"reasoning_effort": "high"}}] + "REASONING": [{"model_name": "claude-opus-5-5", "litellm_params": {"reasoning_effort": "high"}}] } for preset in payload.values(): assert isinstance(preset["label"], str) diff --git a/tests/test_litellm/router_utils/test_auto_router_tuning_baseline.py b/tests/test_litellm/router_utils/test_auto_router_tuning_baseline.py index 75c115cd3ab..9307668d66f 100644 --- a/tests/test_litellm/router_utils/test_auto_router_tuning_baseline.py +++ b/tests/test_litellm/router_utils/test_auto_router_tuning_baseline.py @@ -33,21 +33,6 @@ _HISTORICAL_FINGERPRINTS: Final = ( {"custom_dimensions": [{"name": "sqlDdl", "weight": 0.4, "patterns": [r"\bCREATE\s{1,4}TABLE\b"]}]}, "814ce0017fc7f60a160b262f658d910e9bdf784e6139a4ba4f1e2657aa203950", ), - ( - { - "tiers": _TIERS, - "dimension_weights": {"codePresence": 0.3}, - "custom_dimensions": [ - { - "name": "internalFrameworks", - "weight": 0.2, - "keywords": ["orbitmesh", "fluxgate"], - "patterns": [r"\bALTER\s{1,4}TABLE\b"], - } - ], - }, - "38970dc9224e265ab38c89674563d8d0537822591f9239b45251db6f5ca6cc39", - ), ) @@ -78,11 +63,9 @@ class TestTuningFingerprint: {"tiers": {"SIMPLE": "x"}} ) - @pytest.mark.parametrize("field", sorted(set(HEURISTIC_V1_TUNING_FIELDS) - {"tier_model_configs"})) + @pytest.mark.parametrize("field", HEURISTIC_V1_TUNING_FIELDS) def test_every_tuning_field_changes_the_fingerprint(self, field: str) -> None: samples: dict[str, object] = { - "tiers": _ALT_TIERS, - "classifier_type": "heuristic_first", "tier_boundaries": {"simple_medium": 0.2, "medium_complex": 0.4, "complex_reasoning": 0.7}, "reasoning_override_min_score": 0.05, "token_thresholds": {"simple": 20, "complex": 500}, @@ -97,9 +80,6 @@ class TestTuningFingerprint: "keyword_tier_rules": [{"keywords": ["urgent"], "tier": "COMPLEX"}], } config: dict[str, object] = {field: samples[field]} - if field == "classifier_type": - config["heuristic_first_max_tier"] = "MEDIUM" - config["classifier_llm_config"] = {"model": "judge"} assert tuning_fingerprint(config) != DEFAULT_TUNING_FINGERPRINT def test_explicit_empty_tier_model_configs_follow_omission(self) -> None: @@ -126,12 +106,33 @@ class TestTuningFingerprint: != historical ) - def test_tier_model_overrides_change_the_fingerprint(self) -> None: + def test_tier_model_overrides_do_not_change_the_fingerprint(self) -> None: plain = tuning_fingerprint({"tiers": {"SIMPLE": "x"}}) with_override = tuning_fingerprint( {"tiers": {"SIMPLE": {"model_name": "x", "litellm_params": {"temperature": 0.1}}}} ) - assert plain != with_override + assert plain == with_override == DEFAULT_TUNING_FINGERPRINT + + @pytest.mark.parametrize("classifier_type", ("heuristic", "heuristic_first", "hybrid")) + def test_model_selection_and_classifier_switching_do_not_claim_tuning(self, classifier_type: str) -> None: + config: Final = { + "classifier_type": classifier_type, + **({"classifier_llm_config": {"model": "judge"}} if classifier_type != "heuristic" else {}), + **({"heuristic_first_max_tier": "MEDIUM"} if classifier_type == "heuristic_first" else {}), + **({"hybrid_boundary_margin": 0.1} if classifier_type == "hybrid" else {}), + "tiers": _ALT_TIERS, + "escalation_keywords": ["LITELLM ESCALATE"], + "tier_model_configs": {"COMPLEX": [{"model_name": "other-strong", "litellm_params": {"temperature": 0.1}}]}, + } + tuned: Final = _router("tuned", {"dimension_weights": {"codePresence": 0.9}}) + model_only: Final = _router("model-only", {"tiers": _TIERS}) + candidate: Final = _router("another", config) + assert tuning_fingerprint(config) == DEFAULT_TUNING_FINGERPRINT + assert tuning_quota_violation(candidate=candidate, others=(tuned, model_only), baselines={}, limit=1) is None + + def test_disabling_or_replacing_escalation_is_still_a_custom_rule(self) -> None: + assert tuning_fingerprint({"escalation_keywords": []}) != DEFAULT_TUNING_FINGERPRINT + assert tuning_fingerprint({"escalation_keywords": ["USE A STRONGER MODEL"]}) != DEFAULT_TUNING_FINGERPRINT def test_non_tuning_fields_do_not_change_the_fingerprint(self) -> None: assert ( @@ -230,17 +231,18 @@ class TestQuota: def test_router_added_after_snapshot_is_mutable_only_when_tuned(self) -> None: baselines = snapshot_tuning_baselines([_router("a", {"tiers": _TIERS})]) assert mutable_tuned_identities([_router("new", {})], baselines) == frozenset() - assert mutable_tuned_identities([_router("new", {"tiers": _TIERS})], baselines) == { - router_identity(_router("new", {})) - } + assert mutable_tuned_identities([_router("new", {"tiers": _TIERS})], baselines) == frozenset() + assert mutable_tuned_identities( + [_router("new", {"tiers": _TIERS, "code_keywords": ["internal-api"]})], baselines + ) == {router_identity(_router("new", {}))} def test_quota_matrix(self) -> None: legacy_a = _router("a", {"tiers": _TIERS}) legacy_b = _router("b", {"tiers": _ALT_TIERS}) baselines = snapshot_tuning_baselines([legacy_a, legacy_b]) edited_a = _router("a", {"tiers": _TIERS, "dimension_weights": {"codePresence": 0.9}}) - edited_b = _router("b", {"tiers": _TIERS}) - new_c = _router("c", {"tiers": _TIERS}) + edited_b = _router("b", {"tiers": _TIERS, "code_keywords": ["internal-api"]}) + new_c = _router("c", {"tiers": _TIERS, "code_keywords": ["internal-api"]}) assert tuning_quota_violation(candidate=edited_a, others=[legacy_b], baselines=baselines, limit=1) is None assert ( @@ -260,7 +262,7 @@ class TestQuota: legacy_a = _router("a", {"tiers": _TIERS}) legacy_b = _router("b", {"tiers": _ALT_TIERS}) baselines = snapshot_tuning_baselines([legacy_a, legacy_b]) - edited_b = _router("b", {"tiers": _TIERS}) + edited_b = _router("b", {"tiers": _TIERS, "code_keywords": ["internal-api"]}) assert tuning_quota_violation(candidate=edited_b, others=[legacy_a], baselines=baselines, limit=1) is None assert ( tuning_quota_violation(candidate=edited_b, others=[legacy_a, edited_b], baselines=baselines, limit=1) @@ -304,5 +306,6 @@ class TestQuota: assert message is not None assert "At most 1 auto-router(s)" in message assert "revert the other changed router to its baseline" in message + assert "Selecting models does not use this allowance" in message assert tuning_limit_violation(held=1, limit=1) is None assert tuning_limit_violation(held=5, limit=None) is None diff --git a/ui/litellm-dashboard/src/components/add_model/AutoRouterAvailability.tsx b/ui/litellm-dashboard/src/components/add_model/AutoRouterAvailability.tsx new file mode 100644 index 00000000000..b1233ef03b8 --- /dev/null +++ b/ui/litellm-dashboard/src/components/add_model/AutoRouterAvailability.tsx @@ -0,0 +1,165 @@ +import { createContext, useContext, useEffect, useState } from "react"; +import { useQuery, type UseQueryOptions } from "@tanstack/react-query"; +import { apiClient } from "@/components/networking"; +import { Popover, PopoverContent, PopoverTitle, PopoverTrigger } from "@/components/ui/popover"; +import type { components } from "@/lib/http/schema"; + +type Availability = components["schemas"]["AutoRouterAvailabilityResponse"]; +type Request = components["schemas"]["AutoRouterAvailabilityRequest"]; +export type Allowance = components["schemas"]["AutoRouterAllowance"]; + +type AvailabilityState = { + data?: Availability; + isPending: boolean; + isError: boolean; + isChecking?: boolean; + refetch?: () => unknown; +}; + +export const AutoRouterAvailabilityContext = createContext({ isPending: true, isError: false }); + +export const useAutoRouterAvailability = (accessToken: string, body: Request, enabled = true) => { + const serialized = JSON.stringify(body.complexity_router_config ?? null); + const [debounced, setDebounced] = useState(serialized); + useEffect(() => { + const timeout = setTimeout(() => setDebounced(serialized), 300); + return () => clearTimeout(timeout); + }, [serialized]); + const options: UseQueryOptions = { + queryKey: ["autoRouterAvailability", accessToken, body.team_id, body.saved_model_id, debounced], + queryFn: ({ signal }) => + apiClient.post("/auto_router/availability", { + accessToken, + body: { ...body, complexity_router_config: JSON.parse(debounced) }, + signal, + }), + enabled: enabled && Boolean(accessToken), + placeholderData: (previous, previousQuery) => { + const key = previousQuery?.queryKey; + return key?.[1] === accessToken && key[2] === body.team_id && key[3] === body.saved_model_id + ? previous + : undefined; + }, + refetchOnMount: "always", + staleTime: 0, + retry: false, + }; + const query = useQuery(options); + const isChecking = query.isFetching || query.isPlaceholderData || serialized !== debounced; + const saveBlockedReason = () => { + if (!enabled) return null; + if (query.isPending || isChecking) return "Checking availability"; + if (query.isError || !query.data) return "Could not check availability. Retry before saving."; + return query.data.error ?? null; + }; + return { + ...query, + isPending: query.isPending || (query.isFetching && !query.isFetchedAfterMount), + isChecking, + saveBlockedReason: saveBlockedReason(), + }; +}; + +export const allowanceLabel = (allowance?: Allowance): string | null => { + if (!allowance?.available) return "Availability unavailable"; + if (allowance.limit == null) return null; + if (allowance.used_by_this_router) return "Used by this router"; + return `${allowance.remaining} of ${allowance.limit} available`; +}; + +const availabilityLabel = (state: AvailabilityState, key: string) => { + if (state.isPending || state.isChecking) return "Checking availability"; + if (state.isError) return "Availability unavailable"; + return allowanceLabel(state.data?.allowances.find((entry) => entry.key === key)); +}; + +export const useAllowanceLabel = (key: string) => availabilityLabel(useContext(AutoRouterAvailabilityContext), key); + +export const isAllowanceExhausted = (allowance?: Allowance) => + Boolean(allowance?.available && allowance.limit != null && allowance.remaining === 0) && + !allowance?.used_by_this_router; + +export const AUTO_ROUTER_CONTACT_URL = "https://calendly.com/tin-berri/litellm-auto-router-pricing-discussion"; + +export const AutoRouterContactLink = ({ features, message }: { features?: string[]; message?: string }) => { + const state = useContext(AutoRouterAvailabilityContext); + if (state.isPending || state.isError || state.isChecking) return null; + const exhausted = state.data?.allowances.some( + (entry) => (!features || features.includes(entry.key)) && isAllowanceExhausted(entry), + ); + if (!exhausted) return null; + return ( + + {message} + + Talk to our team + + + ); +}; + +export const AutoRouterAllowanceLabel = ({ feature }: { feature: string }) => { + const label = useAllowanceLabel(feature); + return label ? ( + {label} + ) : null; +}; + +export const AutoRouterAllowanceNote = ({ feature, label }: { feature: string; label: string }) => { + const availability = useAllowanceLabel(feature); + return availability ? ( +

+ {label}: {availability} +

+ ) : null; +}; + +export const AutoRouterLimits = () => { + const state = useContext(AutoRouterAvailabilityContext); + const limits = [ + ["heuristic_v2", "Heuristic v2 routers"], + ["capability", "Capability routers"], + ["llm_v2", "Fuse v2 routers"], + ["tier_or_classifier_prompt", "Custom tiers or prompts"], + ["heuristic_tuning", "Rule-based tuning"], + ]; + return ( + + + View limits + + + Routing and customization limits +

+ Rule-based, Complexity, and Jev are unlimited with built-in settings. Choose or change tier models freely. + Customization allowances are shared across this proxy. +

+
+ {limits.map(([key, label]) => ( +
+
{label}
+
+ {availabilityLabel(state, key) ?? "Unlimited"} +
+
+ ))} +
+

+ Custom tier definitions and written classifier instructions share one allowance. Built-in prompts and + display-name changes do not use it. +

+

+ Changing scoring rules, such as weights, thresholds, keywords, or custom dimensions, uses the Rule-based + tuning allowance. It also applies to Heuristic first and Hybrid. Recorded settings on existing routers are + preserved; new routers start from built-in rules. +

+ +
+
+ ); +}; diff --git a/ui/litellm-dashboard/src/components/add_model/AutoRouterClassifierTabs.integration.test.tsx b/ui/litellm-dashboard/src/components/add_model/AutoRouterClassifierTabs.integration.test.tsx index ac6851349ea..f843a472d15 100644 --- a/ui/litellm-dashboard/src/components/add_model/AutoRouterClassifierTabs.integration.test.tsx +++ b/ui/litellm-dashboard/src/components/add_model/AutoRouterClassifierTabs.integration.test.tsx @@ -1,7 +1,9 @@ import React, { useState } from "react"; import { describe, expect, it, vi } from "vitest"; -import { fireEvent, renderWithProviders, screen } from "../../../tests/test-utils"; +import { fireEvent, renderWithProviders, screen, waitFor, within } from "../../../tests/test-utils"; +import { selectAutoRouterOption } from "../../../tests/autoRouterSetup"; import AutoRouterClassifierTabs from "./AutoRouterClassifierTabs"; +import { AutoRouterAllowanceNote, AutoRouterAvailabilityContext } from "./AutoRouterAvailability"; import type { ComplexityRouterConfigValue } from "./ComplexityRouterConfig"; const initial: ComplexityRouterConfigValue = { @@ -9,28 +11,67 @@ const initial: ComplexityRouterConfigValue = { tiers: { SIMPLE: ["efficient"], MEDIUM: [], COMPLEX: [], REASONING: ["capable"] }, }; -function Form({ initialValue = initial }: { initialValue?: ComplexityRouterConfigValue }) { +function Form({ + initialValue = initial, + remaining = 1, + limit = 1, + ownedFeature, + availabilityState, +}: { + initialValue?: ComplexityRouterConfigValue; + remaining?: number; + limit?: number | null; + ownedFeature?: string; + availabilityState?: Partial>; +}) { const [value, setValue] = useState(initialValue); return ( - - {value.classifier_type} - + ({ + key, + limit, + remaining, + available: true, + used_by_this_router: key === ownedFeature, + }), + ), + error: null, + }, + ...availabilityState, + }} + > + + {value.classifier_type} + + ); } -describe("AutoRouterClassifierTabs", () => { - it.each(["heuristic", "heuristic_v2", "llm", "heuristic_first", "hybrid"] as const)( - "groups %s under Complexity without resetting its configuration", - (classifier_type) => { +describe("Auto-router classifier selection", () => { + it.each(["heuristic", "heuristic_v2", "llm", "heuristic_first", "hybrid", "jev"] as const)( + "shows saved %s without changing its configuration", + async (classifier_type) => { const onChange = vi.fn(); renderWithProviders( - Existing classifier settings + Existing settings , ); - expect(screen.getByRole("tab", { name: "Complexity" })).toHaveAttribute("aria-selected", "true"); - expect(screen.getByRole("tabpanel", { name: "Complexity" })).toHaveTextContent("Existing classifier settings"); - fireEvent.click(screen.getByRole("tab", { name: "Complexity" })); + const family = { + heuristic: "Heuristics", + heuristic_v2: "Heuristics", + llm: "LLM", + heuristic_first: "LLM", + hybrid: "LLM", + jev: "Jev", + }[classifier_type]; + expect(screen.getByRole("radio", { name: new RegExp(`^${family}$`) })).toBeChecked(); + fireEvent.click(screen.getByRole("radio", { name: new RegExp(`^${family}$`) })); expect(onChange).not.toHaveBeenCalled(); }, ); @@ -38,38 +79,186 @@ describe("AutoRouterClassifierTabs", () => { it.each([ ["capability", "Capability"], ["llm_v2", "Fuse v2"], - ] as const)("opens saved %s settings and switches back to local Complexity", (classifier_type, label) => { - renderWithProviders(
); - expect(screen.getByRole("tab", { name: label })).toHaveAttribute("aria-selected", "true"); - fireEvent.click(screen.getByRole("tab", { name: "Complexity" })); - expect(screen.getByRole("tab", { name: "Complexity" })).toHaveAttribute("aria-selected", "true"); - expect(screen.getByRole("status", { name: "Classifier type" })).toHaveTextContent("heuristic"); + ] as const)( + "opens saved %s and retains the LLM family when switching to Complexity", + async (classifier_type, label) => { + renderWithProviders(); + expect(screen.getByRole("button", { name: "Routing approach" })).toHaveTextContent(label); + await selectAutoRouterOption("Routing approach", "Complexity"); + expect(screen.getByRole("status", { name: "Classifier type" })).toHaveTextContent("llm"); + }, + ); + + it.each([ + [1, "heuristic"], + [0, "heuristic"], + ])("defaults to Rule-based when %s v2 slots remain", async (remaining, classifier) => { + renderWithProviders(); + fireEvent.click(screen.getByRole("radio", { name: /^Heuristics$/ })); + expect(screen.getByRole("status", { name: "Classifier type" })).toHaveTextContent(String(classifier)); + fireEvent.click(screen.getByRole("button", { name: "Heuristic" })); + expect(screen.getByRole("menuitemradio", { name: /^Heuristic v2/ })).toHaveTextContent( + `${remaining} of 1 available`, + ); }); - it("keeps custom tiers editable under Complexity and explains why forecast tabs are disabled", () => { + it.each([ + { data: undefined }, + { isPending: true }, + { isError: true }, + { isChecking: true }, + { data: { allowances: [], error: null } }, + { data: { allowances: [{ key: "heuristic_v2", limit: 1, remaining: null, available: false }], error: null } }, + ])("uses Rule-based when v2 availability is unverified: %j", async (availabilityState) => { + renderWithProviders(); + fireEvent.click(screen.getByRole("radio", { name: "Heuristics" })); + expect(screen.getByRole("status", { name: "Classifier type" })).toHaveTextContent(/^heuristic$/); + }); + + it("does not present Rule-based as having a classifier quota", () => { + renderWithProviders(); + expect(screen.getByRole("button", { name: "Heuristic" })).toHaveTextContent(/^Rule-based/); + fireEvent.click(screen.getByRole("button", { name: "Heuristic" })); + expect(screen.getAllByRole("menuitemradio")[0]).toHaveTextContent(/^Rule-based/); + expect(screen.getByRole("menuitemradio", { name: /^Rule-based/ })).not.toHaveTextContent("of 1 available"); + expect(screen.getByRole("menuitemradio", { name: /^Heuristic v2/ })).toHaveTextContent("0 of 1 available"); + }); + + it("omits allowance labels with an unlimited entitlement", async () => { + renderWithProviders(); + fireEvent.click(screen.getByRole("button", { name: "Heuristic" })); + expect(screen.getByRole("menuitemradio", { name: /^Heuristic v2/ })).not.toHaveTextContent("available"); + }); + + it.each([ + ["heuristic", "Heuristic", "Heuristic v2"], + ["llm", "Routing approach", "Capability"], + ["llm", "Routing approach", "Fuse v2"], + ] as const)("blocks exhausted %s options: %s / %s", (classifier_type, field, option) => { + renderWithProviders(); + fireEvent.click(screen.getByRole("button", { name: field })); + const unavailable = screen.getByRole("menuitemradio", { name: new RegExp(`^${option}`) }); + expect(unavailable).toHaveAttribute("aria-disabled", "true"); + fireEvent.click(unavailable); + expect(screen.getByRole("status", { name: "Classifier type" })).toHaveTextContent(classifier_type); + }); + + it.each([ + ["heuristic_v2", "heuristic", "Heuristic", "Heuristic v2"], + ["capability", "llm", "Routing approach", "Capability"], + ["llm_v2", "llm", "Routing approach", "Fuse v2"], + ] as const)("lets a saved router reselect its own %s allowance", async (feature, classifier_type, field, option) => { + renderWithProviders(); + await selectAutoRouterOption(field, option); + expect(screen.getByRole("status", { name: "Classifier type" })).toHaveTextContent(feature); + expect(screen.getByRole("button", { name: field })).toHaveTextContent("Used by this router"); + }); + + it("shows Jev's single Complexity approach without changing saved configuration", () => { const onChange = vi.fn(); renderWithProviders( - + Existing settings + , + ); + expect(screen.getByRole("button", { name: "Routing approach" })).toHaveTextContent("ComplexityUnlimited"); + fireEvent.click(screen.getByRole("button", { name: "Routing approach" })); + expect(screen.getAllByRole("menuitemradio")).toHaveLength(1); + fireEvent.click(screen.getByRole("menuitemradio", { name: /^Complexity/ })); + expect(onChange).not.toHaveBeenCalled(); + }); + + it("keeps custom tiers editable and disables incompatible choices", async () => { + renderWithProviders( + - Custom tiers - , + />, ); - expect(screen.getByRole("tabpanel", { name: "Complexity" })).toHaveTextContent("Custom tiers"); + expect(screen.getByRole("radio", { name: /^Heuristics$/ })).toHaveAttribute("aria-disabled", "true"); + fireEvent.click(screen.getByRole("button", { name: "Routing approach" })); for (const name of ["Capability", "Fuse v2"]) { - const tab = screen.getByRole("tab", { name }); - expect(tab).toHaveAttribute("aria-disabled", "true"); - expect(tab).toHaveAccessibleDescription("Restore standard tiers to use Capability or Fuse v2."); - fireEvent.click(tab); + expect(screen.getByRole("menuitemradio", { name: new RegExp(`^${name}`) })).toHaveAttribute( + "aria-disabled", + "true", + ); } - expect(onChange).not.toHaveBeenCalled(); - expect(screen.getByText("Restore standard tiers to use Capability or Fuse v2.")).toBeVisible(); + expect(screen.getByRole("status", { name: "Classifier type" })).toHaveTextContent("llm"); + }); +}); + +describe("Gated routing contact action", () => { + it("offers a pricing discussion in View limits", async () => { + renderWithProviders(); + expect(screen.queryByRole("link", { name: "Talk to our team" })).not.toBeInTheDocument(); + fireEvent.click(screen.getByRole("button", { name: "View limits" })); + const link = within(screen.getByRole("dialog")).getByRole("link", { name: "Talk to our team" }); + await waitFor(() => expect(link).toBeVisible()); + expect(link).toHaveAttribute("href", "https://calendly.com/tin-berri/litellm-auto-router-pricing-discussion"); + expect(link).toHaveAttribute("target", "_blank"); + expect(link).toHaveAttribute("rel", "noopener noreferrer"); + }); + + it.each([ + ["heuristic", "Heuristic", "Heuristic v2"], + ["llm", "Routing approach", "Capability"], + ] as const)( + "keeps the contact action available beside the disabled %s choice", + async (classifier_type, field, option) => { + renderWithProviders(); + expect(screen.queryByText(/Need more/)).not.toBeInTheDocument(); + fireEvent.click(screen.getByRole("button", { name: field })); + const disabled = screen.getByRole("menuitemradio", { name: new RegExp(`^${option}`) }); + expect(disabled).toHaveAttribute("aria-disabled", "true"); + const link = screen.getByRole("menuitem", { name: `Talk to our team about ${option}` }); + await waitFor(() => expect(link).toBeVisible()); + expect(link).toHaveAttribute("href", "https://calendly.com/tin-berri/litellm-auto-router-pricing-discussion"); + expect(link).toHaveAttribute("target", "_blank"); + fireEvent.click(link); + expect(screen.getByRole("status", { name: "Classifier type" })).toHaveTextContent(classifier_type); + }, + ); + + it.each([ + { remaining: 1 }, + { remaining: 0, limit: null }, + { remaining: 0, availabilityState: { isPending: true } }, + { remaining: 0, availabilityState: { isError: true } }, + { remaining: 0, availabilityState: { isChecking: true } }, + ])("does not pitch an upgrade for a free or unverified option: %j", (props) => { + renderWithProviders(); + fireEvent.click(screen.getByRole("button", { name: "Routing approach" })); + expect(screen.queryByRole("menuitem", { name: /Talk to our team/ })).not.toBeInTheDocument(); + }); + + it("does not pitch an upgrade for the saved heuristic's own slot", () => { + renderWithProviders( + , + ); + fireEvent.click(screen.getByRole("button", { name: "Heuristic" })); + expect(screen.queryByRole("menuitem", { name: /Talk to our team/ })).not.toBeInTheDocument(); + }); + + it("includes the sales action beside customization limits and blocked changes", () => { + const allowance = { key: "tier_or_classifier_prompt", limit: 1, remaining: 0, available: true }; + const state = { + isPending: false, + isError: false, + data: { allowances: [allowance], error: "Custom tiers have no available allowance" }, + }; + renderWithProviders( + + + + + , + ); + expect(screen.getByText(/Custom tiers: 0 of 1 available/)).toHaveTextContent("Talk to our team"); + expect(within(screen.getByRole("alert")).getByRole("link", { name: "Talk to our team" })).toBeVisible(); }); }); diff --git a/ui/litellm-dashboard/src/components/add_model/AutoRouterClassifierTabs.tsx b/ui/litellm-dashboard/src/components/add_model/AutoRouterClassifierTabs.tsx index 98c0d4aab2f..92fc8d2a335 100644 --- a/ui/litellm-dashboard/src/components/add_model/AutoRouterClassifierTabs.tsx +++ b/ui/litellm-dashboard/src/components/add_model/AutoRouterClassifierTabs.tsx @@ -1,8 +1,119 @@ -import React, { useId } from "react"; -import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs"; -import { effectiveClassifierType, type ComplexityRouterConfigValue } from "./ComplexityRouterConfig"; +import React, { useContext, useId } from "react"; +import { Label } from "@/components/ui/label"; +import { RadioGroup, RadioGroupItem } from "@/components/ui/radio-group"; +import { ChevronDownIcon } from "lucide-react"; +import { Button } from "@/components/ui/button"; +import { + DropdownMenu, + DropdownMenuContent, + DropdownMenuItem, + DropdownMenuRadioGroup, + DropdownMenuRadioItem, + DropdownMenuTrigger, +} from "@/components/ui/dropdown-menu"; +import { + effectiveClassifierType, + type ClassifierType, + type ComplexityRouterConfigValue, +} from "./ComplexityRouterConfig"; import { transitionClassifierType } from "./classifier_type_transition"; import { isForecastClassifier } from "./forecast_classifier_config"; +import { + AutoRouterAllowanceLabel, + AutoRouterAvailabilityContext, + AutoRouterLimits, + AutoRouterContactLink, + isAllowanceExhausted, + AUTO_ROUTER_CONTACT_URL, +} from "./AutoRouterAvailability"; + +function ClassifierOption({ + value, + label, + description, + feature, + disabled, + unlimited = true, +}: { + value: string; + label: string; + description: string; + feature?: string; + disabled?: boolean; + unlimited?: boolean; +}) { + const state = useContext(AutoRouterAvailabilityContext); + const allowance = state.data?.allowances.find((entry) => entry.key === feature); + const fresh = !state.isPending && !state.isError && !state.isChecking; + const exhausted = isAllowanceExhausted(allowance); + return ( +
+ + + + {label} + {feature ? ( + + ) : ( + unlimited && Unlimited + )} + + {description} + + + {fresh && exhausted && ( + } + aria-label={`Talk to our team about ${label}`} + className="absolute top-9 right-8 cursor-pointer px-0 py-0 text-xs leading-5 font-medium text-blue-600 focus:text-blue-600 hover:underline dark:text-blue-400 dark:focus:text-blue-400" + > + Talk to our team + + )} +
+ ); +} + +function ClassifierMenu({ + id, + label, + value, + selectedLabel, + feature, + onValueChange, + children, +}: { + id: string; + label: string; + value: string; + selectedLabel: string; + feature?: string; + onValueChange: (value: string) => void; + children: React.ReactNode; +}) { + return ( + + } + > + {selectedLabel} + {feature ? ( + + ) : ( + Unlimited + )} + + + + + {children} + + + + ); +} interface AutoRouterClassifierTabsProps { value: ComplexityRouterConfigValue; @@ -11,47 +122,174 @@ interface AutoRouterClassifierTabsProps { } const AutoRouterClassifierTabs: React.FC = ({ value, onChange, children }) => { - const restrictionId = useId(); + const id = useId(); + const availability = useContext(AutoRouterAvailabilityContext); const classifierType = effectiveClassifierType(value); - const selected = isForecastClassifier(classifierType) ? classifierType : "complexity"; + const familyByType: Record = { + heuristic: "heuristics", + heuristic_v2: "heuristics", + llm: "llm", + heuristic_first: "llm", + hybrid: "llm", + capability: "llm", + llm_v2: "llm", + jev: "jev", + custom: "custom", + }; + const family = familyByType[classifierType]; const hasCustomTiers = Boolean(value.custom_tier_set); - - const handleChange = (tab: unknown) => { - if (tab === selected) return; - if (tab === "complexity") { - onChange(transitionClassifierType(value, isForecastClassifier(classifierType) ? "heuristic" : classifierType)); - } else if (!hasCustomTiers && (tab === "capability" || tab === "llm_v2")) { - onChange(transitionClassifierType(value, tab)); - } + const changeType = (next: ClassifierType) => { + if (next !== classifierType) onChange(transitionClassifierType(value, next)); }; + const changeFamily = (next: unknown) => { + if (next === family) return; + if (next === "heuristics") changeType("heuristic"); + if (next === "llm") changeType("llm"); + if (next === "jev") changeType("jev"); + }; + const approachLabels: Partial> = { capability: "Capability", llm_v2: "Fuse v2" }; + const approachDescription: Partial> = { + capability: "Use the efficient model when it is likely to succeed", + llm_v2: "Use the efficient model when its predicted quality is close enough to the capable model", + }; return ( - -

Classifier type

- - Complexity - - Capability - - - Fuse v2 - - +
+
+ + What classifies your requests? + + + + {[ + { value: "heuristics", label: "Heuristics", description: "Classify locally, with no API call" }, + { value: "llm", label: "LLM", description: "Use a judge model to choose a solver" }, + { value: "jev", label: "Jev", description: "Use TypeSafe System One Choice to choose a tier" }, + ].map((option) => ( + + ))} + +
+ {family === "custom" && ( +

This router uses a custom classifier plugin

+ )} + {family === "heuristics" && ( +
+ + { + if (next === "heuristic" || next === "heuristic_v2") changeType(next); + }} + > + + + +

+ {classifierType === "heuristic_v2" + ? "Use calibrated probabilities to match requests to a tier" + : "Match requests using scoring rules. Choose or change tier models freely"} +

+
+ )} + {(family === "llm" || family === "jev") && ( +
+ + { + if (next === "llm" || next === "capability" || next === "llm_v2") { + if (next === "llm" && !isForecastClassifier(classifierType)) return; + changeType(next); + } + }} + > + + {family === "llm" && ( + <> + + + + )} + +

+ {approachDescription[classifierType] ?? "Match task difficulty to a tier"} +

+
+ )} {hasCustomTiers && ( -

- Restore standard tiers to use Capability or Fuse v2. +

+ Restore standard tiers to use Heuristics, Capability, or Fuse v2

)} - {children} - + {availability.data?.error && !availability.isChecking && ( +
+

{availability.data.error}

+ +
+ )} + {availability.isError && ( +

+ Could not check availability.{" "} + +

+ )} + {children} +
); }; diff --git a/ui/litellm-dashboard/src/components/add_model/ClassificationMethodConfig.tsx b/ui/litellm-dashboard/src/components/add_model/ClassificationMethodConfig.tsx index b7a0fd67443..ccd204aa521 100644 --- a/ui/litellm-dashboard/src/components/add_model/ClassificationMethodConfig.tsx +++ b/ui/litellm-dashboard/src/components/add_model/ClassificationMethodConfig.tsx @@ -1,9 +1,10 @@ +import ClassifierPrimarySettings from "./ClassifierPrimarySettings"; +import { AutoRouterAllowanceNote } from "./AutoRouterAvailability"; import { transitionClassifierType } from "./classifier_type_transition"; import JevClassifierConfig from "./JevClassifierConfig"; import { Info } from "lucide-react"; import { SimpleTooltip } from "@/components/ui/tooltip"; import { MultiSelect } from "@/components/shared/MultiSelect"; -import { SearchSelect } from "@/components/shared/SearchSelect"; import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select"; import { Card, CardContent } from "@/components/ui/card"; import { Button } from "@/components/ui/button"; @@ -25,13 +26,10 @@ import ClassifierTypeRadios from "./ClassifierTypeRadios"; import type { ReasoningEffort } from "./complexity_router_tiers"; import { useComplexityScorerDefaults } from "@/app/(dashboard)/hooks/autoRouter/useComplexityScorerDefaults"; import { - ClassificationFrequency, ClassifierFallback, ClassifierLLMConfig, ClassifierType, ComplexityRouterConfigValue, - classificationFrequency, - withClassificationFrequency, DEFAULT_CLASSIFIER_CONTEXT_BUDGET_CHARS, MIN_QUOTED_CONTEXT_TURN_CHARS, DEFAULT_CLASSIFIER_CONTEXT_WINDOW_SIZE, @@ -171,6 +169,7 @@ interface ClassificationMethodConfigProps { showValidationErrors?: boolean; /** The resolved default model - see resolveComplexityDefaultModel. Names and gates the radio. */ defaultModel?: string; + advancedOnly?: boolean; } export const InactiveHeuristicV2Threshold: React.FC> = ({ @@ -215,13 +214,11 @@ const ClassificationMethodConfig: React.FC = ({ onCustomTechnicalKeywordsChange, showValidationErrors = false, defaultModel, + advancedOnly = false, }) => { const [draft, setDraft] = React.useState<{ id: string; raw: string } | null>(null); const hasDefaultModel = Boolean(defaultModel); const classifierType = effectiveClassifierType(value); - const sessionFrequencyRestriction = restrictedBy(value, "sessionAffinity"); - const classifierModelMissing = - showValidationErrors && usesLlmClassifier(classifierType) && !value.classifier_llm_config?.model; const usesCustomPrompt = Boolean(value.classifier_llm_config?.system_prompt?.trim()); const contextBudget = value.classifier_context_budget_chars ?? DEFAULT_CLASSIFIER_CONTEXT_BUDGET_CHARS; const contextBudgetQuotesNothing = contextBudget > 0 && contextBudget < MIN_QUOTED_CONTEXT_TURN_CHARS; @@ -282,23 +279,6 @@ const ClassificationMethodConfig: React.FC = ({ onChange(nextValue); }; - const handleClassifierModelChange = (model: string | null) => { - if (model === null) return; - if (model === value.classifier_llm_config?.model) return; - const { reasoning_effort: _reasoningEffort, ...classifierLlmConfig } = value.classifier_llm_config ?? { - model: "", - timeout_ms: DEFAULT_CLASSIFIER_TIMEOUT_MS, - }; - onChange({ - ...value, - classifier_llm_config: { - ...classifierLlmConfig, - model, - timeout_ms: classifierLlmConfig.timeout_ms, - }, - }); - }; - const handleClassifierReasoningEffortChange = (reasoningEffort: ReasoningEffort | undefined) => { if (!value.classifier_llm_config) return; const { reasoning_effort: _reasoningEffort, ...classifierLlmConfig } = value.classifier_llm_config; @@ -350,10 +330,6 @@ const ClassificationMethodConfig: React.FC = ({ onChange({ ...value, classifier_fallback: fallback }); }; - const handleClassificationFrequencyChange = (frequency: ClassificationFrequency) => { - onChange(withClassificationFrequency(value, frequency)); - }; - const handleClassifierContextWindowSizeChange = (windowSize: number) => { onChange({ ...value, @@ -389,7 +365,50 @@ const ClassificationMethodConfig: React.FC = ({ return ( <> - + {!advancedOnly && ( + <> + + + + )} + {advancedOnly && ["llm", "heuristic_first", "hybrid"].includes(classifierType) && ( +
+ + +
+ )} {classifierType === "custom" && ( @@ -474,66 +493,9 @@ const ClassificationMethodConfig: React.FC = ({ )} -
- How often to classify - - handleClassificationFrequencyChange(frequency as ClassificationFrequency) - } - > -
- - - -
-
-

- Holding the tier keeps an agent on one model for a whole tool loop and cuts scoring cost. A turn the router - cannot match to a held decision, such as one with no session id or an expired one, is scored again -

-
- {classifierType === "jev" && } {usesLlmClassifier(classifierType) && (
-
- Classifier Model - - {classifierModelMissing && A classifier model is required} -
= ({
+ {!value.custom_tier_set && usesCustomPrompt ? ( = ({ /> Number of prior user turns sent to the classifier provider, excluding tool output and harness reminders. - LLM and JEV default to 3 turns; JEV sends them to the configured TypeSafe endpoint. Set to 0 to omit + LLM and Jev default to 3 turns; Jev sends them to the configured TypeSafe endpoint. Set to 0 to omit conversation history. The current message and selected system text are still sent. @@ -769,6 +735,9 @@ const ClassificationMethodConfig: React.FC = ({ )} + {["heuristic", "heuristic_first", "hybrid"].includes(classifierType) && ( + + )} diff --git a/ui/litellm-dashboard/src/components/add_model/ClassifierPrimarySettings.tsx b/ui/litellm-dashboard/src/components/add_model/ClassifierPrimarySettings.tsx new file mode 100644 index 00000000000..32cb4851580 --- /dev/null +++ b/ui/litellm-dashboard/src/components/add_model/ClassifierPrimarySettings.tsx @@ -0,0 +1,98 @@ +import React from "react"; +import { Label } from "@/components/ui/label"; +import { SearchSelect } from "@/components/shared/SearchSelect"; +import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select"; +import { + classificationFrequency, + withClassificationFrequency, + effectiveClassifierType, + usesLlmClassifier, + DEFAULT_CLASSIFIER_TIMEOUT_MS, + type ComplexityRouterConfigValue, + type ClassificationFrequency, +} from "./ComplexityRouterConfig"; +import { restrictedBy } from "./TierRestrictions"; + +export default function ClassifierPrimarySettings({ + value, + onChange, + modelOptions, + showValidationErrors = false, +}: { + value: ComplexityRouterConfigValue; + onChange: (value: ComplexityRouterConfigValue) => void; + modelOptions: { value: string; label: string }[]; + showValidationErrors?: boolean; +}) { + const id = React.useId(); + const restriction = restrictedBy(value, "sessionAffinity"); + const frequency = classificationFrequency(value); + const frequencyDescription = { + every_request: "Choose a model again for every request", + user_turn: "Reclassify when the user sends a new message", + session: "Keep the same tier for the session. Requires a client session ID", + }[frequency]; + const usesJudge = usesLlmClassifier(effectiveClassifierType(value)); + const missingJudge = showValidationErrors && usesJudge && !value.classifier_llm_config?.model; + return ( +
+
+ + +

{restriction?.reason ?? frequencyDescription}

+
+ {usesJudge && ( +
+ + { + if (!model || model === value.classifier_llm_config?.model) return; + onChange({ + ...value, + classifier_llm_config: { + ...value.classifier_llm_config, + model, + timeout_ms: value.classifier_llm_config?.timeout_ms ?? DEFAULT_CLASSIFIER_TIMEOUT_MS, + reasoning_effort: undefined, + }, + }); + }} + /> + {missingJudge && ( +

+ A judge model is required +

+ )} +
+ )} +
+ ); +} diff --git a/ui/litellm-dashboard/src/components/add_model/ClassifierTypeRadios.tsx b/ui/litellm-dashboard/src/components/add_model/ClassifierTypeRadios.tsx index 1602e19069a..1fd6dfa6a20 100644 --- a/ui/litellm-dashboard/src/components/add_model/ClassifierTypeRadios.tsx +++ b/ui/litellm-dashboard/src/components/add_model/ClassifierTypeRadios.tsx @@ -54,7 +54,7 @@ const ClassifierTypeRadios: React.FC = ({ value, clas diff --git a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterAdvancedSections.tsx b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterAdvancedSections.tsx index 44adecb9e1d..a4e833152a9 100644 --- a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterAdvancedSections.tsx +++ b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterAdvancedSections.tsx @@ -6,6 +6,7 @@ import type { ModelGroup } from "@/components/llm_calls/fetch_models"; import type { ComplexityRouterConfigValue } from "./ComplexityRouterConfig"; import AdaptiveRoutingConfig from "./AdaptiveRoutingConfig"; import ClassificationMethodConfig from "./ClassificationMethodConfig"; +import ForecastClassifierConfig from "./ForecastClassifierConfig"; import ContextWindowEscalationConfig from "./ContextWindowEscalationConfig"; import ResponseFormatControls from "./ResponseFormatControls"; import StallEscalationConfig from "./StallEscalationConfig"; @@ -79,13 +80,31 @@ const ComplexityRouterAdvancedSections: React.FC { const sections = [ + ...(forecast + ? [ + { + key: "classifier", + label: Classifier tuning, + children: ( + + ), + }, + ] + : []), ...(!forecast ? [ { key: "classifier", - label: Advanced: Classification Method, + label: Classification Method, children: ( Advanced: Heuristic Keyword Overrides, + label: Heuristic Keyword Overrides, children: , }, ] : []), { key: "adaptive", - label: Advanced: Adaptive Routing, + label: Adaptive Routing, children: ( @@ -119,39 +138,39 @@ const ComplexityRouterAdvancedSections: React.FCAdvanced: Affinity, + label: Affinity, children: , }, { key: "modality", - label: Advanced: Modality Routing, + label: Modality Routing, children: , }, { key: "plan-mode", - label: Advanced: Plan-Mode Override, + label: Plan-Mode Override, children: ( ), }, { key: "housekeeping", - label: Advanced: Housekeeping Routing, + label: Housekeeping Routing, children: , }, { key: "reminder-markers", - label: Advanced: Ignore Custom Tags, + label: Ignore Custom Tags, children: , }, { key: "context-window", - label: Advanced: Context Window Escalation, + label: Context Window Escalation, children: , }, { key: "stall-escalation", - label: Advanced: Stalled Task Escalation, + label: Stalled Task Escalation, children: ( @@ -160,14 +179,14 @@ const ComplexityRouterAdvancedSections: React.FCAdvanced: Response Format, + label: Response Format, children: , }, ...(onEscalationKeywordsChange ? [ { key: "escalation", - label: Advanced: Escalation Keywords, + label: Escalation Keywords, children: ( @@ -180,7 +199,7 @@ const ComplexityRouterAdvancedSections: React.FCAdvanced: Compression, + label: Compression, children: , }, ] @@ -189,7 +208,7 @@ const ComplexityRouterAdvancedSections: React.FCAdvanced: Keyword/Semantic Matching, + label: Keyword/Semantic Matching, children: ( <> {onKeywordTierRulesChange && ( @@ -220,20 +239,65 @@ const ComplexityRouterAdvancedSections: React.FC(() => + showValidationErrors ? groups.map((group) => group.label) : [], + ); + const [previousValidation, setPreviousValidation] = React.useState(showValidationErrors); + if (previousValidation !== showValidationErrors) { + setPreviousValidation(showValidationErrors); + if (showValidationErrors) setOpenGroups(groups.map((group) => group.label)); + } return ( - <> - {sections - .filter(({ key }) => !forecast || !["adaptive", "context-window", "escalation"].includes(key)) - .map(({ key, label, children }) => ( - - - - {label} - - {children} - - ))} - +
+ {groups.map((group) => ( + + setOpenGroups((current) => + open ? [...current, group.label] : current.filter((label) => label !== group.label), + ) + } + className="border-b border-border last:border-b-0" + > + + + {group.label} + + + {sections + .filter( + ({ key }) => + group.keys.includes(key) && + (!forecast || !["adaptive", "context-window", "escalation"].includes(key)), + ) + .map(({ key, label, children }) => ( +
+ {key !== "classifier" &&

{label}

} + {children} +
+ ))} +
+
+ ))} +
); }; diff --git a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.test.tsx b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.integration.test.tsx similarity index 85% rename from ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.test.tsx rename to ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.integration.test.tsx index 756e505997c..4a00a469f97 100644 --- a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.test.tsx +++ b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.integration.test.tsx @@ -1,8 +1,15 @@ +import { openAutoRouterAdvanced, selectAutoRouterOption } from "../../../tests/autoRouterSetup"; import { fireEvent, renderWithProviders, screen, within } from "../../../tests/test-utils"; import userEvent from "@testing-library/user-event"; import React from "react"; -import { vi, type Mock } from "vitest"; -import ComplexityRouterConfig, { ComplexityRouterConfigValue } from "./ComplexityRouterConfig"; +import { describe, it, expect, vi, type Mock } from "vitest"; +import ComplexityRouterConfigView, { ComplexityRouterConfigValue } from "./ComplexityRouterConfig"; +import AutoRouterClassifierTabs from "./AutoRouterClassifierTabs"; +const ComplexityRouterConfig = (props: React.ComponentProps) => ( + + + +); vi.mock( "@/app/(dashboard)/hooks/autoRouter/useComplexityScorerDefaults", async () => await import("../../../tests/mocks/complexityScorerDefaults"), @@ -45,12 +52,12 @@ const baseProps = { }; describe("ComplexityRouterConfig", () => { - it("should render", () => { + it("should render", async () => { renderWithProviders(); - expect(screen.getByText("Complexity Tier Configuration")).toBeInTheDocument(); + expect(screen.getByText("Models by tier")).toBeInTheDocument(); }); - it("should display all four tier labels", () => { + it("should display all four tier labels", async () => { renderWithProviders(); expect(screen.getByText("Simple Tier")).toBeInTheDocument(); expect(screen.getByText("Medium Tier")).toBeInTheDocument(); @@ -58,7 +65,7 @@ describe("ComplexityRouterConfig", () => { expect(screen.getByText("Reasoning Tier")).toBeInTheDocument(); }); - it("should show example queries for each tier", () => { + it("should show example queries for each tier", async () => { renderWithProviders(); expect(screen.getByText(/Hello!/)).toBeInTheDocument(); expect(screen.getByText(/Explain how REST APIs work/)).toBeInTheDocument(); @@ -66,46 +73,51 @@ describe("ComplexityRouterConfig", () => { expect(screen.getByText(/Think step by step/)).toBeInTheDocument(); }); - it("should display the how classification works section", () => { + it("should display the how classification works section", async () => { renderWithProviders(); - fireEvent.click(screen.getByText("Advanced: Classification Method")); + openAutoRouterAdvanced("Classification Method"); expect(screen.getByText("How Classification Works")).toBeInTheDocument(); }); - it("should show score thresholds in the classification section", () => { + it("should show score thresholds in the classification section", async () => { renderWithProviders(); - fireEvent.click(screen.getByText("Advanced: Classification Method")); + openAutoRouterAdvanced("Classification Method"); expect(screen.getByText(/Score < 0.15/)).toBeInTheDocument(); expect(screen.getByText(/Score 0.15 - 0.35/)).toBeInTheDocument(); expect(screen.getByText(/Score 0.35 - 0.60/)).toBeInTheDocument(); expect(screen.getByText(/Score > 0.60/)).toBeInTheDocument(); }); - it("leaves the score threshold list color to the theme instead of an inline style", () => { + it("leaves the score threshold list color to the theme instead of an inline style", async () => { renderWithProviders(); - fireEvent.click(screen.getByText("Advanced: Classification Method")); + openAutoRouterAdvanced("Classification Method"); const list = screen.getByText(/Score < 0.15/).closest("ul"); expect(list).toBeInTheDocument(); expect(list).toHaveClass("text-muted-foreground"); expect(list?.style.color).toBe(""); }); - it("should default to heuristic and hide classifier model/timeout fields", () => { + it("should default to heuristic and hide classifier model/timeout fields", async () => { renderWithProviders(); - expect(screen.getByText("Advanced: Classification Method")).toBeInTheDocument(); - expect(screen.queryByText("Classifier Model")).not.toBeInTheDocument(); + openAutoRouterAdvanced("Classification Method"); + expect(screen.getByText("Classifier tuning")).toBeInTheDocument(); + expect(screen.queryByText("Judge model")).not.toBeInTheDocument(); }); - it("shows heuristic advanced sections and hides keyword overrides for capability classifiers", () => { + it("shows heuristic advanced sections and hides keyword overrides for capability classifiers", async () => { const { rerender } = renderWithProviders(); - expect(screen.getByText("Advanced: Heuristic Keyword Overrides")).toBeInTheDocument(); - expect(screen.getByText("Advanced: Housekeeping Routing")).toBeInTheDocument(); - expect(screen.getByText("Advanced: Ignore Custom Tags")).toBeInTheDocument(); + openAutoRouterAdvanced("Heuristic Keyword Overrides"); + + expect(screen.getByText("Heuristic Keyword Overrides")).toBeInTheDocument(); + openAutoRouterAdvanced("Housekeeping Routing"); + expect(screen.getByText("Housekeeping Routing")).toBeInTheDocument(); + openAutoRouterAdvanced("Ignore Custom Tags"); + expect(screen.getByText("Ignore Custom Tags")).toBeInTheDocument(); const capabilityValue = { ...defaultValue, classifier_type: "capability" as const }; rerender(); - expect(screen.queryByText("Advanced: Heuristic Keyword Overrides")).not.toBeInTheDocument(); + expect(screen.queryByText("Heuristic Keyword Overrides")).not.toBeInTheDocument(); }); it.each([ @@ -115,7 +127,7 @@ describe("ComplexityRouterConfig", () => { renderWithProviders( , ); - fireEvent.click(screen.getByText("Advanced: Classification Method")); + openAutoRouterAdvanced("Classification Method"); if (visible) { expect(screen.getByLabelText("Classifier plugin timeout (ms)")).toBeInTheDocument(); } else { @@ -128,7 +140,7 @@ describe("ComplexityRouterConfig", () => { renderWithProviders( , ); - fireEvent.click(screen.getByText("Advanced: Ignore Custom Tags")); + openAutoRouterAdvanced("Ignore Custom Tags"); const validation = screen.queryByText(/needs both/i); if (showValidationErrors) { expect(validation).toBeInTheDocument(); @@ -137,11 +149,11 @@ describe("ComplexityRouterConfig", () => { } }); - it("disables housekeeping sentinels when cheapest-tier routing is off", () => { + it("disables housekeeping sentinels when cheapest-tier routing is off", async () => { renderWithProviders( , ); - fireEvent.click(screen.getByText("Advanced: Housekeeping Routing")); + openAutoRouterAdvanced("Housekeeping Routing"); const sentinelInput = screen.getByRole("combobox", { name: "e.g., conversation title" }); expect(sentinelInput).toBeDisabled(); }); @@ -151,7 +163,7 @@ describe("ComplexityRouterConfig", () => { const onChange = vi.fn(); renderWithProviders(); - await user.click(screen.getByText("Advanced: Response Format")); + openAutoRouterAdvanced("Response Format"); await user.click(screen.getByRole("switch", { name: "Return raw model name" })); expect(onChange).toHaveBeenCalledWith({ @@ -160,13 +172,13 @@ describe("ComplexityRouterConfig", () => { }); }); - it("should reveal classifier model and timeout fields when llm is selected", () => { + it("should reveal classifier model and timeout fields when llm is selected", async () => { const onChange = vi.fn(); renderWithProviders(); // Collapse panel content isn't rendered until first expanded. - fireEvent.click(screen.getByText("Advanced: Classification Method")); - fireEvent.click(screen.getByText("LLM Classifier")); + openAutoRouterAdvanced("Classification Method"); + fireEvent.click(screen.getByRole("radio", { name: /^LLM$/ })); const expectedValue: ComplexityRouterConfigValue = { ...defaultValue, @@ -178,14 +190,14 @@ describe("ComplexityRouterConfig", () => { expect(onChange).toHaveBeenCalledWith(expectedValue); }); - it("selects heuristic v2 without requiring a classifier model or showing weighted scoring", () => { + it("selects heuristic v2 without requiring a classifier model or showing weighted scoring", async () => { const onChange = vi.fn(); const { rerender } = renderWithProviders( , ); - fireEvent.click(screen.getByText("Advanced: Classification Method")); - fireEvent.click(screen.getByText("Heuristic v2")); + openAutoRouterAdvanced("Classification Method"); + await selectAutoRouterOption("Heuristic", "Heuristic v2"); expect(onChange).toHaveBeenCalledWith( expect.objectContaining({ @@ -197,7 +209,7 @@ describe("ComplexityRouterConfig", () => { const heuristicV2Value: ComplexityRouterConfigValue = { ...defaultValue, classifier_type: "heuristic_v2" }; rerender(); - expect(screen.queryByText("Classifier Model")).not.toBeInTheDocument(); + expect(screen.queryByText("Judge model")).not.toBeInTheDocument(); expect(screen.queryByText("Advanced scoring")).not.toBeInTheDocument(); expect(screen.getByText(/estimates success probability for all four tiers/)).toBeInTheDocument(); expect(screen.queryByText(/Score < 0.15/)).not.toBeInTheDocument(); @@ -233,7 +245,7 @@ describe("ComplexityRouterConfig", () => { expect(onChange).toHaveBeenCalledWith({ ...value, heuristic_v2_success_threshold: undefined }); }); - it("shows an inactive zero threshold until explicitly cleared and hides the summary for active or absent values", () => { + it("shows an inactive zero threshold until explicitly cleared and hides the summary for active or absent values", async () => { const onChange = vi.fn(); const value = { ...defaultValue, heuristic_v2_success_threshold: 0 }; const { rerender } = renderWithProviders( @@ -248,7 +260,7 @@ describe("ComplexityRouterConfig", () => { expect(screen.queryByRole("region", { name: "Inactive Heuristic v2 threshold" })).not.toBeInTheDocument(); }); - it("should show classifier fields and use the configured values when classifier_type is llm", () => { + it("should show classifier fields and use the configured values when classifier_type is llm", async () => { const llmValue: ComplexityRouterConfigValue = { ...defaultValue, classifier_type: "llm", @@ -258,9 +270,9 @@ describe("ComplexityRouterConfig", () => { }; renderWithProviders(); - fireEvent.click(screen.getByText("Advanced: Classification Method")); + openAutoRouterAdvanced("Classification Method"); - expect(screen.getByText("Classifier Model")).toBeInTheDocument(); + expect(screen.getByText("Judge model")).toBeInTheDocument(); expect(screen.getByLabelText("Timeout (ms)")).toHaveValue("750"); expect(screen.getByRole("switch", { name: "Classifier circuit breaker" })).toBeChecked(); expect(screen.getByLabelText("Circuit breaker cooldown (seconds)")).toHaveValue("30"); @@ -268,7 +280,7 @@ describe("ComplexityRouterConfig", () => { expect(screen.queryByText("Context Per-Turn Character Limit")).not.toBeInTheDocument(); }); - it("should allow the default-on classifier circuit breaker to be disabled", () => { + it("should allow the default-on classifier circuit breaker to be disabled", async () => { const onChange = vi.fn(); const llmValue: ComplexityRouterConfigValue = { ...defaultValue, @@ -276,7 +288,7 @@ describe("ComplexityRouterConfig", () => { classifier_llm_config: { model: "gpt-3.5-turbo", timeout_ms: 3000 }, }; renderWithProviders(); - fireEvent.click(screen.getByText("Advanced: Classification Method")); + openAutoRouterAdvanced("Classification Method"); fireEvent.click(screen.getByRole("switch", { name: "Classifier circuit breaker" })); @@ -287,7 +299,7 @@ describe("ComplexityRouterConfig", () => { ); }); - it("should default the context window and budget when llm is selected", () => { + it("should default the context window and budget when llm is selected", async () => { const llmValue: ComplexityRouterConfigValue = { ...defaultValue, classifier_type: "llm", @@ -295,13 +307,13 @@ describe("ComplexityRouterConfig", () => { }; renderWithProviders(); - fireEvent.click(screen.getByText("Advanced: Classification Method")); + openAutoRouterAdvanced("Classification Method"); expect(screen.getByLabelText("Context Window Size")).toHaveValue("3"); expect(screen.getByLabelText("Context Character Budget")).toHaveValue("8000"); }); - it("should warn when the budget is too small to quote any turn that does not already fit", () => { + it("should warn when the budget is too small to quote any turn that does not already fit", async () => { const llmValue: ComplexityRouterConfigValue = { ...defaultValue, classifier_type: "llm", @@ -310,12 +322,12 @@ describe("ComplexityRouterConfig", () => { }; renderWithProviders(); - fireEvent.click(screen.getByText("Advanced: Classification Method")); + openAutoRouterAdvanced("Classification Method"); expect(screen.getByText(/no room to quote a turn/i)).toBeInTheDocument(); }); - it("should not warn on a budget large enough to quote a turn, nor on a deliberate zero", () => { + it("should not warn on a budget large enough to quote a turn, nor on a deliberate zero", async () => { for (const budget of [120, 8000, 0]) { const { unmount } = renderWithProviders( { onChange={vi.fn()} />, ); - fireEvent.click(screen.getByText("Advanced: Classification Method")); + openAutoRouterAdvanced("Classification Method"); expect(screen.queryByText(/no room to quote a turn/i)).not.toBeInTheDocument(); unmount(); } }); - it("should show the assistant-turns switch with its configured value when classifier_type is llm", () => { + it("should show the assistant-turns switch with its configured value when classifier_type is llm", async () => { const llmValue: ComplexityRouterConfigValue = { ...defaultValue, classifier_type: "llm", @@ -344,13 +356,13 @@ describe("ComplexityRouterConfig", () => { }; renderWithProviders(); - fireEvent.click(screen.getByText("Advanced: Classification Method")); + openAutoRouterAdvanced("Classification Method"); expect(screen.getByText("Include Assistant Turns")).toBeInTheDocument(); expect(screen.getByRole("switch", { name: "Include Assistant Turns" })).toBeChecked(); }); - it("should render the assistant-turns switch off when it is not set", () => { + it("should render the assistant-turns switch off when it is not set", async () => { const llmValue: ComplexityRouterConfigValue = { ...defaultValue, classifier_type: "llm", @@ -358,18 +370,18 @@ describe("ComplexityRouterConfig", () => { }; renderWithProviders(); - fireEvent.click(screen.getByText("Advanced: Classification Method")); + openAutoRouterAdvanced("Classification Method"); expect(screen.getByRole("switch", { name: "Include Assistant Turns" })).not.toBeChecked(); }); - it("should hide the assistant-turns switch when classifier_type is heuristic", () => { + it("should hide the assistant-turns switch when classifier_type is heuristic", async () => { renderWithProviders(); - fireEvent.click(screen.getByText("Advanced: Classification Method")); + openAutoRouterAdvanced("Classification Method"); expect(screen.queryByText("Include Assistant Turns")).not.toBeInTheDocument(); }); - it("should call onChange when the assistant-turns switch is toggled", () => { + it("should call onChange when the assistant-turns switch is toggled", async () => { const onChange = vi.fn(); const llmValue: ComplexityRouterConfigValue = { ...defaultValue, @@ -378,7 +390,7 @@ describe("ComplexityRouterConfig", () => { }; renderWithProviders(); - fireEvent.click(screen.getByText("Advanced: Classification Method")); + openAutoRouterAdvanced("Classification Method"); fireEvent.click(screen.getByRole("switch", { name: "Include Assistant Turns" })); expect(onChange).toHaveBeenCalledWith( @@ -386,9 +398,9 @@ describe("ComplexityRouterConfig", () => { ); }); - it("should hide classifier context fields when classifier_type is heuristic", () => { + it("should hide classifier context fields when classifier_type is heuristic", async () => { renderWithProviders(); - fireEvent.click(screen.getByText("Advanced: Classification Method")); + openAutoRouterAdvanced("Classification Method"); expect(screen.queryByText("Context Window Size")).not.toBeInTheDocument(); expect(screen.queryByText("Context Per-Turn Character Limit")).not.toBeInTheDocument(); }); @@ -416,7 +428,7 @@ describe("ComplexityRouterConfig", () => { classifier_llm_config: { model: "gpt-3.5-turbo", timeout_ms: 3000 }, }; renderWithProviders(); - fireEvent.click(screen.getByText("Advanced: Classification Method")); + openAutoRouterAdvanced("Classification Method"); const input = screen.getByLabelText(label); fireEvent.change(input, { target: { value: "" } }); @@ -429,14 +441,14 @@ describe("ComplexityRouterConfig", () => { expect(onChange).toHaveBeenLastCalledWith({ ...llmValue, ...expected }); }); - it("restores the committed context window size after an empty field loses focus", () => { + it("restores the committed context window size after an empty field loses focus", async () => { const llmValue: ComplexityRouterConfigValue = { ...defaultValue, classifier_type: "llm", classifier_llm_config: { model: "gpt-3.5-turbo", timeout_ms: 3000 }, }; renderWithProviders(); - fireEvent.click(screen.getByText("Advanced: Classification Method")); + openAutoRouterAdvanced("Classification Method"); const input = screen.getByLabelText("Context Window Size"); fireEvent.change(input, { target: { value: "" } }); @@ -445,13 +457,13 @@ describe("ComplexityRouterConfig", () => { expect(input).toHaveValue("3"); }); - it("should render the custom technical keywords field", () => { + it("should render the custom technical keywords field", async () => { renderWithProviders(); - fireEvent.click(screen.getByText("Advanced: Classification Method")); + openAutoRouterAdvanced("Classification Method"); expect(screen.getByText("Custom Technical Keywords")).toBeInTheDocument(); }); - it("should display existing custom technical keywords as tags", () => { + it("should display existing custom technical keywords as tags", async () => { renderWithProviders( { onCustomTechnicalKeywordsChange={vi.fn()} />, ); - fireEvent.click(screen.getByText("Advanced: Classification Method")); + openAutoRouterAdvanced("Classification Method"); expect(screen.getByText("udp")).toBeInTheDocument(); expect(screen.getByText("kafka")).toBeInTheDocument(); }); @@ -474,7 +486,7 @@ describe("ComplexityRouterConfig", () => { onCustomTechnicalKeywordsChange={onCustomTechnicalKeywordsChange} />, ); - fireEvent.click(screen.getByText("Advanced: Classification Method")); + openAutoRouterAdvanced("Classification Method"); const keywordsSection = screen.getByText("Custom Technical Keywords").closest("div")?.parentElement as HTMLElement; await user.type(within(keywordsSection).getByRole("combobox"), "udp"); await user.click(await screen.findByText('Create "udp"')); @@ -491,35 +503,35 @@ describe("ComplexityRouterConfig", () => { onCustomTechnicalKeywordsChange={onCustomTechnicalKeywordsChange} />, ); - fireEvent.click(screen.getByText("Advanced: Classification Method")); + openAutoRouterAdvanced("Classification Method"); const keywordsSection = screen.getByText("Custom Technical Keywords").closest("div")?.parentElement as HTMLElement; await user.type(within(keywordsSection).getByRole("combobox"), "udp, kafka ,terraform"); await user.click(await screen.findByText('Create "udp, kafka ,terraform"')); expect(onCustomTechnicalKeywordsChange).toHaveBeenCalledWith(["udp", "kafka", "terraform"]); }); - it("should render an empty state when no keyword tier rules exist", () => { + it("should render an empty state when no keyword tier rules exist", async () => { renderWithProviders(); - fireEvent.click(screen.getByText("Advanced: Keyword/Semantic Matching")); + openAutoRouterAdvanced("Keyword/Semantic Matching"); expect(screen.getByText("Keyword Tier Overrides")).toBeInTheDocument(); expect(screen.getByText("No keyword tier overrides configured")).toBeInTheDocument(); }); - it("hides the keyword-tier and semantic sections when their change handlers are absent (edit modal)", () => { + it("hides the keyword-tier and semantic sections when their change handlers are absent (edit modal)", async () => { // The edit-auto-router modal renders ComplexityRouterConfig without these handlers; // the sections must stay hidden rather than render interactive-but-dead controls. renderWithProviders(); expect(screen.queryByText("Keyword Tier Overrides")).not.toBeInTheDocument(); expect(screen.queryByText("Semantic keyword matching")).not.toBeInTheDocument(); // Core tier config still renders. - expect(screen.getByText("Complexity Tier Configuration")).toBeInTheDocument(); + expect(screen.getByText("Models by tier")).toBeInTheDocument(); }); it("should call onKeywordTierRulesChange with a new rule when 'Add keyword rule' is clicked", async () => { const user = userEvent.setup(); const onKeywordTierRulesChange = vi.fn(); renderWithProviders(); - fireEvent.click(screen.getByText("Advanced: Keyword/Semantic Matching")); + openAutoRouterAdvanced("Keyword/Semantic Matching"); await user.click(screen.getByRole("button", { name: /add keyword rule/i })); expect(onKeywordTierRulesChange).toHaveBeenCalledTimes(1); const newRules = onKeywordTierRulesChange.mock.calls[0][0]; @@ -537,7 +549,7 @@ describe("ComplexityRouterConfig", () => { onKeywordTierRulesChange={onKeywordTierRulesChange} />, ); - fireEvent.click(screen.getByText("Advanced: Keyword/Semantic Matching")); + openAutoRouterAdvanced("Keyword/Semantic Matching"); const field = screen.getByText("Keywords 1").closest("div") as HTMLElement; await user.type(within(field).getByRole("combobox"), "invoice"); @@ -556,7 +568,7 @@ describe("ComplexityRouterConfig", () => { onKeywordTierRulesChange={onKeywordTierRulesChange} />, ); - fireEvent.click(screen.getByText("Advanced: Keyword/Semantic Matching")); + openAutoRouterAdvanced("Keyword/Semantic Matching"); expect(screen.getByText("invoice")).toBeInTheDocument(); expect(screen.getByText("refund")).toBeInTheDocument(); @@ -564,17 +576,17 @@ describe("ComplexityRouterConfig", () => { expect(onKeywordTierRulesChange).toHaveBeenCalledWith([]); }); - it("should not show embedding model or match score fields when semantic matching is disabled", () => { + it("should not show embedding model or match score fields when semantic matching is disabled", async () => { renderWithProviders(); - fireEvent.click(screen.getByText("Advanced: Keyword/Semantic Matching")); + openAutoRouterAdvanced("Keyword/Semantic Matching"); expect(screen.getByText("Semantic keyword matching")).toBeInTheDocument(); expect(screen.queryByText("Embedding model")).not.toBeInTheDocument(); expect(screen.queryByText("Minimum match score")).not.toBeInTheDocument(); }); - it("should show embedding model and match score fields when semantic matching is enabled", () => { + it("should show embedding model and match score fields when semantic matching is enabled", async () => { renderWithProviders(); - fireEvent.click(screen.getByText("Advanced: Keyword/Semantic Matching")); + openAutoRouterAdvanced("Keyword/Semantic Matching"); expect(screen.getByText("Embedding model")).toBeInTheDocument(); expect(screen.getByText("Minimum match score")).toBeInTheDocument(); }); @@ -589,7 +601,7 @@ describe("ComplexityRouterConfig", () => { onSemanticMatchingEnabledChange={onSemanticMatchingEnabledChange} />, ); - fireEvent.click(screen.getByText("Advanced: Keyword/Semantic Matching")); + openAutoRouterAdvanced("Keyword/Semantic Matching"); await user.click(screen.getByRole("switch", { name: "Semantic keyword matching" })); expect(onSemanticMatchingEnabledChange).toHaveBeenCalledWith(true, expect.anything()); }); @@ -606,34 +618,34 @@ describe("ComplexityRouterConfig", () => { expect(screen.queryAllByText("text-embedding-3-small")).toHaveLength(0); }); - it("does not show tier validation errors by default", () => { + it("does not show tier validation errors by default", async () => { renderWithProviders(); expect(screen.queryByText("This tier is required")).not.toBeInTheDocument(); }); - it("shows an inline error on the classifier model select when llm is selected without a model", () => { + it("shows an inline error on the classifier model select when llm is selected without a model", async () => { const llmValue: ComplexityRouterConfigValue = { ...defaultValue, classifier_type: "llm", classifier_llm_config: { model: "", timeout_ms: 3000 }, }; renderWithProviders(); - fireEvent.click(screen.getByText("Advanced: Classification Method")); - expect(screen.getByText("A classifier model is required")).toBeInTheDocument(); + openAutoRouterAdvanced("Classification Method"); + expect(screen.getByText("A judge model is required")).toBeInTheDocument(); }); - it("does not show the classifier model error once a classifier model is set", () => { + it("does not show the classifier model error once a classifier model is set", async () => { const llmValue: ComplexityRouterConfigValue = { ...defaultValue, classifier_type: "llm", classifier_llm_config: { model: "gpt-3.5-turbo", timeout_ms: 3000 }, }; renderWithProviders(); - fireEvent.click(screen.getByText("Advanced: Classification Method")); - expect(screen.queryByText("A classifier model is required")).not.toBeInTheDocument(); + openAutoRouterAdvanced("Classification Method"); + expect(screen.queryByText("A judge model is required")).not.toBeInTheDocument(); }); - it("shows a validation error only under unfilled tiers when showValidationErrors is true", () => { + it("shows a validation error only under unfilled tiers when showValidationErrors is true", async () => { renderWithProviders( { expect(screen.getAllByText(/tier is required/)).toHaveLength(1); }); - it("renders the escalation keywords section with current keywords when the handler is provided", () => { + it("renders the escalation keywords section with current keywords when the handler is provided", async () => { renderWithProviders( { onEscalationKeywordsChange={vi.fn()} />, ); - fireEvent.click(screen.getByText("Advanced: Escalation Keywords")); - expect(screen.getByText("Escalation Keywords")).toBeInTheDocument(); + openAutoRouterAdvanced("Escalation Keywords"); + expect(screen.getAllByText("Escalation Keywords")).not.toHaveLength(0); expect(screen.getByText("LITELLM ESCALATE")).toBeInTheDocument(); }); - it("hides the escalation keywords section when no handler is provided", () => { + it("hides the escalation keywords section when no handler is provided", async () => { renderWithProviders(); - expect(screen.queryByText("Advanced: Escalation Keywords")).not.toBeInTheDocument(); + expect(screen.queryByText("Escalation Keywords")).not.toBeInTheDocument(); }); }); @@ -671,21 +683,21 @@ describe("ComplexityRouterConfig classifier fallback", () => { classifier_llm_config: { model: "gpt-3.5-turbo", timeout_ms: 3000 }, }; - it("defaults the fallback to the heuristic, matching the backend field default", () => { + it("defaults the fallback to the heuristic, matching the backend field default", async () => { renderWithProviders(); - fireEvent.click(screen.getByText("Advanced: Classification Method")); + openAutoRouterAdvanced("Classification Method"); expect(screen.getByRole("radio", { name: /Score with the heuristic/ })).toBeChecked(); }); - it("records a switch to the default model fallback", () => { + it("records a switch to the default model fallback", async () => { const onChange = vi.fn(); renderWithProviders(); - fireEvent.click(screen.getByText("Advanced: Classification Method")); + openAutoRouterAdvanced("Classification Method"); fireEvent.click(screen.getByRole("radio", { name: /Route to the default model/ })); expect(onChange).toHaveBeenCalledWith(expect.objectContaining({ classifier_fallback: "default_model" })); }); - it("disables the default model fallback when no tier would produce one", () => { + it("disables the default model fallback when no tier would produce one", async () => { // The deployment's default model is derived from the tiers on submit, so offering the option // with no tiers picked would save a config the backend rejects at startup. const noTiers: ComplexityRouterConfigValue = { @@ -693,17 +705,17 @@ describe("ComplexityRouterConfig classifier fallback", () => { tiers: { SIMPLE: [], MEDIUM: [], COMPLEX: [], REASONING: [] }, }; renderWithProviders(); - fireEvent.click(screen.getByText("Advanced: Classification Method")); + openAutoRouterAdvanced("Classification Method"); expect(screen.getByRole("radio", { name: /Route to the default model/ })).toHaveAttribute("aria-disabled", "true"); }); - it("hides the fallback choice for the heuristic classifier, which has nothing to fall back from", () => { + it("hides the fallback choice for the heuristic classifier, which has nothing to fall back from", async () => { renderWithProviders(); - fireEvent.click(screen.getByText("Advanced: Classification Method")); + openAutoRouterAdvanced("Classification Method"); expect(screen.queryByText("If the classifier fails")).not.toBeInTheDocument(); }); - it("stops describing the heuristic as the fallback once a custom prompt routes failures to the default model", () => { + it("stops describing the heuristic as the fallback once a custom prompt routes failures to the default model", async () => { // With both set, the heuristic scorer never runs, so the panel must not keep implying a // score decides anything on this router. renderWithProviders( @@ -717,11 +729,11 @@ describe("ComplexityRouterConfig classifier fallback", () => { onChange={vi.fn()} />, ); - fireEvent.click(screen.getByText("Advanced: Classification Method")); + openAutoRouterAdvanced("Classification Method"); expect(screen.getByText(/no longer runs at all/)).toBeInTheDocument(); }); - it("still describes the heuristic as the fallback when a custom prompt keeps heuristic fallback", () => { + it("still describes the heuristic as the fallback when a custom prompt keeps heuristic fallback", async () => { renderWithProviders( { onChange={vi.fn()} />, ); - fireEvent.click(screen.getByText("Advanced: Classification Method")); + openAutoRouterAdvanced("Classification Method"); expect(screen.getByText(/only when the classifier call fails/)).toBeInTheDocument(); }); - it("clears a stored fallback when switching back to the heuristic classifier", () => { + it("clears a stored fallback when switching back to the heuristic classifier", async () => { const onChange = vi.fn(); renderWithProviders( { onChange={onChange} />, ); - fireEvent.click(screen.getByText("Advanced: Classification Method")); - fireEvent.click(screen.getByRole("radio", { name: /rule-based scoring/ })); + openAutoRouterAdvanced("Classification Method"); + fireEvent.click(screen.getByRole("radio", { name: /^Heuristics$/ })); expect(onChange).toHaveBeenCalledWith(expect.objectContaining({ classifier_fallback: undefined })); }); }); @@ -758,19 +770,21 @@ describe("ComplexityRouterConfig classification frequency", () => { classifier_llm_config: { model: "gpt-3.5-turbo", timeout_ms: 3000 }, }; - it("defaults to every request, matching both backend field defaults", () => { + it("defaults to every request, matching both backend field defaults", async () => { renderWithProviders(); - fireEvent.click(screen.getByText("Advanced: Classification Method")); - expect(screen.getByRole("radio", { name: /Every request/ })).toBeChecked(); - expect(screen.getByRole("radio", { name: /Every new user message/ })).not.toBeChecked(); - expect(screen.getByRole("radio", { name: /Once per session/ })).not.toBeChecked(); + openAutoRouterAdvanced("Classification Method"); + expect(screen.getByRole("combobox", { name: "How often to classify" })).toHaveTextContent("Every request"); + expect(screen.getByRole("combobox", { name: "How often to classify" })).not.toHaveTextContent( + "Every new user message", + ); + expect(screen.getByRole("combobox", { name: "How often to classify" })).not.toHaveTextContent("Once per session"); }); - it("writes both wire fields when the frequency moves to every new user message", () => { + it("writes both wire fields when the frequency moves to every new user message", async () => { const onChange = vi.fn(); renderWithProviders(); - fireEvent.click(screen.getByText("Advanced: Classification Method")); - fireEvent.click(screen.getByRole("radio", { name: /Every new user message/ })); + openAutoRouterAdvanced("Classification Method"); + await selectAutoRouterOption("How often to classify", "Every new user message"); expect(onChange).toHaveBeenCalledWith({ ...llmValue, classification_mode: "user_turn", @@ -778,11 +792,11 @@ describe("ComplexityRouterConfig classification frequency", () => { }); }); - it("writes session affinity, not a classification mode, when the frequency moves to once per session", () => { + it("writes session affinity, not a classification mode, when the frequency moves to once per session", async () => { const onChange = vi.fn(); renderWithProviders(); - fireEvent.click(screen.getByText("Advanced: Classification Method")); - fireEvent.click(screen.getByRole("radio", { name: /Once per session/ })); + openAutoRouterAdvanced("Classification Method"); + await selectAutoRouterOption("How often to classify", "Once per session"); expect(onChange).toHaveBeenCalledWith({ ...llmValue, classification_mode: "every_request", @@ -790,7 +804,7 @@ describe("ComplexityRouterConfig classification frequency", () => { }); }); - it("shows a hand-authored config that sets both fields as once per session, matching the backend", () => { + it("shows a hand-authored config that sets both fields as once per session, matching the backend", async () => { renderWithProviders( { onChange={vi.fn()} />, ); - fireEvent.click(screen.getByText("Advanced: Classification Method")); - expect(screen.getByRole("radio", { name: /Once per session/ })).toBeChecked(); - expect(screen.getByRole("radio", { name: /Every new user message/ })).not.toBeChecked(); + openAutoRouterAdvanced("Classification Method"); + expect(screen.getByRole("combobox", { name: "How often to classify" })).toHaveTextContent("Once per session"); + expect(screen.getByRole("combobox", { name: "How often to classify" })).not.toHaveTextContent( + "Every new user message", + ); }); - it("records a switch back to every request", () => { + it("records a switch back to every request", async () => { const onChange = vi.fn(); renderWithProviders( { onChange={onChange} />, ); - fireEvent.click(screen.getByText("Advanced: Classification Method")); - expect(screen.getByRole("radio", { name: /Every new user message/ })).toBeChecked(); - fireEvent.click(screen.getByRole("radio", { name: /Every request/ })); + openAutoRouterAdvanced("Classification Method"); + expect(screen.getByRole("combobox", { name: "How often to classify" })).toHaveTextContent("Every new user message"); + await selectAutoRouterOption("How often to classify", "Every request"); expect(onChange).toHaveBeenCalledWith(expect.objectContaining({ classification_mode: "every_request" })); }); - it("offers the frequency on a heuristic router, where holding the tier still pins the model", () => { + it("offers the frequency on a heuristic router, where holding the tier still pins the model", async () => { // The backend pin is gated on the two fields alone, so a heuristic router that switches models // mid tool loop is fixed by this control too. renderWithProviders(); - fireEvent.click(screen.getByText("Advanced: Classification Method")); - expect(screen.getByRole("radio", { name: /Every new user message/ })).toBeInTheDocument(); + openAutoRouterAdvanced("Classification Method"); + expect(screen.getByRole("combobox", { name: "How often to classify" })).toBeVisible(); }); }); @@ -836,11 +852,11 @@ describe("ComplexityRouterConfig classifier rubric", () => { const openClassificationPanel = (value: ComplexityRouterConfigValue, onChange = vi.fn()) => { renderWithProviders(); - fireEvent.click(screen.getByText("Advanced: Classification Method")); + openAutoRouterAdvanced("Classification Method"); return onChange; }; - it("shows an existing router with no stored preset as legacy in the prompt control", () => { + it("shows an existing router with no stored preset as legacy in the prompt control", async () => { // This router predates the setting. Displaying a calibrated preset it does not have would tell the // operator their traffic is graded by examples the classifier never receives, and saving the form // unchanged would then move its tier decisions. @@ -849,19 +865,19 @@ describe("ComplexityRouterConfig classifier rubric", () => { expect(screen.getByRole("button", { name: "Customize prompt" })).toBeInTheDocument(); }); - it("stamps the calibrated preset on a classifier being switched on for the first time", () => { + it("stamps the calibrated preset on a classifier being switched on for the first time", async () => { // A heuristic router turning on the LLM classifier has no prior tier behaviour to preserve, so a // newly configured classifier starts on the calibrated rubric rather than the legacy one. const onChange = vi.fn(); renderWithProviders(); - fireEvent.click(screen.getByText("Advanced: Classification Method")); - fireEvent.click(screen.getByText("LLM Classifier")); + openAutoRouterAdvanced("Classification Method"); + fireEvent.click(screen.getByRole("radio", { name: /^LLM$/ })); expect(onChange).toHaveBeenCalledWith( expect.objectContaining({ classifier_llm_config: expect.objectContaining({ classification_rubric: "agentic" }) }), ); }); - it("shows the calibrated preset when a router stores one", () => { + it("shows the calibrated preset when a router stores one", async () => { openClassificationPanel({ ...llmValue, classifier_llm_config: { model: "gpt-3.5-turbo", timeout_ms: 3000, classification_rubric: "agentic" }, @@ -906,7 +922,7 @@ describe("ComplexityRouterConfig classifier rubric", () => { ); }); - it("keeps the rubric out of the legacy whole-prompt editor, which replaces it entirely", () => { + it("keeps the rubric out of the legacy whole-prompt editor, which replaces it entirely", async () => { // The backend rejects both together, so the legacy editor must not offer a rubric to pick. openClassificationPanel({ ...llmValue, @@ -916,7 +932,7 @@ describe("ComplexityRouterConfig classifier rubric", () => { expect(screen.queryByRole("button", { name: "Customize prompt" })).not.toBeInTheDocument(); }); - it("hides the prompt control for the heuristic classifier, which sends no prompt at all", () => { + it("hides the prompt control for the heuristic classifier, which sends no prompt at all", async () => { openClassificationPanel(defaultValue); expect(screen.queryByRole("button", { name: "Customize prompt" })).not.toBeInTheDocument(); }); @@ -928,7 +944,7 @@ describe("ComplexityRouterConfig tier labels", () => { tier_labels: { SIMPLE: "Cheap", MEDIUM: "Standard", COMPLEX: "Premium", REASONING: "Deep" }, }; - it("shows the operator's names in the tier headers instead of the defaults", () => { + it("shows the operator's names in the tier headers instead of the defaults", async () => { renderWithProviders(); expect(screen.getByText("Cheap Tier")).toBeInTheDocument(); expect(screen.getByText("Deep Tier")).toBeInTheDocument(); @@ -936,13 +952,13 @@ describe("ComplexityRouterConfig tier labels", () => { expect(screen.queryByText("Reasoning Tier")).not.toBeInTheDocument(); }); - it("keeps the rung ordinal and canonical name visible under a rename", () => { + it("keeps the rung ordinal and canonical name visible under a rename", async () => { renderWithProviders(); expect(screen.getByText(/Tier 1 of 4/)).toHaveTextContent("Tier 1 of 4 · SIMPLE"); expect(screen.getByText(/Tier 4 of 4/)).toHaveTextContent("Tier 4 of 4 · REASONING"); }); - it("names the renamed tier in the required-field error", () => { + it("names the renamed tier in the required-field error", async () => { renderWithProviders( { expect(screen.getByText("The Deep tier is required")).toBeInTheDocument(); }); - it("reports a typed label back to the caller under its canonical tier key", () => { + it("reports a typed label back to the caller under its canonical tier key", async () => { const onChange = vi.fn(); renderWithProviders(); fireEvent.change(screen.getByLabelText("Display name for the Simple tier"), { target: { value: "Cheap" } }); expect(onChange).toHaveBeenCalledWith(expect.objectContaining({ tier_labels: { SIMPLE: "Cheap" } })); }); - it("shows a stored label in its input so an edit round-trips", () => { + it("shows a stored label in its input so an edit round-trips", async () => { renderWithProviders(); expect(screen.getByLabelText("Display name for the Reasoning tier")).toHaveValue("Deep"); }); - it("leaves the label inputs empty when nothing was renamed", () => { + it("leaves the label inputs empty when nothing was renamed", async () => { renderWithProviders(); expect(screen.getByLabelText("Display name for the Simple tier")).toHaveValue(""); }); - it("uses the operator's names in the classification score table", () => { + it("uses the operator's names in the classification score table", async () => { renderWithProviders(); - fireEvent.click(screen.getByText("Advanced: Classification Method")); + openAutoRouterAdvanced("Classification Method"); expect(screen.getByText("Cheap")).toBeInTheDocument(); expect(screen.getByText("Deep")).toBeInTheDocument(); }); - it("uses the operator's names in the keyword rule tier picker", () => { + it("uses the operator's names in the keyword rule tier picker", async () => { renderWithProviders( { keywordTierRules={[{ id: "r1", keywords: ["invoice"], tier: "REASONING" }]} />, ); - fireEvent.click(screen.getByText("Advanced: Keyword/Semantic Matching")); + openAutoRouterAdvanced("Keyword/Semantic Matching"); expect(screen.getByRole("combobox", { name: "Route keyword rule 1 to tier" })).toHaveTextContent("Deep"); }); }); describe("ComplexityRouterConfig modality panel", () => { - it("defaults the image-routing switch off and writes modality_routing through onChange", () => { + it("defaults the image-routing switch off and writes modality_routing through onChange", async () => { const onChange = vi.fn(); renderWithProviders(); - fireEvent.click(screen.getByText("Advanced: Modality Routing")); + openAutoRouterAdvanced("Modality Routing"); const toggle = screen.getByRole("switch", { name: "Route image requests to vision-capable models" }); expect(toggle).not.toBeChecked(); @@ -1003,19 +1019,19 @@ describe("ComplexityRouterConfig modality panel", () => { expect(onChange).toHaveBeenCalledWith({ ...defaultValue, modality_routing: true }); }); - it("renders a stored modality_routing=true as on", () => { + it("renders a stored modality_routing=true as on", async () => { renderWithProviders(); - fireEvent.click(screen.getByText("Advanced: Modality Routing")); + openAutoRouterAdvanced("Modality Routing"); expect(screen.getByRole("switch", { name: "Route image requests to vision-capable models" })).toBeChecked(); }); // The backend ignores modality_pin_override unless modality_routing is on, so offering it while // image routing is off would let an operator save a flag that does nothing. - it("disables the pin-override switch while image routing is off", () => { + it("disables the pin-override switch while image routing is off", async () => { const onChange = vi.fn(); renderWithProviders(); - fireEvent.click(screen.getByText("Advanced: Modality Routing")); + openAutoRouterAdvanced("Modality Routing"); const override = screen.getByRole("switch", { name: "Override session pin for image requests" }); expect(override).toHaveAttribute("aria-disabled", "true"); @@ -1023,11 +1039,11 @@ describe("ComplexityRouterConfig modality panel", () => { expect(onChange).not.toHaveBeenCalled(); }); - it("writes modality_pin_override through onChange once image routing is on", () => { + it("writes modality_pin_override through onChange once image routing is on", async () => { const onChange = vi.fn(); const value = { ...defaultValue, modality_routing: true }; renderWithProviders(); - fireEvent.click(screen.getByText("Advanced: Modality Routing")); + openAutoRouterAdvanced("Modality Routing"); const override = screen.getByRole("switch", { name: "Override session pin for image requests" }); expect(override).not.toBeChecked(); @@ -1036,51 +1052,51 @@ describe("ComplexityRouterConfig modality panel", () => { expect(onChange).toHaveBeenCalledWith({ ...value, modality_pin_override: true }); }); - it("renders a stored modality_pin_override=true as on", () => { + it("renders a stored modality_pin_override=true as on", async () => { renderWithProviders( , ); - fireEvent.click(screen.getByText("Advanced: Modality Routing")); + openAutoRouterAdvanced("Modality Routing"); expect(screen.getByRole("switch", { name: "Override session pin for image requests" })).toBeChecked(); }); }); describe("ComplexityRouterConfig affinity panel", () => { - it("holds the deployment switch at its backend default, session pinning having moved to the frequency choice", () => { + it("holds the deployment switch at its backend default, session pinning having moved to the frequency choice", async () => { renderWithProviders(); - fireEvent.click(screen.getByText("Advanced: Affinity")); + openAutoRouterAdvanced("Affinity"); expect(screen.getByRole("switch", { name: "Pin one model deployment per tier" })).toBeChecked(); expect(screen.queryByRole("switch", { name: "Pin a session to its first model" })).not.toBeInTheDocument(); }); - it("writes deployment_affinity through onChange without touching other keys", () => { + it("writes deployment_affinity through onChange without touching other keys", async () => { const onChange = vi.fn(); renderWithProviders(); - fireEvent.click(screen.getByText("Advanced: Affinity")); + openAutoRouterAdvanced("Affinity"); fireEvent.click(screen.getByRole("switch", { name: "Pin one model deployment per tier" })); expect(onChange).toHaveBeenCalledWith({ ...defaultValue, deployment_affinity: false }); }); - it("renders a stored deployment_affinity=false as off", () => { + it("renders a stored deployment_affinity=false as off", async () => { renderWithProviders( , ); - fireEvent.click(screen.getByText("Advanced: Affinity")); + openAutoRouterAdvanced("Affinity"); expect(screen.getByRole("switch", { name: "Pin one model deployment per tier" })).not.toBeChecked(); }); - it("writes an idle TTL on blur and keeps the partial input as a draft while typing", () => { + it("writes an idle TTL on blur and keeps the partial input as a draft while typing", async () => { const onChange = vi.fn(); renderWithProviders(); - fireEvent.click(screen.getByText("Advanced: Affinity")); + openAutoRouterAdvanced("Affinity"); const ttl = screen.getByLabelText("How long a pin survives idle (seconds)"); expect(ttl).toHaveAttribute("placeholder", "3600"); @@ -1091,11 +1107,11 @@ describe("ComplexityRouterConfig affinity panel", () => { expect(onChange).toHaveBeenCalledWith({ ...defaultValue, session_affinity_ttl_seconds: 300 }); }); - it("clearing the idle TTL returns the router to its backend default", () => { + it("clearing the idle TTL returns the router to its backend default", async () => { const onChange = vi.fn(); const value = { ...defaultValue, session_affinity_ttl_seconds: 300 }; renderWithProviders(); - fireEvent.click(screen.getByText("Advanced: Affinity")); + openAutoRouterAdvanced("Affinity"); const ttl = screen.getByLabelText("How long a pin survives idle (seconds)"); expect(ttl).toHaveValue("300"); @@ -1105,10 +1121,10 @@ describe("ComplexityRouterConfig affinity panel", () => { expect(onChange).toHaveBeenCalledWith({ ...value, session_affinity_ttl_seconds: undefined }); }); - it("clamps a non-positive idle TTL to the backend's minimum", () => { + it("clamps a non-positive idle TTL to the backend's minimum", async () => { const onChange = vi.fn(); renderWithProviders(); - fireEvent.click(screen.getByText("Advanced: Affinity")); + openAutoRouterAdvanced("Affinity"); const ttl = screen.getByLabelText("How long a pin survives idle (seconds)"); fireEvent.change(ttl, { target: { value: "0" } }); @@ -1121,12 +1137,12 @@ describe("ComplexityRouterConfig affinity panel", () => { describe("ComplexityRouterConfig default model", () => { const getDefaultModelSelect = () => screen.getByRole("combobox", { name: "Default model" }); - it("shows what the tiers currently imply, so an untouched router still names its default", () => { + it("shows what the tiers currently imply, so an untouched router still names its default", async () => { renderWithProviders(); expect(getDefaultModelSelect()).toHaveAttribute("placeholder", "Derived from tiers: gpt-3.5-turbo"); }); - it("asks for a model rather than naming a derived one when no tier holds one", () => { + it("asks for a model rather than naming a derived one when no tier holds one", async () => { const noTiers: ComplexityRouterConfigValue = { ...defaultValue, tiers: { SIMPLE: [], MEDIUM: [], COMPLEX: [], REASONING: [] }, @@ -1157,13 +1173,13 @@ describe("ComplexityRouterConfig default model", () => { expect(onChange).toHaveBeenCalledWith(expect.objectContaining({ default_model: undefined })); }); - it("shows a pinned model as the selection instead of the tier-derived one", () => { + it("shows a pinned model as the selection instead of the tier-derived one", async () => { const pinned: ComplexityRouterConfigValue = { ...defaultValue, default_model: "claude-3-opus" }; renderWithProviders(); expect(getDefaultModelSelect()).toHaveValue("claude-3-opus"); }); - it("unlocks the default model fallback on a pin alone, with no tier to derive from", () => { + it("unlocks the default model fallback on a pin alone, with no tier to derive from", async () => { const pinnedNoTiers: ComplexityRouterConfigValue = { ...defaultValue, classifier_type: "llm", @@ -1172,11 +1188,11 @@ describe("ComplexityRouterConfig default model", () => { default_model: "claude-3-opus", }; renderWithProviders(); - fireEvent.click(screen.getByText("Advanced: Classification Method")); + openAutoRouterAdvanced("Classification Method"); expect(screen.getByRole("radio", { name: /Route to the default model/ })).not.toHaveAttribute("aria-disabled"); }); - it("names the resolved default on the fallback option, so the destination is not a guess", () => { + it("names the resolved default on the fallback option, so the destination is not a guess", async () => { const pinned: ComplexityRouterConfigValue = { ...defaultValue, classifier_type: "llm", @@ -1184,13 +1200,13 @@ describe("ComplexityRouterConfig default model", () => { default_model: "claude-3-opus", }; renderWithProviders(); - fireEvent.click(screen.getByText("Advanced: Classification Method")); + openAutoRouterAdvanced("Classification Method"); expect(screen.getByRole("radio", { name: /Route to the default model \(claude-3-opus\)/ })).toBeInTheDocument(); }); }); describe("plan-mode override", () => { - const openPanel = () => fireEvent.click(screen.getByText("Advanced: Plan-Mode Override")); + const openPanel = () => openAutoRouterAdvanced("Plan-Mode Override"); const switchName = "Route plan-mode requests to a minimum tier"; it("toggling on floors at the highest tier that has models", async () => { @@ -1248,13 +1264,13 @@ describe("plan-mode override", () => { }); describe("ComplexityRouterConfig per-model reasoning effort", () => { - it("renders one effort select per selected model, defaulting to Default", () => { + it("renders one effort select per selected model, defaulting to Default", async () => { renderWithProviders(); const select = screen.getByRole("combobox", { name: "Reasoning effort for gpt-4 in the Complex tier" }); expect(select).toHaveTextContent("Default"); }); - it("shows the hydrated effort for a model that has one stored", () => { + it("shows the hydrated effort for a model that has one stored", async () => { renderWithProviders( { const renderClassifier = (value: ComplexityRouterConfigValue = llmValue, onChange = vi.fn()) => { renderWithProviders(); - fireEvent.click(screen.getByText("Advanced: Classification Method")); + openAutoRouterAdvanced("Classification Method"); return onChange; }; @@ -1348,7 +1364,7 @@ describe("ComplexityRouterConfig classifier reasoning effort", () => { classifier_llm_config: { model: "gpt-4", timeout_ms: 3000, reasoning_effort: "high" }, }); const user = userEvent.setup(); - await user.click(screen.getByRole("combobox", { name: "Classifier Model" })); + await user.click(screen.getByRole("combobox", { name: "Judge model" })); await user.click(await screen.findByRole("option", { name: "gpt-3.5-turbo" })); expect(onChange).toHaveBeenCalledWith({ ...llmValue, @@ -1364,7 +1380,7 @@ describe("ComplexityRouterConfig classifier reasoning effort", () => { classifier_llm_config: { model: "gpt-4", timeout_ms: 3000, reasoning_effort: "high" }, }); const user = userEvent.setup(); - await user.click(screen.getByRole("combobox", { name: "Classifier Model" })); + await user.click(screen.getByRole("combobox", { name: "Judge model" })); if (action === "click") await user.click(await screen.findByRole("option", { name: "gpt-4" })); else await user.keyboard("{Enter}"); expect(onChange).not.toHaveBeenCalled(); @@ -1397,7 +1413,7 @@ describe("ComplexityRouterConfig classifier reasoning effort", () => { }); describe("ComplexityRouterConfig reasoning effort gating", () => { - it("offers no effort select for a model group without reasoning support", () => { + it("offers no effort select for a model group without reasoning support", async () => { renderWithProviders(); expect( screen.queryByRole("combobox", { name: "Reasoning effort for gpt-3.5-turbo in the Simple tier" }), @@ -1406,7 +1422,7 @@ describe("ComplexityRouterConfig reasoning effort gating", () => { // A stored effort on a model the group info calls non-reasoning must stay visible, or the // operator has no way to clear it. - it("keeps the select for a non-reasoning model that already has a stored effort", () => { + it("keeps the select for a non-reasoning model that already has a stored effort", async () => { renderWithProviders( { // An empty list is the group's own answer that its deployments share no level, which is different // from the field being absent, so the control is dropped rather than falling back to every level. - it("offers no effort at all when the group intersects to nothing", () => { + it("offers no effort at all when the group intersects to nothing", async () => { renderWithProviders( { // Hand-authored configs can carry a level outside the supported set (e.g. max); it must render // and stay clearable rather than being masked as Default. - it("keeps showing a stored effort outside the supported set", () => { + it("keeps showing a stored effort outside the supported set", async () => { renderWithProviders( { describe("ComplexityRouterConfig custom technical keywords", () => { const openClassificationPanel = (value: ComplexityRouterConfigValue) => { renderWithProviders(); - fireEvent.click(screen.getByText("Advanced: Classification Method")); + openAutoRouterAdvanced("Classification Method"); }; const llmConfig = { model: "gpt-3.5-turbo", timeout_ms: 3000 }; @@ -1503,7 +1519,7 @@ describe("ComplexityRouterConfig custom technical keywords", () => { expect(screen.getByText("Custom Technical Keywords")).toBeInTheDocument(); }); - it("hides the keywords when the scorer never runs, so they cannot imply an effect they have none", () => { + it("hides the keywords when the scorer never runs, so they cannot imply an effect they have none", async () => { const llmWithDefaultFallback = { ...defaultValue, classifier_type: "llm" as const, @@ -1547,19 +1563,19 @@ describe("ComplexityRouterConfig tier editing", () => { }, }; - it("offers Edit tiers only when the parent owns the editor flag", () => { + it("offers Edit tiers only when the parent owns the editor flag", async () => { renderWithProviders(); expect(screen.queryByRole("button", { name: "Edit tiers" })).not.toBeInTheDocument(); }); - it("surfaces the caller's orphaned-rule verdict while editing, so Done is not a silent exit", () => { + it("surfaces the caller's orphaned-rule verdict while editing, so Done is not a silent exit", async () => { renderEditor(customValue, { keywordRulesError: "Keyword rule(s) 1 route to a tier this router no longer has" }); expect( screen.getByText("Keyword rule(s) 1 route to a tier this router no longer has", { exact: false }), ).toBeInTheDocument(); }); - it("keeps the orphaned-rule verdict out of the collapsed view, where the submit tooltip owns it", () => { + it("keeps the orphaned-rule verdict out of the collapsed view, where the submit tooltip owns it", async () => { renderWithProviders( { expect(screen.queryByText("route to a tier this router no longer has", { exact: false })).not.toBeInTheDocument(); }); - it("renders the four built-in tiers before any edit, unchanged", () => { + it("renders the four built-in tiers before any edit, unchanged", async () => { renderWithProviders(); expect(screen.getByRole("button", { name: "Edit tiers" })).toBeInTheDocument(); expect(screen.getByText("Tier 1 of 4", { exact: false })).toHaveTextContent("SIMPLE"); }); - it("adds a row and moves the form into an edited tier set, which the built-in record never leaves", () => { + it("adds a row and moves the form into an edited tier set, which the built-in record never leaves", async () => { const { committed } = renderEditor(); fireEvent.click(screen.getByRole("button", { name: "Add tier" })); const next = committed(); @@ -1585,7 +1601,7 @@ describe("ComplexityRouterConfig tier editing", () => { expect(next.tiers).toEqual(defaultValue.tiers); }); - it("renames a built-in tier straight from the editor, which is what makes the set custom", () => { + it("renames a built-in tier straight from the editor, which is what makes the set custom", async () => { const { committed } = renderEditor(); fireEvent.change(screen.getByLabelText("Name for tier 3"), { target: { value: "SECURITY_REVIEW" } }); const next = committed(); @@ -1598,13 +1614,13 @@ describe("ComplexityRouterConfig tier editing", () => { expect(next.tiers).toEqual(defaultValue.tiers); }); - it("opening the editor and changing nothing leaves the router on the built-in tiers", () => { + it("opening the editor and changing nothing leaves the router on the built-in tiers", async () => { const { onChange } = renderEditor(); expect(screen.getByRole("button", { name: "Done" })).toBeEnabled(); expect(onChange).not.toHaveBeenCalled(); }); - it("swaps the display-name field for the tier-name field while the editor is open", () => { + it("swaps the display-name field for the tier-name field while the editor is open", async () => { const { rerender } = renderWithProviders(); expect(screen.getByLabelText("Display name for the Simple tier")).toBeInTheDocument(); rerender(); @@ -1612,23 +1628,23 @@ describe("ComplexityRouterConfig tier editing", () => { expect(screen.getByLabelText("Name for tier 1")).toBeInTheDocument(); }); - it("drops the scorer card entirely once an edited tier set replaces the heuristic", () => { + it("drops the scorer card entirely once an edited tier set replaces the heuristic", async () => { renderWithProviders(); - fireEvent.click(screen.getByText("Advanced: Classification Method")); + openAutoRouterAdvanced("Classification Method"); expect(screen.queryByText("How Classification Works")).not.toBeInTheDocument(); expect( screen.queryByText("scores each request across 7 built-in dimensions", { exact: false }), ).not.toBeInTheDocument(); }); - it("keeps the scorer card on a built-in router, whose tiers the score still decides", () => { + it("keeps the scorer card on a built-in router, whose tiers the score still decides", async () => { renderWithProviders(); - fireEvent.click(screen.getByText("Advanced: Classification Method")); + openAutoRouterAdvanced("Classification Method"); expect(screen.getByText("How Classification Works")).toBeInTheDocument(); expect(screen.getByText("scores each request across 7 built-in dimensions", { exact: false })).toBeInTheDocument(); }); - it("says why a custom row is blocked instead of only reddening its border", () => { + it("says why a custom row is blocked instead of only reddening its border", async () => { const missingDefinition: ComplexityRouterConfigValue = { ...customValue, custom_tier_set: { @@ -1652,24 +1668,24 @@ describe("ComplexityRouterConfig tier editing", () => { expect(screen.getByRole("button", { name: "Done" })).toBeDisabled(); }); - it("enables Done once every row carries a name, a definition and a model", () => { + it("enables Done once every row carries a name, a definition and a model", async () => { renderEditor(customValue); expect(screen.getByRole("button", { name: "Done" })).toBeEnabled(); }); - it("refuses to remove a row that would take the set below the backend's minimum", () => { + it("refuses to remove a row that would take the set below the backend's minimum", async () => { renderEditor(customValue); expect(screen.getByRole("button", { name: "Remove the CASUAL tier" })).toBeDisabled(); }); - it("keeps a definition on one line, because the backend rejects a newline in it", () => { + it("keeps a definition on one line, because the backend rejects a newline in it", async () => { const { committed } = renderEditor(customValue); fireEvent.change(screen.getByLabelText("Definition for tier 2"), { target: { value: "audits\nand reviews" } }); const next = committed(); expect(next.custom_tier_set?.tiers[1].definition).toBe("audits and reviews"); }); - it("moves a keyword rule with the tier it points at when that tier is renamed", () => { + it("moves a keyword rule with the tier it points at when that tier is renamed", async () => { const onKeywordTierRulesChange = vi.fn(); renderWithProviders( { expect(onKeywordTierRulesChange).toHaveBeenCalledWith([{ id: "r1", keywords: ["audit"], tier: "AUDIT" }]); }); - it("re-points the fallback tier when the row it named is removed, never leaving it dangling", () => { + it("re-points the fallback tier when the row it named is removed, never leaving it dangling", async () => { const threeRows: ComplexityRouterConfigValue = { ...customValue, custom_tier_set: { @@ -1702,7 +1718,7 @@ describe("ComplexityRouterConfig tier editing", () => { expect(next.custom_tier_set?.tiers.some((row) => row.id === next.custom_tier_set?.fallback_tier_id)).toBe(true); }); - it("turns off a plan-mode floor whose row was removed, rather than leaving it pointing at nothing", () => { + it("turns off a plan-mode floor whose row was removed, rather than leaving it pointing at nothing", async () => { const withFloor: ComplexityRouterConfigValue = { ...customValue, plan_mode_min_tier: "sec", @@ -1719,14 +1735,14 @@ describe("ComplexityRouterConfig tier editing", () => { expect(committed().plan_mode_min_tier).toBeUndefined(); }); - it("replaces the display-name inputs with the reason an edited tier set forbids them", () => { + it("replaces the display-name inputs with the reason an edited tier set forbids them", async () => { renderWithProviders(); expect(screen.queryByLabelText("Display name for the Simple tier")).not.toBeInTheDocument(); expect(screen.getByText("Display names rename the built-in tiers", { exact: false })).toBeInTheDocument(); expect(screen.getByLabelText("Fallback tier")).toBeInTheDocument(); }); - it("disables the once-per-session frequency and says why, rather than letting a stripped value look saved", () => { + it("disables the once-per-session frequency and says why, rather than letting a stripped value look saved", async () => { renderWithProviders( { onEditingTiersChange={vi.fn()} />, ); - fireEvent.click(screen.getByText("Advanced: Classification Method")); - const sessionOption = screen.getByRole("radio", { name: /Once per session/ }); + openAutoRouterAdvanced("Classification Method"); + fireEvent.click(screen.getByRole("combobox", { name: "How often to classify" })); + const sessionOption = screen.getByRole("option", { name: "Once per session" }); expect(sessionOption).toHaveAttribute("aria-disabled", "true"); expect(sessionOption).not.toBeChecked(); expect( @@ -1743,15 +1760,15 @@ describe("ComplexityRouterConfig tier editing", () => { ).toBeInTheDocument(); }); - it("lets an edited tier set write its own opening instructions instead of refusing a prompt outright", () => { + it("lets an edited tier set write its own opening instructions instead of refusing a prompt outright", async () => { renderWithProviders(); - fireEvent.click(screen.getByText("Advanced: Classification Method")); + openAutoRouterAdvanced("Classification Method"); expect(screen.getByText("your own calibration examples", { exact: false })).toBeInTheDocument(); expect(screen.getByRole("button", { name: "Customize prompt" })).toBeInTheDocument(); expect(screen.queryByText("A replacement prompt drops the tier bullets", { exact: false })).not.toBeInTheDocument(); }); - it("gives built-in routers the opening-only editor, keeping the tier definitions derived", () => { + it("gives built-in routers the opening-only editor, keeping the tier definitions derived", async () => { renderWithProviders( { onEditingTiersChange={vi.fn()} />, ); - fireEvent.click(screen.getByText("Advanced: Classification Method")); + openAutoRouterAdvanced("Classification Method"); expect(screen.getByText("The base rubric supplies", { exact: false })).toBeInTheDocument(); expect(screen.getByRole("button", { name: "Customize prompt" })).toBeInTheDocument(); expect(screen.queryByText("Replace the built-in complexity rubric", { exact: false })).not.toBeInTheDocument(); }); - it("keeps the legacy whole-prompt editor only on a router that already stored a replacement prompt", () => { + it("keeps the legacy whole-prompt editor only on a router that already stored a replacement prompt", async () => { renderWithProviders( { onEditingTiersChange={vi.fn()} />, ); - fireEvent.click(screen.getByText("Advanced: Classification Method")); + openAutoRouterAdvanced("Classification Method"); expect(screen.getByRole("button", { name: "Edit custom prompt" })).toBeInTheDocument(); expect(screen.queryByRole("button", { name: "Customize prompt" })).not.toBeInTheDocument(); }); - it("leaves built-in routers with their display-name inputs and no restriction copy", () => { + it("leaves built-in routers with their display-name inputs and no restriction copy", async () => { renderWithProviders(); expect(screen.getByLabelText("Display name for the Simple tier")).toBeInTheDocument(); expect(screen.queryByText("Display names rename the built-in tiers", { exact: false })).not.toBeInTheDocument(); @@ -1810,9 +1827,9 @@ describe("classifier vision settings", () => { ); }; - it("starts off and reveals the default cap when enabled", () => { + it("starts off and reveals the default cap when enabled", async () => { renderWithProviders(); - fireEvent.click(screen.getByText("Advanced: Classification Method")); + openAutoRouterAdvanced("Classification Method"); const vision = screen.getByRole("switch", { name: "Use images for classification" }); expect(vision).not.toBeChecked(); @@ -1823,10 +1840,10 @@ describe("classifier vision settings", () => { expect(screen.getByLabelText("Maximum images per request")).toHaveValue("1"); }); - it("writes the switch and a clamped image cap into the classifier config", () => { + it("writes the switch and a clamped image cap into the classifier config", async () => { const onChange = vi.fn(); renderWithProviders(); - fireEvent.click(screen.getByText("Advanced: Classification Method")); + openAutoRouterAdvanced("Classification Method"); fireEvent.click(screen.getByRole("switch", { name: "Use images for classification" })); expect(onChange).toHaveBeenLastCalledWith({ @@ -1841,10 +1858,10 @@ describe("classifier vision settings", () => { }); }); - it("keeps the image cap draft empty until a valid value is entered", () => { + it("keeps the image cap draft empty until a valid value is entered", async () => { const onChange = vi.fn(); renderWithProviders(); - fireEvent.click(screen.getByText("Advanced: Classification Method")); + openAutoRouterAdvanced("Classification Method"); fireEvent.click(screen.getByRole("switch", { name: "Use images for classification" })); onChange.mockClear(); @@ -1861,9 +1878,9 @@ describe("classifier vision settings", () => { }); }); - it("is absent when the classifier is heuristic", () => { + it("is absent when the classifier is heuristic", async () => { renderWithProviders(); - fireEvent.click(screen.getByText("Advanced: Classification Method")); + openAutoRouterAdvanced("Classification Method"); expect(screen.queryByText("Use images for classification")).not.toBeInTheDocument(); }); diff --git a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx index acf6b62a95a..32f9ebf97ad 100644 --- a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx +++ b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx @@ -1,4 +1,6 @@ import RoutingOptions from "./RoutingOptions"; +import ClassifierPrimarySettings from "./ClassifierPrimarySettings"; +import { AutoRouterAllowanceNote } from "./AutoRouterAvailability"; import type { JevClassifierConfig } from "./jev_classifier_config"; import { type ClassifierType } from "./classifier_types"; export { type ClassifierType, usesLlmClassifier, usesClassifierContext } from "./classifier_types"; @@ -6,11 +8,11 @@ import ForecastClassifierConfig, { ForecastSolverModels } from "./ForecastClassi import { isForecastClassifier, type CapabilitySettings, type FuseSettings } from "./forecast_classifier_config"; import { SimpleTooltip } from "@/components/ui/tooltip"; import { MultiSelect } from "@/components/shared/MultiSelect"; +import TierConfigIntro from "./TierConfigIntro"; import DefaultModelField from "./DefaultModelField"; import { Info, Plus, Trash2, X } from "lucide-react"; import NonReasoningTierToggle from "./NonReasoningTierToggle"; -import TierConfigIntro from "./TierConfigIntro"; import TierRowSelect from "./TierRowSelect"; import { Card, CardContent } from "@/components/ui/card"; import { InputGroup, InputGroupAddon, InputGroupButton, InputGroupInput } from "@/components/ui/input-group"; @@ -226,10 +228,16 @@ const TierSetToolbar: React.FC<{ ) )} + {editing && ( + + )} {editing && ( Add or remove tiers to define your own set. Every custom tier needs a definition the classifier routes on, and - an edited set requires the LLM or JEV classification method + an edited set requires the LLM or Jev classification method )} {editing && keywordRulesError && ( @@ -596,10 +604,14 @@ const ComplexityRouterConfig: React.FC = ({ return (
+
-

- {forecast ? "Solver models" : "Complexity Tier Configuration"} -

+

{forecast ? "Solver models" : "Models by tier"}

{!forecast && ( @@ -619,6 +631,7 @@ const ComplexityRouterConfig: React.FC = ({ fastModeByModel={fastModeByModel} /> = ({ ) : ( <> - {!customTierSet && ( @@ -750,13 +762,19 @@ const ComplexityRouterConfig: React.FC = ({ )} - {!forecast && } + - + {forecast && ( <> - void>(); renderWithProviders(); - await user.click(screen.getByRole("button", { name: "Advanced routing options" })); + openAutoRouterAdvanced("Keyword/Semantic Matching"); expect(screen.getByRole("switch", { name: "Fast mode for secondary in the Medium routing pool tier" })).toBeChecked(); expect(onChange).not.toHaveBeenCalled(); await user.click(screen.getByRole("combobox", { name: "Select medium routing pool models" })); @@ -245,7 +246,7 @@ it.each(["capability", "llm_v2"] as const)( ); const view = renderWithProviders(editor(hydrateComplexityRouterConfig(stored, undefined))); - await user.click(screen.getByRole("button", { name: "Advanced routing options" })); + openAutoRouterAdvanced("Keyword/Semantic Matching"); const select = () => screen.getByRole("combobox", { name: "Default model" }); expect(select()).toHaveValue("legacy-default"); expect(onChange).not.toHaveBeenCalled(); @@ -284,8 +285,8 @@ it.each(["capability", "llm_v2"] as const)("offers only populated keyword target /> ); const view = renderWithProviders(editor([])); - await user.click(screen.getByRole("button", { name: "Advanced routing options" })); - await user.click(screen.getByText("Advanced: Keyword/Semantic Matching")); + openAutoRouterAdvanced("Keyword/Semantic Matching"); + openAutoRouterAdvanced("Keyword/Semantic Matching"); await user.click(screen.getByRole("button", { name: "Add keyword rule" })); const rules = onRulesChange.mock.lastCall![0]; expect(rules[0].tier).toBe("SIMPLE"); diff --git a/ui/litellm-dashboard/src/components/add_model/ForecastClassifierConfig.integration.test.tsx b/ui/litellm-dashboard/src/components/add_model/ForecastClassifierConfig.integration.test.tsx index a03ccb11456..f243e63d387 100644 --- a/ui/litellm-dashboard/src/components/add_model/ForecastClassifierConfig.integration.test.tsx +++ b/ui/litellm-dashboard/src/components/add_model/ForecastClassifierConfig.integration.test.tsx @@ -1,3 +1,4 @@ +import { selectAutoRouterApproach } from "../../../tests/autoRouterSetup"; import React, { useState } from "react"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import userEvent from "@testing-library/user-event"; @@ -306,7 +307,7 @@ describe("forecast classifier form", () => { ); }); - it("switches a populated standard router to Capability without saving hidden pools or their overrides", () => { + it("switches a populated standard router to Capability without saving hidden pools or their overrides", async () => { renderWithProviders( { }} />, ); - fireEvent.click(screen.getByRole("tab", { name: "Capability" })); + await selectAutoRouterApproach("Capability"); fireEvent.change(screen.getByLabelText("Solve probability threshold"), { target: { value: "0.7" } }); expect(screen.getByRole("button", { name: "Save configuration" })).toBeEnabled(); fireEvent.click(screen.getByRole("button", { name: "Save configuration" })); @@ -347,7 +348,7 @@ describe("forecast classifier form", () => { it.each(["capability", "llm_v2"] as const)( "carries non-default solver assignments when switching away from %s", - (source) => { + async (source) => { const pair = { efficient_tier: "MEDIUM", capable_tier: "COMPLEX" }; const previous: ComplexityRouterConfigValue = { ...(source === "capability" ? initial : fuseInitial), @@ -359,7 +360,7 @@ describe("forecast classifier form", () => { tier_model_params: { MEDIUM: { efficient: { max_tokens: 128 } }, COMPLEX: { capable: { speed: "fast" } } }, }; renderWithProviders(); - fireEvent.click(screen.getByRole("tab", { name: source === "capability" ? "Fuse v2" : "Capability" })); + await selectAutoRouterApproach(source === "capability" ? "Fuse v2" : "Capability"); if (source === "capability") { fireEvent.change(screen.getByLabelText("Efficient solver profile"), { target: { value: "Small solver" } }); fireEvent.change(screen.getByLabelText("Capable solver profile"), { target: { value: "Large solver" } }); @@ -402,20 +403,22 @@ describe("forecast classifier form", () => { ] as const)("restores the current rubric when switching %s through Complexity to %s", async (source, target) => { const user = userEvent.setup(); renderWithProviders(); - fireEvent.click(screen.getByRole("tab", { name: "Complexity" })); + await selectAutoRouterApproach("Complexity"); fireEvent.click(screen.getByRole("radio", { name: new RegExp(`^${target}`) })); - await user.click(screen.getByRole("combobox", { name: "Classifier Model" })); + await user.click(screen.getByRole("combobox", { name: "Judge model" })); await user.click(screen.getByRole("option", { name: "judge" })); fireEvent.click(screen.getByRole("button", { name: "Save configuration" })); const output = screen.getByRole("status", { name: "Saved configuration" }); expect(output).toHaveTextContent('"classification_rubric":"agentic"'); expect(output).toHaveTextContent('"model":"judge"'); - expect(output).toHaveTextContent('"timeout_ms":3000'); + expect(output).toHaveTextContent( + `"timeout_ms":${(source === "capability" ? initial : fuseInitial).classifier_llm_config?.timeout_ms}`, + ); expect(output).not.toHaveTextContent('"capability_classifier_config"'); expect(output).not.toHaveTextContent('"llm_v2_config"'); }); - it("saves capability threshold edits together with fitted calibration", () => { + it("saves capability threshold edits together with fitted calibration", async () => { renderWithProviders(); fireEvent.change(screen.getByLabelText("Solve probability threshold"), { target: { value: "0.6" } }); fireEvent.click(screen.getByRole("button", { name: "Classifier options" })); @@ -432,9 +435,9 @@ describe("forecast classifier form", () => { expect(screen.getByRole("button", { name: "Save configuration" })).toBeDisabled(); }); - it("switches to Fuse, requires solver context, and saves the filled fields", () => { + it("switches to Fuse, requires solver context, and saves the filled fields", async () => { renderWithProviders(); - fireEvent.click(screen.getByRole("tab", { name: "Fuse v2" })); + await selectAutoRouterApproach("Fuse v2"); expect(screen.queryByLabelText("Solve probability threshold")).not.toBeInTheDocument(); expect(screen.getByRole("button", { name: "Save configuration" })).toBeDisabled(); fireEvent.change(screen.getByLabelText("Efficient solver profile"), { diff --git a/ui/litellm-dashboard/src/components/add_model/ForecastClassifierConfig.tsx b/ui/litellm-dashboard/src/components/add_model/ForecastClassifierConfig.tsx index c336423fe39..d759b870209 100644 --- a/ui/litellm-dashboard/src/components/add_model/ForecastClassifierConfig.tsx +++ b/ui/litellm-dashboard/src/components/add_model/ForecastClassifierConfig.tsx @@ -36,6 +36,7 @@ interface Props { onChange: (value: ComplexityRouterConfigValue) => void; modelOptions: { value: string; label: string }[]; effortOptionsByModel: Record; + section?: "all" | "required" | "advanced"; } const NumberField = ({ @@ -188,7 +189,7 @@ const CalibrationFields = ({ const emptyCoefficients = () => ({ slope: Number.NaN, intercept: Number.NaN }); -const ForecastClassifierConfig = ({ value, onChange, modelOptions, effortOptionsByModel }: Props) => { +const ForecastClassifierConfig = ({ value, onChange, modelOptions, effortOptionsByModel, section = "all" }: Props) => { const id = React.useId(); const isCapability = value.classifier_type === "capability"; const capability = value.capability_classifier_config ?? newCapabilitySettings(); @@ -212,72 +213,195 @@ const ForecastClassifierConfig = ({ value, onChange, modelOptions, effortOptions ? "Forecasts whether the efficient solver can complete the task using the bundled capability card" : "Forecasts success for both solvers and selects efficient when the estimated quality gap is within your allowance"}

-
- - { - if (model === llm.model) return; - onChange({ ...value, classifier_llm_config: { ...llm, model: model ?? "", reasoning_effort: undefined } }); - }} - /> -
- {isCapability ? ( + {section === "all" && ( <> - updateCapability({ ...capability, base_threshold })} - /> - - ) : ( - <> - - updateFuse({ ...fuse, max_quality_gap })} - /> +
+ + { + if (model === llm.model) return; + onChange({ + ...value, + classifier_llm_config: { ...llm, model: model ?? "", reasoning_effort: undefined }, + }); + }} + /> +
)} - - - - Classifier options - - - onChange({ ...value, classifier_llm_config: { ...llm, reasoning_effort } })} - /> - onChange({ ...value, classifier_llm_config: { ...llm, timeout_ms } })} - /> - onChange({ ...value, classifier_llm_config })} - /> - onChange({ ...value, classifier_llm_config })} - /> + {section !== "advanced" && ( + <> + {isCapability ? ( + <> + updateCapability({ ...capability, base_threshold })} + /> + + ) : ( + <> + + updateFuse({ ...fuse, max_quality_gap })} + /> + + )} + + )} + {section !== "required" && ( + + {section === "all" && ( + + + Classifier options + + )} + + + onChange({ ...value, classifier_llm_config: { ...llm, reasoning_effort } }) + } + /> + onChange({ ...value, classifier_llm_config: { ...llm, timeout_ms } })} + /> + onChange({ ...value, classifier_llm_config })} + /> + onChange({ ...value, classifier_llm_config })} + /> + {isCapability && ( + updateCapability({ ...capability, threshold_step })} + /> + )} + updateTransport({ max_output_tokens })} + /> +
+ + { + if (response_format === "json_schema" || response_format === "json_object") + updateTransport({ response_format }); + }} + /> +
+
+ +

+ Optional coefficients fitted for your judge, solvers, and harness. Leave off to use raw forecasts +

+ {config.calibration && ( +
+ + setCalibrationVersion(event.target.value)} + /> +
+ )} + {isCapability && capability.calibration && ( + + updateCapability({ + ...capability, + calibration: { version: capability.calibration?.version ?? "", ...next }, + }) + } + /> + )} + {!isCapability && + fuse.calibration && + (["efficient", "capable"] as const).map((role) => ( + { + if (fuse.calibration) updateFuse({ ...fuse, calibration: { ...fuse.calibration, [role]: next } }); + }} + /> + ))} +
+
+
+ )} + {section === "all" && ( + <>
- {isCapability && ( - updateCapability({ ...capability, threshold_step })} - /> - )} - updateTransport({ max_output_tokens })} - /> -
- - { - if (response_format === "json_schema" || response_format === "json_object") - updateTransport({ response_format }); - }} - /> -
-
- -

- Optional coefficients fitted for your judge, solvers, and harness. Leave off to use raw forecasts -

- {config.calibration && ( -
- - setCalibrationVersion(event.target.value)} - /> -
- )} - {isCapability && capability.calibration && ( - - updateCapability({ - ...capability, - calibration: { version: capability.calibration?.version ?? "", ...next }, - }) - } - /> - )} - {!isCapability && - fuse.calibration && - (["efficient", "capable"] as const).map((role) => ( - { - if (fuse.calibration) updateFuse({ ...fuse, calibration: { ...fuse.calibration, [role]: next } }); - }} - /> - ))} -
-
-
+ + )}

The classifier uses its bundled prompt and always falls back to the capable solver

- {error && ( + {section !== "advanced" && error && (

{error}

diff --git a/ui/litellm-dashboard/src/components/add_model/JevClassifierConfig.integration.test.tsx b/ui/litellm-dashboard/src/components/add_model/JevClassifierConfig.integration.test.tsx index 896fde3a446..7da8b12c2d7 100644 --- a/ui/litellm-dashboard/src/components/add_model/JevClassifierConfig.integration.test.tsx +++ b/ui/litellm-dashboard/src/components/add_model/JevClassifierConfig.integration.test.tsx @@ -98,28 +98,28 @@ describe("JEV classifier editor", () => { afterEach(() => vi.mocked(useAuthorized).mockReset()); it("uses built-in JEV without a license and preserves custom tiers and context through reload", () => { renderWithProviders(); - expect(screen.getByLabelText("Classifier Model")).toBeInTheDocument(); + expect(screen.getByLabelText("Judge model")).toBeInTheDocument(); expect(screen.getByText("Reasoning Effort")).toBeInTheDocument(); expect(screen.getByText("Classifier Prompt")).toBeInTheDocument(); expect(screen.getByRole("switch", { name: "Use images for classification" })).toBeInTheDocument(); - fireEvent.click(screen.getByRole("radio", { name: /JEV Classifier/ })); - expect(screen.getByRole("tab", { name: "Complexity" })).toHaveAttribute("aria-selected", "true"); - expect(screen.getByLabelText("JEV Model")).toHaveValue("jev-latest"); - expect(screen.getByLabelText("JEV Instructions")).toBeDisabled(); - expect(screen.queryByLabelText("Classifier Model")).not.toBeInTheDocument(); + fireEvent.click(screen.getByRole("radio", { name: /Jev Classifier/ })); + expect(screen.getByRole("radio", { name: /^Jev Classifier/ })).toBeChecked(); + expect(screen.getByLabelText("Jev Model")).toHaveValue("jev-latest"); + expect(screen.getByLabelText("Jev Instructions")).toBeEnabled(); + expect(screen.queryByLabelText("Judge model")).not.toBeInTheDocument(); expect(screen.queryByText("Reasoning Effort")).not.toBeInTheDocument(); expect(screen.queryByText("Classifier Prompt")).not.toBeInTheDocument(); expect(screen.queryByRole("switch", { name: "Use images for classification" })).not.toBeInTheDocument(); - fireEvent.change(screen.getByLabelText("JEV Model"), { target: { value: "jev-test" } }); - fireEvent.change(screen.getByLabelText("JEV Timeout (ms)"), { target: { value: "4200" } }); + fireEvent.change(screen.getByLabelText("Jev Model"), { target: { value: "jev-test" } }); + fireEvent.change(screen.getByLabelText("Jev Timeout (ms)"), { target: { value: "4200" } }); fireEvent.change(screen.getByLabelText("Context Window Size"), { target: { value: "6" } }); fireEvent.change(screen.getByLabelText("Circuit breaker cooldown (seconds)"), { target: { value: "50" } }); fireEvent.click(screen.getByRole("switch", { name: "Classifier circuit breaker" })); fireEvent.click(screen.getByRole("button", { name: "Customize tiers" })); fireEvent.click(screen.getByRole("button", { name: "Save and reload" })); - expect(screen.getByRole("radio", { name: /JEV Classifier/ })).toBeChecked(); - expect(screen.getByLabelText("JEV Model")).toHaveValue("jev-test"); - expect(screen.getByLabelText("JEV Timeout (ms)")).toHaveValue(4200); + expect(screen.getByRole("radio", { name: /Jev Classifier/ })).toBeChecked(); + expect(screen.getByLabelText("Jev Model")).toHaveValue("jev-test"); + expect(screen.getByLabelText("Jev Timeout (ms)")).toHaveValue(4200); expect(screen.getByLabelText("Context Window Size")).toHaveValue("6"); expect(screen.getByRole("switch", { name: "Classifier circuit breaker" })).not.toBeChecked(); fireEvent.click(screen.getByRole("button", { name: "Probe current config" })); @@ -152,10 +152,10 @@ describe("JEV classifier editor", () => { return ; }; renderWithProviders(); - expect(screen.getByLabelText("JEV Instructions")).toBeEnabled(); - fireEvent.change(screen.getByLabelText("JEV Instructions"), { target: { value: "New instructions" } }); - expect(screen.getByLabelText("JEV Instructions")).toHaveValue("New instructions"); - fireEvent.click(screen.getByRole("button", { name: "Restore built-in JEV instructions" })); - expect(screen.getByLabelText("JEV Instructions")).toHaveValue(""); + expect(screen.getByLabelText("Jev Instructions")).toBeEnabled(); + fireEvent.change(screen.getByLabelText("Jev Instructions"), { target: { value: "New instructions" } }); + expect(screen.getByLabelText("Jev Instructions")).toHaveValue("New instructions"); + fireEvent.click(screen.getByRole("button", { name: "Restore built-in Jev instructions" })); + expect(screen.getByLabelText("Jev Instructions")).toHaveValue(""); }); }); diff --git a/ui/litellm-dashboard/src/components/add_model/JevClassifierConfig.tsx b/ui/litellm-dashboard/src/components/add_model/JevClassifierConfig.tsx index 25286eaef07..97609bcd8c3 100644 --- a/ui/litellm-dashboard/src/components/add_model/JevClassifierConfig.tsx +++ b/ui/litellm-dashboard/src/components/add_model/JevClassifierConfig.tsx @@ -1,10 +1,9 @@ import React, { useId } from "react"; -import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; +import { AutoRouterAllowanceNote } from "./AutoRouterAvailability"; import { Button } from "@/components/ui/button"; import { Input } from "@/components/ui/input"; import { Label } from "@/components/ui/label"; import { Textarea } from "@/components/ui/textarea"; -import { SimpleTooltip } from "@/components/ui/tooltip"; import ClassifierCircuitBreakerConfig from "./ClassifierCircuitBreakerConfig"; import type { ComplexityRouterConfigValue } from "./ComplexityRouterConfig"; import { defaultJevClassifierConfig } from "./jev_classifier_config"; @@ -17,7 +16,6 @@ export default function JevClassifierConfig({ onChange: (value: ComplexityRouterConfigValue) => void; }) { const id = useId(); - const { premiumUser } = useAuthorized(); const config = value.jev_classifier_config ?? defaultJevClassifierConfig(); const update = (patch: Partial) => onChange({ ...value, jev_classifier_config: { ...config, ...patch } }); @@ -28,11 +26,11 @@ export default function JevClassifierConfig({ Uses TypeSafe System One Choice evaluation with your configured tiers

- + update({ model: event.target.value })} />
- +
- - -
-