From 3c714ed7a6d1491b3950fa9d563e3b30a016831b Mon Sep 17 00:00:00 2001 From: Abhimanyu Kapur <38531241+akapur99@users.noreply.github.com> Date: Fri, 10 Jul 2026 14:52:13 -0700 Subject: [PATCH 1/8] feat(auto_router): keyword tier overrides and semantic keyword matching for the complexity router Add deterministic keyword-to-tier overrides and optional embedding-based (semantic) keyword matching to the complexity router, and surface both in the Add Auto Router UI behind a Router Type selector: "Auto-Router v2 [Recommended]" (complexity tiers + keyword overrides + semantic matching, the default) and "Semantic Router [to be deprecated]" (the existing utterance-based router, unchanged). Keyword-to-tier overrides resolve to the highest tier matched rather than the first keyword matched, so match order no longer affects the routing decision. Backend: - config: KeywordTierRule model plus keyword_tier_rules, semantic_keyword_matching, embedding_model, and match_threshold on ComplexityRouterConfig, with a validator requiring an embedding model and rules when semantic matching is on - complexity_router: evaluate keyword rules before scoring; lexical matches escalate to the most-severe matched tier (order-independent), and semantic mode reuses LiteLLMRouterEncoder + SemanticRouter to match paraphrases by cosine similarity, falling back to the scorer when nothing matches - model management: clear complexity_routers on cache reload so config edits take effect Frontend: - Add Auto Router tab restores the Router Type radio (Auto-Router v2 recommended by default, Semantic Router still available) and sends keyword_tier_rules plus the semantic settings on the recommended path, instead of flattening keywords into custom_technical_keywords - client-side guard blocks submit when semantic matching is enabled without an embedding model or without any keyword tier rules, mirroring the backend validator - moved the "How Classification Works" explainer below Custom Technical Keywords and above Keyword Tier Overrides - remove the Test Connection action from the recommended flow, which can't build a valid pre-save payload for a router (leaves a TODO for a JSON preview / config test follow-up) Tests cover lexical escalation, semantic matching via the real library with injected embeddings, the semantic config guard, config validation, the reload-clear regression, and the frontend payload builder --- .../model_management_endpoints.py | 28 +- .../complexity_router/complexity_router.py | 143 ++++- .../complexity_router/config.py | 58 ++ .../test_model_management_endpoints.py | 179 +++++- .../router_strategy/test_complexity_router.py | 544 +++++++++++++++++- ui/litellm-dashboard/eslint-metrics.json | 2 +- .../add_model/ComplexityRouterConfig.test.tsx | 95 ++- .../add_model/ComplexityRouterConfig.tsx | 39 +- .../components/add_model/KeywordTierRules.tsx | 112 ++++ .../add_model/SemanticKeywordMatching.tsx | 84 +++ .../add_model/add_auto_router_tab.tsx | 393 ++++++------- .../add_model/add_model_tab.test.tsx | 9 +- .../build_complexity_router_config.test.ts | 141 +++++ .../build_complexity_router_config.ts | 70 +++ .../handle_add_auto_router_submit.tsx | 24 +- 15 files changed, 1635 insertions(+), 286 deletions(-) create mode 100644 ui/litellm-dashboard/src/components/add_model/KeywordTierRules.tsx create mode 100644 ui/litellm-dashboard/src/components/add_model/SemanticKeywordMatching.tsx create mode 100644 ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.test.ts create mode 100644 ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.ts diff --git a/litellm/proxy/management_endpoints/model_management_endpoints.py b/litellm/proxy/management_endpoints/model_management_endpoints.py index afe71084a45..0f0dedc40e6 100644 --- a/litellm/proxy/management_endpoints/model_management_endpoints.py +++ b/litellm/proxy/management_endpoints/model_management_endpoints.py @@ -1111,7 +1111,16 @@ async def delete_model( ## DELETE FROM ROUTER ## if llm_router is not None: - llm_router.delete_deployment(id=model_info.id) + deleted_deployment = llm_router.delete_deployment(id=model_info.id) + # delete_deployment only drops the deployment from model_list; the auto/ + # complexity router registries are keyed by model_name and would otherwise + # retain a stale (now unbacked) entry, so evict it here too. + deleted_name = getattr(deleted_deployment, "model_name", None) + if deleted_name is None and isinstance(deleted_deployment, dict): + deleted_name = deleted_deployment.get("model_name") + if deleted_name is not None: + llm_router.auto_routers.pop(deleted_name, None) + llm_router.complexity_routers.pop(deleted_name, None) # Runs after the row delete so the sibling check sees post-delete state. if model_params.model_info.team_id is not None: @@ -1715,8 +1724,21 @@ async def clear_cache(): for model_id in db_model_ids: llm_router.delete_deployment(id=model_id) - # Clear auto routers - llm_router.auto_routers.clear() + # Clear only DB-backed auto/complexity routers, keyed by model_name, so the reload + # below rebuilds them fresh. A blanket .clear() would also drop config-defined + # routers, which are never re-added below (add_deployment only reloads DB models), + # leaving them permanently unroutable until a full proxy restart for every tenant. + # Restrict to deployments whose model is actually an auto_router/* so a config + # router that merely shares a model_name with a regular DB model isn't evicted. + db_router_names = { + model.get("model_name") + for model in current_models + if model.get("model_info", {}).get("db_model", False) + and str(model.get("litellm_params", {}).get("model", "")).startswith("auto_router/") + } + for model_name in db_router_names: + llm_router.auto_routers.pop(model_name, None) + llm_router.complexity_routers.pop(model_name, None) # Reload only DB models await proxy_config.add_deployment(prisma_client=prisma_client, proxy_logging_obj=proxy_logging_obj) diff --git a/litellm/router_strategy/complexity_router/complexity_router.py b/litellm/router_strategy/complexity_router/complexity_router.py index 16bf9a87a41..f5decbaa670 100644 --- a/litellm/router_strategy/complexity_router/complexity_router.py +++ b/litellm/router_strategy/complexity_router/complexity_router.py @@ -7,12 +7,15 @@ to classify requests by complexity and route them to appropriate models. By default, scoring is local (regex/keyword-based) with no external API calls and <1ms latency. Optionally, classifier_type="llm" routes classification through a configured model instead, trading that latency/cost guarantee for potentially better accuracy. +keyword_tier_rules (lexical or, with semantic_keyword_matching, embedding-based) are +evaluated before either classification strategy and force a tier outright when matched. Inspired by ClawRouter: https://github.com/BlockRunAI/ClawRouter """ +import asyncio import re -from typing import TYPE_CHECKING, Any, Dict, List, Literal, Optional, Tuple, Union +from typing import TYPE_CHECKING, Any, Dict, List, Literal, Optional, Tuple, Union, cast from pydantic import BaseModel @@ -25,16 +28,20 @@ from .config import ( DEFAULT_REASONING_KEYWORDS, DEFAULT_SIMPLE_KEYWORDS, DEFAULT_TECHNICAL_KEYWORDS, + TIER_SEVERITY_ORDER, ComplexityRouterConfig, ComplexityTier, ) if TYPE_CHECKING: + from semantic_router.routers import SemanticRouter + from litellm.router import Router from litellm.types.router import PreRoutingHookResponse else: Router = Any PreRoutingHookResponse = Any + SemanticRouter = Any class TierClassification(BaseModel): @@ -141,6 +148,10 @@ class ComplexityRouter(CustomLogger): ) self.simple_keywords = self.config.simple_keywords or DEFAULT_SIMPLE_KEYWORDS + # Lazily built on first semantic request and cached for reuse (route + # embeddings are static, only the prompt is embedded per request). + self._semantic_routelayer: Optional[SemanticRouter] = None + # Pre-compile regex patterns for efficiency # Use non-greedy .*? to prevent ReDoS on pathological inputs self._multi_step_patterns = [ @@ -419,6 +430,125 @@ class ComplexityRouter(CustomLogger): raise ValueError(f"No model configured for tier {tier_key} and no default_model set") + def _lexical_tier_override(self, user_message: str) -> Optional[ComplexityTier]: + """When keyword_tier_rules match literally, the most-severe matched tier wins. + + Escalating to the highest tier (rather than the first rule in the list) keeps + routing independent of the order rules were authored in: a prompt hitting both a + SIMPLE and a REASONING keyword routes to REASONING. + """ + rules = self.config.keyword_tier_rules + if not rules: + return None + text = user_message.lower() + matched_tiers = [ + rule.tier for rule in rules if any(self._keyword_matches(text, keyword) for keyword in rule.keywords) + ] + if not matched_tiers: + return None + return max(matched_tiers, key=TIER_SEVERITY_ORDER.index) + + def _get_or_create_semantic_routelayer(self) -> "SemanticRouter": + """Build (once) a SemanticRouter with one route per tier, utterances = that tier's keywords.""" + if self._semantic_routelayer is not None: + return self._semantic_routelayer + + from semantic_router.routers import SemanticRouter + from semantic_router.routers.base import Route + + from litellm.router_strategy.auto_router.litellm_encoder import ( + LiteLLMRouterEncoder, + ) + + embedding_model = self.config.embedding_model + if embedding_model is None: + raise ValueError("embedding_model is required for semantic keyword matching") + + rules = self.config.keyword_tier_rules or [] + ordered_tiers = tuple(dict.fromkeys(rule.tier.value for rule in rules)) + routes = [ + Route( + name=tier, + utterances=[keyword for rule in rules if rule.tier.value == tier for keyword in rule.keywords], + score_threshold=self.config.match_threshold, + ) + for tier in ordered_tiers + ] + routelayer = SemanticRouter( + routes=routes, + encoder=LiteLLMRouterEncoder( + litellm_router_instance=self.litellm_router_instance, + model_name=embedding_model, + score_threshold=self.config.match_threshold, + ), + auto_sync="local", + ) + self._semantic_routelayer = routelayer + return routelayer + + async def _semantic_tier_override(self, user_message: str, request_kwargs: Dict) -> Optional[ComplexityTier]: + """Match the prompt against keyword_tier_rules by embedding similarity. + + Embeds the query ourselves (instead of letting SemanticRouter.acall embed it + internally) so the caller's metadata/litellm_metadata flows into aembedding() + and this spend is attributed and budget-checked against the originating key/team, + the same as any other litellm call. SemanticRouter.acall() has no parameter to + pass such kwargs through to the encoder, so it's bypassed for the query embedding; + the route index itself (static utterances, embedded once at build time with no + caller context) is unaffected and still reused via the precomputed `vector=` path. + """ + from semantic_router.schema import RouteChoice + + from litellm.router_strategy.auto_router.litellm_encoder import ( + LiteLLMRouterEncoder, + ) + + # Building the SemanticRouter embeds the (static) route utterances via the + # encoder's *synchronous* path; run it in a worker thread so that one-time, + # per-router-instance provider I/O never blocks the async event loop and stalls + # other requests. Once cached, subsequent calls return immediately (no I/O). + routelayer = await asyncio.to_thread(self._get_or_create_semantic_routelayer) + encoder = cast(LiteLLMRouterEncoder, routelayer.encoder) # cast-ok: always the encoder we built above + # Strip the parent request's budget reservation before forwarding: the reservation + # belongs to the routed completion this embedding is helping select, not to the + # embedding call. Forwarding it would let the embedding's cost callback finalize the + # reservation, so the routed completion's own callback then skips incrementing the + # key/team budget. Key/team attribution fields are preserved for spend logging. + metadata = _classifier_call_metadata(request_kwargs.get("metadata")) or {} + litellm_metadata = _classifier_call_metadata(request_kwargs.get("litellm_metadata")) or {} + query_vector = ( + await encoder.aencode_queries([user_message], metadata=metadata, litellm_metadata=litellm_metadata) + )[0] + route_choice = await routelayer.acall(vector=query_vector) + + if isinstance(route_choice, list): + route_choice = route_choice[0] if route_choice else None + if not isinstance(route_choice, RouteChoice) or not route_choice.name: + return None + try: + return ComplexityTier(route_choice.name) + except ValueError: + return None + + async def _resolve_keyword_tier_override(self, user_message: str, request_kwargs: Dict) -> Optional[ComplexityTier]: + """Resolve a keyword_tier_rule override, semantically or lexically per config. + + Returns None (no override -> fall through to the scorer) not only when no rule + matches, but also when the semantic path fails: the embedding call can error or + time out, and a routing helper must never turn that into a failed user request. + """ + if not self.config.keyword_tier_rules: + return None + if not self.config.semantic_keyword_matching: + return self._lexical_tier_override(user_message) + try: + return await self._semantic_tier_override(user_message, request_kwargs) + except Exception as e: # noqa: BLE001 -- embedding call can fail many ways (timeout, provider/network/parse error); any failure must fall back to scoring, never fail the request + verbose_router_logger.warning( + f"ComplexityRouter: semantic keyword matching failed ({e}), falling back to complexity scoring" + ) + return None + def _resolve_messages( self, messages: Optional[List[Dict[str, Any]]], @@ -539,6 +669,17 @@ class ComplexityRouter(CustomLogger): messages=messages if has_original_messages else None, ) + override_tier = await self._resolve_keyword_tier_override(user_message, request_kwargs) + if override_tier is not None: + routed_model = self.get_model_for_tier(override_tier) + verbose_router_logger.info( + f"ComplexityRouter: keyword rule fired, tier={override_tier.value}, routed_model={routed_model}" + ) + return PreRoutingHookResponse( + model=routed_model, + messages=messages if has_original_messages else None, + ) + tier, score, signals = await self.aclassify(user_message, system_prompt, request_kwargs) routed_model = self.get_model_for_tier(tier) diff --git a/litellm/router_strategy/complexity_router/config.py b/litellm/router_strategy/complexity_router/config.py index 6407c9a4590..9901ae6f813 100644 --- a/litellm/router_strategy/complexity_router/config.py +++ b/litellm/router_strategy/complexity_router/config.py @@ -20,6 +20,32 @@ class ComplexityTier(str, Enum): REASONING = "REASONING" +TIER_SEVERITY_ORDER: tuple[ComplexityTier, ...] = ( + ComplexityTier.SIMPLE, + ComplexityTier.MEDIUM, + ComplexityTier.COMPLEX, + ComplexityTier.REASONING, +) + + +class KeywordTierRule(BaseModel): + """A deterministic override: if any keyword matches, route to this tier.""" + + keywords: List[str] = Field( + min_length=1, + description="Keywords/phrases that trigger this rule (lexical or semantic match)", + ) + tier: ComplexityTier = Field( + description="Tier to route to when this rule matches", + ) + + @model_validator(mode="after") + def _validate_non_empty_keywords(self) -> "KeywordTierRule": + if not any(keyword.strip() for keyword in self.keywords): + raise ValueError("keyword_tier_rules entries must contain at least one non-empty keyword") + return self + + # ─── Default Keyword Lists ─── # Note: Keywords should be full words/phrases to avoid substring false positives. # The matching logic uses word boundary detection for single-word keywords. @@ -279,6 +305,28 @@ class ComplexityRouterConfig(BaseModel): description="Configuration for the LLM classifier; required when classifier_type is 'llm'", ) + # Deterministic keyword -> tier overrides, evaluated before weighted scoring + keyword_tier_rules: Optional[List[KeywordTierRule]] = Field( + default=None, + description="Rules that force a specific tier when their keywords match the prompt", + ) + + # Semantic (embedding) matching for keyword_tier_rules instead of literal text matching + semantic_keyword_matching: bool = Field( + default=False, + description="Match keyword_tier_rules by embedding similarity instead of literal text", + ) + embedding_model: Optional[str] = Field( + default=None, + description="Embedding model (LiteLLM model name) used when semantic_keyword_matching is enabled", + ) + match_threshold: float = Field( + default=0.5, + ge=0.0, + le=1.0, + description="Minimum cosine similarity for a semantic keyword match", + ) + model_config = ConfigDict(extra="allow") # Allow additional fields @model_validator(mode="after") @@ -287,6 +335,16 @@ class ComplexityRouterConfig(BaseModel): raise ValueError("classifier_llm_config is required when classifier_type is 'llm'") return self + @model_validator(mode="after") + def _validate_semantic_matching(self) -> "ComplexityRouterConfig": + if not self.semantic_keyword_matching: + return self + if not self.embedding_model: + raise ValueError("embedding_model is required when semantic_keyword_matching is enabled") + if not self.keyword_tier_rules: + raise ValueError("keyword_tier_rules must be non-empty when semantic_keyword_matching is enabled") + return self + # Combined default config DEFAULT_COMPLEXITY_CONFIG = ComplexityRouterConfig() diff --git a/tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py index 6a81b1b613b..51411c4ae84 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py @@ -436,28 +436,31 @@ class TestClearCache: clear_cache, ) - # Create mock router with mixed DB and config models + # Create mock router with mixed DB and config router deployments. The two DB + # entries are auto_router/* deployments (so their router-map entries should be + # cleared for reload); the config-defined router is preserved. mock_router = MagicMock() mock_router.model_list = [ { - "model_name": "gpt-4", + "model_name": "db-auto-router", "model_info": {"id": "db-model-1", "db_model": True}, - "litellm_params": {"model": "gpt-4"}, + "litellm_params": {"model": "auto_router/db-auto-router"}, }, { - "model_name": "gpt-3.5-turbo", + "model_name": "config-router", "model_info": {"id": "config-model-1", "db_model": False}, - "litellm_params": {"model": "gpt-3.5-turbo"}, + "litellm_params": {"model": "auto_router/complexity_router"}, }, { - "model_name": "claude-3", + "model_name": "db-complexity-router", "model_info": {"id": "db-model-2", "db_model": True}, - "litellm_params": {"model": "claude-3"}, + "litellm_params": {"model": "auto_router/complexity_router"}, }, ] mock_router.delete_deployment = MagicMock(return_value=True) - mock_router.auto_routers = MagicMock() - mock_router.auto_routers.clear = MagicMock() + # Real dicts (not MagicMock) so we can assert on their actual contents below. + mock_router.auto_routers = {"db-auto-router": MagicMock(), "config-router": MagicMock()} + mock_router.complexity_routers = {"db-complexity-router": MagicMock(), "config-router": MagicMock()} mock_config = MagicMock() mock_config.add_deployment = AsyncMock(return_value=True) @@ -479,8 +482,14 @@ class TestClearCache: mock_router.delete_deployment.assert_any_call(id="db-model-1") mock_router.delete_deployment.assert_any_call(id="db-model-2") - # Should have cleared auto routers - mock_router.auto_routers.clear.assert_called_once() + # DB-backed router entries are cleared so they can be re-populated by the + # reload below; the config-backed router must survive, since add_deployment() + # only reloads DB models and would otherwise leave it permanently unroutable + # (see TestClearCachePreservesConfigRouters). + assert "db-auto-router" not in mock_router.auto_routers + assert "db-complexity-router" not in mock_router.complexity_routers + assert "config-router" in mock_router.auto_routers + assert "config-router" in mock_router.complexity_routers # Should have called add_deployment to reload DB models mock_config.add_deployment.assert_called_once_with( @@ -488,6 +497,154 @@ class TestClearCache: ) +class TestClearCachePreservesConfigRouters: + """ + Regression test: clear_cache() must not wipe config-defined auto/complexity + routers. + + clear_cache() runs after any DB model write (e.g. a team admin patching a + team-owned model via PATCH /model/{id}/update). Before this fix, it called + auto_routers.clear() / complexity_routers.clear() unconditionally, which also + dropped routers defined in config.yaml belonging to *other* tenants. Those + entries are never restored, because the reload below only re-adds DB models + (proxy_config.add_deployment), so a config-defined router would stay + permanently unroutable until a full proxy restart - a cross-tenant + denial-of-service triggerable by any team admin's unrelated model update. + """ + + @pytest.mark.asyncio + async def test_config_backed_routers_survive_unrelated_db_model_update(self): + from litellm.proxy.management_endpoints.model_management_endpoints import ( + clear_cache, + ) + + mock_router = MagicMock() + mock_router.model_list = [ + { + "model_name": "team-a-db-router", + "model_info": {"id": "db-model-1", "db_model": True}, + "litellm_params": {"model": "auto_router/complexity_router"}, + }, + ] + mock_router.delete_deployment = MagicMock(return_value=True) + mock_router.auto_routers = {"config-semantic-router": MagicMock()} + mock_router.complexity_routers = { + "team-a-db-router": MagicMock(), + "config-defined-complexity-router": MagicMock(), + } + + mock_config = MagicMock() + mock_config.add_deployment = AsyncMock(return_value=True) + + with ( + patch("litellm.proxy.proxy_server.llm_router", mock_router), + patch("litellm.proxy.proxy_server.proxy_config", mock_config), + patch("litellm.proxy.proxy_server.prisma_client", MagicMock()), + patch("litellm.proxy.proxy_server.proxy_logging_obj", MagicMock()), + patch("litellm.proxy.proxy_server.verbose_proxy_logger"), + ): + await clear_cache() + + # The DB-backed router for the model that was actually updated is cleared + # so the reload below can re-populate it. + assert "team-a-db-router" not in mock_router.complexity_routers + # Config-defined routers for unrelated tenants must survive untouched. + assert "config-defined-complexity-router" in mock_router.complexity_routers + assert "config-semantic-router" in mock_router.auto_routers + + @pytest.mark.asyncio + async def test_config_router_sharing_name_with_regular_db_model_is_preserved(self): + """A config router must not be evicted just because a regular (non-router) DB + model happens to share its model_name; only DB deployments that are themselves + auto_router/* deployments should have their router entry cleared. + """ + from litellm.proxy.management_endpoints.model_management_endpoints import clear_cache + + mock_router = MagicMock() + mock_router.model_list = [ + { + "model_name": "shared-name", + "model_info": {"id": "db-model-1", "db_model": True}, + "litellm_params": {"model": "openai/gpt-4o"}, # a regular model, NOT a router + }, + ] + mock_router.delete_deployment = MagicMock(return_value=True) + # A config-defined complexity router registered under the same name as the DB model. + mock_router.auto_routers = {} + mock_router.complexity_routers = {"shared-name": MagicMock()} + + mock_config = MagicMock() + mock_config.add_deployment = AsyncMock(return_value=True) + + with ( + patch("litellm.proxy.proxy_server.llm_router", mock_router), + patch("litellm.proxy.proxy_server.proxy_config", mock_config), + patch("litellm.proxy.proxy_server.prisma_client", MagicMock()), + patch("litellm.proxy.proxy_server.proxy_logging_obj", MagicMock()), + patch("litellm.proxy.proxy_server.verbose_proxy_logger"), + ): + await clear_cache() + + # The DB model isn't a router, so the same-named config router must be left intact. + assert "shared-name" in mock_router.complexity_routers + + +class TestDeleteModelClearsRouterRegistry: + """delete_model must evict the deleted deployment from the auto/complexity router maps, + not just from model_list, or a stale (now unbacked) router entry lingers until restart. + """ + + @pytest.mark.asyncio + async def test_delete_model_pops_router_registries(self): + from litellm.proxy.management_endpoints.model_management_endpoints import ( + delete_model as delete_model_endpoint, + ) + from litellm.proxy.management_endpoints.model_management_endpoints import ModelInfoDelete + + model_id = "router-del-1" + admin_user = UserAPIKeyAuth(user_id="admin", user_role=LitellmUserRoles.PROXY_ADMIN) + db_row = LiteLLM_ProxyModelTable( + model_id=model_id, + model_name="smart-router", + litellm_params={"model": "auto_router/complexity_router"}, + model_info={"id": model_id}, + created_by="admin", + updated_by="admin", + ) + + mock_prisma = MagicMock() + mock_prisma.db = MagicMock() + mock_prisma.db.litellm_proxymodeltable = AsyncMock() + mock_prisma.db.litellm_proxymodeltable.find_unique = AsyncMock(return_value=db_row) + mock_prisma.db.litellm_proxymodeltable.delete = AsyncMock(return_value=db_row) + + mock_router = MagicMock() + mock_router.delete_deployment = MagicMock( + return_value={"model_name": "smart-router", "model_info": {"id": model_id}} + ) + mock_router.auto_routers = {"smart-router": MagicMock()} + mock_router.complexity_routers = {"smart-router": MagicMock()} + + _PS = "litellm.proxy.proxy_server" + with ( + patch(f"{_PS}.prisma_client", mock_prisma), + patch(f"{_PS}.store_model_in_db", True), + patch(f"{_PS}.proxy_config", MagicMock()), + patch(f"{_PS}.proxy_logging_obj", MagicMock()), + patch(f"{_PS}.general_settings", {}), + patch(f"{_PS}.premium_user", True), + patch(f"{_PS}.llm_router", mock_router), + ): + await delete_model_endpoint( + model_info=ModelInfoDelete(id=model_id), + user_api_key_dict=admin_user, + ) + + mock_router.delete_deployment.assert_called_once_with(id=model_id) + assert "smart-router" not in mock_router.auto_routers + assert "smart-router" not in mock_router.complexity_routers + + class TestUpdateModel: """ Tests for the update_model (POST /model/update) handler. diff --git a/tests/test_litellm/router_strategy/test_complexity_router.py b/tests/test_litellm/router_strategy/test_complexity_router.py index c40b362333a..f4da1018f3e 100644 --- a/tests/test_litellm/router_strategy/test_complexity_router.py +++ b/tests/test_litellm/router_strategy/test_complexity_router.py @@ -6,7 +6,7 @@ Tests the rule-based complexity scoring and tier assignment logic. import os import sys -from typing import Dict +from typing import Dict, List from unittest.mock import AsyncMock, MagicMock, patch import pytest @@ -16,6 +16,7 @@ sys.path.insert( 0, os.path.abspath("../../..") ) # Adds the parent directory to the system path +import litellm from litellm import Router from litellm.router_strategy.complexity_router.complexity_router import ( ComplexityRouter, @@ -1309,3 +1310,544 @@ class TestLLMClassifier: assert result.model == "o1-preview" # REASONING tier model call_kwargs = mock_router_instance.acompletion.call_args.kwargs assert call_kwargs["metadata"] == request_metadata + + +class TestLexicalKeywordTierRules: + """Test deterministic (literal) keyword_tier_rules overrides.""" + + @pytest.fixture + def rule_config(self, basic_config) -> Dict: + return { + **basic_config, + "keyword_tier_rules": [ + {"keywords": ["deploy to k8s"], "tier": "REASONING"}, + ], + } + + @pytest.mark.asyncio + async def test_matching_rule_overrides_scoring( + self, mock_router_instance, rule_config + ): + """A prompt hitting a rule keyword routes to that tier, not the scored tier.""" + router = ComplexityRouter( + model_name="test-router", + litellm_router_instance=mock_router_instance, + complexity_router_config=rule_config, + ) + prompt = "please deploy to k8s now" + # Without the rule this short prompt would not score into REASONING. + scored_tier, _, _ = router.classify(prompt) + assert scored_tier != ComplexityTier.REASONING + + result = await router.async_pre_routing_hook( + model="test-model", + request_kwargs={}, + messages=[{"role": "user", "content": prompt}], + ) + assert result is not None + assert result.model == "o1-preview" # REASONING tier model + + @pytest.mark.asyncio + async def test_most_severe_tier_wins_regardless_of_rule_order(self, mock_router_instance, basic_config): + """When several rules match, the highest-severity tier wins, independent of list order.""" + config = { + **basic_config, + "keyword_tier_rules": [ + {"keywords": ["database"], "tier": "SIMPLE"}, # listed first, lower tier + {"keywords": ["database"], "tier": "REASONING"}, # listed later, higher tier + ], + } + router = ComplexityRouter( + model_name="test-router", + litellm_router_instance=mock_router_instance, + complexity_router_config=config, + ) + result = await router.async_pre_routing_hook( + model="test-model", + request_kwargs={}, + messages=[{"role": "user", "content": "tell me about the database"}], + ) + assert result is not None + assert result.model == "o1-preview" # REASONING wins over the earlier SIMPLE rule + + @pytest.mark.asyncio + async def test_distinct_keywords_escalate_to_highest_tier(self, mock_router_instance, basic_config): + """A prompt hitting keywords across tiers routes to the most complex one.""" + config = { + **basic_config, + "keyword_tier_rules": [ + {"keywords": ["hi"], "tier": "SIMPLE"}, + {"keywords": ["advise"], "tier": "COMPLEX"}, + {"keywords": ["kubernetes"], "tier": "REASONING"}, + ], + } + router = ComplexityRouter( + model_name="test-router", + litellm_router_instance=mock_router_instance, + complexity_router_config=config, + ) + result = await router.async_pre_routing_hook( + model="test-model", + request_kwargs={}, + messages=[{"role": "user", "content": "hi, advise me on kubernetes"}], + ) + assert result is not None + assert result.model == "o1-preview" # REASONING, the highest of SIMPLE/COMPLEX/REASONING + + def test_lexical_override_returns_most_severe_matched_tier(self, mock_router_instance, basic_config): + """Unit-level check of the escalation helper across mixed matches.""" + config = { + **basic_config, + "keyword_tier_rules": [ + {"keywords": ["hi"], "tier": "SIMPLE"}, + {"keywords": ["advise"], "tier": "COMPLEX"}, + ], + } + router = ComplexityRouter( + model_name="test-router", + litellm_router_instance=mock_router_instance, + complexity_router_config=config, + ) + assert router._lexical_tier_override("hi there, please advise") == ComplexityTier.COMPLEX + assert router._lexical_tier_override("just saying hi") == ComplexityTier.SIMPLE + assert router._lexical_tier_override("nothing relevant here") is None + + @pytest.mark.asyncio + async def test_no_rule_match_falls_back_to_scoring( + self, mock_router_instance, basic_config + ): + """A prompt that matches no rule is classified by the scorer as usual.""" + config = { + **basic_config, + "keyword_tier_rules": [ + {"keywords": ["zzznomatch"], "tier": "REASONING"}, + ], + } + router = ComplexityRouter( + model_name="test-router", + litellm_router_instance=mock_router_instance, + complexity_router_config=config, + ) + result = await router.async_pre_routing_hook( + model="test-model", + request_kwargs={}, + messages=[{"role": "user", "content": "Hello!"}], + ) + assert result is not None + assert result.model == "gpt-4o-mini" # SIMPLE via scoring, rule did not fire + + def test_word_boundary_avoids_substring_false_positive( + self, mock_router_instance, basic_config + ): + """A single-word rule keyword must not match inside a larger word.""" + config = { + **basic_config, + "keyword_tier_rules": [{"keywords": ["k8s"], "tier": "REASONING"}], + } + router = ComplexityRouter( + model_name="test-router", + litellm_router_instance=mock_router_instance, + complexity_router_config=config, + ) + assert router._lexical_tier_override("running my k8s cluster") == ComplexityTier.REASONING + assert router._lexical_tier_override("what is a k8scluster thing") is None + + +def _make_embedding_response(vectors: List[List[float]]) -> "litellm.EmbeddingResponse": + return litellm.EmbeddingResponse( + model="fake-embed", + data=[ + {"embedding": vec, "index": idx, "object": "embedding"} + for idx, vec in enumerate(vectors) + ], + object="list", + ) + + +class FakeEmbeddingRouter: + """A stand-in router whose embeddings are deterministic 2D unit vectors. + + Any text mentioning a cluster/container concept maps to [1, 0]; everything + else maps to [0, 1]. This lets the real SemanticRouter compute exact cosine + similarities (1.0 or 0.0) so threshold behavior is testable without a network call. + """ + + _CLUSTER_MARKERS = ("k8s", "kube", "container", "cluster", "orchestrat") + + def __init__(self): + self.async_embedding_calls: List[List[str]] = [] + self.async_embedding_kwargs: List[Dict] = [] + self.sync_embedding_thread_ids: List[int] = [] + + def _vectors(self, docs: List[str]) -> List[List[float]]: + return [ + [1.0, 0.0] if any(marker in doc.lower() for marker in self._CLUSTER_MARKERS) else [0.0, 1.0] + for doc in docs + ] + + @staticmethod + def _as_list(text) -> List[str]: + return text if isinstance(text, list) else [text] + + def embedding(self, input, model, **kwargs): + # Record the thread this synchronous (route-index build) call ran on, so tests can + # assert it is offloaded off the event-loop thread. + import threading + + self.sync_embedding_thread_ids.append(threading.get_ident()) + return _make_embedding_response(self._vectors(self._as_list(input))) + + async def aembedding(self, input, model, **kwargs): + docs = self._as_list(input) + self.async_embedding_calls.append(docs) + self.async_embedding_kwargs.append(kwargs) + return _make_embedding_response(self._vectors(docs)) + + +class TestSemanticKeywordTierRules: + """Test embedding-based keyword_tier_rules matching.""" + + @pytest.mark.asyncio + async def test_semantic_match_routes_to_rule_tier(self, basic_config): + """A paraphrase (no literal keyword) still routes via embedding similarity.""" + fake_router = FakeEmbeddingRouter() + config = { + **basic_config, + "keyword_tier_rules": [ + {"keywords": ["kubernetes deployment", "container orchestration"], "tier": "REASONING"}, + {"keywords": ["hello", "thanks"], "tier": "SIMPLE"}, + ], + "semantic_keyword_matching": True, + "embedding_model": "fake-embed", + "match_threshold": 0.5, + } + router = ComplexityRouter( + model_name="test-router", + litellm_router_instance=fake_router, + complexity_router_config=config, + ) + result = await router.async_pre_routing_hook( + model="test-model", + request_kwargs={}, + messages=[{"role": "user", "content": "help me roll out my k8s cluster today"}], + ) + assert result is not None + assert result.model == "o1-preview" # REASONING via semantic match + assert fake_router.async_embedding_calls, "expected an embedding call for the prompt" + + @pytest.mark.asyncio + async def test_semantic_embedding_call_carries_caller_metadata(self, basic_config): + """The query embedding call must carry the caller's metadata/litellm_metadata + so embedding spend is attributed and budget-checked against the originating + key/team, instead of being logged as an untracked, unattributed cost. + """ + fake_router = FakeEmbeddingRouter() + config = { + **basic_config, + "keyword_tier_rules": [{"keywords": ["kubernetes deployment"], "tier": "REASONING"}], + "semantic_keyword_matching": True, + "embedding_model": "fake-embed", + "match_threshold": 0.5, + } + router = ComplexityRouter( + model_name="test-router", + litellm_router_instance=fake_router, + complexity_router_config=config, + ) + caller_metadata = {"user_api_key_hash": "hash-abc", "user_api_key_team_id": "team-1"} + caller_litellm_metadata = {"user_api_key": "hash-abc"} + result = await router.async_pre_routing_hook( + model="test-model", + request_kwargs={"metadata": caller_metadata, "litellm_metadata": caller_litellm_metadata}, + messages=[{"role": "user", "content": "roll out my k8s cluster"}], + ) + assert result is not None + assert fake_router.async_embedding_kwargs, "expected an embedding call for the prompt" + assert fake_router.async_embedding_kwargs[0]["metadata"] == caller_metadata + assert fake_router.async_embedding_kwargs[0]["litellm_metadata"] == caller_litellm_metadata + + @pytest.mark.asyncio + async def test_semantic_embedding_call_strips_budget_reservation(self, basic_config): + """The embedding call must not carry the parent request's budget reservation. + + The reservation belongs to the routed completion this embedding helps select, not + to the embedding call. Forwarding it would let the embedding's cost callback + finalize the reservation, so the routed completion's callback then skips + incrementing the key/team budget - letting a caller run completions while only the + embedding cost is enforced. Key/team attribution fields must still be forwarded. + """ + fake_router = FakeEmbeddingRouter() + config = { + **basic_config, + "keyword_tier_rules": [{"keywords": ["kubernetes deployment"], "tier": "REASONING"}], + "semantic_keyword_matching": True, + "embedding_model": "fake-embed", + "match_threshold": 0.5, + } + router = ComplexityRouter( + model_name="test-router", + litellm_router_instance=fake_router, + complexity_router_config=config, + ) + caller_metadata = { + "user_api_key_hash": "hash-abc", + "user_api_key_team_id": "team-1", + "user_api_key_budget_reservation": {"reserved_cost": 1.0}, + "user_api_key_auth": {"budget_reservation": {"reserved_cost": 1.0}}, + } + await router.async_pre_routing_hook( + model="test-model", + request_kwargs={"metadata": caller_metadata, "litellm_metadata": dict(caller_metadata)}, + messages=[{"role": "user", "content": "roll out my k8s cluster"}], + ) + assert fake_router.async_embedding_kwargs, "expected an embedding call for the prompt" + expected = {"user_api_key_hash": "hash-abc", "user_api_key_team_id": "team-1"} + assert fake_router.async_embedding_kwargs[0]["metadata"] == expected + assert fake_router.async_embedding_kwargs[0]["litellm_metadata"] == expected + + @pytest.mark.asyncio + async def test_semantic_routelayer_build_runs_off_event_loop(self, basic_config): + """Building the SemanticRouter embeds route utterances via a synchronous provider + call; it must run in a worker thread, not block the async event loop. + """ + import threading + + fake_router = FakeEmbeddingRouter() + config = { + **basic_config, + "keyword_tier_rules": [{"keywords": ["kubernetes deployment"], "tier": "REASONING"}], + "semantic_keyword_matching": True, + "embedding_model": "fake-embed", + "match_threshold": 0.5, + } + router = ComplexityRouter( + model_name="test-router", + litellm_router_instance=fake_router, + complexity_router_config=config, + ) + loop_thread_id = threading.get_ident() + await router.async_pre_routing_hook( + model="test-model", + request_kwargs={}, + messages=[{"role": "user", "content": "roll out my k8s cluster"}], + ) + # The route-index build did a synchronous embedding call... + assert fake_router.sync_embedding_thread_ids, "expected the route-index build to embed utterances" + # ...and none of it ran on the event-loop thread. + assert all(tid != loop_thread_id for tid in fake_router.sync_embedding_thread_ids) + + @pytest.mark.asyncio + async def test_below_threshold_falls_back_to_scoring(self, basic_config): + """When no route clears the threshold, scoring decides the tier.""" + fake_router = FakeEmbeddingRouter() + config = { + **basic_config, + "keyword_tier_rules": [ + {"keywords": ["kubernetes deployment"], "tier": "REASONING"}, + ], + "semantic_keyword_matching": True, + "embedding_model": "fake-embed", + "match_threshold": 0.9, + } + router = ComplexityRouter( + model_name="test-router", + litellm_router_instance=fake_router, + complexity_router_config=config, + ) + # "hello there friend" embeds orthogonal to the REASONING route (cos 0 < 0.9). + result = await router.async_pre_routing_hook( + model="test-model", + request_kwargs={}, + messages=[{"role": "user", "content": "hello there friend"}], + ) + assert result is not None + assert result.model == "gpt-4o-mini" # SIMPLE via scoring fallback + + @pytest.mark.asyncio + async def test_route_embeddings_cached_across_requests(self, basic_config): + """The route layer is built once and reused on subsequent requests.""" + fake_router = FakeEmbeddingRouter() + config = { + **basic_config, + "keyword_tier_rules": [ + {"keywords": ["kubernetes deployment"], "tier": "REASONING"}, + ], + "semantic_keyword_matching": True, + "embedding_model": "fake-embed", + "match_threshold": 0.5, + } + router = ComplexityRouter( + model_name="test-router", + litellm_router_instance=fake_router, + complexity_router_config=config, + ) + assert router._semantic_routelayer is None + await router.async_pre_routing_hook( + model="test-model", + request_kwargs={}, + messages=[{"role": "user", "content": "roll out my k8s cluster"}], + ) + first_layer = router._semantic_routelayer + assert first_layer is not None + await router.async_pre_routing_hook( + model="test-model", + request_kwargs={}, + messages=[{"role": "user", "content": "scale my container cluster"}], + ) + assert router._semantic_routelayer is first_layer + + +class TestSemanticConfigValidation: + """Test config validation for semantic_keyword_matching.""" + + def test_semantic_without_embedding_model_raises(self): + with pytest.raises(ValidationError): + ComplexityRouterConfig( + semantic_keyword_matching=True, + keyword_tier_rules=[{"keywords": ["k8s"], "tier": "REASONING"}], + ) + + def test_semantic_without_rules_raises(self): + with pytest.raises(ValidationError): + ComplexityRouterConfig( + semantic_keyword_matching=True, + embedding_model="fake-embed", + ) + + def test_semantic_disabled_needs_no_embedding_model(self): + config = ComplexityRouterConfig( + keyword_tier_rules=[{"keywords": ["k8s"], "tier": "REASONING"}], + ) + assert config.semantic_keyword_matching is False + assert config.match_threshold == 0.5 + + def test_keyword_tier_rule_rejects_empty_keywords(self): + """A rule with no keywords is meaningless (and yields a zero-utterance semantic route).""" + with pytest.raises(ValidationError): + ComplexityRouterConfig(keyword_tier_rules=[{"keywords": [], "tier": "SIMPLE"}]) + + def test_keyword_tier_rule_rejects_blank_only_keywords(self): + """Whitespace-only keywords don't count as content.""" + with pytest.raises(ValidationError): + ComplexityRouterConfig(keyword_tier_rules=[{"keywords": [" ", ""], "tier": "SIMPLE"}]) + + +class _StubEncoder: + """Minimal stand-in for LiteLLMRouterEncoder.aencode_queries, capturing the kwargs it was called with.""" + + def __init__(self): + self.aencode_queries_calls: List[Dict] = [] + + async def aencode_queries(self, docs, **kwargs): + self.aencode_queries_calls.append(kwargs) + return [[0.0]] + + +class _StubRouteLayer: + """Returns a fixed acall result so _semantic_tier_override branches can be exercised.""" + + def __init__(self, result): + self._result = result + self.encoder = _StubEncoder() + + async def acall(self, text=None, vector=None): + return self._result + + +class _RaisingEncoder: + """Simulates an embedding-provider failure during semantic matching.""" + + async def aencode_queries(self, docs, **kwargs): + raise RuntimeError("embedding provider unavailable") + + +class _RaisingRouteLayer: + def __init__(self): + self.encoder = _RaisingEncoder() + + async def acall(self, text=None, vector=None): + raise AssertionError("acall should not be reached when the encoder fails") + + +class TestKeywordOverrideEdgeCases: + """Cover the defensive branches of the lexical and semantic override helpers.""" + + def _semantic_router(self, mock_router_instance, basic_config): + config = { + **basic_config, + "keyword_tier_rules": [{"keywords": ["kubernetes"], "tier": "REASONING"}], + "semantic_keyword_matching": True, + "embedding_model": "fake-embed", + "match_threshold": 0.5, + } + return ComplexityRouter( + model_name="test-router", + litellm_router_instance=mock_router_instance, + complexity_router_config=config, + ) + + def test_lexical_override_none_when_no_rules(self, mock_router_instance, basic_config): + """No keyword_tier_rules configured -> lexical override is a no-op.""" + router = ComplexityRouter( + model_name="test-router", + litellm_router_instance=mock_router_instance, + complexity_router_config=basic_config, + ) + assert router._lexical_tier_override("deploy to k8s and reason step by step") is None + + def test_semantic_routelayer_requires_embedding_model(self, mock_router_instance, basic_config): + """Building the route layer without an embedding model raises (defensive invariant).""" + config = {**basic_config, "keyword_tier_rules": [{"keywords": ["k8s"], "tier": "REASONING"}]} + router = ComplexityRouter( + model_name="test-router", + litellm_router_instance=mock_router_instance, + complexity_router_config=config, + ) + assert router.config.embedding_model is None + with pytest.raises(ValueError, match="embedding_model is required"): + router._get_or_create_semantic_routelayer() + + @pytest.mark.asyncio + async def test_semantic_override_maps_first_of_list(self, mock_router_instance, basic_config): + """A list RouteChoice result maps to the first entry's tier.""" + from semantic_router.schema import RouteChoice + + router = self._semantic_router(mock_router_instance, basic_config) + router._semantic_routelayer = _StubRouteLayer([RouteChoice(name="COMPLEX"), RouteChoice(name="SIMPLE")]) + assert await router._semantic_tier_override("anything", {}) == ComplexityTier.COMPLEX + + @pytest.mark.asyncio + async def test_semantic_override_empty_list_returns_none(self, mock_router_instance, basic_config): + """An empty list result falls through to scoring.""" + router = self._semantic_router(mock_router_instance, basic_config) + router._semantic_routelayer = _StubRouteLayer([]) + assert await router._semantic_tier_override("anything", {}) is None + + @pytest.mark.asyncio + async def test_semantic_override_unknown_route_name_returns_none(self, mock_router_instance, basic_config): + """A matched route whose name is not a ComplexityTier is ignored.""" + from semantic_router.schema import RouteChoice + + router = self._semantic_router(mock_router_instance, basic_config) + router._semantic_routelayer = _StubRouteLayer(RouteChoice(name="NOT_A_TIER")) + assert await router._semantic_tier_override("anything", {}) is None + + @pytest.mark.asyncio + async def test_semantic_embedding_error_falls_back_to_scoring(self, mock_router_instance, basic_config): + """An embedding failure must not fail the request: the override yields None so + async_pre_routing_hook falls through to the complexity scorer. + """ + router = self._semantic_router(mock_router_instance, basic_config) + router._semantic_routelayer = _RaisingRouteLayer() + + # _resolve_keyword_tier_override swallows the error and returns None (no override). + assert await router._resolve_keyword_tier_override("roll out my k8s cluster", {}) is None + + # End-to-end, the hook still returns a routed model (from scoring) rather than raising. + result = await router.async_pre_routing_hook( + model="test-model", + request_kwargs={}, + messages=[{"role": "user", "content": "roll out my k8s cluster"}], + ) + assert result is not None + assert result.model in {"gpt-4o-mini", "gpt-4o", "claude-sonnet-4-20250514", "o1-preview"} diff --git a/ui/litellm-dashboard/eslint-metrics.json b/ui/litellm-dashboard/eslint-metrics.json index 7abbb186771..fe6f9f5b482 100644 --- a/ui/litellm-dashboard/eslint-metrics.json +++ b/ui/litellm-dashboard/eslint-metrics.json @@ -1,5 +1,5 @@ { - "@typescript-eslint/no-explicit-any": 1978, + "@typescript-eslint/no-explicit-any": 1976, "complexity": 130, "local/no-large-inline-object-arg": 509, "local/no-long-condition-chain": 234, diff --git a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.test.tsx b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.test.tsx index 37ec7742d65..5bfcd94398d 100644 --- a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.test.tsx +++ b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.test.tsx @@ -19,14 +19,28 @@ const defaultValue: ComplexityRouterConfigValue = { classifier_type: "heuristic", }; +const baseProps = { + modelInfo: mockModelInfo, + value: defaultValue, + onChange: vi.fn(), + keywordTierRules: [], + onKeywordTierRulesChange: vi.fn(), + semanticMatchingEnabled: false, + onSemanticMatchingEnabledChange: vi.fn(), + embeddingModel: undefined, + onEmbeddingModelChange: vi.fn(), + matchThreshold: 0.5, + onMatchThresholdChange: vi.fn(), +}; + describe("ComplexityRouterConfig", () => { it("should render", () => { - renderWithProviders(); + renderWithProviders(); expect(screen.getByText("Complexity Tier Configuration")).toBeInTheDocument(); }); it("should display all four tier labels", () => { - renderWithProviders(); + renderWithProviders(); expect(screen.getByText("Simple Tier")).toBeInTheDocument(); expect(screen.getByText("Medium Tier")).toBeInTheDocument(); expect(screen.getByText("Complex Tier")).toBeInTheDocument(); @@ -34,7 +48,7 @@ describe("ComplexityRouterConfig", () => { }); it("should show example queries for each tier", () => { - renderWithProviders(); + renderWithProviders(); expect(screen.getByText(/Hello!/)).toBeInTheDocument(); expect(screen.getByText(/Explain how REST APIs work/)).toBeInTheDocument(); expect(screen.getByText(/Design a microservices architecture/)).toBeInTheDocument(); @@ -42,12 +56,12 @@ describe("ComplexityRouterConfig", () => { }); it("should display the how classification works section", () => { - renderWithProviders(); + renderWithProviders(); expect(screen.getByText("How Classification Works")).toBeInTheDocument(); }); it("should show score thresholds in the classification section", () => { - renderWithProviders(); + renderWithProviders(); expect(screen.getByText(/Score < 0.15/)).toBeInTheDocument(); expect(screen.getByText(/Score 0.15 - 0.35/)).toBeInTheDocument(); expect(screen.getByText(/Score 0.35 - 0.60/)).toBeInTheDocument(); @@ -91,16 +105,14 @@ describe("ComplexityRouterConfig", () => { }); it("should render the custom technical keywords field", () => { - renderWithProviders(); + renderWithProviders(); expect(screen.getByText("Custom Technical Keywords")).toBeInTheDocument(); }); it("should display existing custom technical keywords as tags", () => { renderWithProviders( , @@ -114,9 +126,7 @@ describe("ComplexityRouterConfig", () => { const onCustomTechnicalKeywordsChange = vi.fn(); renderWithProviders( , @@ -126,4 +136,65 @@ describe("ComplexityRouterConfig", () => { await user.type(input, "udp,"); expect(onCustomTechnicalKeywordsChange).toHaveBeenCalledWith(["udp"]); }); + + it("should render an empty state when no keyword tier rules exist", () => { + renderWithProviders(); + expect(screen.getByText("Keyword Tier Overrides")).toBeInTheDocument(); + expect(screen.getByText("No keyword tier overrides configured")).toBeInTheDocument(); + }); + + it("should call onKeywordTierRulesChange with a new rule when 'Add keyword rule' is clicked", async () => { + const user = userEvent.setup(); + const onKeywordTierRulesChange = vi.fn(); + renderWithProviders(); + await user.click(screen.getByRole("button", { name: /add keyword rule/i })); + expect(onKeywordTierRulesChange).toHaveBeenCalledTimes(1); + const newRules = onKeywordTierRulesChange.mock.calls[0][0]; + expect(newRules).toHaveLength(1); + expect(newRules[0]).toMatchObject({ keywords: [], tier: "COMPLEX" }); + }); + + it("should render an existing keyword tier rule and remove it when the delete button is clicked", async () => { + const user = userEvent.setup(); + const onKeywordTierRulesChange = vi.fn(); + renderWithProviders( + , + ); + expect(screen.getByText("invoice")).toBeInTheDocument(); + expect(screen.getByText("refund")).toBeInTheDocument(); + + await user.click(screen.getByRole("button", { name: /remove keyword rule 1/i })); + expect(onKeywordTierRulesChange).toHaveBeenCalledWith([]); + }); + + it("should not show embedding model or match score fields when semantic matching is disabled", () => { + renderWithProviders(); + expect(screen.getByText("Semantic keyword matching")).toBeInTheDocument(); + expect(screen.queryByText("Embedding model")).not.toBeInTheDocument(); + expect(screen.queryByText("Minimum match score")).not.toBeInTheDocument(); + }); + + it("should show embedding model and match score fields when semantic matching is enabled", () => { + renderWithProviders(); + expect(screen.getByText("Embedding model")).toBeInTheDocument(); + expect(screen.getByText("Minimum match score")).toBeInTheDocument(); + }); + + it("should call onSemanticMatchingEnabledChange when the semantic matching switch is toggled", async () => { + const user = userEvent.setup(); + const onSemanticMatchingEnabledChange = vi.fn(); + renderWithProviders( + , + ); + await user.click(screen.getByRole("switch")); + expect(onSemanticMatchingEnabledChange).toHaveBeenCalledWith(true, expect.anything()); + }); }); diff --git a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx index 2088e7183b9..ae2e03e576f 100644 --- a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx +++ b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx @@ -2,6 +2,8 @@ import { InfoCircleOutlined } from "@ant-design/icons"; import { Select as AntdSelect, Card, Collapse, Divider, InputNumber, Radio, Space, Tooltip, Typography } from "antd"; import React from "react"; import { ModelGroup } from "@/components/llm_calls/fetch_models"; +import KeywordTierRules, { KeywordTierRule } from "./KeywordTierRules"; +import SemanticKeywordMatching from "./SemanticKeywordMatching"; const { Text } = Typography; @@ -33,6 +35,16 @@ interface ComplexityRouterConfigProps { onChange: (value: ComplexityRouterConfigValue) => void; customTechnicalKeywords?: string[]; onCustomTechnicalKeywordsChange?: (keywords: string[]) => void; + // Optional: the edit-auto-router modal doesn't yet support editing keyword tier + // rules or semantic matching, so it renders this component without them. + keywordTierRules?: KeywordTierRule[]; + onKeywordTierRulesChange?: (rules: KeywordTierRule[]) => void; + semanticMatchingEnabled?: boolean; + onSemanticMatchingEnabledChange?: (enabled: boolean) => void; + embeddingModel?: string; + onEmbeddingModelChange?: (model: string) => void; + matchThreshold?: number; + onMatchThresholdChange?: (threshold: number) => void; } const TIER_DESCRIPTIONS: Record = { @@ -64,6 +76,14 @@ const ComplexityRouterConfig: React.FC = ({ onChange, customTechnicalKeywords, onCustomTechnicalKeywordsChange, + keywordTierRules = [], + onKeywordTierRulesChange = () => {}, + semanticMatchingEnabled = false, + onSemanticMatchingEnabledChange = () => {}, + embeddingModel, + onEmbeddingModelChange = () => {}, + matchThreshold = 0.5, + onMatchThresholdChange = () => {}, }) => { // Prepare model options for dropdowns const modelOptions = modelInfo.map((model) => ({ @@ -239,7 +259,8 @@ const ComplexityRouterConfig: React.FC = ({ - Optional: add terms the built-in list misses (e.g., udp, kafka, terraform) + Optional: Add terms to the built-in list to improve classification accuracy on the technical dimension. (e.g., + udp, kafka, terraform). = ({ + + + + + + + + ); }; diff --git a/ui/litellm-dashboard/src/components/add_model/KeywordTierRules.tsx b/ui/litellm-dashboard/src/components/add_model/KeywordTierRules.tsx new file mode 100644 index 00000000000..4f647c710f2 --- /dev/null +++ b/ui/litellm-dashboard/src/components/add_model/KeywordTierRules.tsx @@ -0,0 +1,112 @@ +import { DeleteOutlined, InfoCircleOutlined, PlusOutlined } from "@ant-design/icons"; +import { Button, Card, Empty, Select as AntdSelect, Tooltip, Typography } from "antd"; +import React from "react"; + +const { Text } = Typography; + +export type ComplexityTier = "SIMPLE" | "MEDIUM" | "COMPLEX" | "REASONING"; + +export interface KeywordTierRule { + id: string; + keywords: string[]; + tier: ComplexityTier; +} + +interface KeywordTierRulesProps { + rules: KeywordTierRule[]; + onChange: (rules: KeywordTierRule[]) => void; +} + +const TIER_OPTIONS: { value: ComplexityTier; label: string }[] = [ + { value: "SIMPLE", label: "Simple" }, + { value: "MEDIUM", label: "Medium" }, + { value: "COMPLEX", label: "Complex" }, + { value: "REASONING", label: "Reasoning" }, +]; + +const KeywordTierRules: React.FC = ({ rules, onChange }) => { + const addRule = () => { + onChange([...rules, { id: `${Date.now()}`, keywords: [], tier: "COMPLEX" }]); + }; + + const updateRule = (id: string, updates: Partial>) => { + onChange(rules.map((rule) => (rule.id === id ? { ...rule, ...updates } : rule))); + }; + + const removeRule = (id: string) => { + onChange(rules.filter((rule) => rule.id !== id)); + }; + + return ( +
+
+
+ + Keyword Tier Overrides + + + + +
+ +
+ + Optional: route requests containing specific keywords directly to a tier, e.g. route "invoice, refund, + billing" to the medium tier. + + + {rules.length === 0 ? ( + + + + ) : ( +
+ {rules.map((rule, index) => ( + +
+
+ + Keywords {index + 1} + + updateRule(rule.id, { keywords })} + placeholder="e.g., invoice, refund, billing" + tokenSeparators={[","]} + open={false} + suffixIcon={null} + style={{ width: "100%" }} + allowClear + /> +
+
+ + Route to tier + + updateRule(rule.id, { tier })} + options={TIER_OPTIONS} + style={{ width: "100%" }} + /> +
+
+ + ))} +
+ )} +
+ ); +}; + +export default KeywordTierRules; diff --git a/ui/litellm-dashboard/src/components/add_model/SemanticKeywordMatching.tsx b/ui/litellm-dashboard/src/components/add_model/SemanticKeywordMatching.tsx new file mode 100644 index 00000000000..0f9907ac6c9 --- /dev/null +++ b/ui/litellm-dashboard/src/components/add_model/SemanticKeywordMatching.tsx @@ -0,0 +1,84 @@ +import { InfoCircleOutlined } from "@ant-design/icons"; +import { Card, InputNumber, Select as AntdSelect, Switch, Tooltip, Typography } from "antd"; +import React from "react"; +import { ModelGroup } from "@/components/llm_calls/fetch_models"; + +const { Text } = Typography; + +const DEFAULT_MATCH_THRESHOLD = 0.5; + +interface SemanticKeywordMatchingProps { + enabled: boolean; + onEnabledChange: (enabled: boolean) => void; + embeddingModel: string | undefined; + onEmbeddingModelChange: (model: string) => void; + matchThreshold: number; + onMatchThresholdChange: (threshold: number) => void; + modelInfo: ModelGroup[]; +} + +const SemanticKeywordMatching: React.FC = ({ + enabled, + onEnabledChange, + embeddingModel, + onEmbeddingModelChange, + matchThreshold, + onMatchThresholdChange, + modelInfo, +}) => { + const modelOptions = Array.from(new Set(modelInfo.map((model) => model.model_group))).map((model_group) => ({ + value: model_group, + label: model_group, + })); + + return ( + +
+
+
+ Semantic keyword matching + + + +
+ + Uses same keyword-tier pairs as above and overrides direct keyword matching. Adds latency based on embedding + model network request. + +
+ +
+ + {enabled && ( +
+
+ Embedding model + +
+
+ Minimum match score + onMatchThresholdChange(value ?? DEFAULT_MATCH_THRESHOLD)} + min={0} + max={1} + step={0.05} + style={{ width: "100%" }} + /> + Match only at or above this similarity score. +
+
+ )} + + ); +}; + +export default SemanticKeywordMatching; +export { DEFAULT_MATCH_THRESHOLD }; diff --git a/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx b/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx index f8047a3d938..79c07040210 100644 --- a/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx +++ b/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx @@ -1,16 +1,18 @@ import React, { useEffect, useState } from "react"; -import { Card, Form, Button, Tooltip, Typography, Select as AntdSelect, Modal, Radio, Badge, Space } from "antd"; +import { Card, Form, Button, Tooltip, Typography, Select as AntdSelect, Radio, Badge, Space } from "antd"; import type { FormInstance } from "antd"; +import { ThunderboltOutlined, BranchesOutlined } from "@ant-design/icons"; import { Text, TextInput } from "@tremor/react"; import { modelAvailableCall } from "../networking"; -import ConnectionErrorDisplay from "./model_connection_test"; import { all_admin_roles } from "@/utils/roles"; import { handleAddAutoRouterSubmit } from "./handle_add_auto_router_submit"; import { fetchAvailableModels, ModelGroup } from "@/components/llm_calls/fetch_models"; import RouterConfigBuilder from "./RouterConfigBuilder"; import ComplexityRouterConfig, { ComplexityRouterConfigValue } from "./ComplexityRouterConfig"; +import { KeywordTierRule } from "./KeywordTierRules"; +import { DEFAULT_MATCH_THRESHOLD } from "./SemanticKeywordMatching"; +import { buildComplexityRouterConfig, getSemanticConfigError } from "./build_complexity_router_config"; import NotificationManager from "../molecules/notifications_manager"; -import { ThunderboltOutlined, BranchesOutlined } from "@ant-design/icons"; interface AddAutoRouterTabProps { form: FormInstance; @@ -19,34 +21,29 @@ interface AddAutoRouterTabProps { userRole: string; } -type RouterType = "complexity" | "semantic"; +type RouterType = "recommended" | "semantic"; -const { Title, Link } = Typography; +const { Title } = Typography; const AddAutoRouterTab: React.FC = ({ form, handleOk, accessToken, userRole }) => { - // State for connection testing - const [isResultModalVisible, setIsResultModalVisible] = useState(false); - const [isTestingConnection, setIsTestingConnection] = useState(false); - const [connectionTestId, setConnectionTestId] = useState(""); - const [modelAccessGroups, setModelAccessGroups] = useState([]); const [modelInfo, setModelInfo] = useState([]); - const [showCustomDefaultModel, setShowCustomDefaultModel] = useState(false); - const [showCustomEmbeddingModel, setShowCustomEmbeddingModel] = useState(false); - // Router type state - default to complexity router - const [routerType, setRouterType] = useState("complexity"); + const [routerType, setRouterType] = useState("recommended"); - // Semantic router config (existing) - const [routerConfig, setRouterConfig] = useState(null); - - // Complexity router config (new) const [complexityRouterConfig, setComplexityRouterConfig] = useState({ tiers: { SIMPLE: "", MEDIUM: "", COMPLEX: "", REASONING: "" }, classifier_type: "heuristic", }); const [customTechnicalKeywords, setCustomTechnicalKeywords] = useState([]); + const [keywordTierRules, setKeywordTierRules] = useState([]); + const [semanticMatchingEnabled, setSemanticMatchingEnabled] = useState(false); + const [embeddingModel, setEmbeddingModel] = useState(undefined); + const [matchThreshold, setMatchThreshold] = useState(DEFAULT_MATCH_THRESHOLD); + + // Semantic router config (existing) + const [routerConfig, setRouterConfig] = useState(null); useEffect(() => { const fetchModelAccessGroups = async () => { @@ -70,135 +67,130 @@ const AddAutoRouterTab: React.FC = ({ form, handleOk, acc const isAdmin = all_admin_roles.includes(userRole); - // Test connection when button is clicked - const handleTestConnection = async () => { - setIsTestingConnection(true); - setConnectionTestId(`test-${Date.now()}`); - setIsResultModalVisible(true); + const modelGroupOptions = Array.from(new Set(modelInfo.map((option) => option.model_group))).map((model_group) => ({ + value: model_group, + label: model_group, + })); + + const submitRecommendedRouter = (name: string) => { + const { + tiers, + classifier_type: classifierType, + classifier_llm_config: classifierLlmConfig, + } = complexityRouterConfig; + + const filledTiers = Object.values(tiers).filter(Boolean); + if (filledTiers.length === 0) { + NotificationManager.fromBackend("Please select at least one model for a complexity tier"); + return; + } + + if (classifierType === "llm" && !classifierLlmConfig?.model) { + NotificationManager.fromBackend("Please select a classifier model, or switch back to Heuristic"); + return; + } + + const semanticError = getSemanticConfigError({ semanticMatchingEnabled, embeddingModel, keywordTierRules }); + if (semanticError) { + NotificationManager.fromBackend(semanticError); + return; + } + + const defaultModel = tiers.MEDIUM || tiers.SIMPLE || tiers.COMPLEX || tiers.REASONING; + + form.setFieldsValue({ + custom_llm_provider: "auto_router", + model: name, + api_key: "not_required_for_auto_router", + auto_router_default_model: defaultModel, + }); + + form + .validateFields(["auto_router_name"]) + .then((values) => { + const complexityRouterConfigParams = { + tiers, + classifierType, + classifierLlmConfig, + customTechnicalKeywords, + keywordTierRules, + semanticMatchingEnabled, + embeddingModel, + matchThreshold, + }; + + const submitValues = { + ...values, + auto_router_name: name, + auto_router_default_model: defaultModel, + model_type: "complexity_router", + complexity_router_config: buildComplexityRouterConfig(complexityRouterConfigParams), + model_access_group: form.getFieldValue("model_access_group"), + }; + + handleAddAutoRouterSubmit(submitValues, accessToken, form, handleOk); + }) + .catch((error) => { + console.error("Validation failed:", error); + NotificationManager.fromBackend("Please fill in all required fields"); + }); }; - // Auto router specific form submit handler - const handleAutoRouterSubmit = () => { - const currentFormValues = form.getFieldsValue(); + const submitSemanticRouter = (name: string) => { + if (!form.getFieldValue("auto_router_default_model")) { + NotificationManager.fromBackend("Please select a Default Model"); + return; + } - // Check basic required fields first - if (!currentFormValues.auto_router_name) { + if (!routerConfig || !routerConfig.routes || routerConfig.routes.length === 0) { + NotificationManager.fromBackend("Please configure at least one route for the auto router"); + return; + } + + const invalidRoutes = routerConfig.routes.filter( + (route: any) => !route.name || !route.description || route.utterances.length === 0, + ); + if (invalidRoutes.length > 0) { + NotificationManager.fromBackend( + "Please ensure all routes have a target model, description, and at least one utterance", + ); + return; + } + + form.setFieldsValue({ + custom_llm_provider: "auto_router", + model: name, + api_key: "not_required_for_auto_router", + }); + + form + .validateFields() + .then((values) => { + const submitValues = { + ...values, + auto_router_name: name, + auto_router_config: routerConfig, + model_type: "semantic_router", + }; + handleAddAutoRouterSubmit(submitValues, accessToken, form, handleOk); + }) + .catch((error) => { + console.error("Validation failed:", error); + NotificationManager.fromBackend("Please fill in all required fields"); + }); + }; + + const handleAutoRouterSubmit = () => { + const name = form.getFieldValue("auto_router_name"); + if (!name) { NotificationManager.fromBackend("Please enter an Auto Router Name"); return; } - // Validation differs based on router type - if (routerType === "complexity") { - // Complexity Router validation - const { tiers, classifier_type, classifier_llm_config } = complexityRouterConfig; - const filledTiers = Object.values(tiers).filter(Boolean); - if (filledTiers.length === 0) { - NotificationManager.fromBackend("Please select at least one model for a complexity tier"); - return; - } - - if (classifier_type === "llm" && !classifier_llm_config?.model) { - NotificationManager.fromBackend("Please select a classifier model, or switch back to Heuristic"); - return; - } - - // For complexity router, use the first non-empty tier as default - const defaultModel = tiers.MEDIUM || tiers.SIMPLE || tiers.COMPLEX || tiers.REASONING; - - // Set form values for complexity router - form.setFieldsValue({ - custom_llm_provider: "auto_router", - model: currentFormValues.auto_router_name, - api_key: "not_required_for_auto_router", - auto_router_default_model: defaultModel, - }); - - form - .validateFields(["auto_router_name"]) - .then((values) => { - // Build the complexity router config - const submitValues = { - ...values, - auto_router_name: currentFormValues.auto_router_name, - auto_router_default_model: defaultModel, - // Use special model prefix for complexity router - model_type: "complexity_router", - complexity_router_config: { - tiers, - classifier_type, - ...(classifier_type === "llm" ? { classifier_llm_config } : {}), - ...(customTechnicalKeywords.length > 0 && { custom_technical_keywords: customTechnicalKeywords }), - }, - model_access_group: currentFormValues.model_access_group, - }; - - handleAddAutoRouterSubmit(submitValues, accessToken, form, handleOk); - }) - .catch((error) => { - console.error("Validation failed:", error); - NotificationManager.fromBackend("Please fill in all required fields"); - }); + if (routerType === "recommended") { + submitRecommendedRouter(name); } else { - // Semantic Router validation (existing logic) - if (!currentFormValues.auto_router_default_model) { - NotificationManager.fromBackend("Please select a Default Model"); - return; - } - - form.setFieldsValue({ - custom_llm_provider: "auto_router", - model: currentFormValues.auto_router_name, - api_key: "not_required_for_auto_router", - }); - - // Custom validation for router config - if (!routerConfig || !routerConfig.routes || routerConfig.routes.length === 0) { - NotificationManager.fromBackend("Please configure at least one route for the auto router"); - return; - } - - // Check if all routes have required fields - const invalidRoutes = routerConfig.routes.filter( - (route: any) => !route.name || !route.description || route.utterances.length === 0, - ); - - if (invalidRoutes.length > 0) { - NotificationManager.fromBackend( - "Please ensure all routes have a target model, description, and at least one utterance", - ); - return; - } - - form - .validateFields() - .then((values) => { - const submitValues = { - ...values, - auto_router_config: routerConfig, - model_type: "semantic_router", - }; - handleAddAutoRouterSubmit(submitValues, accessToken, form, handleOk); - }) - .catch((error) => { - console.error("Validation failed:", error); - const fieldErrors = error.errorFields || []; - if (fieldErrors.length > 0) { - const missingFields = fieldErrors.map((field: any) => { - const fieldName = field.name[0]; - const friendlyNames: { [key: string]: string } = { - auto_router_name: "Auto Router Name", - auto_router_default_model: "Default Model", - auto_router_embedding_model: "Embedding Model", - }; - return friendlyNames[fieldName] || fieldName; - }); - NotificationManager.fromBackend( - `Please fill in the following required fields: ${missingFields.join(", ")}`, - ); - } else { - NotificationManager.fromBackend("Please fill in all required fields"); - } - }); + submitSemanticRouter(name); } }; @@ -207,7 +199,7 @@ const AddAutoRouterTab: React.FC = ({ form, handleOk, acc Add Auto Router Create an auto router that automatically selects the best model based on request complexity or semantic - matching. + matching. Use in place of a single default model. @@ -215,35 +207,28 @@ const AddAutoRouterTab: React.FC = ({ form, handleOk, acc Router Type setRouterType(e.target.value)} className="w-full"> - +
- Complexity Router + Auto-Router v2
- Automatically routes based on request complexity. No training data needed — just pick 4 models and go. -
- ✓ Zero API calls ·{" "} - ✓ <1ms latency ·{" "} - ✓ No cost + Routes by request complexity across four tiers, with optional keyword-to-tier overrides and semantic + keyword matching. No training data needed.
- Semantic Router + Semantic Router [to be deprecated]
- Routes based on semantic similarity to example utterances. Requires embedding model and training - examples. + Routes based on semantic similarity to example utterances. Requires an embedding model and example + utterances.
@@ -259,7 +244,6 @@ const AddAutoRouterTab: React.FC = ({ form, handleOk, acc wrapperCol={{ span: 16 }} labelAlign="left" > - {/* Auto Router Name */} = ({ form, handleOk, acc - {/* Conditional rendering based on router type */} - {routerType === "complexity" ? ( - /* Complexity Router Configuration */ + {routerType === "recommended" ? (
{ - setComplexityRouterConfig(config); - }} + onChange={setComplexityRouterConfig} customTechnicalKeywords={customTechnicalKeywords} onCustomTechnicalKeywordsChange={setCustomTechnicalKeywords} + keywordTierRules={keywordTierRules} + onKeywordTierRulesChange={setKeywordTierRules} + semanticMatchingEnabled={semanticMatchingEnabled} + onSemanticMatchingEnabledChange={setSemanticMatchingEnabled} + embeddingModel={embeddingModel} + onEmbeddingModelChange={setEmbeddingModel} + matchThreshold={matchThreshold} + onMatchThresholdChange={setMatchThreshold} />
) : ( - /* Semantic Router Configuration (existing) */ <> - {/* Router Configuration Builder */}
= ({ form, handleOk, acc />
- {/* Auto Router Default Model */} = ({ form, handleOk, acc > { - setShowCustomDefaultModel(value === "custom"); - }} - options={[ - ...Array.from(new Set(modelInfo.map((option) => option.model_group))).map((model_group) => ({ - value: model_group, - label: model_group, - })), - { value: "custom", label: "Enter custom model name" }, - ]} + options={modelGroupOptions} style={{ width: "100%" }} - showSearch={true} + showSearch /> - {/* Auto Router Embedding Model */} { - setShowCustomEmbeddingModel(value === "custom"); - form.setFieldValue("auto_router_embedding_model", value); - }} - options={[ - ...Array.from(new Set(modelInfo.map((option) => option.model_group))).map((model_group) => ({ - value: model_group, - label: model_group, - })), - { value: "custom", label: "Enter custom model name" }, - ]} + options={modelGroupOptions} style={{ width: "100%" }} - showSearch={true} + showSearch allowClear /> @@ -391,9 +355,10 @@ const AddAutoRouterTab: React.FC = ({ form, handleOk, acc Need Help?
- + {/* TODO: add back a Test Connection or JSON preview action here. Test Connection was removed + because prepareModelAddRequest can't build a valid pre-save payload for an auto router + (tiers are model-group references, not litellm_params); a JSON preview of the + complexity_router_config would be a good alternative. */}
- - {/* Test Connection Results Modal */} - { - setIsResultModalVisible(false); - setIsTestingConnection(false); - }} - footer={[ - , - ]} - width={700} - > - {/* Only render the ConnectionErrorDisplay when modal is visible and we have a test ID */} - {isResultModalVisible && ( - { - setIsResultModalVisible(false); - setIsTestingConnection(false); - }} - onTestComplete={() => setIsTestingConnection(false)} - /> - )} - ); }; diff --git a/ui/litellm-dashboard/src/components/add_model/add_model_tab.test.tsx b/ui/litellm-dashboard/src/components/add_model/add_model_tab.test.tsx index 0a8d2d9124e..816cd1bf97b 100644 --- a/ui/litellm-dashboard/src/components/add_model/add_model_tab.test.tsx +++ b/ui/litellm-dashboard/src/components/add_model/add_model_tab.test.tsx @@ -1,5 +1,5 @@ import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; -import { render, renderHook, screen, waitFor } from "@testing-library/react"; +import { render, renderHook, screen, waitFor, within } from "@testing-library/react"; import userEvent from "@testing-library/user-event"; import { Form } from "antd"; import type { UploadProps } from "antd/es/upload"; @@ -299,8 +299,11 @@ describe("Add Model Tab", () => { // Wait for component to load await screen.findByText("Provider"); - // Find the team-BYOK switch by its role - const teamSwitch = screen.getByRole("switch"); + // Scope to the Team-BYOK Model Form.Item: the Add Auto Router tab, mounted alongside + // this one, also renders a "Semantic keyword matching" switch, so a bare + // getByRole("switch") would match more than one element. + const teamByokFormItem = screen.getByText("Team-BYOK Model").closest(".ant-form-item") as HTMLElement; + const teamSwitch = within(teamByokFormItem).getByRole("switch"); expect(teamSwitch).toBeInTheDocument(); // Initially, team selection should not be visible diff --git a/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.test.ts b/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.test.ts new file mode 100644 index 00000000000..250b7d13a5b --- /dev/null +++ b/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.test.ts @@ -0,0 +1,141 @@ +import { + buildComplexityRouterConfig, + getSemanticConfigError, + BuildComplexityRouterConfigParams, +} from "./build_complexity_router_config"; + +const tiers = { + SIMPLE: "gpt-4o-mini", + MEDIUM: "gpt-4o", + COMPLEX: "claude-sonnet-4", + REASONING: "o1-preview", +}; + +const baseParams: BuildComplexityRouterConfigParams = { + tiers, + classifierType: "heuristic", + classifierLlmConfig: undefined, + customTechnicalKeywords: [], + keywordTierRules: [], + semanticMatchingEnabled: false, + embeddingModel: undefined, + matchThreshold: 0.5, +}; + +describe("buildComplexityRouterConfig", () => { + it("emits only tiers and classifier_type when nothing else is configured", () => { + const config = buildComplexityRouterConfig(baseParams); + expect(config).toEqual({ tiers, classifier_type: "heuristic" }); + }); + + it("includes classifier_llm_config only when classifier_type is llm", () => { + const config = buildComplexityRouterConfig({ + ...baseParams, + classifierType: "llm", + classifierLlmConfig: { model: "gpt-4o-mini", timeout_ms: 3000 }, + }); + expect(config.classifier_type).toBe("llm"); + expect(config.classifier_llm_config).toEqual({ model: "gpt-4o-mini", timeout_ms: 3000 }); + }); + + it("omits classifier_llm_config when classifier_type is heuristic even if config lingers in state", () => { + const config = buildComplexityRouterConfig({ + ...baseParams, + classifierType: "heuristic", + classifierLlmConfig: { model: "gpt-4o-mini", timeout_ms: 3000 }, + }); + expect(config.classifier_llm_config).toBeUndefined(); + }); + + it("sends keyword_tier_rules with their per-tier targeting preserved (not flattened)", () => { + const params: BuildComplexityRouterConfigParams = { + ...baseParams, + customTechnicalKeywords: ["udp"], + keywordTierRules: [ + { id: "r1", keywords: ["deploy to k8s"], tier: "REASONING" }, + { id: "r2", keywords: ["invoice", "refund"], tier: "SIMPLE" }, + ], + }; + const config = buildComplexityRouterConfig(params); + expect(config.keyword_tier_rules).toEqual([ + { keywords: ["deploy to k8s"], tier: "REASONING" }, + { keywords: ["invoice", "refund"], tier: "SIMPLE" }, + ]); + // custom technical keywords stay their own list, not merged with rule keywords + expect(config.custom_technical_keywords).toEqual(["udp"]); + expect(config.semantic_keyword_matching).toBeUndefined(); + }); + + it("includes semantic fields only when semantic matching is enabled", () => { + const params: BuildComplexityRouterConfigParams = { + ...baseParams, + keywordTierRules: [{ id: "r1", keywords: ["k8s"], tier: "REASONING" }], + semanticMatchingEnabled: true, + embeddingModel: "openai/text-embedding-3-small", + matchThreshold: 0.42, + }; + const config = buildComplexityRouterConfig(params); + expect(config.semantic_keyword_matching).toBe(true); + expect(config.embedding_model).toBe("openai/text-embedding-3-small"); + expect(config.match_threshold).toBe(0.42); + }); + + it("omits semantic fields when the toggle is off even if an embedding model lingers in state", () => { + const params: BuildComplexityRouterConfigParams = { + ...baseParams, + keywordTierRules: [{ id: "r1", keywords: ["k8s"], tier: "REASONING" }], + semanticMatchingEnabled: false, + embeddingModel: "openai/text-embedding-3-small", + matchThreshold: 0.42, + }; + const config = buildComplexityRouterConfig(params); + expect(config.semantic_keyword_matching).toBeUndefined(); + expect(config.embedding_model).toBeUndefined(); + expect(config.match_threshold).toBeUndefined(); + }); + + it("omits empty optional lists", () => { + const config = buildComplexityRouterConfig(baseParams); + expect(config.custom_technical_keywords).toBeUndefined(); + expect(config.keyword_tier_rules).toBeUndefined(); + }); +}); + +describe("getSemanticConfigError", () => { + const rule = { id: "r1", keywords: ["k8s"], tier: "REASONING" as const }; + + it("returns null when semantic matching is disabled (even with gaps)", () => { + expect( + getSemanticConfigError({ semanticMatchingEnabled: false, embeddingModel: undefined, keywordTierRules: [] }), + ).toBeNull(); + }); + + it("errors when enabled without an embedding model", () => { + expect( + getSemanticConfigError({ semanticMatchingEnabled: true, embeddingModel: undefined, keywordTierRules: [rule] }), + ).toMatch(/embedding model/i); + }); + + it("errors when enabled with an embedding model but no keyword tier rules", () => { + expect( + getSemanticConfigError({ semanticMatchingEnabled: true, embeddingModel: "voyage-3-5", keywordTierRules: [] }), + ).toMatch(/keyword tier rule/i); + }); + + it("errors when a rule has no non-empty keywords", () => { + const emptyRule = { id: "r2", keywords: ["", " "], tier: "SIMPLE" as const }; + expect( + getSemanticConfigError({ + semanticMatchingEnabled: true, + embeddingModel: "voyage-3-5", + keywordTierRules: [emptyRule], + }), + ).toMatch(/at least one keyword/i); + }); + + it("returns null when enabled with both an embedding model and rules", () => { + expect( + getSemanticConfigError({ semanticMatchingEnabled: true, embeddingModel: "voyage-3-5", keywordTierRules: [rule] }), + ).toBeNull(); + }); +}); diff --git a/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.ts b/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.ts new file mode 100644 index 00000000000..00cc19cde0c --- /dev/null +++ b/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.ts @@ -0,0 +1,70 @@ +import { KeywordTierRule } from "./KeywordTierRules"; +import { ClassifierLLMConfig, ClassifierType } from "./ComplexityRouterConfig"; + +export interface ComplexityTiers { + SIMPLE: string; + MEDIUM: string; + COMPLEX: string; + REASONING: string; +} + +export interface BuildComplexityRouterConfigParams { + tiers: ComplexityTiers; + classifierType: ClassifierType; + classifierLlmConfig: ClassifierLLMConfig | undefined; + customTechnicalKeywords: string[]; + keywordTierRules: KeywordTierRule[]; + semanticMatchingEnabled: boolean; + embeddingModel: string | undefined; + matchThreshold: number; +} + +export interface ComplexityRouterConfigPayload { + tiers: ComplexityTiers; + classifier_type: ClassifierType; + classifier_llm_config?: ClassifierLLMConfig; + custom_technical_keywords?: string[]; + keyword_tier_rules?: { keywords: string[]; tier: KeywordTierRule["tier"] }[]; + semantic_keyword_matching?: boolean; + embedding_model?: string; + match_threshold?: number; +} + +export const getSemanticConfigError = ({ + semanticMatchingEnabled, + embeddingModel, + keywordTierRules, +}: Pick): + | string + | null => { + if (!semanticMatchingEnabled) return null; + if (!embeddingModel) return "Select an embedding model to use semantic keyword matching"; + if (keywordTierRules.length === 0) return "Add at least one keyword tier rule to use semantic keyword matching"; + if (keywordTierRules.some((rule) => !rule.keywords.some((keyword) => keyword.trim()))) + return "Every keyword tier rule needs at least one keyword"; + return null; +}; + +export const buildComplexityRouterConfig = ({ + tiers, + classifierType, + classifierLlmConfig, + customTechnicalKeywords, + keywordTierRules, + semanticMatchingEnabled, + embeddingModel, + matchThreshold, +}: BuildComplexityRouterConfigParams): ComplexityRouterConfigPayload => ({ + tiers, + classifier_type: classifierType, + ...(classifierType === "llm" && classifierLlmConfig && { classifier_llm_config: classifierLlmConfig }), + ...(customTechnicalKeywords.length > 0 && { custom_technical_keywords: customTechnicalKeywords }), + ...(keywordTierRules.length > 0 && { + keyword_tier_rules: keywordTierRules.map((rule) => ({ keywords: rule.keywords, tier: rule.tier })), + }), + ...(semanticMatchingEnabled && { + semantic_keyword_matching: true, + embedding_model: embeddingModel, + match_threshold: matchThreshold, + }), +}); diff --git a/ui/litellm-dashboard/src/components/add_model/handle_add_auto_router_submit.tsx b/ui/litellm-dashboard/src/components/add_model/handle_add_auto_router_submit.tsx index 5939a4a5844..ef774d870b5 100644 --- a/ui/litellm-dashboard/src/components/add_model/handle_add_auto_router_submit.tsx +++ b/ui/litellm-dashboard/src/components/add_model/handle_add_auto_router_submit.tsx @@ -6,62 +6,46 @@ export const handleAddAutoRouterSubmit = async (values: any, accessToken: string let autoRouterConfig: any; if (values.model_type === "complexity_router") { - // Complexity Router configuration - autoRouterConfig = { model_name: values.auto_router_name, litellm_params: { - // Use special prefix for complexity router model: `auto_router/complexity_router`, - // Pass the complexity router config as a JSON object (not stringified) complexity_router_config: values.complexity_router_config, - // Default model for fallback (use MEDIUM or first available tier) complexity_router_default_model: values.auto_router_default_model, }, model_info: {}, }; } else { - // Semantic Router configuration (existing behavior) - autoRouterConfig = { model_name: values.auto_router_name, litellm_params: { model: `auto_router/${values.auto_router_name}`, - auto_router_config: JSON.stringify(values.auto_router_config), // Convert JSON object to string as expected by backend + auto_router_config: JSON.stringify(values.auto_router_config), auto_router_default_model: values.auto_router_default_model, }, model_info: {}, }; - // Add optional embedding model if provided - if (values.auto_router_embedding_model && values.auto_router_embedding_model !== "custom") { + if (values.auto_router_embedding_model) { autoRouterConfig.litellm_params.auto_router_embedding_model = values.auto_router_embedding_model; - } else if (values.custom_embedding_model) { - autoRouterConfig.litellm_params.auto_router_embedding_model = values.custom_embedding_model; } } - // Add team information if provided if (values.team_id) { autoRouterConfig.model_info.team_id = values.team_id; } - // Add model access groups if provided if (values.model_access_group && values.model_access_group.length > 0) { autoRouterConfig.model_info.access_groups = values.model_access_group; } - // Create the auto router using the same model creation endpoint - const response: any = await modelCreateCall(accessToken, autoRouterConfig as Model); + await modelCreateCall(accessToken, autoRouterConfig as Model); - // Show success notification - const routerTypeName = values.model_type === "complexity_router" ? "Complexity Router" : "Semantic Router"; + const routerTypeName = values.model_type === "complexity_router" ? "Auto Router" : "Semantic Router"; NotificationManager.success(`Successfully created ${routerTypeName}: ${values.auto_router_name}`); - // Reset the form form.resetFields(); - // Call the callback if provided (e.g., to close modal) if (callback) { callback(); } From 186d083bc7ac5a17368f43535a5c8cbd040b1535 Mon Sep 17 00:00:00 2001 From: Abhimanyu Kapur <38531241+akapur99@users.noreply.github.com> Date: Fri, 10 Jul 2026 23:18:38 -0700 Subject: [PATCH 2/8] fix(complexity_router): build semantic route index once under concurrent cold-start Concurrent first requests each hit asyncio.to_thread to build the SemanticRouter index, firing duplicate embedding calls for the static route utterances. Guard the lazy build with a per-router asyncio.Lock (double-checked) so the index is constructed exactly once regardless of how many callers race in cold. Adds a regression test asserting ten simultaneous cold-start requests build the index the same number of times as a single request, and reworks the fake embedding router to count builds by how often a route utterance is embedded (robust to which embedding path the library uses) while still recording sync-call thread ids for the off-event-loop assertion. --- .../complexity_router/complexity_router.py | 27 +++++++--- .../router_strategy/test_complexity_router.py | 53 +++++++++++++++++-- 2 files changed, 71 insertions(+), 9 deletions(-) diff --git a/litellm/router_strategy/complexity_router/complexity_router.py b/litellm/router_strategy/complexity_router/complexity_router.py index f5decbaa670..2dae9393510 100644 --- a/litellm/router_strategy/complexity_router/complexity_router.py +++ b/litellm/router_strategy/complexity_router/complexity_router.py @@ -149,8 +149,11 @@ class ComplexityRouter(CustomLogger): self.simple_keywords = self.config.simple_keywords or DEFAULT_SIMPLE_KEYWORDS # Lazily built on first semantic request and cached for reuse (route - # embeddings are static, only the prompt is embedded per request). + # embeddings are static, only the prompt is embedded per request). The lock + # serializes the one-time build so concurrent cold-start requests don't each + # construct the index and fire duplicate embedding calls. self._semantic_routelayer: Optional[SemanticRouter] = None + self._semantic_routelayer_lock = asyncio.Lock() # Pre-compile regex patterns for efficiency # Use non-greedy .*? to prevent ReDoS on pathological inputs @@ -486,6 +489,22 @@ class ComplexityRouter(CustomLogger): self._semantic_routelayer = routelayer return routelayer + async def _ensure_semantic_routelayer(self) -> "SemanticRouter": + """Return the cached route layer, building it once under a lock if needed. + + The build embeds the static route utterances via the encoder's synchronous path, + so it runs in a worker thread to avoid blocking the event loop. A double-checked + asyncio lock ensures concurrent cold-start requests build it exactly once rather + than each firing duplicate embedding calls. + """ + if self._semantic_routelayer is not None: + return self._semantic_routelayer + async with self._semantic_routelayer_lock: + routelayer = self._semantic_routelayer + if routelayer is None: + routelayer = await asyncio.to_thread(self._get_or_create_semantic_routelayer) + return routelayer + async def _semantic_tier_override(self, user_message: str, request_kwargs: Dict) -> Optional[ComplexityTier]: """Match the prompt against keyword_tier_rules by embedding similarity. @@ -503,11 +522,7 @@ class ComplexityRouter(CustomLogger): LiteLLMRouterEncoder, ) - # Building the SemanticRouter embeds the (static) route utterances via the - # encoder's *synchronous* path; run it in a worker thread so that one-time, - # per-router-instance provider I/O never blocks the async event loop and stalls - # other requests. Once cached, subsequent calls return immediately (no I/O). - routelayer = await asyncio.to_thread(self._get_or_create_semantic_routelayer) + routelayer = await self._ensure_semantic_routelayer() encoder = cast(LiteLLMRouterEncoder, routelayer.encoder) # cast-ok: always the encoder we built above # Strip the parent request's budget reservation before forwarding: the reservation # belongs to the routed completion this embedding is helping select, not to the diff --git a/tests/test_litellm/router_strategy/test_complexity_router.py b/tests/test_litellm/router_strategy/test_complexity_router.py index f4da1018f3e..eda73f7e7b6 100644 --- a/tests/test_litellm/router_strategy/test_complexity_router.py +++ b/tests/test_litellm/router_strategy/test_complexity_router.py @@ -4,6 +4,7 @@ Tests for the ComplexityRouter. Tests the rule-based complexity scoring and tier assignment logic. """ +import asyncio import os import sys from typing import Dict, List @@ -1477,6 +1478,11 @@ class FakeEmbeddingRouter: def __init__(self): self.async_embedding_calls: List[List[str]] = [] self.async_embedding_kwargs: List[Dict] = [] + # Every embedded batch (sync route-index build AND async query), so tests can count + # builds independently of which embedding path the library happens to use. + self.embedded_batches: List[List[str]] = [] + # Thread ids of the synchronous (route-index build) embedding calls, so a test can + # assert the build is offloaded off the event-loop thread. self.sync_embedding_thread_ids: List[int] = [] def _vectors(self, docs: List[str]) -> List[List[float]]: @@ -1490,19 +1496,24 @@ class FakeEmbeddingRouter: return text if isinstance(text, list) else [text] def embedding(self, input, model, **kwargs): - # Record the thread this synchronous (route-index build) call ran on, so tests can - # assert it is offloaded off the event-loop thread. import threading + docs = self._as_list(input) + self.embedded_batches.append(docs) self.sync_embedding_thread_ids.append(threading.get_ident()) - return _make_embedding_response(self._vectors(self._as_list(input))) + return _make_embedding_response(self._vectors(docs)) async def aembedding(self, input, model, **kwargs): docs = self._as_list(input) + self.embedded_batches.append(docs) self.async_embedding_calls.append(docs) self.async_embedding_kwargs.append(kwargs) return _make_embedding_response(self._vectors(docs)) + def utterance_embedding_count(self, utterance: str) -> int: + """How many times the given route utterance was embedded == number of route-index builds.""" + return sum(1 for batch in self.embedded_batches if utterance in batch) + class TestSemanticKeywordTierRules: """Test embedding-based keyword_tier_rules matching.""" @@ -1636,6 +1647,42 @@ class TestSemanticKeywordTierRules: # ...and none of it ran on the event-loop thread. assert all(tid != loop_thread_id for tid in fake_router.sync_embedding_thread_ids) + @pytest.mark.asyncio + async def test_concurrent_cold_start_builds_routelayer_once(self, basic_config): + """Concurrent first requests must not each construct the route index (which would + fire duplicate embedding calls); the lazy build happens exactly once. + """ + config = { + **basic_config, + "keyword_tier_rules": [{"keywords": ["kubernetes deployment"], "tier": "REASONING"}], + "semantic_keyword_matching": True, + "embedding_model": "fake-embed", + "match_threshold": 0.5, + } + + def _make_router(fake): + return ComplexityRouter( + model_name="test-router", + litellm_router_instance=fake, + complexity_router_config=config, + ) + + # Baseline: a single cold request's route-index build embeds the route utterance once. + route_utterance = "kubernetes deployment" + baseline_fake = FakeEmbeddingRouter() + await _make_router(baseline_fake)._semantic_tier_override("roll out my k8s cluster", {}) + baseline_builds = baseline_fake.utterance_embedding_count(route_utterance) + assert baseline_builds >= 1 + + # Ten simultaneous cold-start requests must build the index the same number of + # times as one request - i.e. exactly once, not once per concurrent caller. + concurrent_fake = FakeEmbeddingRouter() + concurrent_router = _make_router(concurrent_fake) + await asyncio.gather( + *(concurrent_router._semantic_tier_override("roll out my k8s cluster", {}) for _ in range(10)) + ) + assert concurrent_fake.utterance_embedding_count(route_utterance) == baseline_builds + @pytest.mark.asyncio async def test_below_threshold_falls_back_to_scoring(self, basic_config): """When no route clears the threshold, scoring decides the tier.""" From 02137b2d12f8f5f7ae88ab084d5bb31f829d1a81 Mon Sep 17 00:00:00 2001 From: Abhimanyu Kapur <38531241+akapur99@users.noreply.github.com> Date: Sat, 11 Jul 2026 00:08:47 -0700 Subject: [PATCH 3/8] fix(proxy): guard delete_model router eviction on auto_router/ prefix delete_model popped the auto_routers/complexity_routers registries by the deleted deployment's model_name without checking it was actually an auto_router/* deployment. Deleting a regular DB model that merely shares a name with a config-defined router therefore evicted that router, which add_deployment never restores, leaving it unroutable until a proxy restart. This is the same cross-tenant DoS clear_cache was hardened against; mirror its auto_router/ prefix guard here. Extracts _deployment_name_and_model to read model_name and litellm_params.model from the deployment (delete_deployment returns the raw model_list dict at runtime despite its Deployment annotation), and adds a regression test asserting a same-named config router survives deletion of an unrelated regular model. --- .../model_management_endpoints.py | 27 ++++++-- .../test_model_management_endpoints.py | 66 ++++++++++++++++++- 2 files changed, 87 insertions(+), 6 deletions(-) diff --git a/litellm/proxy/management_endpoints/model_management_endpoints.py b/litellm/proxy/management_endpoints/model_management_endpoints.py index 0f0dedc40e6..23a3b3ce4f2 100644 --- a/litellm/proxy/management_endpoints/model_management_endpoints.py +++ b/litellm/proxy/management_endpoints/model_management_endpoints.py @@ -1040,6 +1040,22 @@ class ModelManagementAuthChecks: return True +def _deployment_name_and_model(deployment: Optional[Union[Deployment, Dict[str, object]]]) -> Tuple[Optional[str], str]: + """Return (model_name, litellm_params.model) for a deployment. + + delete_deployment is annotated to return a Deployment but hands back the raw + model_list dict at runtime, so both shapes are handled; the model defaults to "". + """ + if deployment is None: + return None, "" + if isinstance(deployment, dict): + name = deployment.get("model_name") + params = deployment.get("litellm_params") + model = params.get("model") if isinstance(params, dict) else None + return (name if isinstance(name, str) else None), (model if isinstance(model, str) else "") + return deployment.model_name, str(getattr(deployment.litellm_params, "model", "") or "") + + #### [BETA] - This is a beta endpoint, format might change based on user feedback. - https://github.com/BerriAI/litellm/issues/964 @router.post( "/model/delete", @@ -1114,11 +1130,12 @@ async def delete_model( deleted_deployment = llm_router.delete_deployment(id=model_info.id) # delete_deployment only drops the deployment from model_list; the auto/ # complexity router registries are keyed by model_name and would otherwise - # retain a stale (now unbacked) entry, so evict it here too. - deleted_name = getattr(deleted_deployment, "model_name", None) - if deleted_name is None and isinstance(deleted_deployment, dict): - deleted_name = deleted_deployment.get("model_name") - if deleted_name is not None: + # retain a stale (now unbacked) entry, so evict it here too. Guard on the + # auto_router/ prefix (as clear_cache does): a regular DB model that merely + # shares a model_name with a config-defined router must not evict that router, + # since add_deployment never restores config-defined routers. + deleted_name, deleted_model = _deployment_name_and_model(deleted_deployment) + if deleted_name is not None and deleted_model.startswith("auto_router/"): llm_router.auto_routers.pop(deleted_name, None) llm_router.complexity_routers.pop(deleted_name, None) diff --git a/tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py index 51411c4ae84..d9b5745807a 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py @@ -620,7 +620,11 @@ class TestDeleteModelClearsRouterRegistry: mock_router = MagicMock() mock_router.delete_deployment = MagicMock( - return_value={"model_name": "smart-router", "model_info": {"id": model_id}} + return_value={ + "model_name": "smart-router", + "litellm_params": {"model": "auto_router/complexity_router"}, + "model_info": {"id": model_id}, + } ) mock_router.auto_routers = {"smart-router": MagicMock()} mock_router.complexity_routers = {"smart-router": MagicMock()} @@ -644,6 +648,66 @@ class TestDeleteModelClearsRouterRegistry: assert "smart-router" not in mock_router.auto_routers assert "smart-router" not in mock_router.complexity_routers + @pytest.mark.asyncio + async def test_delete_regular_model_preserves_config_router_sharing_name(self): + """Deleting a regular (non-router) DB model must not evict a config-defined router + that merely shares its model_name. delete_deployment pops the DB model, but the + auto/complexity registries hold a config router under the same name that + add_deployment never restores, so an unguarded pop would make it permanently + unroutable (the same cross-tenant DoS clear_cache was hardened against). + """ + from litellm.proxy.management_endpoints.model_management_endpoints import ( + delete_model as delete_model_endpoint, + ) + from litellm.proxy.management_endpoints.model_management_endpoints import ModelInfoDelete + + model_id = "regular-del-1" + admin_user = UserAPIKeyAuth(user_id="admin", user_role=LitellmUserRoles.PROXY_ADMIN) + db_row = LiteLLM_ProxyModelTable( + model_id=model_id, + model_name="shared-name", + litellm_params={"model": "openai/gpt-4o"}, + model_info={"id": model_id}, + created_by="admin", + updated_by="admin", + ) + + mock_prisma = MagicMock() + mock_prisma.db = MagicMock() + mock_prisma.db.litellm_proxymodeltable = AsyncMock() + mock_prisma.db.litellm_proxymodeltable.find_unique = AsyncMock(return_value=db_row) + mock_prisma.db.litellm_proxymodeltable.delete = AsyncMock(return_value=db_row) + + mock_router = MagicMock() + mock_router.delete_deployment = MagicMock( + return_value={ + "model_name": "shared-name", + "litellm_params": {"model": "openai/gpt-4o"}, + "model_info": {"id": model_id}, + } + ) + config_router = MagicMock() + mock_router.auto_routers = {} + mock_router.complexity_routers = {"shared-name": config_router} + + _PS = "litellm.proxy.proxy_server" + with ( + patch(f"{_PS}.prisma_client", mock_prisma), + patch(f"{_PS}.store_model_in_db", True), + patch(f"{_PS}.proxy_config", MagicMock()), + patch(f"{_PS}.proxy_logging_obj", MagicMock()), + patch(f"{_PS}.general_settings", {}), + patch(f"{_PS}.premium_user", True), + patch(f"{_PS}.llm_router", mock_router), + ): + await delete_model_endpoint( + model_info=ModelInfoDelete(id=model_id), + user_api_key_dict=admin_user, + ) + + mock_router.delete_deployment.assert_called_once_with(id=model_id) + assert mock_router.complexity_routers.get("shared-name") is config_router + class TestUpdateModel: """ From eb0c9662550a674f296c2ff8830af84c37ebcd49 Mon Sep 17 00:00:00 2001 From: Abhimanyu Kapur <38531241+akapur99@users.noreply.github.com> Date: Sat, 11 Jul 2026 11:14:59 -0700 Subject: [PATCH 4/8] fix(complexity_router): use max aggregation for semantic keyword route scoring MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit SemanticRouter defaults to mean aggregation across a route's utterances. Since each tier's route holds one utterance per configured keyword, a real semantic match on one keyword was averaged together with the tier's other, unrelated keywords and dragged below match_threshold — e.g. a MEDIUM tier with keywords [beep, boop, new york] never fired for a genuine "new york" paraphrase, because mean(sim_to_beep, sim_to_boop, sim_to_new_york) landed well under the threshold even though sim_to_new_york alone cleared it. Pass aggregation="max" so a tier matches if the query is close enough to any one of its keywords, not the average of all of them. Verified against live Voyage embeddings: raw cosine similarity for "new york" vs a paraphrase was 0.54 (above a 0.5 threshold), but the route scored 0.28 under mean aggregation and never matched; max aggregation fixes it. Adds a regression test with a tier holding one matching and two unrelated keywords, asserting the tier still fires; fails without aggregation="max". --- .../complexity_router/complexity_router.py | 1 + .../router_strategy/test_complexity_router.py | 34 +++++++++++++++++++ 2 files changed, 35 insertions(+) diff --git a/litellm/router_strategy/complexity_router/complexity_router.py b/litellm/router_strategy/complexity_router/complexity_router.py index 2dae9393510..2eb69aad2de 100644 --- a/litellm/router_strategy/complexity_router/complexity_router.py +++ b/litellm/router_strategy/complexity_router/complexity_router.py @@ -485,6 +485,7 @@ class ComplexityRouter(CustomLogger): score_threshold=self.config.match_threshold, ), auto_sync="local", + aggregation="max", ) self._semantic_routelayer = routelayer return routelayer diff --git a/tests/test_litellm/router_strategy/test_complexity_router.py b/tests/test_litellm/router_strategy/test_complexity_router.py index eda73f7e7b6..3d92700032b 100644 --- a/tests/test_litellm/router_strategy/test_complexity_router.py +++ b/tests/test_litellm/router_strategy/test_complexity_router.py @@ -1546,6 +1546,40 @@ class TestSemanticKeywordTierRules: assert result.model == "o1-preview" # REASONING via semantic match assert fake_router.async_embedding_calls, "expected an embedding call for the prompt" + @pytest.mark.asyncio + async def test_tier_matches_on_best_utterance_not_diluted_by_others(self, basic_config): + """A tier with several keywords must match if the query is close to ANY of them, + not the average across all of them. A tier's route holds one utterance per keyword; + mean aggregation (the semantic_router library default) scores the query against the + *average* similarity across every utterance in the route, so a real match on one + keyword gets dragged below threshold by the tier's other, unrelated keywords. + """ + fake_router = FakeEmbeddingRouter() + config = { + **basic_config, + "keyword_tier_rules": [ + {"keywords": ["kubernetes deployment", "thanks", "goodbye"], "tier": "REASONING"}, + ], + "semantic_keyword_matching": True, + "embedding_model": "fake-embed", + "match_threshold": 0.5, + } + router = ComplexityRouter( + model_name="test-router", + litellm_router_instance=fake_router, + complexity_router_config=config, + ) + # Only "kubernetes deployment" is close to this query (cos 1.0); "thanks" and + # "goodbye" are orthogonal (cos 0.0). Mean over the three would be ~0.33, below the + # 0.5 threshold; the best (max) utterance alone clears it. + result = await router.async_pre_routing_hook( + model="test-model", + request_kwargs={}, + messages=[{"role": "user", "content": "help me roll out my k8s cluster today"}], + ) + assert result is not None + assert result.model == "o1-preview" # REASONING via best-utterance semantic match + @pytest.mark.asyncio async def test_semantic_embedding_call_carries_caller_metadata(self, basic_config): """The query embedding call must carry the caller's metadata/litellm_metadata From 4a4262aa0d0490ead1859fa34cf999d6dec53736 Mon Sep 17 00:00:00 2001 From: Abhimanyu Kapur <38531241+akapur99@users.noreply.github.com> Date: Sat, 11 Jul 2026 11:57:32 -0700 Subject: [PATCH 5/8] fix(complexity_router): preserve user_api_key_auth in sub-call metadata Removing user_api_key_auth entirely from classifier/embedding sub-call metadata (as _BUDGET_RESERVATION_METADATA_KEYS previously did) prevented _filter_deployments_by_model_access_groups from scoping those sub-calls to the caller's authorized access groups. An access-group-scoped caller could therefore reach embedding/classifier deployments outside their group. Only strip user_api_key_budget_reservation, which is the actual budget- reservation state that must not reach sub-calls. user_api_key_auth is now kept so access-group filtering works correctly for both the embedding path and the LLM classifier path. --- .../complexity_router/complexity_router.py | 18 ++++++++++++------ .../router_strategy/test_complexity_router.py | 17 ++++++++++++++--- 2 files changed, 26 insertions(+), 9 deletions(-) diff --git a/litellm/router_strategy/complexity_router/complexity_router.py b/litellm/router_strategy/complexity_router/complexity_router.py index 2eb69aad2de..e14c1091d3e 100644 --- a/litellm/router_strategy/complexity_router/complexity_router.py +++ b/litellm/router_strategy/complexity_router/complexity_router.py @@ -70,12 +70,18 @@ def _append_custom_keywords(base_keywords: list[str], custom_keywords: Optional[ return [*base_keywords, *deduped_custom.values()] -# Metadata keys that carry the parent request's budget reservation. These must not -# reach the classifier's internal acompletion call: the reservation belongs to the -# routed completion that the classifier is deciding on, not to the classifier call -# itself, and forwarding it would let the classifier's cost-tracking reconcile -# against a reservation it isn't responsible for. -_BUDGET_RESERVATION_METADATA_KEYS = frozenset({"user_api_key_budget_reservation", "user_api_key_auth"}) +# Metadata keys that carry only the parent request's budget reservation state. These +# must not reach internal sub-calls (classifier, embedding): the reservation belongs to +# the routed completion being decided on, not to the sub-call itself, and forwarding it +# would let the sub-call's cost callback finalize the reservation, causing the routed +# completion's callback to skip incrementing key/team budget counters. +# +# Note: user_api_key_auth itself is intentionally kept; it is required by +# _filter_deployments_by_model_access_groups to scope embedding/classifier model +# selection to the caller's authorized access groups. Only the budget reservation +# sub-field inside it is the problem -- stripping the whole object would allow +# an access-group-scoped caller to reach embedding deployments outside their group. +_BUDGET_RESERVATION_METADATA_KEYS = frozenset({"user_api_key_budget_reservation"}) def _classifier_call_metadata(metadata: Optional[dict[str, Any]]) -> Optional[dict[str, Any]]: diff --git a/tests/test_litellm/router_strategy/test_complexity_router.py b/tests/test_litellm/router_strategy/test_complexity_router.py index 3d92700032b..5f408d1eae4 100644 --- a/tests/test_litellm/router_strategy/test_complexity_router.py +++ b/tests/test_litellm/router_strategy/test_complexity_router.py @@ -1254,15 +1254,19 @@ class TestLLMClassifier: "user_api_key": "sk-abc", "user_api_key_team_id": "team-1", "user_api_key_budget_reservation": {"reserved_cost": 1.0}, - "user_api_key_auth": {"budget_reservation": {"reserved_cost": 1.0}}, + "user_api_key_auth": {"models": ["gpt-4o"], "budget_reservation": {"reserved_cost": 1.0}}, } await llm_complexity_router.aclassify( "hi", request_kwargs={"litellm_metadata": request_metadata} ) call_kwargs = mock_router_instance.acompletion.call_args.kwargs + # user_api_key_budget_reservation is stripped (budget enforcement) but + # user_api_key_auth is kept so _filter_deployments_by_model_access_groups + # can scope the classifier's model selection to the caller's access groups. assert call_kwargs["metadata"] == { "user_api_key": "sk-abc", "user_api_key_team_id": "team-1", + "user_api_key_auth": {"models": ["gpt-4o"], "budget_reservation": {"reserved_cost": 1.0}}, } @pytest.mark.asyncio @@ -1638,7 +1642,7 @@ class TestSemanticKeywordTierRules: "user_api_key_hash": "hash-abc", "user_api_key_team_id": "team-1", "user_api_key_budget_reservation": {"reserved_cost": 1.0}, - "user_api_key_auth": {"budget_reservation": {"reserved_cost": 1.0}}, + "user_api_key_auth": {"models": ["voyage-3-5"], "budget_reservation": {"reserved_cost": 1.0}}, } await router.async_pre_routing_hook( model="test-model", @@ -1646,7 +1650,14 @@ class TestSemanticKeywordTierRules: messages=[{"role": "user", "content": "roll out my k8s cluster"}], ) assert fake_router.async_embedding_kwargs, "expected an embedding call for the prompt" - expected = {"user_api_key_hash": "hash-abc", "user_api_key_team_id": "team-1"} + # user_api_key_budget_reservation is stripped to prevent budget-bypass. + # user_api_key_auth is kept so _filter_deployments_by_model_access_groups + # scopes the embedding model selection to the caller's authorized groups. + expected = { + "user_api_key_hash": "hash-abc", + "user_api_key_team_id": "team-1", + "user_api_key_auth": {"models": ["voyage-3-5"], "budget_reservation": {"reserved_cost": 1.0}}, + } assert fake_router.async_embedding_kwargs[0]["metadata"] == expected assert fake_router.async_embedding_kwargs[0]["litellm_metadata"] == expected From 9388d030895433f5ff3b053d3aecf3c993ba5410 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 11 Jul 2026 19:10:10 +0000 Subject: [PATCH 6/8] fix(complexity_router): sanitize budget reservation inside forwarded user_api_key_auth --- .../complexity_router/complexity_router.py | 22 +++++- .../router_strategy/test_complexity_router.py | 74 +++++++++++++++++-- 2 files changed, 87 insertions(+), 9 deletions(-) diff --git a/litellm/router_strategy/complexity_router/complexity_router.py b/litellm/router_strategy/complexity_router/complexity_router.py index e14c1091d3e..8dea8144dc2 100644 --- a/litellm/router_strategy/complexity_router/complexity_router.py +++ b/litellm/router_strategy/complexity_router/complexity_router.py @@ -78,16 +78,30 @@ def _append_custom_keywords(base_keywords: list[str], custom_keywords: Optional[ # # Note: user_api_key_auth itself is intentionally kept; it is required by # _filter_deployments_by_model_access_groups to scope embedding/classifier model -# selection to the caller's authorized access groups. Only the budget reservation -# sub-field inside it is the problem -- stripping the whole object would allow -# an access-group-scoped caller to reach embedding deployments outside their group. +# selection to the caller's authorized access groups. It is forwarded as a sanitized +# copy with its budget_reservation sub-field removed, because the proxy cost callback +# (_get_budget_reservation_from_metadata) falls back to reading the reservation from +# inside the auth object when the top-level key is absent; forwarding it unsanitized +# would re-create the exact double-finalization this stripping exists to prevent. _BUDGET_RESERVATION_METADATA_KEYS = frozenset({"user_api_key_budget_reservation"}) +def _sanitize_user_api_key_auth(auth: Any) -> Any: + if isinstance(auth, dict): + return {k: v for k, v in auth.items() if k != "budget_reservation"} + if getattr(auth, "budget_reservation", None) is not None and hasattr(auth, "model_copy"): + return auth.model_copy(update={"budget_reservation": None}) + return auth + + def _classifier_call_metadata(metadata: Optional[dict[str, Any]]) -> Optional[dict[str, Any]]: if not metadata: return metadata - return {k: v for k, v in metadata.items() if k not in _BUDGET_RESERVATION_METADATA_KEYS} + return { + k: _sanitize_user_api_key_auth(v) if k == "user_api_key_auth" else v + for k, v in metadata.items() + if k not in _BUDGET_RESERVATION_METADATA_KEYS + } class DimensionScore: diff --git a/tests/test_litellm/router_strategy/test_complexity_router.py b/tests/test_litellm/router_strategy/test_complexity_router.py index 5f408d1eae4..fdc12b6f25b 100644 --- a/tests/test_litellm/router_strategy/test_complexity_router.py +++ b/tests/test_litellm/router_strategy/test_complexity_router.py @@ -1260,13 +1260,20 @@ class TestLLMClassifier: "hi", request_kwargs={"litellm_metadata": request_metadata} ) call_kwargs = mock_router_instance.acompletion.call_args.kwargs - # user_api_key_budget_reservation is stripped (budget enforcement) but + # user_api_key_budget_reservation is stripped (budget enforcement) while # user_api_key_auth is kept so _filter_deployments_by_model_access_groups - # can scope the classifier's model selection to the caller's access groups. + # can scope the classifier's model selection to the caller's access groups, + # but only as a sanitized copy without its budget_reservation sub-field: + # the cost callback falls back to reading the reservation from inside the + # auth object when the top-level key is absent. assert call_kwargs["metadata"] == { "user_api_key": "sk-abc", "user_api_key_team_id": "team-1", - "user_api_key_auth": {"models": ["gpt-4o"], "budget_reservation": {"reserved_cost": 1.0}}, + "user_api_key_auth": {"models": ["gpt-4o"]}, + } + assert request_metadata["user_api_key_auth"] == { + "models": ["gpt-4o"], + "budget_reservation": {"reserved_cost": 1.0}, } @pytest.mark.asyncio @@ -1652,14 +1659,20 @@ class TestSemanticKeywordTierRules: assert fake_router.async_embedding_kwargs, "expected an embedding call for the prompt" # user_api_key_budget_reservation is stripped to prevent budget-bypass. # user_api_key_auth is kept so _filter_deployments_by_model_access_groups - # scopes the embedding model selection to the caller's authorized groups. + # scopes the embedding model selection to the caller's authorized groups, + # but its budget_reservation sub-field is removed because the cost callback + # falls back to reading the reservation from inside the auth object. expected = { "user_api_key_hash": "hash-abc", "user_api_key_team_id": "team-1", - "user_api_key_auth": {"models": ["voyage-3-5"], "budget_reservation": {"reserved_cost": 1.0}}, + "user_api_key_auth": {"models": ["voyage-3-5"]}, } assert fake_router.async_embedding_kwargs[0]["metadata"] == expected assert fake_router.async_embedding_kwargs[0]["litellm_metadata"] == expected + assert caller_metadata["user_api_key_auth"] == { + "models": ["voyage-3-5"], + "budget_reservation": {"reserved_cost": 1.0}, + } @pytest.mark.asyncio async def test_semantic_routelayer_build_runs_off_event_loop(self, basic_config): @@ -1943,3 +1956,54 @@ class TestKeywordOverrideEdgeCases: ) assert result is not None assert result.model in {"gpt-4o-mini", "gpt-4o", "claude-sonnet-4-20250514", "o1-preview"} + + +class TestSubCallMetadataSanitization: + """The proxy cost callback must not be able to recover the parent budget reservation + from sub-call metadata, in either of the shapes it knows how to read.""" + + def test_cost_callback_cannot_recover_reservation_from_sanitized_metadata(self): + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.hooks.proxy_track_cost_callback import ( + _get_budget_reservation_from_metadata, + ) + from litellm.router_strategy.complexity_router.complexity_router import ( + _classifier_call_metadata, + ) + + reservation = {"reserved_cost": 1.0} + auth_shapes = ( + {"models": ["gpt-4o"], "budget_reservation": dict(reservation)}, + UserAPIKeyAuth(api_key="sk-abc", budget_reservation=dict(reservation)), + ) + for auth in auth_shapes: + metadata = { + "user_api_key_hash": "hash-abc", + "user_api_key_budget_reservation": dict(reservation), + "user_api_key_auth": auth, + } + assert _get_budget_reservation_from_metadata(metadata) == reservation + + sanitized = _classifier_call_metadata(metadata) + assert sanitized is not None + assert sanitized["user_api_key_auth"] is not None + assert _get_budget_reservation_from_metadata(sanitized) is None + + def test_sanitized_auth_keeps_access_group_fields_and_leaves_original_untouched(self): + from litellm.proxy._types import UserAPIKeyAuth + from litellm.router_strategy.complexity_router.complexity_router import ( + _classifier_call_metadata, + ) + + auth = UserAPIKeyAuth( + api_key="sk-abc", + team_id="team-1", + budget_reservation={"reserved_cost": 1.0}, + ) + sanitized = _classifier_call_metadata({"user_api_key_auth": auth}) + assert sanitized is not None + sanitized_auth = sanitized["user_api_key_auth"] + assert sanitized_auth.budget_reservation is None + assert sanitized_auth.team_id == "team-1" + assert sanitized_auth.api_key == auth.api_key + assert auth.budget_reservation == {"reserved_cost": 1.0} From 952127647c10c14fb3025f1c939ca5338545e067 Mon Sep 17 00:00:00 2001 From: Abhimanyu Kapur <38531241+akapur99@users.noreply.github.com> Date: Sat, 11 Jul 2026 12:15:24 -0700 Subject: [PATCH 7/8] fix(complexity_router): review hardening - blank keywords, router registry eviction, edit-modal controls - config: KeywordTierRule now strips and drops blank/whitespace keywords (a stray "" makes _keyword_matches match every prompt, silently forcing that tier for all traffic); still requires at least one real keyword to remain - frontend build_complexity_router_config: trim keywords and drop rules left empty so an unfilled "Add keyword rule" row no longer ships a rule the backend rejects with a 400 in the heuristic (non-semantic) flow, where the client-side semantic guard doesn't run - proxy clear_cache / delete_model: the auto_router/ prefix also covers quality_router/ and adaptive_router/, so pop the model_name from all four router registries (no-op where absent) instead of only auto/complexity; otherwise a DB quality_router's stale entry made reload raise "already exists" and abort, and adaptive left a leak - frontend ComplexityRouterConfig: only render the Keyword Tier Overrides and Semantic keyword matching sections when their change handlers are provided, so the edit-auto- router modal (which omits them) no longer shows interactive-but-dead controls --- litellm/proxy/dev_config.yaml | 6 +++ .../model_management_endpoints.py | 13 ++++-- .../complexity_router/config.py | 10 ++++- .../test_model_management_endpoints.py | 42 +++++++++++++++++++ .../router_strategy/test_complexity_router.py | 12 ++++++ .../add_model/ComplexityRouterConfig.test.tsx | 10 +++++ .../add_model/ComplexityRouterConfig.tsx | 41 +++++++++++------- .../build_complexity_router_config.test.ts | 23 ++++++++++ .../build_complexity_router_config.ts | 36 +++++++++------- 9 files changed, 158 insertions(+), 35 deletions(-) diff --git a/litellm/proxy/dev_config.yaml b/litellm/proxy/dev_config.yaml index 65a4b8e7cbf..4e9c710d446 100644 --- a/litellm/proxy/dev_config.yaml +++ b/litellm/proxy/dev_config.yaml @@ -33,6 +33,12 @@ model_list: model: anthropic/claude-sonnet-5 api_key: os.environ/ANTHROPIC_API_KEY + # ---------- Embeddings (for complexity router semantic keyword matching) ---------- + - model_name: voyage-4-large + litellm_params: + model: voyage/voyage-4-large + api_key: os.environ/VOYAGE_API_KEY + # ---------- Bedrock Invoke ---------- - model_name: bedrock-invoke-haiku-4-5 litellm_params: diff --git a/litellm/proxy/management_endpoints/model_management_endpoints.py b/litellm/proxy/management_endpoints/model_management_endpoints.py index 23a3b3ce4f2..18d95d4e083 100644 --- a/litellm/proxy/management_endpoints/model_management_endpoints.py +++ b/litellm/proxy/management_endpoints/model_management_endpoints.py @@ -1138,6 +1138,8 @@ async def delete_model( if deleted_name is not None and deleted_model.startswith("auto_router/"): llm_router.auto_routers.pop(deleted_name, None) llm_router.complexity_routers.pop(deleted_name, None) + llm_router.adaptive_routers.pop(deleted_name, None) + llm_router.quality_routers.pop(deleted_name, None) # Runs after the row delete so the sibling check sees post-delete state. if model_params.model_info.team_id is not None: @@ -1741,12 +1743,15 @@ async def clear_cache(): for model_id in db_model_ids: llm_router.delete_deployment(id=model_id) - # Clear only DB-backed auto/complexity routers, keyed by model_name, so the reload - # below rebuilds them fresh. A blanket .clear() would also drop config-defined + # Clear only DB-backed auto-router-family entries, keyed by model_name, so the + # reload below rebuilds them fresh. A blanket .clear() would also drop config-defined # routers, which are never re-added below (add_deployment only reloads DB models), # leaving them permanently unroutable until a full proxy restart for every tenant. # Restrict to deployments whose model is actually an auto_router/* so a config - # router that merely shares a model_name with a regular DB model isn't evicted. + # router that merely shares a model_name with a regular DB model isn't evicted. The + # auto_router/ prefix also covers quality_router/ and adaptive_router/, so pop the + # name from every router registry (no-op where absent); missing quality/adaptive + # entries would otherwise make init raise "already exists" on reload and abort it. db_router_names = { model.get("model_name") for model in current_models @@ -1756,6 +1761,8 @@ async def clear_cache(): for model_name in db_router_names: llm_router.auto_routers.pop(model_name, None) llm_router.complexity_routers.pop(model_name, None) + llm_router.adaptive_routers.pop(model_name, None) + llm_router.quality_routers.pop(model_name, None) # Reload only DB models await proxy_config.add_deployment(prisma_client=prisma_client, proxy_logging_obj=proxy_logging_obj) diff --git a/litellm/router_strategy/complexity_router/config.py b/litellm/router_strategy/complexity_router/config.py index 9901ae6f813..125de6f7489 100644 --- a/litellm/router_strategy/complexity_router/config.py +++ b/litellm/router_strategy/complexity_router/config.py @@ -40,9 +40,15 @@ class KeywordTierRule(BaseModel): ) @model_validator(mode="after") - def _validate_non_empty_keywords(self) -> "KeywordTierRule": - if not any(keyword.strip() for keyword in self.keywords): + def _normalize_keywords(self) -> "KeywordTierRule": + # Strip and drop blank keywords. An empty/whitespace keyword is a routing foot-gun: + # _keyword_matches treats "" / " " as a substring that matches essentially every + # prompt, so a single stray blank would silently force this rule's tier for all + # traffic. Require at least one real keyword to remain. + cleaned = [stripped for keyword in self.keywords if (stripped := keyword.strip())] + if not cleaned: raise ValueError("keyword_tier_rules entries must contain at least one non-empty keyword") + self.keywords = cleaned return self diff --git a/tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py index d9b5745807a..8c6bdefedae 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py @@ -588,6 +588,48 @@ class TestClearCachePreservesConfigRouters: # The DB model isn't a router, so the same-named config router must be left intact. assert "shared-name" in mock_router.complexity_routers + @pytest.mark.asyncio + async def test_db_quality_and_adaptive_routers_are_evicted(self): + """The auto_router/ prefix also covers quality_router/ and adaptive_router/. Their + registry entries must be popped too, or reload's init raises 'already exists' + (quality) or leaves a stale entry (adaptive). + """ + from litellm.proxy.management_endpoints.model_management_endpoints import clear_cache + + mock_router = MagicMock() + mock_router.model_list = [ + { + "model_name": "q1", + "model_info": {"id": "db-q", "db_model": True}, + "litellm_params": {"model": "auto_router/quality_router/q1"}, + }, + { + "model_name": "a1", + "model_info": {"id": "db-a", "db_model": True}, + "litellm_params": {"model": "auto_router/adaptive_router/a1"}, + }, + ] + mock_router.delete_deployment = MagicMock(return_value=True) + mock_router.auto_routers = {} + mock_router.complexity_routers = {} + mock_router.quality_routers = {"q1": MagicMock()} + mock_router.adaptive_routers = {"a1": MagicMock()} + + mock_config = MagicMock() + mock_config.add_deployment = AsyncMock(return_value=True) + + with ( + patch("litellm.proxy.proxy_server.llm_router", mock_router), + patch("litellm.proxy.proxy_server.proxy_config", mock_config), + patch("litellm.proxy.proxy_server.prisma_client", MagicMock()), + patch("litellm.proxy.proxy_server.proxy_logging_obj", MagicMock()), + patch("litellm.proxy.proxy_server.verbose_proxy_logger"), + ): + await clear_cache() + + assert "q1" not in mock_router.quality_routers + assert "a1" not in mock_router.adaptive_routers + class TestDeleteModelClearsRouterRegistry: """delete_model must evict the deleted deployment from the auto/complexity router maps, diff --git a/tests/test_litellm/router_strategy/test_complexity_router.py b/tests/test_litellm/router_strategy/test_complexity_router.py index fdc12b6f25b..12a8a47e1f1 100644 --- a/tests/test_litellm/router_strategy/test_complexity_router.py +++ b/tests/test_litellm/router_strategy/test_complexity_router.py @@ -1836,6 +1836,18 @@ class TestSemanticConfigValidation: with pytest.raises(ValidationError): ComplexityRouterConfig(keyword_tier_rules=[{"keywords": [" ", ""], "tier": "SIMPLE"}]) + def test_keyword_tier_rule_strips_and_drops_blank_keywords(self): + """Blank keywords mixed with real ones are dropped (not kept), and survivors trimmed. + + A stray "" would otherwise match-all in _keyword_matches and silently force this + tier for every request. + """ + config = ComplexityRouterConfig( + keyword_tier_rules=[{"keywords": ["", " deploy to k8s ", " ", "kubernetes"], "tier": "REASONING"}] + ) + assert config.keyword_tier_rules is not None + assert config.keyword_tier_rules[0].keywords == ["deploy to k8s", "kubernetes"] + class _StubEncoder: """Minimal stand-in for LiteLLMRouterEncoder.aencode_queries, capturing the kwargs it was called with.""" diff --git a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.test.tsx b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.test.tsx index 5bfcd94398d..a433808085b 100644 --- a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.test.tsx +++ b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.test.tsx @@ -143,6 +143,16 @@ describe("ComplexityRouterConfig", () => { expect(screen.getByText("No keyword tier overrides configured")).toBeInTheDocument(); }); + it("hides the keyword-tier and semantic sections when their change handlers are absent (edit modal)", () => { + // The edit-auto-router modal renders ComplexityRouterConfig without these handlers; + // the sections must stay hidden rather than render interactive-but-dead controls. + renderWithProviders(); + expect(screen.queryByText("Keyword Tier Overrides")).not.toBeInTheDocument(); + expect(screen.queryByText("Semantic keyword matching")).not.toBeInTheDocument(); + // Core tier config still renders. + expect(screen.getByText("Complexity Tier Configuration")).toBeInTheDocument(); + }); + it("should call onKeywordTierRulesChange with a new rule when 'Add keyword rule' is clicked", async () => { const user = userEvent.setup(); const onKeywordTierRulesChange = vi.fn(); diff --git a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx index ae2e03e576f..555648db8ad 100644 --- a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx +++ b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx @@ -77,9 +77,9 @@ const ComplexityRouterConfig: React.FC = ({ customTechnicalKeywords, onCustomTechnicalKeywordsChange, keywordTierRules = [], - onKeywordTierRulesChange = () => {}, + onKeywordTierRulesChange, semanticMatchingEnabled = false, - onSemanticMatchingEnabledChange = () => {}, + onSemanticMatchingEnabledChange, embeddingModel, onEmbeddingModelChange = () => {}, matchThreshold = 0.5, @@ -302,21 +302,30 @@ const ComplexityRouterConfig: React.FC = ({ - + {/* Keyword-tier and semantic sections only render when their change handlers are + wired (the add-router flow). The edit-auto-router modal doesn't pass them yet, so + they stay hidden there rather than rendering interactive-but-dead controls. */} + {onKeywordTierRulesChange && ( + <> + + + + )} - - - - - + {onSemanticMatchingEnabledChange && ( + <> + + + + )} ); }; diff --git a/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.test.ts b/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.test.ts index 250b7d13a5b..3c252646b57 100644 --- a/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.test.ts +++ b/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.test.ts @@ -99,6 +99,29 @@ describe("buildComplexityRouterConfig", () => { expect(config.custom_technical_keywords).toBeUndefined(); expect(config.keyword_tier_rules).toBeUndefined(); }); + + it("trims keywords and drops rules left empty, so unfilled rows never 400 the backend", () => { + const params: BuildComplexityRouterConfigParams = { + ...baseParams, + keywordTierRules: [ + { id: "r1", keywords: [" deploy to k8s ", "", " "], tier: "REASONING" }, + { id: "r2", keywords: [], tier: "COMPLEX" }, // seeded by "Add keyword rule", never filled + { id: "r3", keywords: [" "], tier: "SIMPLE" }, // whitespace only + ], + }; + const config = buildComplexityRouterConfig(params); + // r1 keeps only its real keyword (trimmed); r2 and r3 are dropped entirely. + expect(config.keyword_tier_rules).toEqual([{ keywords: ["deploy to k8s"], tier: "REASONING" }]); + }); + + it("omits keyword_tier_rules entirely when every rule is empty", () => { + const params: BuildComplexityRouterConfigParams = { + ...baseParams, + keywordTierRules: [{ id: "r1", keywords: ["", " "], tier: "COMPLEX" }], + }; + const config = buildComplexityRouterConfig(params); + expect(config.keyword_tier_rules).toBeUndefined(); + }); }); describe("getSemanticConfigError", () => { diff --git a/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.ts b/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.ts index 00cc19cde0c..82ea4f8c12f 100644 --- a/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.ts +++ b/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.ts @@ -54,17 +54,25 @@ export const buildComplexityRouterConfig = ({ semanticMatchingEnabled, embeddingModel, matchThreshold, -}: BuildComplexityRouterConfigParams): ComplexityRouterConfigPayload => ({ - tiers, - classifier_type: classifierType, - ...(classifierType === "llm" && classifierLlmConfig && { classifier_llm_config: classifierLlmConfig }), - ...(customTechnicalKeywords.length > 0 && { custom_technical_keywords: customTechnicalKeywords }), - ...(keywordTierRules.length > 0 && { - keyword_tier_rules: keywordTierRules.map((rule) => ({ keywords: rule.keywords, tier: rule.tier })), - }), - ...(semanticMatchingEnabled && { - semantic_keyword_matching: true, - embedding_model: embeddingModel, - match_threshold: matchThreshold, - }), -}); +}: BuildComplexityRouterConfigParams): ComplexityRouterConfigPayload => { + // Trim keywords and drop empty ones; drop any rule left with no keywords. Clicking + // "Add keyword rule" seeds a rule with an empty keywords list, so without this an + // unfilled row (common in the heuristic flow, where getSemanticConfigError doesn't run) + // would ship keyword_tier_rules the backend validator rejects with a 400. + const cleanedKeywordTierRules = keywordTierRules + .map((rule) => ({ keywords: rule.keywords.map((k) => k.trim()).filter(Boolean), tier: rule.tier })) + .filter((rule) => rule.keywords.length > 0); + + return { + tiers, + classifier_type: classifierType, + ...(classifierType === "llm" && classifierLlmConfig && { classifier_llm_config: classifierLlmConfig }), + ...(customTechnicalKeywords.length > 0 && { custom_technical_keywords: customTechnicalKeywords }), + ...(cleanedKeywordTierRules.length > 0 && { keyword_tier_rules: cleanedKeywordTierRules }), + ...(semanticMatchingEnabled && { + semantic_keyword_matching: true, + embedding_model: embeddingModel, + match_threshold: matchThreshold, + }), + }; +}; From 8982e1cef0aec64fba2a41deb82941fe0b5efec2 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 11 Jul 2026 19:31:57 +0000 Subject: [PATCH 8/8] fix(proxy): skip None model_name in clear_cache router eviction set --- .../proxy/management_endpoints/model_management_endpoints.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/litellm/proxy/management_endpoints/model_management_endpoints.py b/litellm/proxy/management_endpoints/model_management_endpoints.py index 18d95d4e083..d458d0f7c4a 100644 --- a/litellm/proxy/management_endpoints/model_management_endpoints.py +++ b/litellm/proxy/management_endpoints/model_management_endpoints.py @@ -1755,7 +1755,8 @@ async def clear_cache(): db_router_names = { model.get("model_name") for model in current_models - if model.get("model_info", {}).get("db_model", False) + if model.get("model_name") is not None + and model.get("model_info", {}).get("db_model", False) and str(model.get("litellm_params", {}).get("model", "")).startswith("auto_router/") } for model_name in db_router_names: