From 01639a8e00493af576bd04c3cfe44b1c54f0a484 Mon Sep 17 00:00:00 2001 From: Tin Chi Lo Date: Thu, 27 Aug 2026 17:43:37 -0700 Subject: [PATCH] fix(complexity_router): route client housekeeping calls to the cheapest tier A coding agent names each conversation by quoting the whole session and asking for a title. The classifier rated the quoted session rather than the request, so the cheapest call the client makes routed to the most expensive tier: 11 of 17 title generations in one day of real traffic came back COMPLEX. Recognize those prompts by literal sentinel on the newest ask and route them to the cheapest configured tier without classifying them, so the call costs nothing to route. The placement is scoped to the one request that carries the sentinel: it never displaces an operator's classifier plugin, the bandit cannot reach above the tier as raised, it never becomes the session pin, and the sentinel that matched is recorded on the routing decision. Detection reads the newest ask alone, so a title request quoted into a later turn cannot cheapen the work that follows it, and a keyword rule, an escalation keyword or the plan-mode floor all still decide over it. Regenerates the lazy OpenAPI snapshot, which was already stale on the base for an unrelated Presidio guardrail field and failed the schema check on every PR. Resolves LIT-6349 --- litellm/proxy/_lazy_openapi_snapshot.json | 12 + .../complexity_router/complexity_router.py | 91 +++- .../complexity_router/config.py | 44 ++ litellm/types/utils.py | 5 + .../router_strategy/test_complexity_router.py | 417 ++++++++++++++++++ .../RoutingDecisionCard.test.tsx | 26 ++ .../LogDetailsDrawer/RoutingDecisionCard.tsx | 12 + ui/litellm-dashboard/src/lib/http/schema.d.ts | 18 +- 8 files changed, 618 insertions(+), 7 deletions(-) diff --git a/litellm/proxy/_lazy_openapi_snapshot.json b/litellm/proxy/_lazy_openapi_snapshot.json index 0930febc449..4ccfde18d36 100644 --- a/litellm/proxy/_lazy_openapi_snapshot.json +++ b/litellm/proxy/_lazy_openapi_snapshot.json @@ -11364,6 +11364,18 @@ "description": "Path to a JSON file containing ad-hoc recognizers for Presidio", "title": "Presidio Ad Hoc Recognizers" }, + "presidio_analyze_chunk_size_bytes": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "description": "Maximum UTF-8 bytes of text sent in a single Presidio /analyze call. Longer texts are split into overlapping chunks of at most this size and the merged results are remapped onto the original text. Defaults to 500000; set it below your analyzer deployment's request body limit, leaving headroom for the rest of the analyze payload.", + "title": "Presidio Analyze Chunk Size Bytes" + }, "presidio_analyzer_api_base": { "anyOf": [ { diff --git a/litellm/router_strategy/complexity_router/complexity_router.py b/litellm/router_strategy/complexity_router/complexity_router.py index f1f791ba72e..2a725125f50 100644 --- a/litellm/router_strategy/complexity_router/complexity_router.py +++ b/litellm/router_strategy/complexity_router/complexity_router.py @@ -48,6 +48,7 @@ from .config import ( DEFAULT_REASONING_KEYWORDS, DEFAULT_SIMPLE_KEYWORDS, DEFAULT_TECHNICAL_KEYWORDS, + HOUSEKEEPING_ASK_SENTINELS, PLAN_MODE_SYSTEM_SENTINELS, PLAN_MODE_TAIL_SENTINELS, PLAN_MODE_TOOL_NAME, @@ -679,8 +680,17 @@ def _decision_is_pinnable(decision: StandardLoggingRoutingDecision | None) -> bo on the floor's premium model after the user exits plan mode; leaving it unpinned means the floor re-detects while plan mode lasts and the first ordinary turn classifies and pins as if plan mode had never happened. + + A housekeeping call is transient in the same way, and pinning it is the most expensive mistake + of the three: an agent names the conversation on its first turn, so the cheapest tier would be + the pin every session starts with, and the real work that follows would run there for the whole + TTL. It describes what that one call is, never what the session's traffic looks like. """ - return decision is None or decision.get("cause") not in ("default_model_fallback", "plan_mode") + return decision is None or decision.get("cause") not in ( + "default_model_fallback", + "plan_mode", + "housekeeping", + ) class DimensionScore: @@ -720,6 +730,7 @@ class ClassificationOutcome(NamedTuple): "reasoning_override", "llm_classifier", "heuristic_first_short_circuit", + "housekeeping", "classifier_plugin", "classifier_fallback", "default_model_fallback", @@ -1738,12 +1749,20 @@ class ComplexityRouter(CustomLogger): user_message: str, request_kwargs: dict[str, Any] | None = None, hard_floor: ComplexityTier | str | None = None, + hard_ceiling: ComplexityTier | str | None = None, ) -> str: """hard_floor excludes every candidate whose tiers all sit below it, turning this pick's soft floors (a distance penalty a high-scoring cheap model can outweigh) into a hard minimum for requests that carry one, e.g. the plan-mode floor. classified_tier arrives already clamped to the floor, so the cold-start pool and the classified_tier eligibility - mode satisfy it by construction; only the "all" eligibility mode can reach below.""" + mode satisfy it by construction; only the "all" eligibility mode can reach below. + + hard_ceiling is the same bound in the other direction, for a request whose tier was decided + by what it IS rather than by how hard it is: a housekeeping call is placed at the cheapest + tier because that is all it is worth, so a bandit trading cost for quality has nothing to + win and must not reach above it. Without it the distance penalty is the only thing holding + the tier, and a deployment that lowers tier_distance_penalty silently gets the expensive + model back while the routing decision still reads as the cheapest tier.""" from litellm.router_strategy.adaptive_router.bandit import ( normalized_cost, thompson_sample, @@ -1799,6 +1818,7 @@ class ComplexityRouter(CustomLogger): penalty_weight: Final = self.config.tier_distance_penalty floor_severity: Final = self._active_tier_severity(hard_floor) if hard_floor is not None else None + ceiling_severity: Final = self._active_tier_severity(hard_ceiling) if hard_ceiling is not None else None best_model: str | None = None best_score = float("-inf") candidate_scores: Final[list[dict[str, Any]]] = [] @@ -1808,6 +1828,11 @@ class ComplexityRouter(CustomLogger): for model_tier in self._model_tiers.get(model, (classified_tier,)) ): continue + if ceiling_severity is not None and all( + self._active_tier_severity(model_tier) > ceiling_severity + for model_tier in self._model_tiers.get(model, (classified_tier,)) + ): + continue cell = adaptive._cells[(request_type, model)] quality_sample = thompson_sample(cell) cost_score = normalized_cost(adaptive.model_to_cost.get(model, 0.0), all_costs) @@ -1881,6 +1906,44 @@ class ComplexityRouter(CustomLogger): self._reminder_markers, ) + def _matched_housekeeping_sentinel(self, newest_ask: str | None) -> str | None: + """The client housekeeping sentinel on this request's newest ask, or None. + + Read from the newest ask alone, never the whole history, for the reason `_newest_turn_ask` + exists: a title request quoted into a later turn's context would otherwise keep matching and + route real work to the cheapest tier for the rest of the session. + + Declines whenever an operator's classifier plugin owns the decision. The sentinels are + caller-controlled text, and displacing the built-in classifier with them only ever spends + less; displacing a plugin is different in kind, because a plugin is where an operator + encodes policy the tier ladder does not express, so a caller pasting a title prompt could + route a request past a sensitivity or identity rule to a pool that rule would have refused. + """ + if self.config.classifier_type == "custom" or not self.config.route_housekeeping_to_cheapest_tier: + return None + if not newest_ask: + return None + return next( + ( + sentinel + for sentinel in (*HOUSEKEEPING_ASK_SENTINELS, *(self.config.housekeeping_patterns or ())) + if sentinel in newest_ask + ), + None, + ) + + def _cheapest_configured_tier(self) -> ComplexityTier | str | None: + """The least severe tier that has models, or None when none does. + + Tiers can be declared without a pool, so this cannot assume the first name in the severity + order is routable; routing to an empty pool is what `default_fallback` exists to catch. + """ + pools: Final = self._tier_pools() + name: Final = next((name for name in self.config.tier_names() if pools.get(name)), None) + if name is None: + return None + return name if self.config.has_custom_tiers else ComplexityTier(name) + def _apply_plan_mode_floor(self, tier: ComplexityTier | str) -> ComplexityTier | str: """The higher of the decided tier and the plan-mode floor; identity when the floor is unset.""" floor: Final = self._resolve_plan_mode_floor() @@ -2476,8 +2539,14 @@ class ComplexityRouter(CustomLogger): ), ) - outcome: Final = await self.aclassify( - user_message, system_prompt, request_kwargs, resolved_messages, raw_messages=messages + housekeeping_sentinel: Final = self._matched_housekeeping_sentinel(newest_ask) + housekeeping_tier: Final = self._cheapest_configured_tier() if housekeeping_sentinel is not None else None + outcome: Final = ( + ClassificationOutcome(tier=housekeeping_tier, score=None, signals=("housekeeping",), cause="housekeeping") + if housekeeping_tier is not None + else await self.aclassify( + user_message, system_prompt, request_kwargs, resolved_messages, raw_messages=messages + ) ) tier, score, signals = outcome.tier, outcome.score, outcome.signals classified_tier: Final = tier @@ -2533,7 +2602,14 @@ class ComplexityRouter(CustomLogger): # has plan_floored False, yet adaptive_eligible="all" scores every model and only # penalizes tier distance, so without the floor the bandit could still route below # it -- and a floor a bandit can slide under is not a floor. - routed_model = self._soft_floor_pick(tier, user_message, request_kwargs, hard_floor=plan_floor) + # The ceiling tracks the tier as raised, never the placement it started from: escalation + # and the plan-mode floor both move a housekeeping call up, and a ceiling still naming + # the cheapest tier would then contradict the floor and bound the pick below the tier + # the decision reports. + housekeeping_ceiling: Final = tier if outcome.cause == "housekeeping" else None + routed_model = self._soft_floor_pick( + tier, user_message, request_kwargs, hard_floor=plan_floor, hard_ceiling=housekeeping_ceiling + ) adaptive: Final = self._ensure_adaptive_router() if adaptive is not None: kwargs_metadata: Final = request_kwargs.setdefault("metadata", {}) @@ -2582,6 +2658,9 @@ class ComplexityRouter(CustomLogger): else signals ) decision_cause: Final[RoutingDecisionCause] = "plan_mode" if plan_floored else outcome.cause + decision_keyword: Final = ( + plan_mode_sentinel if plan_floored else (housekeeping_sentinel if outcome.cause == "housekeeping" else None) + ) return PreRoutingHookResponse( model=routed_model, messages=messages if has_original_messages else None, @@ -2593,7 +2672,7 @@ class ComplexityRouter(CustomLogger): tier=classified_pool_tier, score=score, signals=decision_signals, - matched_keyword=plan_mode_sentinel if plan_floored else None, + matched_keyword=decision_keyword, escalation_keyword=escalation_keyword, escalated=escalated, classifier_model=classifier_model, diff --git a/litellm/router_strategy/complexity_router/config.py b/litellm/router_strategy/complexity_router/config.py index 9b2a25f5d28..db02242f95f 100644 --- a/litellm/router_strategy/complexity_router/config.py +++ b/litellm/router_strategy/complexity_router/config.py @@ -321,6 +321,18 @@ PLAN_MODE_TAIL_SENTINELS: Final[tuple[str, ...]] = ( "Plan mode still active", ) PLAN_MODE_SYSTEM_SENTINELS: Final[tuple[str, ...]] = ('You are currently running in "Plan" mode.',) + +# Taken verbatim from classifier payloads captured on a live gateway, 789 calls over one day: the +# first appears on 17 of them and the second on 2. A coding agent names the conversation by quoting +# the session and asking for a title, so the ask carries the session's engineering vocabulary while +# the task is the cheapest one the client performs. Only wording observed on the wire belongs here, +# never a paraphrase: a sentinel that matches nothing costs a substring scan per request and reads +# as coverage the router does not have. These are client-owned strings that drift with client +# releases, so operators extend coverage via housekeeping_patterns rather than editing these. +HOUSEKEEPING_ASK_SENTINELS: Final[tuple[str, ...]] = ( + "Write the title in the predominant language of the session", + "You are coming up with a succinct title for a coding session", +) PLAN_MODE_TOOL_NAME: Final[str] = "exit_plan_mode" @@ -770,6 +782,29 @@ class ComplexityRouterConfig(BaseModel): "wording the built-ins don't cover, or after a client release changes its strings." ), ) + route_housekeeping_to_cheapest_tier: bool = Field( + default=True, + description=( + "Route a coding agent's own housekeeping calls to the cheapest configured tier " + "without classifying them. A client names the conversation by quoting the whole " + "session and asking for a title, so the ask reads as the session's engineering work " + "and lands on the most expensive tier, which is the reverse of what the call is " + "worth. Detection is a literal match against client-owned sentinels on the newest " + "ask only, so it cannot fire on an earlier turn, and it never lowers what anyone " + "else asked for: a keyword_tier_rule or a session pin still decides instead, and an " + "escalation keyword or the plan-mode floor still raises the tier from here. Only the " + "classifier is displaced, and its call is skipped, so a matched request costs " + "nothing to route. Set false to classify these calls like any other." + ), + ) + housekeeping_patterns: tuple[str, ...] | None = Field( + default=None, + description=( + "Additional case-sensitive literal sentinels that mark a request as client " + "housekeeping, on top of the built-in conversation-title ones. For clients whose " + "wording the built-ins don't cover, or after a client release changes its strings." + ), + ) # Semantic (embedding) matching for keyword_tier_rules instead of literal text matching semantic_keyword_matching: bool = Field( @@ -939,6 +974,15 @@ class ComplexityRouterConfig(BaseModel): return None return tuple(stripped for pattern in value if (stripped := pattern.strip())) + @field_validator("housekeeping_patterns") + @classmethod + def _normalize_housekeeping_patterns(cls, value: tuple[str, ...] | None) -> tuple[str, ...] | None: + """Blank patterns are dropped: an empty string substring-matches every request, which would + silently route all traffic to the cheapest tier.""" + if value is None: + return None + return tuple(stripped for pattern in value if (stripped := pattern.strip())) + @model_validator(mode="after") def _validate_plan_mode_min_tier(self) -> "ComplexityRouterConfig": if self.plan_mode_min_tier is None: diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 95429e899c9..dec9f4c5c77 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -2832,6 +2832,11 @@ RoutingDecisionCause = Literal[ # keyword rule, or session pin), or the floor was already the top configured tier and the # classifier was skipped. The matched sentinel rides in matched_keyword. "plan_mode", + # A client housekeeping sentinel (a coding agent's conversation-title prompt) was detected on + # the newest ask, so the request routed to the cheapest configured tier and the classifier was + # never called. The matched sentinel rides in matched_keyword. Distinct from the keyword causes, + # which are operator-authored rules; these sentinels ship with the router. + "housekeeping", "session_affinity_pin", "session_affinity_escalation", "default_fallback", diff --git a/tests/test_litellm/router_strategy/test_complexity_router.py b/tests/test_litellm/router_strategy/test_complexity_router.py index ae9e3907aeb..a135383880a 100644 --- a/tests/test_litellm/router_strategy/test_complexity_router.py +++ b/tests/test_litellm/router_strategy/test_complexity_router.py @@ -6783,6 +6783,423 @@ class TestContextAwareClassifier: assert "LITELLM ESCALATE" in user_payload +# The shape a coding agent actually sends, taken from a captured classifier payload: the session +# quoted whole, then one line asking for a title. The engineering vocabulary is all inside the +# quoted block, which is what used to decide the tier. +TITLE_ASK = ( + "\nthe retry path livelocks under contention, find and fix the root cause\n" + "\n\nWrite the title in the predominant language of the session, a stray word or code token in " + "another language does not change it, and neither does the English of these instructions." +) + + +class TestClientHousekeepingCalls: + """A coding agent's own title generation is the cheapest call it makes, and must route that way.""" + + @pytest.mark.asyncio + async def test_a_title_request_routes_to_the_cheapest_tier_without_classifying( + self, mock_router_instance, llm_classifier_config + ): + """The regression: title generation quoted the session, so the classifier rated the session. + + Skipping the classifier is half the fix. Paying for a classification whose answer is fixed + is the same waste as routing the call to the top tier, only smaller. + """ + router = ComplexityRouter( + model_name="test-router", + litellm_router_instance=mock_router_instance, + complexity_router_config=llm_classifier_config, + ) + mock_router_instance.acompletion = AsyncMock(return_value=_llm_response('{"tier": "REASONING"}')) + + result = await router.async_pre_routing_hook( + model="test-model", + request_kwargs={}, + messages=[{"role": "user", "content": TITLE_ASK}], + ) + + assert result is not None + assert result.model == "gpt-4o-mini" + assert result.routing_decision["cause"] == "housekeeping" + mock_router_instance.acompletion.assert_not_called() + + @pytest.mark.asyncio + async def test_the_sentinel_only_counts_on_the_newest_ask(self, mock_router_instance, llm_classifier_config): + """A title request quoted into a later turn must not cheapen the real work that follows it. + + `_newest_turn_ask` exists for this: reading the newest ask in history instead would keep + matching for the rest of the session, which is how one escalate request once walked a whole + session to the top tier. + """ + router = ComplexityRouter( + model_name="test-router", + litellm_router_instance=mock_router_instance, + complexity_router_config=llm_classifier_config, + ) + mock_router_instance.acompletion = AsyncMock(return_value=_llm_response('{"tier": "REASONING"}')) + + result = await router.async_pre_routing_hook( + model="test-model", + request_kwargs={}, + messages=[ + {"role": "user", "content": TITLE_ASK}, + {"role": "assistant", "content": "Retry path livelock"}, + {"role": "user", "content": "now design the fix and prove it cannot livelock"}, + ], + ) + + assert result is not None + assert result.model == "o1-preview" + mock_router_instance.acompletion.assert_called_once() + + @pytest.mark.asyncio + async def test_an_escalation_keyword_beats_the_cheapest_tier(self, mock_router_instance, llm_classifier_config): + """A caller who explicitly escalated asked for something; the cap must not silently undo it.""" + router = ComplexityRouter( + model_name="test-router", + litellm_router_instance=mock_router_instance, + complexity_router_config=llm_classifier_config, + ) + mock_router_instance.acompletion = AsyncMock(return_value=_llm_response('{"tier": "SIMPLE"}')) + + result = await router.async_pre_routing_hook( + model="test-model", + request_kwargs={}, + messages=[{"role": "user", "content": f"LITELLM ESCALATE {TITLE_ASK}"}], + ) + + assert result is not None + assert result.model != "gpt-4o-mini" + + @pytest.mark.asyncio + async def test_an_operator_keyword_rule_beats_the_cheapest_tier(self, mock_router_instance, llm_classifier_config): + """keyword_tier_rules are the operator's own instruction, decided before this ever runs.""" + router = ComplexityRouter( + model_name="test-router", + litellm_router_instance=mock_router_instance, + complexity_router_config={ + **llm_classifier_config, + "keyword_tier_rules": [{"keywords": ["livelocks under contention"], "tier": "REASONING"}], + }, + ) + + result = await router.async_pre_routing_hook( + model="test-model", request_kwargs={}, messages=[{"role": "user", "content": TITLE_ASK}] + ) + + assert result is not None + assert result.model == "o1-preview" + + @pytest.mark.asyncio + async def test_the_plan_mode_floor_still_raises_a_housekeeping_call( + self, mock_router_instance, llm_classifier_config + ): + """The floor is an operator guarantee about what plan-mode turns may run on, so it wins.""" + router = ComplexityRouter( + model_name="test-router", + litellm_router_instance=mock_router_instance, + complexity_router_config={**llm_classifier_config, "plan_mode_min_tier": "COMPLEX"}, + ) + + result = await router.async_pre_routing_hook( + model="test-model", + request_kwargs={}, + messages=[ + {"role": "system", "content": 'You are currently running in "Plan" mode.'}, + {"role": "user", "content": TITLE_ASK}, + ], + ) + + assert result is not None + assert result.model == "claude-sonnet-4-20250514" + + @pytest.mark.asyncio + async def test_turning_it_off_classifies_the_title_request_like_anything_else( + self, mock_router_instance, llm_classifier_config + ): + """An operator who wants these classified keeps the old behaviour, classifier call included.""" + router = ComplexityRouter( + model_name="test-router", + litellm_router_instance=mock_router_instance, + complexity_router_config={**llm_classifier_config, "route_housekeeping_to_cheapest_tier": False}, + ) + mock_router_instance.acompletion = AsyncMock(return_value=_llm_response('{"tier": "REASONING"}')) + + result = await router.async_pre_routing_hook( + model="test-model", request_kwargs={}, messages=[{"role": "user", "content": TITLE_ASK}] + ) + + assert result is not None + assert result.model == "o1-preview" + mock_router_instance.acompletion.assert_called_once() + + @pytest.mark.asyncio + async def test_an_operator_pattern_covers_a_client_the_built_ins_do_not( + self, mock_router_instance, llm_classifier_config + ): + """Client wording drifts with releases, so coverage has to be extensible without a code change.""" + router = ComplexityRouter( + model_name="test-router", + litellm_router_instance=mock_router_instance, + complexity_router_config={ + **llm_classifier_config, + "housekeeping_patterns": ["Summarize this thread for the sidebar"], + }, + ) + mock_router_instance.acompletion = AsyncMock(return_value=_llm_response('{"tier": "REASONING"}')) + + result = await router.async_pre_routing_hook( + model="test-model", + request_kwargs={}, + messages=[{"role": "user", "content": "Summarize this thread for the sidebar\nx"}], + ) + + assert result is not None + assert result.model == "gpt-4o-mini" + mock_router_instance.acompletion.assert_not_called() + + def test_a_blank_operator_pattern_is_dropped(self): + """An empty string substring-matches everything, which would route all traffic to the floor.""" + config = ComplexityRouterConfig(housekeeping_patterns=(" ", "keep me")) + + assert config.housekeeping_patterns == ("keep me",) + + @pytest.mark.asyncio + async def test_the_cheapest_tier_is_the_cheapest_one_that_has_models(self, mock_router_instance): + """A tier can be declared with no pool, and routing to an empty pool is a different bug.""" + router = ComplexityRouter( + model_name="test-router", + litellm_router_instance=mock_router_instance, + complexity_router_config={ + "tiers": {"COMPLEX": "claude-sonnet-4-20250514", "REASONING": "o1-preview"}, + "default_model": "gpt-4o-mini", + "classifier_type": "llm", + "classifier_llm_config": {"model": "haiku-classifier"}, + }, + ) + mock_router_instance.acompletion = AsyncMock(return_value=_llm_response('{"tier": "REASONING"}')) + + result = await router.async_pre_routing_hook( + model="test-model", request_kwargs={}, messages=[{"role": "user", "content": TITLE_ASK}] + ) + + assert result is not None + assert result.model == "claude-sonnet-4-20250514" + + + @pytest.mark.asyncio + async def test_a_classifier_plugin_still_decides_its_own_routers(self, mock_router_instance): + """A plugin is where an operator encodes policy the tier ladder cannot express. + + The sentinels are caller-controlled text. Displacing the built-in classifier with them only + ever spends less, but displacing a plugin is different in kind: a caller pasting a title + prompt could otherwise route past a sensitivity or identity rule to a pool it would refuse. + """ + plugin_calls: list[object] = [] + + class RecordingPlugin: + async def classify(self, context): + plugin_calls.append(context) + return "REASONING" + + router = ComplexityRouter( + model_name="test-router", + litellm_router_instance=mock_router_instance, + complexity_router_config={ + "tiers": {"SIMPLE": "gpt-4o-mini", "REASONING": "o1-preview"}, + "classifier_type": "custom", + "classifier_plugin": RecordingPlugin(), + }, + ) + + result = await router.async_pre_routing_hook( + model="test-model", request_kwargs={}, messages=[{"role": "user", "content": TITLE_ASK}] + ) + + assert len(plugin_calls) == 1 + assert result is not None + assert result.model == "o1-preview" + assert result.routing_decision["cause"] == "classifier_plugin" + + def _adaptive_router( + self, tier_distance_penalty: float, plan_mode_min_tier: str | None = None + ) -> ComplexityRouter: + adaptive_instance = MagicMock() + adaptive_instance.model_list = [ + { + "model_name": "cheap", + "litellm_params": {"model": "openai/gpt-4o-mini", "input_cost_per_token": 0.00000015}, + "model_info": {"adaptive_router_preferences": {"quality_tier": 1, "strengths": []}}, + }, + { + "model_name": "premium", + "litellm_params": {"model": "openai/gpt-4o", "input_cost_per_token": 0.000005}, + "model_info": {"adaptive_router_preferences": {"quality_tier": 3, "strengths": []}}, + }, + ] + adaptive_instance.model_name_to_deployment_indices = {"cheap": [0], "premium": [1]} + router = ComplexityRouter( + model_name="hybrid", + litellm_router_instance=adaptive_instance, + complexity_router_config={ + "adaptive": True, + "adaptive_eligible": "all", + "tiers": {"SIMPLE": ["cheap"], "COMPLEX": ["premium"]}, + "tier_distance_penalty": tier_distance_penalty, + "adaptive_weights": {"quality": 1.0, "cost": 0.0}, + **({"plan_mode_min_tier": plan_mode_min_tier} if plan_mode_min_tier else {}), + }, + ) + from litellm.router_strategy.adaptive_router.bandit import BanditCell + from litellm.types.router import RequestType + + adaptive = router._ensure_adaptive_router() + assert adaptive is not None + adaptive._cells[(RequestType.GENERAL, "cheap")] = BanditCell(alpha=1.0, beta=500.0) + adaptive._cells[(RequestType.GENERAL, "premium")] = BanditCell(alpha=500.0, beta=1.0) + return router + + @pytest.mark.asyncio + async def test_the_bandit_cannot_route_a_housekeeping_call_above_the_cheapest_tier( + self, mock_router_instance + ): + """The tier here is what the request IS, not how hard it is, so the bandit has nothing to win. + + Without a ceiling the tier distance penalty is the only thing holding the tier, so a + deployment that lowers tier_distance_penalty silently gets the expensive model back while + the routing decision still reads as the cheapest tier. Penalty 0 is the honest test. + + The posteriors are far enough apart that the real sampler decides this without patching it. + """ + router = self._adaptive_router(tier_distance_penalty=0.0) + + result = await router.async_pre_routing_hook( + model="test-model", request_kwargs={}, messages=[{"role": "user", "content": TITLE_ASK}] + ) + + assert result is not None + assert result.model == "cheap" + assert result.routing_decision["cause"] == "housekeeping" + + @pytest.mark.asyncio + async def test_the_bandit_is_still_free_on_a_request_that_is_not_housekeeping(self, mock_router_instance): + """The ceiling must bind only where it was set; the negative class proves it is not global.""" + router = self._adaptive_router(tier_distance_penalty=0.0) + + result = await router.async_pre_routing_hook( + model="test-model", + request_kwargs={}, + messages=[{"role": "user", "content": "design a rate limiter that stays correct under concurrency"}], + ) + + assert result is not None + assert result.model == "premium" + + + @pytest.mark.asyncio + async def test_a_housekeeping_call_never_becomes_the_session_pin(self, mock_router_instance): + """Pinning this is the most expensive mistake of the transient causes. + + An agent names the conversation on its first turn, so the cheapest tier would be the pin + every session starts with and the real work that follows would run there for the whole TTL. + """ + mock_router_instance.cache = DualCache() + mock_router_instance.acompletion = AsyncMock(return_value=_llm_response('{"tier": "REASONING"}')) + router = ComplexityRouter( + model_name="test-router", + litellm_router_instance=mock_router_instance, + complexity_router_config={ + "tiers": { + "SIMPLE": "gpt-4o-mini", + "MEDIUM": "gpt-4o", + "COMPLEX": "claude-sonnet-4-20250514", + "REASONING": "o1-preview", + }, + "classifier_type": "llm", + "classifier_llm_config": {"model": "haiku-classifier"}, + "session_affinity": True, + }, + ) + session = {"metadata": {"session_id": "housekeeping-first"}} + + title_turn = await router.async_pre_routing_hook( + model="test-model", request_kwargs=dict(session), messages=[{"role": "user", "content": TITLE_ASK}] + ) + work_turn = await router.async_pre_routing_hook( + model="test-model", + request_kwargs=dict(session), + messages=[{"role": "user", "content": "design a rate limiter and prove it cannot livelock"}], + ) + + assert title_turn is not None and title_turn.model == "gpt-4o-mini" + assert work_turn is not None + assert work_turn.model == "o1-preview" + assert work_turn.routing_decision["cause"] == "llm_classifier" + + @pytest.mark.asyncio + async def test_the_decision_records_which_sentinel_matched( + self, mock_router_instance, llm_classifier_config + ): + """The cause's contract says the sentinel rides in matched_keyword, so it has to be there. + + Without it an operator reading the logs can see that a call was treated as housekeeping but + not which string did it, which is the one fact they need to tune housekeeping_patterns. + """ + router = ComplexityRouter( + model_name="test-router", + litellm_router_instance=mock_router_instance, + complexity_router_config=llm_classifier_config, + ) + + result = await router.async_pre_routing_hook( + model="test-model", request_kwargs={}, messages=[{"role": "user", "content": TITLE_ASK}] + ) + + assert result is not None + assert result.routing_decision["matched_keyword"] == ( + "Write the title in the predominant language of the session" + ) + + + @pytest.mark.asyncio + async def test_the_plan_mode_floor_raises_a_housekeeping_call_under_adaptive(self, mock_router_instance): + """Floor and ceiling must not contradict each other on the same request. + + The ceiling names the tier as raised, not the placement it started from. Naming the cheapest + tier here would bound the pick below the floor, leaving the filters with nothing to choose + from and the decision reporting a tier the routed model does not belong to. + """ + router = self._adaptive_router(tier_distance_penalty=0.0, plan_mode_min_tier="COMPLEX") + + result = await router.async_pre_routing_hook( + model="test-model", + request_kwargs={}, + messages=[ + {"role": "system", "content": 'You are currently running in "Plan" mode.'}, + {"role": "user", "content": TITLE_ASK}, + ], + ) + + assert result is not None + assert result.model == "premium" + assert result.routing_decision["tier"] == "COMPLEX" + + @pytest.mark.asyncio + async def test_an_escalation_keyword_raises_a_housekeeping_call_under_adaptive(self, mock_router_instance): + """Escalating a housekeeping call must move the model too, not just the reported tier.""" + router = self._adaptive_router(tier_distance_penalty=0.0) + + result = await router.async_pre_routing_hook( + model="test-model", + request_kwargs={}, + messages=[{"role": "user", "content": f"LITELLM ESCALATE {TITLE_ASK}"}], + ) + + assert result is not None + assert result.model == "premium" + assert result.routing_decision["tier"] == "COMPLEX" + + class TestClassifierTrustBoundary: """The classifier's system role carries the operator's rubric and nothing a caller supplied.""" diff --git a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/RoutingDecisionCard.test.tsx b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/RoutingDecisionCard.test.tsx index b6803f23f50..e084fdf37e6 100644 --- a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/RoutingDecisionCard.test.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/RoutingDecisionCard.test.tsx @@ -160,6 +160,32 @@ describe("RoutingDecisionCard", () => { expect(screen.getByText('Plan-mode floor: "Plan mode is active"')).toBeInTheDocument(); }); + it("names the housekeeping sentinel so an operator can extend the pattern list", () => { + // The sentinel is the string they would add to housekeeping_patterns to cover another + // client, so the row is only useful if it says which one matched. + render( + , + ); + expect( + screen.getByText('Client housekeeping call: "Write the title in the predominant language of the session"'), + ).toBeInTheDocument(); + }); + + it("still labels a housekeeping row when redaction dropped the sentinel", () => { + // matched_keyword is prompt-quoting, so message-log redaction removes it. The row must + // still read as a housekeeping decision rather than falling back to the raw cause. + render(); + expect(screen.getByText("Client housekeeping call, classifier skipped")).toBeInTheDocument(); + expect(screen.queryByText("housekeeping")).not.toBeInTheDocument(); + }); + it("shows the escalation keyword", () => { render( , diff --git a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/RoutingDecisionCard.tsx b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/RoutingDecisionCard.tsx index 4e1a9b8ee7f..8c77b2db630 100644 --- a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/RoutingDecisionCard.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/RoutingDecisionCard.tsx @@ -66,6 +66,16 @@ function describePlanModeFloor(matchedKeyword: string | undefined): string { return "Plan-mode floor"; } +/** + * The sentinel is the whole reason this row is worth reading: it is the string an operator + * would add to housekeeping_patterns to cover another client, so naming it turns the row into + * the instruction. Without it the drawer says only that the classifier was skipped. + */ +function describeHousekeeping(matchedKeyword: string | undefined): string { + if (matchedKeyword) return `Client housekeeping call: "${matchedKeyword}"`; + return "Client housekeeping call, classifier skipped"; +} + /** Rows logged before the floor was recorded name what it tracked back then instead of a number. */ function describeReasoningOverride(tierLabel: string | undefined, floor: number | undefined): string { const stated = floor === undefined ? "the Simple to Medium boundary" : String(floor); @@ -108,6 +118,8 @@ function describeCause(decision: RoutingDecision): string { return matchedKeyword ? `Keyword match: "${matchedKeyword}"` : "Keyword match"; case "plan_mode": return describePlanModeFloor(matchedKeyword); + case "housekeeping": + return describeHousekeeping(matchedKeyword); default: return cause ?? "Unknown"; } diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 9df7d8baba7..3cc833cfcf9 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -29937,6 +29937,11 @@ export interface components { * @description Path to a JSON file containing ad-hoc recognizers for Presidio */ presidio_ad_hoc_recognizers?: string | null; + /** + * Presidio Analyze Chunk Size Bytes + * @description Maximum UTF-8 bytes of text sent in a single Presidio /analyze call. Longer texts are split into overlapping chunks of at most this size and the merged results are remapped onto the original text. Defaults to 500000; set it below your analyzer deployment's request body limit, leaving headroom for the rest of the analyze payload. + */ + presidio_analyze_chunk_size_bytes?: number | null; /** * Presidio Analyzer Api Base * @description Base URL for the Presidio analyzer API @@ -33805,6 +33810,11 @@ export interface components { * @description The highest tier the local scorer may decide on its own; required when classifier_type is 'heuristic_first' and rejected otherwise. A request whose heuristic tier is at or below this one skips the LLM classifier and routes straight to that heuristic tier, so the classifier call is only paid for on traffic the scorer could not place cheaply. The scorer must also have produced at least one signal: a prompt where no dimension fired scores 0.0 and would otherwise land SIMPLE by default rather than by evidence, which is how a chained router would silently send unclassified traffic to the cheapest model. Names a built-in tier, and may not name the highest one, since that would make the LLM classifier unreachable. */ heuristic_first_max_tier?: string | null; + /** + * Housekeeping Patterns + * @description Additional case-sensitive literal sentinels that mark a request as client housekeeping, on top of the built-in conversation-title ones. For clients whose wording the built-ins don't cover, or after a client release changes its strings. + */ + housekeeping_patterns?: string[] | null; /** * Keyword Tier Rules * @description Rules that force a specific tier when their keywords match the prompt @@ -33852,6 +33862,12 @@ export interface components { * @default false */ return_raw_model_name: boolean; + /** + * Route Housekeeping To Cheapest Tier + * @description Route a coding agent's own housekeeping calls to the cheapest configured tier without classifying them. A client names the conversation by quoting the whole session and asking for a title, so the ask reads as the session's engineering work and lands on the most expensive tier, which is the reverse of what the call is worth. Detection is a literal match against client-owned sentinels on the newest ask only, so it cannot fire on an earlier turn, and it never lowers what anyone else asked for: a keyword_tier_rule or a session pin still decides instead, and an escalation keyword or the plan-mode floor still raises the tier from here. Only the classifier is displaced, and its call is skipped, so a matched request costs nothing to route. Set false to classify these calls like any other. + * @default true + */ + route_housekeeping_to_cheapest_tier: boolean; /** * Semantic Keyword Matching * @description Match keyword_tier_rules by embedding similarity instead of literal text @@ -34908,7 +34924,7 @@ export interface components { * Cause * @enum {string} */ - cause?: "heuristic_scorer" | "reasoning_override" | "llm_classifier" | "heuristic_first_short_circuit" | "classifier_plugin" | "classifier_fallback" | "default_model_fallback" | "literal_keyword_match" | "semantic_keyword_match" | "plan_mode" | "session_affinity_pin" | "session_affinity_escalation" | "default_fallback" | "keyword" | "quality_tier" | "bandit"; + cause?: "heuristic_scorer" | "reasoning_override" | "llm_classifier" | "heuristic_first_short_circuit" | "classifier_plugin" | "classifier_fallback" | "default_model_fallback" | "literal_keyword_match" | "semantic_keyword_match" | "plan_mode" | "housekeeping" | "session_affinity_pin" | "session_affinity_escalation" | "default_fallback" | "keyword" | "quality_tier" | "bandit"; /** Classifier Cost */ classifier_cost?: number; /** Classifier Model */