diff --git a/litellm/integrations/datadog/datadog_llm_obs.py b/litellm/integrations/datadog/datadog_llm_obs.py index 728bf41856f..d2383d35111 100644 --- a/litellm/integrations/datadog/datadog_llm_obs.py +++ b/litellm/integrations/datadog/datadog_llm_obs.py @@ -79,11 +79,20 @@ _ROUTER_SPAN_FIELDS: Final[Mapping[str, str]] = MappingProxyType( "cause": "router_cause", "score": "router_score", "escalated": "router_escalated", + "stall_escalated": "router_stall_escalated", + "stall_escalation_reason": "router_stall_reason", + "stall_escalation_original_tier": "router_stall_original_tier", "signals": "router_signals", "routed_model": "routed_model", } ) -_ROUTER_DIMENSIONS: Final[tuple[str, ...]] = ("router_tier", "router_cause", "router_escalated", "routed_model") +_ROUTER_DIMENSIONS: Final[tuple[str, ...]] = ( + "router_tier", + "router_cause", + "router_escalated", + "router_stall_reason", + "routed_model", +) _COST_DIMENSIONS: Final[tuple[str, ...]] = ("team", "user", "key_alias", "model_group", *_ROUTER_DIMENSIONS) @@ -99,7 +108,11 @@ def _router_span_fields( routing_decision: Final = _mapping_field(_metadata_of(standard_logging_payload), "routing_decision") if not routing_decision: return _EMPTY_MAPPING - escalated: Final = bool(routing_decision.get("escalated") or routing_decision.get("context_escalated")) + escalated: Final = bool( + routing_decision.get("escalated") + or routing_decision.get("context_escalated") + or routing_decision.get("stall_escalated") + ) return MappingProxyType( { _ROUTER_SPAN_FIELDS[record_field]: value diff --git a/litellm/router_strategy/complexity_router/README.md b/litellm/router_strategy/complexity_router/README.md index d605b43e42a..a6a5bfcbc19 100644 --- a/litellm/router_strategy/complexity_router/README.md +++ b/litellm/router_strategy/complexity_router/README.md @@ -322,9 +322,22 @@ There is no state to expire or leak: detection reruns on every classified turn f request's own message list, so the bump lasts only as long as the recent tool calls still look stuck and lifts on its own the moment they don't. This also means it reads the whole conversation rather than only the turns since the newest human ask, so a plain follow-up like -"try again" does not discard evidence from before it. Escalation records `stall_escalation` in -`routing_decision.signals`; unlike `escalation_keywords`, it does not set the -`escalated`/`escalation_keyword` pair, which is reserved for the keyword mechanism specifically. +"try again" does not discard evidence from before it. + +Escalation records three fields of its own on the routing decision: `stall_escalated`, +`stall_escalation_reason` (`repeated_tool_call` or `repeated_tool_error`), and +`stall_escalation_original_tier`, the tier the request was bumped off. They are derived rather +than prompt-quoting, so they survive message redaction, unlike the `stall_escalation` entry in +`routing_decision.signals` that also marks the bump. Datadog LLM Observability carries them as +`router_stall_escalated` / `router_stall_reason` / `router_stall_original_tier`, with +`router_stall_reason` a cost dimension and `router_escalated` true, so escalated spend is a slice +you can chart and the two branches can be told apart. The reason is worth splitting out because a +repeat loop is a model that is genuinely stuck, which a stronger model can break, while repeated +tool errors are usually a broken tool or bad credentials, where the stronger model fails the same +way at a higher price. A stall detected on a request already at the highest configured tier +records the reason with `stall_escalated: false` and no original tier, since nothing moved. +Unlike `escalation_keywords`, it does not set the `escalated`/`escalation_keyword` pair, which is +reserved for the keyword mechanism specifically. `stall_escalation_enabled` cannot be combined with `session_affinity` or `classification_mode: user_turn`: both replay a held routing decision on most turns instead of diff --git a/litellm/router_strategy/complexity_router/complexity_router.py b/litellm/router_strategy/complexity_router/complexity_router.py index faafcea404a..3bc3d84d759 100644 --- a/litellm/router_strategy/complexity_router/complexity_router.py +++ b/litellm/router_strategy/complexity_router/complexity_router.py @@ -61,6 +61,7 @@ from litellm.types.utils import ( AUTOROUTER_CLASSIFIER_CALL_ORIGIN, ModelResponse, RoutingDecisionCause, + StallEscalationReason, StandardLoggingRoutingDecision, StandardLoggingRoutingDecisionTierBoundaries, ) @@ -1547,6 +1548,8 @@ class ComplexityRouter(CustomLogger): conversation_continuing: bool = True, tier_litellm_params: Mapping[str, object] | None = None, context_escalation_original_tier: ComplexityTier | str | None = None, + stall_escalation_reason: StallEscalationReason | None = None, + stall_escalation_original_tier: ComplexityTier | str | None = None, ) -> StandardLoggingRoutingDecision: """Assemble the per-request provenance record for this router's decision. @@ -1602,6 +1605,14 @@ class ComplexityRouter(CustomLogger): # (classifier, keyword rule, or session pin) had placed it before physics did. decision["context_escalated"] = True decision["context_escalation_original_tier"] = _tier_name(context_escalation_original_tier) + if stall_escalation_reason is not None: + # Same two facts the keyword pair records: the detector fired, and whether the tier + # actually moved. A stall detected at the highest configured tier has nowhere to go, + # so it reports the reason with stall_escalated False rather than claiming a bump. + decision["stall_escalated"] = stall_escalation_original_tier is not None + decision["stall_escalation_reason"] = stall_escalation_reason + if stall_escalation_original_tier is not None: + decision["stall_escalation_original_tier"] = _tier_name(stall_escalation_original_tier) if tier_litellm_params: masked_tier_litellm_params: Final = mask_credentials_in_payload(tier_litellm_params) if isinstance(masked_tier_litellm_params, Mapping): @@ -2876,6 +2887,10 @@ class ComplexityRouter(CustomLogger): context_escalation_original_tier=( decision.get("context_escalation_original_tier") if decision is not None else None ), + stall_escalation_reason=decision.get("stall_escalation_reason") if decision is not None else None, + stall_escalation_original_tier=( + decision.get("stall_escalation_original_tier") if decision is not None else None + ), ) from litellm.types.router import PreRoutingHookResponse as HookResponse @@ -3041,6 +3056,8 @@ class ComplexityRouter(CustomLogger): conversation_continuing=bool(decision.get("conversation_continuing", True)), tier_litellm_params=self._litellm_params_for_model(decided_tier, new_model), context_escalation_original_tier=decision.get("context_escalation_original_tier"), + stall_escalation_reason=decision.get("stall_escalation_reason"), + stall_escalation_original_tier=decision.get("stall_escalation_original_tier"), ) return response.model_copy( update={ # mutable-ok: model_copy types update as a plain dict @@ -3596,10 +3613,14 @@ class ComplexityRouter(CustomLogger): # Resolved here rather than beside the classifier because the keyword-override path below # returns before any classification runs, and a forced tier gets stuck for the same reason # a classified one does. - stalled: Final = self.config.stall_escalation_enabled and detect_stalled_task( - resolved_messages, - window=self.config.stall_escalation_window, - repeat_threshold=self.config.stall_escalation_repeat_threshold, + stalled: Final = ( + detect_stalled_task( + resolved_messages, + window=self.config.stall_escalation_window, + repeat_threshold=self.config.stall_escalation_repeat_threshold, + ) + if self.config.stall_escalation_enabled + else None ) plan_mode_sentinel: Final = self._matched_plan_mode_signal(request_kwargs, resolved_messages) @@ -3636,7 +3657,10 @@ class ComplexityRouter(CustomLogger): keyword_bumped_tier: Final = ( self._escalate_tier(override.tier) if escalation_keyword is not None else override.tier ) - escalated_tier: Final = self._escalate_tier(keyword_bumped_tier) if stalled else keyword_bumped_tier + escalated_tier: Final = ( + self._escalate_tier(keyword_bumped_tier) if stalled is not None else keyword_bumped_tier + ) + keyword_stall_original_tier: Final = keyword_bumped_tier if escalated_tier != keyword_bumped_tier else None keyword_escalated: Final = keyword_bumped_tier != override.tier routed_tier: Final = ( self._apply_plan_mode_floor(escalated_tier) if plan_floor is not None else escalated_tier @@ -3665,11 +3689,13 @@ class ComplexityRouter(CustomLogger): conversation_continuing=conversation_continuing, cause=keyword_cause, tier=routed_tier, - signals=("stall_escalation",) if stalled else None, + signals=("stall_escalation",) if stalled is not None else None, matched_keyword=plan_mode_sentinel if keyword_plan_floored else override.matched_keyword, escalation_keyword=escalation_keyword, escalated=keyword_escalated, tier_litellm_params=keyword_tier_litellm_params, + stall_escalation_reason=stalled, + stall_escalation_original_tier=keyword_stall_original_tier, ), ) @@ -3687,9 +3713,11 @@ class ComplexityRouter(CustomLogger): escalated: Final = tier != classified_tier if escalated: signals = (*signals, "escalation") - if stalled: + pre_stall_tier: Final = tier + if stalled is not None: tier = self._escalate_tier(tier) signals = (*signals, "stall_escalation") + stall_original_tier: Final = pre_stall_tier if tier != pre_stall_tier else None pre_floor_tier: Final = tier if plan_floor is not None: tier = self._apply_plan_mode_floor(tier) @@ -3832,5 +3860,7 @@ class ComplexityRouter(CustomLogger): classifier_cost=outcome.classifier_cost, tier_litellm_params=tier_litellm_params, context_escalation_original_tier=context_original_tier, + stall_escalation_reason=stalled, + stall_escalation_original_tier=stall_original_tier, ), ) diff --git a/litellm/router_strategy/complexity_router/stall_detector.py b/litellm/router_strategy/complexity_router/stall_detector.py index 690603f05d5..d5034cc4224 100644 --- a/litellm/router_strategy/complexity_router/stall_detector.py +++ b/litellm/router_strategy/complexity_router/stall_detector.py @@ -2,7 +2,7 @@ Mid-task stall detection for the Complexity Router. Reads the assistant's own recent tool calls, which every agentic client resends on each -turn, and reports whether the task currently looks stuck. No LLM call and no stored state: +turn, and reports which stuck pattern the task looks caught in, if any. No LLM call and no state: the same window is rescanned per classified turn, so the verdict follows the conversation rather than latching. @@ -20,6 +20,8 @@ from collections.abc import Iterator, Mapping, Sequence from itertools import islice from typing import Final, NamedTuple +from litellm.types.utils import StallEscalationReason + _ARGUMENTS_PARSE_FAILED: Final = object() @@ -94,8 +96,8 @@ def detect_stalled_task( *, window: int, repeat_threshold: int, -) -> bool: - """Whether the newest tool call is still part of a stuck pattern: it repeats, or it +) -> StallEscalationReason | None: + """Which stuck pattern the newest tool call is still part of, or None: it repeats, or it errored, at least repeat_threshold times across the last `window` calls. Both tests are anchored on the newest call rather than counting whichever pattern is @@ -105,14 +107,16 @@ def detect_stalled_task( matches, so a retry loop broken up by an unrelated lookup still reads as stuck. """ if not messages or repeat_threshold <= 0: - return False + return None recent: Final = tuple(islice(_iter_tool_call_events_newest_first(messages), window)) if len(recent) < repeat_threshold: - return False + return None newest: Final = recent[0] repeats: Final = sum(1 for event in recent if event.signature == newest.signature) if repeats >= repeat_threshold: - return True + return "repeated_tool_call" if not newest.is_error: - return False - return sum(1 for event in recent if event.is_error) >= repeat_threshold + return None + if sum(1 for event in recent if event.is_error) >= repeat_threshold: + return "repeated_tool_error" + return None diff --git a/litellm/types/utils.py b/litellm/types/utils.py index ab0cc5f959c..70e6b3f9508 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -8,6 +8,7 @@ from typing import ( Any, Final, Literal, + TypeAlias, get_args, ) @@ -2948,6 +2949,12 @@ SHADOW_EVAL_JUDGE_CALL_ORIGIN: Final[InternalCallOrigin] = "shadow_eval_judge" BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN: Final[InternalCallOrigin] = "background_response_cost_poll" +StallEscalationReason: TypeAlias = Literal["repeated_tool_call", "repeated_tool_error"] +"""Which stall branch fired. A repeat loop is a model that is stuck, which a stronger model can +break; repeated tool errors are usually a broken tool or bad credentials, where the stronger +model fails the same way at a higher price. Operationally different, so they are recorded apart.""" + + class StandardLoggingRoutingDecision(TypedDict, total=False): """Per-request provenance for a pre-routing strategy (auto-router) decision.""" @@ -2967,6 +2974,9 @@ class StandardLoggingRoutingDecision(TypedDict, total=False): escalated: bool context_escalated: bool # writable-ok: Pydantic warns on ReadOnly TypedDict fields context_escalation_original_tier: str # writable-ok: Pydantic warns on ReadOnly TypedDict fields + stall_escalated: bool # writable-ok: Pydantic warns on ReadOnly TypedDict fields + stall_escalation_reason: StallEscalationReason # writable-ok: Pydantic warns on ReadOnly TypedDict fields + stall_escalation_original_tier: str # writable-ok: Pydantic warns on ReadOnly TypedDict fields tier_boundaries: StandardLoggingRoutingDecisionTierBoundaries reasoning_override_min_score: float # writable-ok: Pydantic warns on ReadOnly TypedDict fields conversation_continuing: bool @@ -2995,6 +3005,9 @@ DERIVED_ROUTING_DECISION_FIELDS: Final[frozenset[str]] = frozenset( "escalated", "context_escalated", "context_escalation_original_tier", + "stall_escalated", + "stall_escalation_reason", + "stall_escalation_original_tier", "tier_boundaries", "reasoning_override_min_score", "conversation_continuing", diff --git a/tests/test_litellm/integrations/datadog/test_datadog_llm_obs.py b/tests/test_litellm/integrations/datadog/test_datadog_llm_obs.py index 0555447e34f..f16616db5b5 100644 --- a/tests/test_litellm/integrations/datadog/test_datadog_llm_obs.py +++ b/tests/test_litellm/integrations/datadog/test_datadog_llm_obs.py @@ -398,6 +398,69 @@ def test_a_context_escalated_route_reports_as_escalated(logger: DataDogLLMObsLog assert "router_escalated:true" in payload["tags"] +def test_a_stall_escalated_route_reports_as_escalated(logger: DataDogLLMObsLogger) -> None: + """The stall detector bumped the tier, so the spend on this request is escalated spend and + has to land in the same slice as every other escalation.""" + payload = build(logger, metadata={"routing_decision": {"tier": "premium", "stall_escalated": True}}) + + assert payload["meta"]["metadata"]["router_escalated"] is True + assert "router_escalated:true" in payload["tags"] + + +def test_the_stall_branch_is_its_own_cost_dimension(logger: DataDogLLMObsLogger) -> None: + """Charting the two branches apart is what tells an operator whether the bumps are buying + anything: a stuck model can be unstuck by a stronger one, a broken tool cannot.""" + payload = build( + logger, + metadata={ + "routing_decision": { + "tier": "premium", + "stall_escalated": True, + "stall_escalation_reason": "repeated_tool_error", + "stall_escalation_original_tier": "SIMPLE", + } + }, + ) + + assert payload["meta"]["metadata"]["router_stall_reason"] == "repeated_tool_error" + assert payload["meta"]["metadata"]["router_stall_original_tier"] == "SIMPLE" + assert "router_stall_reason:repeated_tool_error" in payload["tags"] + assert "router_stall_reason" in payload["meta"]["metadata"]["_dd"]["cost_tags"] + + +def test_stall_fields_survive_prompt_redaction() -> None: + """They aggregate the prompt rather than quoting it, unlike the signals entry that used to be + the only marker, so a redacted span still says the request was escalated and why.""" + with patch.dict(os.environ, {"DD_API_KEY": "k", "DD_SITE": "us5.datadoghq.com"}, clear=True): + with patch("asyncio.create_task"): + redacted_logger = DataDogLLMObsLogger(turn_off_message_logging=True) + result = json.loads( + safe_dumps( + redacted_logger.create_llm_obs_payload( + build_payload( + metadata={ + "routing_decision": { + "tier": "premium", + "signals": ["secret prompt text", "stall_escalation"], + "stall_escalated": True, + "stall_escalation_reason": "repeated_tool_call", + "stall_escalation_original_tier": "SIMPLE", + } + } + ), + datetime(2026, 9, 1, 12, 0, 0), + datetime(2026, 9, 1, 12, 0, 2), + ) + ) + ) + + assert result["meta"]["metadata"]["router_stall_escalated"] is True + assert result["meta"]["metadata"]["router_stall_reason"] == "repeated_tool_call" + assert result["meta"]["metadata"]["router_stall_original_tier"] == "SIMPLE" + assert result["meta"]["metadata"]["router_escalated"] is True + assert "router_signals" not in result["meta"]["metadata"] + + def test_a_routed_request_that_did_not_escalate_reports_false(logger: DataDogLLMObsLogger) -> None: """Without this the escalation dimension is absent on ordinary traffic, so nothing can group by it.""" payload = build(logger, metadata={"routing_decision": {"tier": "simple", "cause": "heuristic_scorer"}}) diff --git a/tests/test_litellm/router_strategy/test_complexity_router.py b/tests/test_litellm/router_strategy/test_complexity_router.py index 51103297c58..903f2b3d546 100644 --- a/tests/test_litellm/router_strategy/test_complexity_router.py +++ b/tests/test_litellm/router_strategy/test_complexity_router.py @@ -6809,6 +6809,139 @@ class TestStallEscalation: result = await router.async_pre_routing_hook(model="test-model", request_kwargs={}, messages=messages) assert "stall_escalation" in result.routing_decision["signals"] + @pytest.mark.asyncio + async def test_the_escalation_is_recorded_as_its_own_fields(self, mock_router_instance, basic_config): + """`signals` is prompt-quoting and redaction drops it, so a stall recorded only there is + invisible to an operator running with message logging off. These fields aggregate the + prompt rather than quoting it, so they survive and can carry a metric dimension.""" + router = ComplexityRouter( + model_name="test-router", + litellm_router_instance=mock_router_instance, + complexity_router_config={**basic_config, "stall_escalation_enabled": True}, + ) + messages = [*_stalled_tool_history(), {"role": "user", "content": "Hello there!"}] + result = await router.async_pre_routing_hook(model="test-model", request_kwargs={}, messages=messages) + assert result.routing_decision["stall_escalated"] is True + assert result.routing_decision["stall_escalation_original_tier"] == "SIMPLE" + assert result.routing_decision["tier"] == "MEDIUM" + + @pytest.mark.asyncio + async def test_the_two_detector_branches_are_told_apart(self, mock_router_instance, basic_config): + """A repeat loop is a stuck model, which is what the bump is for. Repeated tool errors are + usually a broken tool, where the stronger model fails the same way at a higher price.""" + router = ComplexityRouter( + model_name="test-router", + litellm_router_instance=mock_router_instance, + complexity_router_config={**basic_config, "stall_escalation_enabled": True}, + ) + erroring = [ + turn + for i, cmd in enumerate(("pytest a", "pytest b", "pytest c")) + for turn in ( + { + "role": "assistant", + "content": [{"type": "tool_use", "id": f"e-{i}", "name": "bash", "input": {"cmd": cmd}}], + }, + { + "role": "user", + "content": [ + {"type": "tool_result", "tool_use_id": f"e-{i}", "is_error": True, "content": "fail"} + ], + }, + ) + ] + repeating = await router.async_pre_routing_hook( + model="test-model", + request_kwargs={}, + messages=[*_stalled_tool_history(), {"role": "user", "content": "Hello there!"}], + ) + erroring_result = await router.async_pre_routing_hook( + model="test-model", + request_kwargs={}, + messages=[*erroring, {"role": "user", "content": "Hello there!"}], + ) + assert repeating.routing_decision["stall_escalation_reason"] == "repeated_tool_call" + assert erroring_result.routing_decision["stall_escalation_reason"] == "repeated_tool_error" + + @pytest.mark.asyncio + async def test_an_unstalled_request_carries_no_stall_fields(self, mock_router_instance, basic_config): + """The fields must distinguish an escalated request from one the classifier placed at the + same tier on its own, so they cannot appear on ordinary traffic.""" + router = ComplexityRouter( + model_name="test-router", + litellm_router_instance=mock_router_instance, + complexity_router_config={**basic_config, "stall_escalation_enabled": True}, + ) + result = await router.async_pre_routing_hook( + model="test-model", request_kwargs={}, messages=[{"role": "user", "content": "Hello there!"}] + ) + assert "stall_escalated" not in result.routing_decision + assert "stall_escalation_reason" not in result.routing_decision + assert "stall_escalation_original_tier" not in result.routing_decision + + @pytest.mark.asyncio + async def test_a_stall_with_nowhere_to_go_records_the_detection_without_a_bump( + self, mock_router_instance, basic_config + ): + """Already at the highest configured tier, so no tier moved and no extra money was spent. + Reporting stall_escalated True there would overstate what the feature costs.""" + router = ComplexityRouter( + model_name="test-router", + litellm_router_instance=mock_router_instance, + complexity_router_config={**basic_config, "stall_escalation_enabled": True}, + ) + messages = [ + *_stalled_tool_history(), + {"role": "user", "content": "Let's think step by step and reason through this carefully."}, + ] + result = await router.async_pre_routing_hook(model="test-model", request_kwargs={}, messages=messages) + assert result.routing_decision["stall_escalated"] is False + assert result.routing_decision["stall_escalation_reason"] == "repeated_tool_call" + assert "stall_escalation_original_tier" not in result.routing_decision + + @pytest.mark.asyncio + async def test_the_fields_survive_message_redaction(self, mock_router_instance, basic_config): + """The whole point: with redaction on, `signals` goes and these have to stay, otherwise a + stall-escalated row is identical to one the classifier placed at that tier itself.""" + router = ComplexityRouter( + model_name="test-router", + litellm_router_instance=mock_router_instance, + complexity_router_config={**basic_config, "stall_escalation_enabled": True}, + ) + request_kwargs: Dict = {"metadata": {"headers": {"x-litellm-enable-message-redaction": True}}} + result = await router.async_pre_routing_hook( + model="test-model", + request_kwargs=request_kwargs, + messages=[*_stalled_tool_history(), {"role": "user", "content": "Hello there!"}], + ) + redacted = litellm.Router._redact_prompt_text_if_needed(request_kwargs, result.routing_decision) + assert "signals" not in redacted + assert redacted["stall_escalated"] is True + assert redacted["stall_escalation_reason"] == "repeated_tool_call" + assert redacted["stall_escalation_original_tier"] == "SIMPLE" + + @pytest.mark.asyncio + async def test_a_keyword_forced_tier_records_the_stall_fields_too(self, mock_router_instance, basic_config): + """The keyword-override path returns before classification, so it assembles its own + decision record and would otherwise drop the fields the classified path sets.""" + router = ComplexityRouter( + model_name="test-router", + litellm_router_instance=mock_router_instance, + complexity_router_config={ + **basic_config, + "stall_escalation_enabled": True, + "keyword_tier_rules": [{"keywords": ["billing"], "tier": "SIMPLE"}], + }, + ) + result = await router.async_pre_routing_hook( + model="test-model", + request_kwargs={}, + messages=[*_stalled_tool_history(), {"role": "user", "content": "a billing question"}], + ) + assert result.routing_decision["stall_escalated"] is True + assert result.routing_decision["stall_escalation_reason"] == "repeated_tool_call" + assert result.routing_decision["stall_escalation_original_tier"] == "SIMPLE" + @pytest.mark.asyncio async def test_stall_escalation_caps_at_highest_tier(self, mock_router_instance, basic_config): router = ComplexityRouter( diff --git a/tests/test_litellm/router_strategy/test_stall_detector.py b/tests/test_litellm/router_strategy/test_stall_detector.py index 34067cc3626..2996011ce96 100644 --- a/tests/test_litellm/router_strategy/test_stall_detector.py +++ b/tests/test_litellm/router_strategy/test_stall_detector.py @@ -35,7 +35,7 @@ class TestDetectStalledTask: *_anthropic_call("t2", "bash", {"cmd": "pytest"}, is_error=False), *_anthropic_call("t3", "bash", {"cmd": "pytest"}, is_error=False), ] - assert detect_stalled_task(messages, window=6, repeat_threshold=3) is True + assert detect_stalled_task(messages, window=6, repeat_threshold=3) == "repeated_tool_call" def test_repeated_errors_are_stalled_even_with_varied_arguments(self): messages = [ @@ -43,7 +43,7 @@ class TestDetectStalledTask: *_anthropic_call("t2", "bash", {"cmd": "pytest tests/b.py"}, is_error=True), *_anthropic_call("t3", "bash", {"cmd": "pytest tests/c.py"}, is_error=True), ] - assert detect_stalled_task(messages, window=6, repeat_threshold=3) is True + assert detect_stalled_task(messages, window=6, repeat_threshold=3) == "repeated_tool_error" def test_varied_successful_calls_are_not_stalled(self): messages = [ @@ -51,7 +51,7 @@ class TestDetectStalledTask: *_anthropic_call("t2", "bash", {"cmd": "pytest"}, is_error=False), *_anthropic_call("t3", "grep", {"pattern": "x"}, is_error=False), ] - assert detect_stalled_task(messages, window=6, repeat_threshold=3) is False + assert detect_stalled_task(messages, window=6, repeat_threshold=3) is None def test_chat_completions_repeats_are_stalled(self): messages = [ @@ -59,7 +59,7 @@ class TestDetectStalledTask: *_chat_completions_call("c2", "bash", '{"cmd": "pytest"}'), *_chat_completions_call("c3", "bash", '{"cmd": "pytest"}'), ] - assert detect_stalled_task(messages, window=6, repeat_threshold=3) is True + assert detect_stalled_task(messages, window=6, repeat_threshold=3) == "repeated_tool_call" def test_chat_completions_has_no_structured_error_signal(self): """A chat-completions tool message carries no standard error flag, so varied calls @@ -69,7 +69,7 @@ class TestDetectStalledTask: *_chat_completions_call("c2", "bash", '{"cmd": "b"}'), *_chat_completions_call("c3", "bash", '{"cmd": "c"}'), ] - assert detect_stalled_task(messages, window=6, repeat_threshold=3) is False + assert detect_stalled_task(messages, window=6, repeat_threshold=3) is None def test_dict_and_json_string_arguments_compare_equal_across_surfaces(self): messages = [ @@ -77,14 +77,14 @@ class TestDetectStalledTask: *_chat_completions_call("c2", "bash", '{"cmd": "pytest"}'), *_anthropic_call("t3", "bash", {"cmd": "pytest"}, is_error=False), ] - assert detect_stalled_task(messages, window=6, repeat_threshold=3) is True + assert detect_stalled_task(messages, window=6, repeat_threshold=3) == "repeated_tool_call" def test_below_repeat_threshold_is_not_stalled(self): messages = [ *_anthropic_call("t1", "bash", {"cmd": "pytest"}, is_error=False), *_anthropic_call("t2", "bash", {"cmd": "pytest"}, is_error=False), ] - assert detect_stalled_task(messages, window=6, repeat_threshold=3) is False + assert detect_stalled_task(messages, window=6, repeat_threshold=3) is None def test_evidence_older_than_the_window_does_not_count(self): """Only the most recent `window` tool calls are considered, so a stall the model @@ -96,7 +96,7 @@ class TestDetectStalledTask: *_anthropic_call("t4", "grep", {"pattern": "a"}, is_error=False), *_anthropic_call("t5", "grep", {"pattern": "b"}, is_error=False), ] - assert detect_stalled_task(messages, window=2, repeat_threshold=2) is False + assert detect_stalled_task(messages, window=2, repeat_threshold=2) is None def test_evidence_survives_a_new_human_ask(self): """A follow-up like 'try again' must not erase evidence from before it: detection @@ -107,7 +107,7 @@ class TestDetectStalledTask: *_anthropic_call("t3", "bash", {"cmd": "pytest"}, is_error=False), {"role": "user", "content": [{"type": "text", "text": "try again"}]}, ] - assert detect_stalled_task(messages, window=6, repeat_threshold=3) is True + assert detect_stalled_task(messages, window=6, repeat_threshold=3) == "repeated_tool_call" def test_a_recovered_task_is_not_stalled_while_its_old_failures_sit_in_the_window(self): """The three identical failures stay in the window for a few turns after the model @@ -120,7 +120,7 @@ class TestDetectStalledTask: *_anthropic_call("t4", "read_file", {"path": "conftest.py"}, is_error=False), *_anthropic_call("t5", "edit_file", {"path": "conftest.py"}, is_error=False), ] - assert detect_stalled_task(messages, window=6, repeat_threshold=3) is False + assert detect_stalled_task(messages, window=6, repeat_threshold=3) is None def test_a_retry_loop_broken_up_by_an_unrelated_call_still_counts(self): """Anchoring on the newest call must not require the repeats to be adjacent: a model @@ -131,7 +131,7 @@ class TestDetectStalledTask: *_anthropic_call("t3", "bash", {"cmd": "pytest"}, is_error=True), *_anthropic_call("t4", "bash", {"cmd": "pytest"}, is_error=True), ] - assert detect_stalled_task(messages, window=6, repeat_threshold=3) is True + assert detect_stalled_task(messages, window=6, repeat_threshold=3) == "repeated_tool_call" def test_errors_only_count_while_the_newest_call_is_still_failing(self): messages = [ @@ -140,15 +140,25 @@ class TestDetectStalledTask: *_anthropic_call("t3", "bash", {"cmd": "pytest c"}, is_error=True), *_anthropic_call("t4", "bash", {"cmd": "pytest d"}, is_error=False), ] - assert detect_stalled_task(messages, window=6, repeat_threshold=3) is False + assert detect_stalled_task(messages, window=6, repeat_threshold=3) is None + + def test_a_repeating_call_that_also_errors_reports_the_repeat_branch(self): + """Both branches hold here. The repeat is the more specific fact and the one the + feature exists for, so a stuck model is never filed under 'the tool is broken'.""" + messages = [ + *_anthropic_call("t1", "bash", {"cmd": "make build"}, is_error=True), + *_anthropic_call("t2", "bash", {"cmd": "make build"}, is_error=True), + *_anthropic_call("t3", "bash", {"cmd": "make build"}, is_error=True), + ] + assert detect_stalled_task(messages, window=6, repeat_threshold=3) == "repeated_tool_call" def test_no_messages_is_not_stalled(self): - assert detect_stalled_task(None, window=6, repeat_threshold=3) is False - assert detect_stalled_task([], window=6, repeat_threshold=3) is False + assert detect_stalled_task(None, window=6, repeat_threshold=3) is None + assert detect_stalled_task([], window=6, repeat_threshold=3) is None def test_zero_threshold_never_flags_stalled(self): messages = [ *_anthropic_call("t1", "bash", {"cmd": "pytest"}, is_error=True), *_anthropic_call("t2", "bash", {"cmd": "pytest"}, is_error=True), ] - assert detect_stalled_task(messages, window=6, repeat_threshold=0) is False + assert detect_stalled_task(messages, window=6, repeat_threshold=0) is None diff --git a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/RoutingDecisionCard.test.tsx b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/RoutingDecisionCard.test.tsx index fd1777f802c..1c50c6ed5c9 100644 --- a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/RoutingDecisionCard.test.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/RoutingDecisionCard.test.tsx @@ -226,6 +226,53 @@ describe("RoutingDecisionCard", () => { expect(screen.getByText("Requested; already at the highest tier")).toBeInTheDocument(); }); + it("marks a stall escalation with the branch that fired and the tier it came from", () => { + render( + , + ); + expect(screen.getByText("Yes, from SIMPLE, the same tool call erroring")).toBeInTheDocument(); + }); + + it("tells a stall-escalated row apart from an ordinary one under redaction", () => { + // The signals entry that used to be the only marker is gone with redaction on, so + // without these fields this row is identical to a request classified at MEDIUM. + render( + , + ); + expect(screen.getByText("Yes, from SIMPLE, the same tool call repeating")).toBeInTheDocument(); + }); + + it("says a stall was detected even when there was no higher tier to move to", () => { + render( + , + ); + expect(screen.getByText("Detected, the same tool call repeating; already at the highest tier")).toBeInTheDocument(); + }); + + it("omits the stall row on a request the detector never flagged", () => { + render(); + expect(screen.queryByText("Stall escalated")).not.toBeInTheDocument(); + }); + it("does not claim the score chose the tier on a redacted override row", () => { // `signals` is gone under redaction; the cause alone must suppress the band. render(); diff --git a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/RoutingDecisionCard.tsx b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/RoutingDecisionCard.tsx index cf2c71e64c6..4f27e3802b7 100644 --- a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/RoutingDecisionCard.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/RoutingDecisionCard.tsx @@ -25,6 +25,9 @@ export interface RoutingDecision { escalation_keyword?: string; classifier_model?: string; escalated?: boolean; + stall_escalated?: boolean; + stall_escalation_reason?: string; + stall_escalation_original_tier?: string; tier_boundaries?: RoutingDecisionTierBoundaries; reasoning_override_min_score?: number; } @@ -141,6 +144,23 @@ function describeEscalation(escalated: boolean, keyword: string | undefined): st return keyword ? `Requested via "${keyword}"; already at the highest tier` : "Requested; already at the highest tier"; } +const STALL_REASON_LABELS: Record = { + repeated_tool_call: "the same tool call repeating", + repeated_tool_error: "the same tool call erroring", +}; + +/** + * A stall can be detected on a request already at the highest configured tier, which has + * nowhere to go. That row still has to say the detector fired, so an operator reading a bill + * can tell a real bump from a no-op. Only called when a stall was detected. + */ +function describeStallEscalation(escalated: boolean, reason: string | undefined, originalTier: string | undefined) { + const named = reason ? STALL_REASON_LABELS[reason] ?? reason : undefined; + const because = named ? `, ${named}` : ""; + if (!escalated) return `Detected${because}; already at the highest tier`; + return originalTier ? `Yes, from ${originalTier}${because}` : `Yes${because}`; +} + function Row({ label, children }: { label: string; children: React.ReactNode }) { return (
@@ -170,6 +190,9 @@ export function RoutingDecisionCard({ signals, escalated, escalation_keyword: escalationKeyword, + stall_escalated: stallEscalated, + stall_escalation_reason: stallReason, + stall_escalation_original_tier: stallOriginalTier, tier_boundaries: tierBoundaries, } = decision; @@ -220,6 +243,12 @@ export function RoutingDecisionCard({ {escalated !== undefined && {describeEscalation(escalated, escalationKeyword)}} + {stallEscalated !== undefined && ( + + {describeStallEscalation(stallEscalated, stallReason, stallOriginalTier)} + + )} + {signals && signals.length > 0 && ( diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 6d62ce2b675..53a720c9419 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -36452,6 +36452,15 @@ export interface components { score?: number; /** Signals */ signals?: string[]; + /** Stall Escalated */ + stall_escalated?: boolean; + /** Stall Escalation Original Tier */ + stall_escalation_original_tier?: string; + /** + * Stall Escalation Reason + * @enum {string} + */ + stall_escalation_reason?: "repeated_tool_call" | "repeated_tool_error"; /** Tier */ tier?: string; tier_boundaries?: components["schemas"]["StandardLoggingRoutingDecisionTierBoundaries"];