feat: record stall escalation on the routing decision

Experiment arm: control

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
moe-berri 2026-09-09 21:23:37 -07:00
parent fbed17d567
commit a467ba074f
11 changed files with 399 additions and 35 deletions

View file

@ -79,11 +79,20 @@ _ROUTER_SPAN_FIELDS: Final[Mapping[str, str]] = MappingProxyType(
"cause": "router_cause",
"score": "router_score",
"escalated": "router_escalated",
"stall_escalated": "router_stall_escalated",
"stall_escalation_reason": "router_stall_reason",
"stall_escalation_original_tier": "router_stall_original_tier",
"signals": "router_signals",
"routed_model": "routed_model",
}
)
_ROUTER_DIMENSIONS: Final[tuple[str, ...]] = ("router_tier", "router_cause", "router_escalated", "routed_model")
_ROUTER_DIMENSIONS: Final[tuple[str, ...]] = (
"router_tier",
"router_cause",
"router_escalated",
"router_stall_reason",
"routed_model",
)
_COST_DIMENSIONS: Final[tuple[str, ...]] = ("team", "user", "key_alias", "model_group", *_ROUTER_DIMENSIONS)
@ -99,7 +108,11 @@ def _router_span_fields(
routing_decision: Final = _mapping_field(_metadata_of(standard_logging_payload), "routing_decision")
if not routing_decision:
return _EMPTY_MAPPING
escalated: Final = bool(routing_decision.get("escalated") or routing_decision.get("context_escalated"))
escalated: Final = bool(
routing_decision.get("escalated")
or routing_decision.get("context_escalated")
or routing_decision.get("stall_escalated")
)
return MappingProxyType(
{
_ROUTER_SPAN_FIELDS[record_field]: value

View file

@ -322,9 +322,22 @@ There is no state to expire or leak: detection reruns on every classified turn f
request's own message list, so the bump lasts only as long as the recent tool calls still look
stuck and lifts on its own the moment they don't. This also means it reads the whole
conversation rather than only the turns since the newest human ask, so a plain follow-up like
"try again" does not discard evidence from before it. Escalation records `stall_escalation` in
`routing_decision.signals`; unlike `escalation_keywords`, it does not set the
`escalated`/`escalation_keyword` pair, which is reserved for the keyword mechanism specifically.
"try again" does not discard evidence from before it.
Escalation records three fields of its own on the routing decision: `stall_escalated`,
`stall_escalation_reason` (`repeated_tool_call` or `repeated_tool_error`), and
`stall_escalation_original_tier`, the tier the request was bumped off. They are derived rather
than prompt-quoting, so they survive message redaction, unlike the `stall_escalation` entry in
`routing_decision.signals` that also marks the bump. Datadog LLM Observability carries them as
`router_stall_escalated` / `router_stall_reason` / `router_stall_original_tier`, with
`router_stall_reason` a cost dimension and `router_escalated` true, so escalated spend is a slice
you can chart and the two branches can be told apart. The reason is worth splitting out because a
repeat loop is a model that is genuinely stuck, which a stronger model can break, while repeated
tool errors are usually a broken tool or bad credentials, where the stronger model fails the same
way at a higher price. A stall detected on a request already at the highest configured tier
records the reason with `stall_escalated: false` and no original tier, since nothing moved.
Unlike `escalation_keywords`, it does not set the `escalated`/`escalation_keyword` pair, which is
reserved for the keyword mechanism specifically.
`stall_escalation_enabled` cannot be combined with `session_affinity` or
`classification_mode: user_turn`: both replay a held routing decision on most turns instead of

View file

@ -61,6 +61,7 @@ from litellm.types.utils import (
AUTOROUTER_CLASSIFIER_CALL_ORIGIN,
ModelResponse,
RoutingDecisionCause,
StallEscalationReason,
StandardLoggingRoutingDecision,
StandardLoggingRoutingDecisionTierBoundaries,
)
@ -1547,6 +1548,8 @@ class ComplexityRouter(CustomLogger):
conversation_continuing: bool = True,
tier_litellm_params: Mapping[str, object] | None = None,
context_escalation_original_tier: ComplexityTier | str | None = None,
stall_escalation_reason: StallEscalationReason | None = None,
stall_escalation_original_tier: ComplexityTier | str | None = None,
) -> StandardLoggingRoutingDecision:
"""Assemble the per-request provenance record for this router's decision.
@ -1602,6 +1605,14 @@ class ComplexityRouter(CustomLogger):
# (classifier, keyword rule, or session pin) had placed it before physics did.
decision["context_escalated"] = True
decision["context_escalation_original_tier"] = _tier_name(context_escalation_original_tier)
if stall_escalation_reason is not None:
# Same two facts the keyword pair records: the detector fired, and whether the tier
# actually moved. A stall detected at the highest configured tier has nowhere to go,
# so it reports the reason with stall_escalated False rather than claiming a bump.
decision["stall_escalated"] = stall_escalation_original_tier is not None
decision["stall_escalation_reason"] = stall_escalation_reason
if stall_escalation_original_tier is not None:
decision["stall_escalation_original_tier"] = _tier_name(stall_escalation_original_tier)
if tier_litellm_params:
masked_tier_litellm_params: Final = mask_credentials_in_payload(tier_litellm_params)
if isinstance(masked_tier_litellm_params, Mapping):
@ -2876,6 +2887,10 @@ class ComplexityRouter(CustomLogger):
context_escalation_original_tier=(
decision.get("context_escalation_original_tier") if decision is not None else None
),
stall_escalation_reason=decision.get("stall_escalation_reason") if decision is not None else None,
stall_escalation_original_tier=(
decision.get("stall_escalation_original_tier") if decision is not None else None
),
)
from litellm.types.router import PreRoutingHookResponse as HookResponse
@ -3041,6 +3056,8 @@ class ComplexityRouter(CustomLogger):
conversation_continuing=bool(decision.get("conversation_continuing", True)),
tier_litellm_params=self._litellm_params_for_model(decided_tier, new_model),
context_escalation_original_tier=decision.get("context_escalation_original_tier"),
stall_escalation_reason=decision.get("stall_escalation_reason"),
stall_escalation_original_tier=decision.get("stall_escalation_original_tier"),
)
return response.model_copy(
update={ # mutable-ok: model_copy types update as a plain dict
@ -3596,10 +3613,14 @@ class ComplexityRouter(CustomLogger):
# Resolved here rather than beside the classifier because the keyword-override path below
# returns before any classification runs, and a forced tier gets stuck for the same reason
# a classified one does.
stalled: Final = self.config.stall_escalation_enabled and detect_stalled_task(
resolved_messages,
window=self.config.stall_escalation_window,
repeat_threshold=self.config.stall_escalation_repeat_threshold,
stalled: Final = (
detect_stalled_task(
resolved_messages,
window=self.config.stall_escalation_window,
repeat_threshold=self.config.stall_escalation_repeat_threshold,
)
if self.config.stall_escalation_enabled
else None
)
plan_mode_sentinel: Final = self._matched_plan_mode_signal(request_kwargs, resolved_messages)
@ -3636,7 +3657,10 @@ class ComplexityRouter(CustomLogger):
keyword_bumped_tier: Final = (
self._escalate_tier(override.tier) if escalation_keyword is not None else override.tier
)
escalated_tier: Final = self._escalate_tier(keyword_bumped_tier) if stalled else keyword_bumped_tier
escalated_tier: Final = (
self._escalate_tier(keyword_bumped_tier) if stalled is not None else keyword_bumped_tier
)
keyword_stall_original_tier: Final = keyword_bumped_tier if escalated_tier != keyword_bumped_tier else None
keyword_escalated: Final = keyword_bumped_tier != override.tier
routed_tier: Final = (
self._apply_plan_mode_floor(escalated_tier) if plan_floor is not None else escalated_tier
@ -3665,11 +3689,13 @@ class ComplexityRouter(CustomLogger):
conversation_continuing=conversation_continuing,
cause=keyword_cause,
tier=routed_tier,
signals=("stall_escalation",) if stalled else None,
signals=("stall_escalation",) if stalled is not None else None,
matched_keyword=plan_mode_sentinel if keyword_plan_floored else override.matched_keyword,
escalation_keyword=escalation_keyword,
escalated=keyword_escalated,
tier_litellm_params=keyword_tier_litellm_params,
stall_escalation_reason=stalled,
stall_escalation_original_tier=keyword_stall_original_tier,
),
)
@ -3687,9 +3713,11 @@ class ComplexityRouter(CustomLogger):
escalated: Final = tier != classified_tier
if escalated:
signals = (*signals, "escalation")
if stalled:
pre_stall_tier: Final = tier
if stalled is not None:
tier = self._escalate_tier(tier)
signals = (*signals, "stall_escalation")
stall_original_tier: Final = pre_stall_tier if tier != pre_stall_tier else None
pre_floor_tier: Final = tier
if plan_floor is not None:
tier = self._apply_plan_mode_floor(tier)
@ -3832,5 +3860,7 @@ class ComplexityRouter(CustomLogger):
classifier_cost=outcome.classifier_cost,
tier_litellm_params=tier_litellm_params,
context_escalation_original_tier=context_original_tier,
stall_escalation_reason=stalled,
stall_escalation_original_tier=stall_original_tier,
),
)

View file

@ -2,7 +2,7 @@
Mid-task stall detection for the Complexity Router.
Reads the assistant's own recent tool calls, which every agentic client resends on each
turn, and reports whether the task currently looks stuck. No LLM call and no stored state:
turn, and reports which stuck pattern the task looks caught in, if any. No LLM call and no state:
the same window is rescanned per classified turn, so the verdict follows the conversation
rather than latching.
@ -20,6 +20,8 @@ from collections.abc import Iterator, Mapping, Sequence
from itertools import islice
from typing import Final, NamedTuple
from litellm.types.utils import StallEscalationReason
_ARGUMENTS_PARSE_FAILED: Final = object()
@ -94,8 +96,8 @@ def detect_stalled_task(
*,
window: int,
repeat_threshold: int,
) -> bool:
"""Whether the newest tool call is still part of a stuck pattern: it repeats, or it
) -> StallEscalationReason | None:
"""Which stuck pattern the newest tool call is still part of, or None: it repeats, or it
errored, at least repeat_threshold times across the last `window` calls.
Both tests are anchored on the newest call rather than counting whichever pattern is
@ -105,14 +107,16 @@ def detect_stalled_task(
matches, so a retry loop broken up by an unrelated lookup still reads as stuck.
"""
if not messages or repeat_threshold <= 0:
return False
return None
recent: Final = tuple(islice(_iter_tool_call_events_newest_first(messages), window))
if len(recent) < repeat_threshold:
return False
return None
newest: Final = recent[0]
repeats: Final = sum(1 for event in recent if event.signature == newest.signature)
if repeats >= repeat_threshold:
return True
return "repeated_tool_call"
if not newest.is_error:
return False
return sum(1 for event in recent if event.is_error) >= repeat_threshold
return None
if sum(1 for event in recent if event.is_error) >= repeat_threshold:
return "repeated_tool_error"
return None

View file

@ -8,6 +8,7 @@ from typing import (
Any,
Final,
Literal,
TypeAlias,
get_args,
)
@ -2948,6 +2949,12 @@ SHADOW_EVAL_JUDGE_CALL_ORIGIN: Final[InternalCallOrigin] = "shadow_eval_judge"
BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN: Final[InternalCallOrigin] = "background_response_cost_poll"
StallEscalationReason: TypeAlias = Literal["repeated_tool_call", "repeated_tool_error"]
"""Which stall branch fired. A repeat loop is a model that is stuck, which a stronger model can
break; repeated tool errors are usually a broken tool or bad credentials, where the stronger
model fails the same way at a higher price. Operationally different, so they are recorded apart."""
class StandardLoggingRoutingDecision(TypedDict, total=False):
"""Per-request provenance for a pre-routing strategy (auto-router) decision."""
@ -2967,6 +2974,9 @@ class StandardLoggingRoutingDecision(TypedDict, total=False):
escalated: bool
context_escalated: bool # writable-ok: Pydantic warns on ReadOnly TypedDict fields
context_escalation_original_tier: str # writable-ok: Pydantic warns on ReadOnly TypedDict fields
stall_escalated: bool # writable-ok: Pydantic warns on ReadOnly TypedDict fields
stall_escalation_reason: StallEscalationReason # writable-ok: Pydantic warns on ReadOnly TypedDict fields
stall_escalation_original_tier: str # writable-ok: Pydantic warns on ReadOnly TypedDict fields
tier_boundaries: StandardLoggingRoutingDecisionTierBoundaries
reasoning_override_min_score: float # writable-ok: Pydantic warns on ReadOnly TypedDict fields
conversation_continuing: bool
@ -2995,6 +3005,9 @@ DERIVED_ROUTING_DECISION_FIELDS: Final[frozenset[str]] = frozenset(
"escalated",
"context_escalated",
"context_escalation_original_tier",
"stall_escalated",
"stall_escalation_reason",
"stall_escalation_original_tier",
"tier_boundaries",
"reasoning_override_min_score",
"conversation_continuing",

View file

@ -398,6 +398,69 @@ def test_a_context_escalated_route_reports_as_escalated(logger: DataDogLLMObsLog
assert "router_escalated:true" in payload["tags"]
def test_a_stall_escalated_route_reports_as_escalated(logger: DataDogLLMObsLogger) -> None:
"""The stall detector bumped the tier, so the spend on this request is escalated spend and
has to land in the same slice as every other escalation."""
payload = build(logger, metadata={"routing_decision": {"tier": "premium", "stall_escalated": True}})
assert payload["meta"]["metadata"]["router_escalated"] is True
assert "router_escalated:true" in payload["tags"]
def test_the_stall_branch_is_its_own_cost_dimension(logger: DataDogLLMObsLogger) -> None:
"""Charting the two branches apart is what tells an operator whether the bumps are buying
anything: a stuck model can be unstuck by a stronger one, a broken tool cannot."""
payload = build(
logger,
metadata={
"routing_decision": {
"tier": "premium",
"stall_escalated": True,
"stall_escalation_reason": "repeated_tool_error",
"stall_escalation_original_tier": "SIMPLE",
}
},
)
assert payload["meta"]["metadata"]["router_stall_reason"] == "repeated_tool_error"
assert payload["meta"]["metadata"]["router_stall_original_tier"] == "SIMPLE"
assert "router_stall_reason:repeated_tool_error" in payload["tags"]
assert "router_stall_reason" in payload["meta"]["metadata"]["_dd"]["cost_tags"]
def test_stall_fields_survive_prompt_redaction() -> None:
"""They aggregate the prompt rather than quoting it, unlike the signals entry that used to be
the only marker, so a redacted span still says the request was escalated and why."""
with patch.dict(os.environ, {"DD_API_KEY": "k", "DD_SITE": "us5.datadoghq.com"}, clear=True):
with patch("asyncio.create_task"):
redacted_logger = DataDogLLMObsLogger(turn_off_message_logging=True)
result = json.loads(
safe_dumps(
redacted_logger.create_llm_obs_payload(
build_payload(
metadata={
"routing_decision": {
"tier": "premium",
"signals": ["secret prompt text", "stall_escalation"],
"stall_escalated": True,
"stall_escalation_reason": "repeated_tool_call",
"stall_escalation_original_tier": "SIMPLE",
}
}
),
datetime(2026, 9, 1, 12, 0, 0),
datetime(2026, 9, 1, 12, 0, 2),
)
)
)
assert result["meta"]["metadata"]["router_stall_escalated"] is True
assert result["meta"]["metadata"]["router_stall_reason"] == "repeated_tool_call"
assert result["meta"]["metadata"]["router_stall_original_tier"] == "SIMPLE"
assert result["meta"]["metadata"]["router_escalated"] is True
assert "router_signals" not in result["meta"]["metadata"]
def test_a_routed_request_that_did_not_escalate_reports_false(logger: DataDogLLMObsLogger) -> None:
"""Without this the escalation dimension is absent on ordinary traffic, so nothing can group by it."""
payload = build(logger, metadata={"routing_decision": {"tier": "simple", "cause": "heuristic_scorer"}})

View file

@ -6809,6 +6809,139 @@ class TestStallEscalation:
result = await router.async_pre_routing_hook(model="test-model", request_kwargs={}, messages=messages)
assert "stall_escalation" in result.routing_decision["signals"]
@pytest.mark.asyncio
async def test_the_escalation_is_recorded_as_its_own_fields(self, mock_router_instance, basic_config):
"""`signals` is prompt-quoting and redaction drops it, so a stall recorded only there is
invisible to an operator running with message logging off. These fields aggregate the
prompt rather than quoting it, so they survive and can carry a metric dimension."""
router = ComplexityRouter(
model_name="test-router",
litellm_router_instance=mock_router_instance,
complexity_router_config={**basic_config, "stall_escalation_enabled": True},
)
messages = [*_stalled_tool_history(), {"role": "user", "content": "Hello there!"}]
result = await router.async_pre_routing_hook(model="test-model", request_kwargs={}, messages=messages)
assert result.routing_decision["stall_escalated"] is True
assert result.routing_decision["stall_escalation_original_tier"] == "SIMPLE"
assert result.routing_decision["tier"] == "MEDIUM"
@pytest.mark.asyncio
async def test_the_two_detector_branches_are_told_apart(self, mock_router_instance, basic_config):
"""A repeat loop is a stuck model, which is what the bump is for. Repeated tool errors are
usually a broken tool, where the stronger model fails the same way at a higher price."""
router = ComplexityRouter(
model_name="test-router",
litellm_router_instance=mock_router_instance,
complexity_router_config={**basic_config, "stall_escalation_enabled": True},
)
erroring = [
turn
for i, cmd in enumerate(("pytest a", "pytest b", "pytest c"))
for turn in (
{
"role": "assistant",
"content": [{"type": "tool_use", "id": f"e-{i}", "name": "bash", "input": {"cmd": cmd}}],
},
{
"role": "user",
"content": [
{"type": "tool_result", "tool_use_id": f"e-{i}", "is_error": True, "content": "fail"}
],
},
)
]
repeating = await router.async_pre_routing_hook(
model="test-model",
request_kwargs={},
messages=[*_stalled_tool_history(), {"role": "user", "content": "Hello there!"}],
)
erroring_result = await router.async_pre_routing_hook(
model="test-model",
request_kwargs={},
messages=[*erroring, {"role": "user", "content": "Hello there!"}],
)
assert repeating.routing_decision["stall_escalation_reason"] == "repeated_tool_call"
assert erroring_result.routing_decision["stall_escalation_reason"] == "repeated_tool_error"
@pytest.mark.asyncio
async def test_an_unstalled_request_carries_no_stall_fields(self, mock_router_instance, basic_config):
"""The fields must distinguish an escalated request from one the classifier placed at the
same tier on its own, so they cannot appear on ordinary traffic."""
router = ComplexityRouter(
model_name="test-router",
litellm_router_instance=mock_router_instance,
complexity_router_config={**basic_config, "stall_escalation_enabled": True},
)
result = await router.async_pre_routing_hook(
model="test-model", request_kwargs={}, messages=[{"role": "user", "content": "Hello there!"}]
)
assert "stall_escalated" not in result.routing_decision
assert "stall_escalation_reason" not in result.routing_decision
assert "stall_escalation_original_tier" not in result.routing_decision
@pytest.mark.asyncio
async def test_a_stall_with_nowhere_to_go_records_the_detection_without_a_bump(
self, mock_router_instance, basic_config
):
"""Already at the highest configured tier, so no tier moved and no extra money was spent.
Reporting stall_escalated True there would overstate what the feature costs."""
router = ComplexityRouter(
model_name="test-router",
litellm_router_instance=mock_router_instance,
complexity_router_config={**basic_config, "stall_escalation_enabled": True},
)
messages = [
*_stalled_tool_history(),
{"role": "user", "content": "Let's think step by step and reason through this carefully."},
]
result = await router.async_pre_routing_hook(model="test-model", request_kwargs={}, messages=messages)
assert result.routing_decision["stall_escalated"] is False
assert result.routing_decision["stall_escalation_reason"] == "repeated_tool_call"
assert "stall_escalation_original_tier" not in result.routing_decision
@pytest.mark.asyncio
async def test_the_fields_survive_message_redaction(self, mock_router_instance, basic_config):
"""The whole point: with redaction on, `signals` goes and these have to stay, otherwise a
stall-escalated row is identical to one the classifier placed at that tier itself."""
router = ComplexityRouter(
model_name="test-router",
litellm_router_instance=mock_router_instance,
complexity_router_config={**basic_config, "stall_escalation_enabled": True},
)
request_kwargs: Dict = {"metadata": {"headers": {"x-litellm-enable-message-redaction": True}}}
result = await router.async_pre_routing_hook(
model="test-model",
request_kwargs=request_kwargs,
messages=[*_stalled_tool_history(), {"role": "user", "content": "Hello there!"}],
)
redacted = litellm.Router._redact_prompt_text_if_needed(request_kwargs, result.routing_decision)
assert "signals" not in redacted
assert redacted["stall_escalated"] is True
assert redacted["stall_escalation_reason"] == "repeated_tool_call"
assert redacted["stall_escalation_original_tier"] == "SIMPLE"
@pytest.mark.asyncio
async def test_a_keyword_forced_tier_records_the_stall_fields_too(self, mock_router_instance, basic_config):
"""The keyword-override path returns before classification, so it assembles its own
decision record and would otherwise drop the fields the classified path sets."""
router = ComplexityRouter(
model_name="test-router",
litellm_router_instance=mock_router_instance,
complexity_router_config={
**basic_config,
"stall_escalation_enabled": True,
"keyword_tier_rules": [{"keywords": ["billing"], "tier": "SIMPLE"}],
},
)
result = await router.async_pre_routing_hook(
model="test-model",
request_kwargs={},
messages=[*_stalled_tool_history(), {"role": "user", "content": "a billing question"}],
)
assert result.routing_decision["stall_escalated"] is True
assert result.routing_decision["stall_escalation_reason"] == "repeated_tool_call"
assert result.routing_decision["stall_escalation_original_tier"] == "SIMPLE"
@pytest.mark.asyncio
async def test_stall_escalation_caps_at_highest_tier(self, mock_router_instance, basic_config):
router = ComplexityRouter(

View file

@ -35,7 +35,7 @@ class TestDetectStalledTask:
*_anthropic_call("t2", "bash", {"cmd": "pytest"}, is_error=False),
*_anthropic_call("t3", "bash", {"cmd": "pytest"}, is_error=False),
]
assert detect_stalled_task(messages, window=6, repeat_threshold=3) is True
assert detect_stalled_task(messages, window=6, repeat_threshold=3) == "repeated_tool_call"
def test_repeated_errors_are_stalled_even_with_varied_arguments(self):
messages = [
@ -43,7 +43,7 @@ class TestDetectStalledTask:
*_anthropic_call("t2", "bash", {"cmd": "pytest tests/b.py"}, is_error=True),
*_anthropic_call("t3", "bash", {"cmd": "pytest tests/c.py"}, is_error=True),
]
assert detect_stalled_task(messages, window=6, repeat_threshold=3) is True
assert detect_stalled_task(messages, window=6, repeat_threshold=3) == "repeated_tool_error"
def test_varied_successful_calls_are_not_stalled(self):
messages = [
@ -51,7 +51,7 @@ class TestDetectStalledTask:
*_anthropic_call("t2", "bash", {"cmd": "pytest"}, is_error=False),
*_anthropic_call("t3", "grep", {"pattern": "x"}, is_error=False),
]
assert detect_stalled_task(messages, window=6, repeat_threshold=3) is False
assert detect_stalled_task(messages, window=6, repeat_threshold=3) is None
def test_chat_completions_repeats_are_stalled(self):
messages = [
@ -59,7 +59,7 @@ class TestDetectStalledTask:
*_chat_completions_call("c2", "bash", '{"cmd": "pytest"}'),
*_chat_completions_call("c3", "bash", '{"cmd": "pytest"}'),
]
assert detect_stalled_task(messages, window=6, repeat_threshold=3) is True
assert detect_stalled_task(messages, window=6, repeat_threshold=3) == "repeated_tool_call"
def test_chat_completions_has_no_structured_error_signal(self):
"""A chat-completions tool message carries no standard error flag, so varied calls
@ -69,7 +69,7 @@ class TestDetectStalledTask:
*_chat_completions_call("c2", "bash", '{"cmd": "b"}'),
*_chat_completions_call("c3", "bash", '{"cmd": "c"}'),
]
assert detect_stalled_task(messages, window=6, repeat_threshold=3) is False
assert detect_stalled_task(messages, window=6, repeat_threshold=3) is None
def test_dict_and_json_string_arguments_compare_equal_across_surfaces(self):
messages = [
@ -77,14 +77,14 @@ class TestDetectStalledTask:
*_chat_completions_call("c2", "bash", '{"cmd": "pytest"}'),
*_anthropic_call("t3", "bash", {"cmd": "pytest"}, is_error=False),
]
assert detect_stalled_task(messages, window=6, repeat_threshold=3) is True
assert detect_stalled_task(messages, window=6, repeat_threshold=3) == "repeated_tool_call"
def test_below_repeat_threshold_is_not_stalled(self):
messages = [
*_anthropic_call("t1", "bash", {"cmd": "pytest"}, is_error=False),
*_anthropic_call("t2", "bash", {"cmd": "pytest"}, is_error=False),
]
assert detect_stalled_task(messages, window=6, repeat_threshold=3) is False
assert detect_stalled_task(messages, window=6, repeat_threshold=3) is None
def test_evidence_older_than_the_window_does_not_count(self):
"""Only the most recent `window` tool calls are considered, so a stall the model
@ -96,7 +96,7 @@ class TestDetectStalledTask:
*_anthropic_call("t4", "grep", {"pattern": "a"}, is_error=False),
*_anthropic_call("t5", "grep", {"pattern": "b"}, is_error=False),
]
assert detect_stalled_task(messages, window=2, repeat_threshold=2) is False
assert detect_stalled_task(messages, window=2, repeat_threshold=2) is None
def test_evidence_survives_a_new_human_ask(self):
"""A follow-up like 'try again' must not erase evidence from before it: detection
@ -107,7 +107,7 @@ class TestDetectStalledTask:
*_anthropic_call("t3", "bash", {"cmd": "pytest"}, is_error=False),
{"role": "user", "content": [{"type": "text", "text": "try again"}]},
]
assert detect_stalled_task(messages, window=6, repeat_threshold=3) is True
assert detect_stalled_task(messages, window=6, repeat_threshold=3) == "repeated_tool_call"
def test_a_recovered_task_is_not_stalled_while_its_old_failures_sit_in_the_window(self):
"""The three identical failures stay in the window for a few turns after the model
@ -120,7 +120,7 @@ class TestDetectStalledTask:
*_anthropic_call("t4", "read_file", {"path": "conftest.py"}, is_error=False),
*_anthropic_call("t5", "edit_file", {"path": "conftest.py"}, is_error=False),
]
assert detect_stalled_task(messages, window=6, repeat_threshold=3) is False
assert detect_stalled_task(messages, window=6, repeat_threshold=3) is None
def test_a_retry_loop_broken_up_by_an_unrelated_call_still_counts(self):
"""Anchoring on the newest call must not require the repeats to be adjacent: a model
@ -131,7 +131,7 @@ class TestDetectStalledTask:
*_anthropic_call("t3", "bash", {"cmd": "pytest"}, is_error=True),
*_anthropic_call("t4", "bash", {"cmd": "pytest"}, is_error=True),
]
assert detect_stalled_task(messages, window=6, repeat_threshold=3) is True
assert detect_stalled_task(messages, window=6, repeat_threshold=3) == "repeated_tool_call"
def test_errors_only_count_while_the_newest_call_is_still_failing(self):
messages = [
@ -140,15 +140,25 @@ class TestDetectStalledTask:
*_anthropic_call("t3", "bash", {"cmd": "pytest c"}, is_error=True),
*_anthropic_call("t4", "bash", {"cmd": "pytest d"}, is_error=False),
]
assert detect_stalled_task(messages, window=6, repeat_threshold=3) is False
assert detect_stalled_task(messages, window=6, repeat_threshold=3) is None
def test_a_repeating_call_that_also_errors_reports_the_repeat_branch(self):
"""Both branches hold here. The repeat is the more specific fact and the one the
feature exists for, so a stuck model is never filed under 'the tool is broken'."""
messages = [
*_anthropic_call("t1", "bash", {"cmd": "make build"}, is_error=True),
*_anthropic_call("t2", "bash", {"cmd": "make build"}, is_error=True),
*_anthropic_call("t3", "bash", {"cmd": "make build"}, is_error=True),
]
assert detect_stalled_task(messages, window=6, repeat_threshold=3) == "repeated_tool_call"
def test_no_messages_is_not_stalled(self):
assert detect_stalled_task(None, window=6, repeat_threshold=3) is False
assert detect_stalled_task([], window=6, repeat_threshold=3) is False
assert detect_stalled_task(None, window=6, repeat_threshold=3) is None
assert detect_stalled_task([], window=6, repeat_threshold=3) is None
def test_zero_threshold_never_flags_stalled(self):
messages = [
*_anthropic_call("t1", "bash", {"cmd": "pytest"}, is_error=True),
*_anthropic_call("t2", "bash", {"cmd": "pytest"}, is_error=True),
]
assert detect_stalled_task(messages, window=6, repeat_threshold=0) is False
assert detect_stalled_task(messages, window=6, repeat_threshold=0) is None

View file

@ -226,6 +226,53 @@ describe("RoutingDecisionCard", () => {
expect(screen.getByText("Requested; already at the highest tier")).toBeInTheDocument();
});
it("marks a stall escalation with the branch that fired and the tier it came from", () => {
render(
<RoutingDecisionCard
decision={{
...heuristic,
tier: "MEDIUM",
stall_escalated: true,
stall_escalation_reason: "repeated_tool_error",
stall_escalation_original_tier: "SIMPLE",
}}
/>,
);
expect(screen.getByText("Yes, from SIMPLE, the same tool call erroring")).toBeInTheDocument();
});
it("tells a stall-escalated row apart from an ordinary one under redaction", () => {
// The signals entry that used to be the only marker is gone with redaction on, so
// without these fields this row is identical to a request classified at MEDIUM.
render(
<RoutingDecisionCard
decision={{
...heuristic,
tier: "MEDIUM",
signals: undefined,
stall_escalated: true,
stall_escalation_reason: "repeated_tool_call",
stall_escalation_original_tier: "SIMPLE",
}}
/>,
);
expect(screen.getByText("Yes, from SIMPLE, the same tool call repeating")).toBeInTheDocument();
});
it("says a stall was detected even when there was no higher tier to move to", () => {
render(
<RoutingDecisionCard
decision={{ ...heuristic, stall_escalated: false, stall_escalation_reason: "repeated_tool_call" }}
/>,
);
expect(screen.getByText("Detected, the same tool call repeating; already at the highest tier")).toBeInTheDocument();
});
it("omits the stall row on a request the detector never flagged", () => {
render(<RoutingDecisionCard decision={heuristic} />);
expect(screen.queryByText("Stall escalated")).not.toBeInTheDocument();
});
it("does not claim the score chose the tier on a redacted override row", () => {
// `signals` is gone under redaction; the cause alone must suppress the band.
render(<RoutingDecisionCard decision={{ ...heuristic, cause: "reasoning_override", signals: undefined }} />);

View file

@ -25,6 +25,9 @@ export interface RoutingDecision {
escalation_keyword?: string;
classifier_model?: string;
escalated?: boolean;
stall_escalated?: boolean;
stall_escalation_reason?: string;
stall_escalation_original_tier?: string;
tier_boundaries?: RoutingDecisionTierBoundaries;
reasoning_override_min_score?: number;
}
@ -141,6 +144,23 @@ function describeEscalation(escalated: boolean, keyword: string | undefined): st
return keyword ? `Requested via "${keyword}"; already at the highest tier` : "Requested; already at the highest tier";
}
const STALL_REASON_LABELS: Record<string, string> = {
repeated_tool_call: "the same tool call repeating",
repeated_tool_error: "the same tool call erroring",
};
/**
* A stall can be detected on a request already at the highest configured tier, which has
* nowhere to go. That row still has to say the detector fired, so an operator reading a bill
* can tell a real bump from a no-op. Only called when a stall was detected.
*/
function describeStallEscalation(escalated: boolean, reason: string | undefined, originalTier: string | undefined) {
const named = reason ? STALL_REASON_LABELS[reason] ?? reason : undefined;
const because = named ? `, ${named}` : "";
if (!escalated) return `Detected${because}; already at the highest tier`;
return originalTier ? `Yes, from ${originalTier}${because}` : `Yes${because}`;
}
function Row({ label, children }: { label: string; children: React.ReactNode }) {
return (
<div className="flex gap-3 py-1 text-sm">
@ -170,6 +190,9 @@ export function RoutingDecisionCard({
signals,
escalated,
escalation_keyword: escalationKeyword,
stall_escalated: stallEscalated,
stall_escalation_reason: stallReason,
stall_escalation_original_tier: stallOriginalTier,
tier_boundaries: tierBoundaries,
} = decision;
@ -220,6 +243,12 @@ export function RoutingDecisionCard({
{escalated !== undefined && <Row label="Escalated">{describeEscalation(escalated, escalationKeyword)}</Row>}
{stallEscalated !== undefined && (
<Row label="Stall escalated">
{describeStallEscalation(stallEscalated, stallReason, stallOriginalTier)}
</Row>
)}
{signals && signals.length > 0 && (
<Row label="Signals">
<span className="flex flex-wrap gap-1">

View file

@ -36452,6 +36452,15 @@ export interface components {
score?: number;
/** Signals */
signals?: string[];
/** Stall Escalated */
stall_escalated?: boolean;
/** Stall Escalation Original Tier */
stall_escalation_original_tier?: string;
/**
* Stall Escalation Reason
* @enum {string}
*/
stall_escalation_reason?: "repeated_tool_call" | "repeated_tool_error";
/** Tier */
tier?: string;
tier_boundaries?: components["schemas"]["StandardLoggingRoutingDecisionTierBoundaries"];