mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
feat: record stall escalation on the routing decision
Experiment arm: control Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
parent
fbed17d567
commit
a467ba074f
11 changed files with 399 additions and 35 deletions
|
|
@ -79,11 +79,20 @@ _ROUTER_SPAN_FIELDS: Final[Mapping[str, str]] = MappingProxyType(
|
|||
"cause": "router_cause",
|
||||
"score": "router_score",
|
||||
"escalated": "router_escalated",
|
||||
"stall_escalated": "router_stall_escalated",
|
||||
"stall_escalation_reason": "router_stall_reason",
|
||||
"stall_escalation_original_tier": "router_stall_original_tier",
|
||||
"signals": "router_signals",
|
||||
"routed_model": "routed_model",
|
||||
}
|
||||
)
|
||||
_ROUTER_DIMENSIONS: Final[tuple[str, ...]] = ("router_tier", "router_cause", "router_escalated", "routed_model")
|
||||
_ROUTER_DIMENSIONS: Final[tuple[str, ...]] = (
|
||||
"router_tier",
|
||||
"router_cause",
|
||||
"router_escalated",
|
||||
"router_stall_reason",
|
||||
"routed_model",
|
||||
)
|
||||
_COST_DIMENSIONS: Final[tuple[str, ...]] = ("team", "user", "key_alias", "model_group", *_ROUTER_DIMENSIONS)
|
||||
|
||||
|
||||
|
|
@ -99,7 +108,11 @@ def _router_span_fields(
|
|||
routing_decision: Final = _mapping_field(_metadata_of(standard_logging_payload), "routing_decision")
|
||||
if not routing_decision:
|
||||
return _EMPTY_MAPPING
|
||||
escalated: Final = bool(routing_decision.get("escalated") or routing_decision.get("context_escalated"))
|
||||
escalated: Final = bool(
|
||||
routing_decision.get("escalated")
|
||||
or routing_decision.get("context_escalated")
|
||||
or routing_decision.get("stall_escalated")
|
||||
)
|
||||
return MappingProxyType(
|
||||
{
|
||||
_ROUTER_SPAN_FIELDS[record_field]: value
|
||||
|
|
|
|||
|
|
@ -322,9 +322,22 @@ There is no state to expire or leak: detection reruns on every classified turn f
|
|||
request's own message list, so the bump lasts only as long as the recent tool calls still look
|
||||
stuck and lifts on its own the moment they don't. This also means it reads the whole
|
||||
conversation rather than only the turns since the newest human ask, so a plain follow-up like
|
||||
"try again" does not discard evidence from before it. Escalation records `stall_escalation` in
|
||||
`routing_decision.signals`; unlike `escalation_keywords`, it does not set the
|
||||
`escalated`/`escalation_keyword` pair, which is reserved for the keyword mechanism specifically.
|
||||
"try again" does not discard evidence from before it.
|
||||
|
||||
Escalation records three fields of its own on the routing decision: `stall_escalated`,
|
||||
`stall_escalation_reason` (`repeated_tool_call` or `repeated_tool_error`), and
|
||||
`stall_escalation_original_tier`, the tier the request was bumped off. They are derived rather
|
||||
than prompt-quoting, so they survive message redaction, unlike the `stall_escalation` entry in
|
||||
`routing_decision.signals` that also marks the bump. Datadog LLM Observability carries them as
|
||||
`router_stall_escalated` / `router_stall_reason` / `router_stall_original_tier`, with
|
||||
`router_stall_reason` a cost dimension and `router_escalated` true, so escalated spend is a slice
|
||||
you can chart and the two branches can be told apart. The reason is worth splitting out because a
|
||||
repeat loop is a model that is genuinely stuck, which a stronger model can break, while repeated
|
||||
tool errors are usually a broken tool or bad credentials, where the stronger model fails the same
|
||||
way at a higher price. A stall detected on a request already at the highest configured tier
|
||||
records the reason with `stall_escalated: false` and no original tier, since nothing moved.
|
||||
Unlike `escalation_keywords`, it does not set the `escalated`/`escalation_keyword` pair, which is
|
||||
reserved for the keyword mechanism specifically.
|
||||
|
||||
`stall_escalation_enabled` cannot be combined with `session_affinity` or
|
||||
`classification_mode: user_turn`: both replay a held routing decision on most turns instead of
|
||||
|
|
|
|||
|
|
@ -61,6 +61,7 @@ from litellm.types.utils import (
|
|||
AUTOROUTER_CLASSIFIER_CALL_ORIGIN,
|
||||
ModelResponse,
|
||||
RoutingDecisionCause,
|
||||
StallEscalationReason,
|
||||
StandardLoggingRoutingDecision,
|
||||
StandardLoggingRoutingDecisionTierBoundaries,
|
||||
)
|
||||
|
|
@ -1547,6 +1548,8 @@ class ComplexityRouter(CustomLogger):
|
|||
conversation_continuing: bool = True,
|
||||
tier_litellm_params: Mapping[str, object] | None = None,
|
||||
context_escalation_original_tier: ComplexityTier | str | None = None,
|
||||
stall_escalation_reason: StallEscalationReason | None = None,
|
||||
stall_escalation_original_tier: ComplexityTier | str | None = None,
|
||||
) -> StandardLoggingRoutingDecision:
|
||||
"""Assemble the per-request provenance record for this router's decision.
|
||||
|
||||
|
|
@ -1602,6 +1605,14 @@ class ComplexityRouter(CustomLogger):
|
|||
# (classifier, keyword rule, or session pin) had placed it before physics did.
|
||||
decision["context_escalated"] = True
|
||||
decision["context_escalation_original_tier"] = _tier_name(context_escalation_original_tier)
|
||||
if stall_escalation_reason is not None:
|
||||
# Same two facts the keyword pair records: the detector fired, and whether the tier
|
||||
# actually moved. A stall detected at the highest configured tier has nowhere to go,
|
||||
# so it reports the reason with stall_escalated False rather than claiming a bump.
|
||||
decision["stall_escalated"] = stall_escalation_original_tier is not None
|
||||
decision["stall_escalation_reason"] = stall_escalation_reason
|
||||
if stall_escalation_original_tier is not None:
|
||||
decision["stall_escalation_original_tier"] = _tier_name(stall_escalation_original_tier)
|
||||
if tier_litellm_params:
|
||||
masked_tier_litellm_params: Final = mask_credentials_in_payload(tier_litellm_params)
|
||||
if isinstance(masked_tier_litellm_params, Mapping):
|
||||
|
|
@ -2876,6 +2887,10 @@ class ComplexityRouter(CustomLogger):
|
|||
context_escalation_original_tier=(
|
||||
decision.get("context_escalation_original_tier") if decision is not None else None
|
||||
),
|
||||
stall_escalation_reason=decision.get("stall_escalation_reason") if decision is not None else None,
|
||||
stall_escalation_original_tier=(
|
||||
decision.get("stall_escalation_original_tier") if decision is not None else None
|
||||
),
|
||||
)
|
||||
from litellm.types.router import PreRoutingHookResponse as HookResponse
|
||||
|
||||
|
|
@ -3041,6 +3056,8 @@ class ComplexityRouter(CustomLogger):
|
|||
conversation_continuing=bool(decision.get("conversation_continuing", True)),
|
||||
tier_litellm_params=self._litellm_params_for_model(decided_tier, new_model),
|
||||
context_escalation_original_tier=decision.get("context_escalation_original_tier"),
|
||||
stall_escalation_reason=decision.get("stall_escalation_reason"),
|
||||
stall_escalation_original_tier=decision.get("stall_escalation_original_tier"),
|
||||
)
|
||||
return response.model_copy(
|
||||
update={ # mutable-ok: model_copy types update as a plain dict
|
||||
|
|
@ -3596,10 +3613,14 @@ class ComplexityRouter(CustomLogger):
|
|||
# Resolved here rather than beside the classifier because the keyword-override path below
|
||||
# returns before any classification runs, and a forced tier gets stuck for the same reason
|
||||
# a classified one does.
|
||||
stalled: Final = self.config.stall_escalation_enabled and detect_stalled_task(
|
||||
resolved_messages,
|
||||
window=self.config.stall_escalation_window,
|
||||
repeat_threshold=self.config.stall_escalation_repeat_threshold,
|
||||
stalled: Final = (
|
||||
detect_stalled_task(
|
||||
resolved_messages,
|
||||
window=self.config.stall_escalation_window,
|
||||
repeat_threshold=self.config.stall_escalation_repeat_threshold,
|
||||
)
|
||||
if self.config.stall_escalation_enabled
|
||||
else None
|
||||
)
|
||||
|
||||
plan_mode_sentinel: Final = self._matched_plan_mode_signal(request_kwargs, resolved_messages)
|
||||
|
|
@ -3636,7 +3657,10 @@ class ComplexityRouter(CustomLogger):
|
|||
keyword_bumped_tier: Final = (
|
||||
self._escalate_tier(override.tier) if escalation_keyword is not None else override.tier
|
||||
)
|
||||
escalated_tier: Final = self._escalate_tier(keyword_bumped_tier) if stalled else keyword_bumped_tier
|
||||
escalated_tier: Final = (
|
||||
self._escalate_tier(keyword_bumped_tier) if stalled is not None else keyword_bumped_tier
|
||||
)
|
||||
keyword_stall_original_tier: Final = keyword_bumped_tier if escalated_tier != keyword_bumped_tier else None
|
||||
keyword_escalated: Final = keyword_bumped_tier != override.tier
|
||||
routed_tier: Final = (
|
||||
self._apply_plan_mode_floor(escalated_tier) if plan_floor is not None else escalated_tier
|
||||
|
|
@ -3665,11 +3689,13 @@ class ComplexityRouter(CustomLogger):
|
|||
conversation_continuing=conversation_continuing,
|
||||
cause=keyword_cause,
|
||||
tier=routed_tier,
|
||||
signals=("stall_escalation",) if stalled else None,
|
||||
signals=("stall_escalation",) if stalled is not None else None,
|
||||
matched_keyword=plan_mode_sentinel if keyword_plan_floored else override.matched_keyword,
|
||||
escalation_keyword=escalation_keyword,
|
||||
escalated=keyword_escalated,
|
||||
tier_litellm_params=keyword_tier_litellm_params,
|
||||
stall_escalation_reason=stalled,
|
||||
stall_escalation_original_tier=keyword_stall_original_tier,
|
||||
),
|
||||
)
|
||||
|
||||
|
|
@ -3687,9 +3713,11 @@ class ComplexityRouter(CustomLogger):
|
|||
escalated: Final = tier != classified_tier
|
||||
if escalated:
|
||||
signals = (*signals, "escalation")
|
||||
if stalled:
|
||||
pre_stall_tier: Final = tier
|
||||
if stalled is not None:
|
||||
tier = self._escalate_tier(tier)
|
||||
signals = (*signals, "stall_escalation")
|
||||
stall_original_tier: Final = pre_stall_tier if tier != pre_stall_tier else None
|
||||
pre_floor_tier: Final = tier
|
||||
if plan_floor is not None:
|
||||
tier = self._apply_plan_mode_floor(tier)
|
||||
|
|
@ -3832,5 +3860,7 @@ class ComplexityRouter(CustomLogger):
|
|||
classifier_cost=outcome.classifier_cost,
|
||||
tier_litellm_params=tier_litellm_params,
|
||||
context_escalation_original_tier=context_original_tier,
|
||||
stall_escalation_reason=stalled,
|
||||
stall_escalation_original_tier=stall_original_tier,
|
||||
),
|
||||
)
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@
|
|||
Mid-task stall detection for the Complexity Router.
|
||||
|
||||
Reads the assistant's own recent tool calls, which every agentic client resends on each
|
||||
turn, and reports whether the task currently looks stuck. No LLM call and no stored state:
|
||||
turn, and reports which stuck pattern the task looks caught in, if any. No LLM call and no state:
|
||||
the same window is rescanned per classified turn, so the verdict follows the conversation
|
||||
rather than latching.
|
||||
|
||||
|
|
@ -20,6 +20,8 @@ from collections.abc import Iterator, Mapping, Sequence
|
|||
from itertools import islice
|
||||
from typing import Final, NamedTuple
|
||||
|
||||
from litellm.types.utils import StallEscalationReason
|
||||
|
||||
_ARGUMENTS_PARSE_FAILED: Final = object()
|
||||
|
||||
|
||||
|
|
@ -94,8 +96,8 @@ def detect_stalled_task(
|
|||
*,
|
||||
window: int,
|
||||
repeat_threshold: int,
|
||||
) -> bool:
|
||||
"""Whether the newest tool call is still part of a stuck pattern: it repeats, or it
|
||||
) -> StallEscalationReason | None:
|
||||
"""Which stuck pattern the newest tool call is still part of, or None: it repeats, or it
|
||||
errored, at least repeat_threshold times across the last `window` calls.
|
||||
|
||||
Both tests are anchored on the newest call rather than counting whichever pattern is
|
||||
|
|
@ -105,14 +107,16 @@ def detect_stalled_task(
|
|||
matches, so a retry loop broken up by an unrelated lookup still reads as stuck.
|
||||
"""
|
||||
if not messages or repeat_threshold <= 0:
|
||||
return False
|
||||
return None
|
||||
recent: Final = tuple(islice(_iter_tool_call_events_newest_first(messages), window))
|
||||
if len(recent) < repeat_threshold:
|
||||
return False
|
||||
return None
|
||||
newest: Final = recent[0]
|
||||
repeats: Final = sum(1 for event in recent if event.signature == newest.signature)
|
||||
if repeats >= repeat_threshold:
|
||||
return True
|
||||
return "repeated_tool_call"
|
||||
if not newest.is_error:
|
||||
return False
|
||||
return sum(1 for event in recent if event.is_error) >= repeat_threshold
|
||||
return None
|
||||
if sum(1 for event in recent if event.is_error) >= repeat_threshold:
|
||||
return "repeated_tool_error"
|
||||
return None
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@ from typing import (
|
|||
Any,
|
||||
Final,
|
||||
Literal,
|
||||
TypeAlias,
|
||||
get_args,
|
||||
)
|
||||
|
||||
|
|
@ -2948,6 +2949,12 @@ SHADOW_EVAL_JUDGE_CALL_ORIGIN: Final[InternalCallOrigin] = "shadow_eval_judge"
|
|||
BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN: Final[InternalCallOrigin] = "background_response_cost_poll"
|
||||
|
||||
|
||||
StallEscalationReason: TypeAlias = Literal["repeated_tool_call", "repeated_tool_error"]
|
||||
"""Which stall branch fired. A repeat loop is a model that is stuck, which a stronger model can
|
||||
break; repeated tool errors are usually a broken tool or bad credentials, where the stronger
|
||||
model fails the same way at a higher price. Operationally different, so they are recorded apart."""
|
||||
|
||||
|
||||
class StandardLoggingRoutingDecision(TypedDict, total=False):
|
||||
"""Per-request provenance for a pre-routing strategy (auto-router) decision."""
|
||||
|
||||
|
|
@ -2967,6 +2974,9 @@ class StandardLoggingRoutingDecision(TypedDict, total=False):
|
|||
escalated: bool
|
||||
context_escalated: bool # writable-ok: Pydantic warns on ReadOnly TypedDict fields
|
||||
context_escalation_original_tier: str # writable-ok: Pydantic warns on ReadOnly TypedDict fields
|
||||
stall_escalated: bool # writable-ok: Pydantic warns on ReadOnly TypedDict fields
|
||||
stall_escalation_reason: StallEscalationReason # writable-ok: Pydantic warns on ReadOnly TypedDict fields
|
||||
stall_escalation_original_tier: str # writable-ok: Pydantic warns on ReadOnly TypedDict fields
|
||||
tier_boundaries: StandardLoggingRoutingDecisionTierBoundaries
|
||||
reasoning_override_min_score: float # writable-ok: Pydantic warns on ReadOnly TypedDict fields
|
||||
conversation_continuing: bool
|
||||
|
|
@ -2995,6 +3005,9 @@ DERIVED_ROUTING_DECISION_FIELDS: Final[frozenset[str]] = frozenset(
|
|||
"escalated",
|
||||
"context_escalated",
|
||||
"context_escalation_original_tier",
|
||||
"stall_escalated",
|
||||
"stall_escalation_reason",
|
||||
"stall_escalation_original_tier",
|
||||
"tier_boundaries",
|
||||
"reasoning_override_min_score",
|
||||
"conversation_continuing",
|
||||
|
|
|
|||
|
|
@ -398,6 +398,69 @@ def test_a_context_escalated_route_reports_as_escalated(logger: DataDogLLMObsLog
|
|||
assert "router_escalated:true" in payload["tags"]
|
||||
|
||||
|
||||
def test_a_stall_escalated_route_reports_as_escalated(logger: DataDogLLMObsLogger) -> None:
|
||||
"""The stall detector bumped the tier, so the spend on this request is escalated spend and
|
||||
has to land in the same slice as every other escalation."""
|
||||
payload = build(logger, metadata={"routing_decision": {"tier": "premium", "stall_escalated": True}})
|
||||
|
||||
assert payload["meta"]["metadata"]["router_escalated"] is True
|
||||
assert "router_escalated:true" in payload["tags"]
|
||||
|
||||
|
||||
def test_the_stall_branch_is_its_own_cost_dimension(logger: DataDogLLMObsLogger) -> None:
|
||||
"""Charting the two branches apart is what tells an operator whether the bumps are buying
|
||||
anything: a stuck model can be unstuck by a stronger one, a broken tool cannot."""
|
||||
payload = build(
|
||||
logger,
|
||||
metadata={
|
||||
"routing_decision": {
|
||||
"tier": "premium",
|
||||
"stall_escalated": True,
|
||||
"stall_escalation_reason": "repeated_tool_error",
|
||||
"stall_escalation_original_tier": "SIMPLE",
|
||||
}
|
||||
},
|
||||
)
|
||||
|
||||
assert payload["meta"]["metadata"]["router_stall_reason"] == "repeated_tool_error"
|
||||
assert payload["meta"]["metadata"]["router_stall_original_tier"] == "SIMPLE"
|
||||
assert "router_stall_reason:repeated_tool_error" in payload["tags"]
|
||||
assert "router_stall_reason" in payload["meta"]["metadata"]["_dd"]["cost_tags"]
|
||||
|
||||
|
||||
def test_stall_fields_survive_prompt_redaction() -> None:
|
||||
"""They aggregate the prompt rather than quoting it, unlike the signals entry that used to be
|
||||
the only marker, so a redacted span still says the request was escalated and why."""
|
||||
with patch.dict(os.environ, {"DD_API_KEY": "k", "DD_SITE": "us5.datadoghq.com"}, clear=True):
|
||||
with patch("asyncio.create_task"):
|
||||
redacted_logger = DataDogLLMObsLogger(turn_off_message_logging=True)
|
||||
result = json.loads(
|
||||
safe_dumps(
|
||||
redacted_logger.create_llm_obs_payload(
|
||||
build_payload(
|
||||
metadata={
|
||||
"routing_decision": {
|
||||
"tier": "premium",
|
||||
"signals": ["secret prompt text", "stall_escalation"],
|
||||
"stall_escalated": True,
|
||||
"stall_escalation_reason": "repeated_tool_call",
|
||||
"stall_escalation_original_tier": "SIMPLE",
|
||||
}
|
||||
}
|
||||
),
|
||||
datetime(2026, 9, 1, 12, 0, 0),
|
||||
datetime(2026, 9, 1, 12, 0, 2),
|
||||
)
|
||||
)
|
||||
)
|
||||
|
||||
assert result["meta"]["metadata"]["router_stall_escalated"] is True
|
||||
assert result["meta"]["metadata"]["router_stall_reason"] == "repeated_tool_call"
|
||||
assert result["meta"]["metadata"]["router_stall_original_tier"] == "SIMPLE"
|
||||
assert result["meta"]["metadata"]["router_escalated"] is True
|
||||
assert "router_signals" not in result["meta"]["metadata"]
|
||||
|
||||
|
||||
def test_a_routed_request_that_did_not_escalate_reports_false(logger: DataDogLLMObsLogger) -> None:
|
||||
"""Without this the escalation dimension is absent on ordinary traffic, so nothing can group by it."""
|
||||
payload = build(logger, metadata={"routing_decision": {"tier": "simple", "cause": "heuristic_scorer"}})
|
||||
|
|
|
|||
|
|
@ -6809,6 +6809,139 @@ class TestStallEscalation:
|
|||
result = await router.async_pre_routing_hook(model="test-model", request_kwargs={}, messages=messages)
|
||||
assert "stall_escalation" in result.routing_decision["signals"]
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_the_escalation_is_recorded_as_its_own_fields(self, mock_router_instance, basic_config):
|
||||
"""`signals` is prompt-quoting and redaction drops it, so a stall recorded only there is
|
||||
invisible to an operator running with message logging off. These fields aggregate the
|
||||
prompt rather than quoting it, so they survive and can carry a metric dimension."""
|
||||
router = ComplexityRouter(
|
||||
model_name="test-router",
|
||||
litellm_router_instance=mock_router_instance,
|
||||
complexity_router_config={**basic_config, "stall_escalation_enabled": True},
|
||||
)
|
||||
messages = [*_stalled_tool_history(), {"role": "user", "content": "Hello there!"}]
|
||||
result = await router.async_pre_routing_hook(model="test-model", request_kwargs={}, messages=messages)
|
||||
assert result.routing_decision["stall_escalated"] is True
|
||||
assert result.routing_decision["stall_escalation_original_tier"] == "SIMPLE"
|
||||
assert result.routing_decision["tier"] == "MEDIUM"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_the_two_detector_branches_are_told_apart(self, mock_router_instance, basic_config):
|
||||
"""A repeat loop is a stuck model, which is what the bump is for. Repeated tool errors are
|
||||
usually a broken tool, where the stronger model fails the same way at a higher price."""
|
||||
router = ComplexityRouter(
|
||||
model_name="test-router",
|
||||
litellm_router_instance=mock_router_instance,
|
||||
complexity_router_config={**basic_config, "stall_escalation_enabled": True},
|
||||
)
|
||||
erroring = [
|
||||
turn
|
||||
for i, cmd in enumerate(("pytest a", "pytest b", "pytest c"))
|
||||
for turn in (
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": [{"type": "tool_use", "id": f"e-{i}", "name": "bash", "input": {"cmd": cmd}}],
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "tool_result", "tool_use_id": f"e-{i}", "is_error": True, "content": "fail"}
|
||||
],
|
||||
},
|
||||
)
|
||||
]
|
||||
repeating = await router.async_pre_routing_hook(
|
||||
model="test-model",
|
||||
request_kwargs={},
|
||||
messages=[*_stalled_tool_history(), {"role": "user", "content": "Hello there!"}],
|
||||
)
|
||||
erroring_result = await router.async_pre_routing_hook(
|
||||
model="test-model",
|
||||
request_kwargs={},
|
||||
messages=[*erroring, {"role": "user", "content": "Hello there!"}],
|
||||
)
|
||||
assert repeating.routing_decision["stall_escalation_reason"] == "repeated_tool_call"
|
||||
assert erroring_result.routing_decision["stall_escalation_reason"] == "repeated_tool_error"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_an_unstalled_request_carries_no_stall_fields(self, mock_router_instance, basic_config):
|
||||
"""The fields must distinguish an escalated request from one the classifier placed at the
|
||||
same tier on its own, so they cannot appear on ordinary traffic."""
|
||||
router = ComplexityRouter(
|
||||
model_name="test-router",
|
||||
litellm_router_instance=mock_router_instance,
|
||||
complexity_router_config={**basic_config, "stall_escalation_enabled": True},
|
||||
)
|
||||
result = await router.async_pre_routing_hook(
|
||||
model="test-model", request_kwargs={}, messages=[{"role": "user", "content": "Hello there!"}]
|
||||
)
|
||||
assert "stall_escalated" not in result.routing_decision
|
||||
assert "stall_escalation_reason" not in result.routing_decision
|
||||
assert "stall_escalation_original_tier" not in result.routing_decision
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_stall_with_nowhere_to_go_records_the_detection_without_a_bump(
|
||||
self, mock_router_instance, basic_config
|
||||
):
|
||||
"""Already at the highest configured tier, so no tier moved and no extra money was spent.
|
||||
Reporting stall_escalated True there would overstate what the feature costs."""
|
||||
router = ComplexityRouter(
|
||||
model_name="test-router",
|
||||
litellm_router_instance=mock_router_instance,
|
||||
complexity_router_config={**basic_config, "stall_escalation_enabled": True},
|
||||
)
|
||||
messages = [
|
||||
*_stalled_tool_history(),
|
||||
{"role": "user", "content": "Let's think step by step and reason through this carefully."},
|
||||
]
|
||||
result = await router.async_pre_routing_hook(model="test-model", request_kwargs={}, messages=messages)
|
||||
assert result.routing_decision["stall_escalated"] is False
|
||||
assert result.routing_decision["stall_escalation_reason"] == "repeated_tool_call"
|
||||
assert "stall_escalation_original_tier" not in result.routing_decision
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_the_fields_survive_message_redaction(self, mock_router_instance, basic_config):
|
||||
"""The whole point: with redaction on, `signals` goes and these have to stay, otherwise a
|
||||
stall-escalated row is identical to one the classifier placed at that tier itself."""
|
||||
router = ComplexityRouter(
|
||||
model_name="test-router",
|
||||
litellm_router_instance=mock_router_instance,
|
||||
complexity_router_config={**basic_config, "stall_escalation_enabled": True},
|
||||
)
|
||||
request_kwargs: Dict = {"metadata": {"headers": {"x-litellm-enable-message-redaction": True}}}
|
||||
result = await router.async_pre_routing_hook(
|
||||
model="test-model",
|
||||
request_kwargs=request_kwargs,
|
||||
messages=[*_stalled_tool_history(), {"role": "user", "content": "Hello there!"}],
|
||||
)
|
||||
redacted = litellm.Router._redact_prompt_text_if_needed(request_kwargs, result.routing_decision)
|
||||
assert "signals" not in redacted
|
||||
assert redacted["stall_escalated"] is True
|
||||
assert redacted["stall_escalation_reason"] == "repeated_tool_call"
|
||||
assert redacted["stall_escalation_original_tier"] == "SIMPLE"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_keyword_forced_tier_records_the_stall_fields_too(self, mock_router_instance, basic_config):
|
||||
"""The keyword-override path returns before classification, so it assembles its own
|
||||
decision record and would otherwise drop the fields the classified path sets."""
|
||||
router = ComplexityRouter(
|
||||
model_name="test-router",
|
||||
litellm_router_instance=mock_router_instance,
|
||||
complexity_router_config={
|
||||
**basic_config,
|
||||
"stall_escalation_enabled": True,
|
||||
"keyword_tier_rules": [{"keywords": ["billing"], "tier": "SIMPLE"}],
|
||||
},
|
||||
)
|
||||
result = await router.async_pre_routing_hook(
|
||||
model="test-model",
|
||||
request_kwargs={},
|
||||
messages=[*_stalled_tool_history(), {"role": "user", "content": "a billing question"}],
|
||||
)
|
||||
assert result.routing_decision["stall_escalated"] is True
|
||||
assert result.routing_decision["stall_escalation_reason"] == "repeated_tool_call"
|
||||
assert result.routing_decision["stall_escalation_original_tier"] == "SIMPLE"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_stall_escalation_caps_at_highest_tier(self, mock_router_instance, basic_config):
|
||||
router = ComplexityRouter(
|
||||
|
|
|
|||
|
|
@ -35,7 +35,7 @@ class TestDetectStalledTask:
|
|||
*_anthropic_call("t2", "bash", {"cmd": "pytest"}, is_error=False),
|
||||
*_anthropic_call("t3", "bash", {"cmd": "pytest"}, is_error=False),
|
||||
]
|
||||
assert detect_stalled_task(messages, window=6, repeat_threshold=3) is True
|
||||
assert detect_stalled_task(messages, window=6, repeat_threshold=3) == "repeated_tool_call"
|
||||
|
||||
def test_repeated_errors_are_stalled_even_with_varied_arguments(self):
|
||||
messages = [
|
||||
|
|
@ -43,7 +43,7 @@ class TestDetectStalledTask:
|
|||
*_anthropic_call("t2", "bash", {"cmd": "pytest tests/b.py"}, is_error=True),
|
||||
*_anthropic_call("t3", "bash", {"cmd": "pytest tests/c.py"}, is_error=True),
|
||||
]
|
||||
assert detect_stalled_task(messages, window=6, repeat_threshold=3) is True
|
||||
assert detect_stalled_task(messages, window=6, repeat_threshold=3) == "repeated_tool_error"
|
||||
|
||||
def test_varied_successful_calls_are_not_stalled(self):
|
||||
messages = [
|
||||
|
|
@ -51,7 +51,7 @@ class TestDetectStalledTask:
|
|||
*_anthropic_call("t2", "bash", {"cmd": "pytest"}, is_error=False),
|
||||
*_anthropic_call("t3", "grep", {"pattern": "x"}, is_error=False),
|
||||
]
|
||||
assert detect_stalled_task(messages, window=6, repeat_threshold=3) is False
|
||||
assert detect_stalled_task(messages, window=6, repeat_threshold=3) is None
|
||||
|
||||
def test_chat_completions_repeats_are_stalled(self):
|
||||
messages = [
|
||||
|
|
@ -59,7 +59,7 @@ class TestDetectStalledTask:
|
|||
*_chat_completions_call("c2", "bash", '{"cmd": "pytest"}'),
|
||||
*_chat_completions_call("c3", "bash", '{"cmd": "pytest"}'),
|
||||
]
|
||||
assert detect_stalled_task(messages, window=6, repeat_threshold=3) is True
|
||||
assert detect_stalled_task(messages, window=6, repeat_threshold=3) == "repeated_tool_call"
|
||||
|
||||
def test_chat_completions_has_no_structured_error_signal(self):
|
||||
"""A chat-completions tool message carries no standard error flag, so varied calls
|
||||
|
|
@ -69,7 +69,7 @@ class TestDetectStalledTask:
|
|||
*_chat_completions_call("c2", "bash", '{"cmd": "b"}'),
|
||||
*_chat_completions_call("c3", "bash", '{"cmd": "c"}'),
|
||||
]
|
||||
assert detect_stalled_task(messages, window=6, repeat_threshold=3) is False
|
||||
assert detect_stalled_task(messages, window=6, repeat_threshold=3) is None
|
||||
|
||||
def test_dict_and_json_string_arguments_compare_equal_across_surfaces(self):
|
||||
messages = [
|
||||
|
|
@ -77,14 +77,14 @@ class TestDetectStalledTask:
|
|||
*_chat_completions_call("c2", "bash", '{"cmd": "pytest"}'),
|
||||
*_anthropic_call("t3", "bash", {"cmd": "pytest"}, is_error=False),
|
||||
]
|
||||
assert detect_stalled_task(messages, window=6, repeat_threshold=3) is True
|
||||
assert detect_stalled_task(messages, window=6, repeat_threshold=3) == "repeated_tool_call"
|
||||
|
||||
def test_below_repeat_threshold_is_not_stalled(self):
|
||||
messages = [
|
||||
*_anthropic_call("t1", "bash", {"cmd": "pytest"}, is_error=False),
|
||||
*_anthropic_call("t2", "bash", {"cmd": "pytest"}, is_error=False),
|
||||
]
|
||||
assert detect_stalled_task(messages, window=6, repeat_threshold=3) is False
|
||||
assert detect_stalled_task(messages, window=6, repeat_threshold=3) is None
|
||||
|
||||
def test_evidence_older_than_the_window_does_not_count(self):
|
||||
"""Only the most recent `window` tool calls are considered, so a stall the model
|
||||
|
|
@ -96,7 +96,7 @@ class TestDetectStalledTask:
|
|||
*_anthropic_call("t4", "grep", {"pattern": "a"}, is_error=False),
|
||||
*_anthropic_call("t5", "grep", {"pattern": "b"}, is_error=False),
|
||||
]
|
||||
assert detect_stalled_task(messages, window=2, repeat_threshold=2) is False
|
||||
assert detect_stalled_task(messages, window=2, repeat_threshold=2) is None
|
||||
|
||||
def test_evidence_survives_a_new_human_ask(self):
|
||||
"""A follow-up like 'try again' must not erase evidence from before it: detection
|
||||
|
|
@ -107,7 +107,7 @@ class TestDetectStalledTask:
|
|||
*_anthropic_call("t3", "bash", {"cmd": "pytest"}, is_error=False),
|
||||
{"role": "user", "content": [{"type": "text", "text": "try again"}]},
|
||||
]
|
||||
assert detect_stalled_task(messages, window=6, repeat_threshold=3) is True
|
||||
assert detect_stalled_task(messages, window=6, repeat_threshold=3) == "repeated_tool_call"
|
||||
|
||||
def test_a_recovered_task_is_not_stalled_while_its_old_failures_sit_in_the_window(self):
|
||||
"""The three identical failures stay in the window for a few turns after the model
|
||||
|
|
@ -120,7 +120,7 @@ class TestDetectStalledTask:
|
|||
*_anthropic_call("t4", "read_file", {"path": "conftest.py"}, is_error=False),
|
||||
*_anthropic_call("t5", "edit_file", {"path": "conftest.py"}, is_error=False),
|
||||
]
|
||||
assert detect_stalled_task(messages, window=6, repeat_threshold=3) is False
|
||||
assert detect_stalled_task(messages, window=6, repeat_threshold=3) is None
|
||||
|
||||
def test_a_retry_loop_broken_up_by_an_unrelated_call_still_counts(self):
|
||||
"""Anchoring on the newest call must not require the repeats to be adjacent: a model
|
||||
|
|
@ -131,7 +131,7 @@ class TestDetectStalledTask:
|
|||
*_anthropic_call("t3", "bash", {"cmd": "pytest"}, is_error=True),
|
||||
*_anthropic_call("t4", "bash", {"cmd": "pytest"}, is_error=True),
|
||||
]
|
||||
assert detect_stalled_task(messages, window=6, repeat_threshold=3) is True
|
||||
assert detect_stalled_task(messages, window=6, repeat_threshold=3) == "repeated_tool_call"
|
||||
|
||||
def test_errors_only_count_while_the_newest_call_is_still_failing(self):
|
||||
messages = [
|
||||
|
|
@ -140,15 +140,25 @@ class TestDetectStalledTask:
|
|||
*_anthropic_call("t3", "bash", {"cmd": "pytest c"}, is_error=True),
|
||||
*_anthropic_call("t4", "bash", {"cmd": "pytest d"}, is_error=False),
|
||||
]
|
||||
assert detect_stalled_task(messages, window=6, repeat_threshold=3) is False
|
||||
assert detect_stalled_task(messages, window=6, repeat_threshold=3) is None
|
||||
|
||||
def test_a_repeating_call_that_also_errors_reports_the_repeat_branch(self):
|
||||
"""Both branches hold here. The repeat is the more specific fact and the one the
|
||||
feature exists for, so a stuck model is never filed under 'the tool is broken'."""
|
||||
messages = [
|
||||
*_anthropic_call("t1", "bash", {"cmd": "make build"}, is_error=True),
|
||||
*_anthropic_call("t2", "bash", {"cmd": "make build"}, is_error=True),
|
||||
*_anthropic_call("t3", "bash", {"cmd": "make build"}, is_error=True),
|
||||
]
|
||||
assert detect_stalled_task(messages, window=6, repeat_threshold=3) == "repeated_tool_call"
|
||||
|
||||
def test_no_messages_is_not_stalled(self):
|
||||
assert detect_stalled_task(None, window=6, repeat_threshold=3) is False
|
||||
assert detect_stalled_task([], window=6, repeat_threshold=3) is False
|
||||
assert detect_stalled_task(None, window=6, repeat_threshold=3) is None
|
||||
assert detect_stalled_task([], window=6, repeat_threshold=3) is None
|
||||
|
||||
def test_zero_threshold_never_flags_stalled(self):
|
||||
messages = [
|
||||
*_anthropic_call("t1", "bash", {"cmd": "pytest"}, is_error=True),
|
||||
*_anthropic_call("t2", "bash", {"cmd": "pytest"}, is_error=True),
|
||||
]
|
||||
assert detect_stalled_task(messages, window=6, repeat_threshold=0) is False
|
||||
assert detect_stalled_task(messages, window=6, repeat_threshold=0) is None
|
||||
|
|
|
|||
|
|
@ -226,6 +226,53 @@ describe("RoutingDecisionCard", () => {
|
|||
expect(screen.getByText("Requested; already at the highest tier")).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("marks a stall escalation with the branch that fired and the tier it came from", () => {
|
||||
render(
|
||||
<RoutingDecisionCard
|
||||
decision={{
|
||||
...heuristic,
|
||||
tier: "MEDIUM",
|
||||
stall_escalated: true,
|
||||
stall_escalation_reason: "repeated_tool_error",
|
||||
stall_escalation_original_tier: "SIMPLE",
|
||||
}}
|
||||
/>,
|
||||
);
|
||||
expect(screen.getByText("Yes, from SIMPLE, the same tool call erroring")).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("tells a stall-escalated row apart from an ordinary one under redaction", () => {
|
||||
// The signals entry that used to be the only marker is gone with redaction on, so
|
||||
// without these fields this row is identical to a request classified at MEDIUM.
|
||||
render(
|
||||
<RoutingDecisionCard
|
||||
decision={{
|
||||
...heuristic,
|
||||
tier: "MEDIUM",
|
||||
signals: undefined,
|
||||
stall_escalated: true,
|
||||
stall_escalation_reason: "repeated_tool_call",
|
||||
stall_escalation_original_tier: "SIMPLE",
|
||||
}}
|
||||
/>,
|
||||
);
|
||||
expect(screen.getByText("Yes, from SIMPLE, the same tool call repeating")).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("says a stall was detected even when there was no higher tier to move to", () => {
|
||||
render(
|
||||
<RoutingDecisionCard
|
||||
decision={{ ...heuristic, stall_escalated: false, stall_escalation_reason: "repeated_tool_call" }}
|
||||
/>,
|
||||
);
|
||||
expect(screen.getByText("Detected, the same tool call repeating; already at the highest tier")).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("omits the stall row on a request the detector never flagged", () => {
|
||||
render(<RoutingDecisionCard decision={heuristic} />);
|
||||
expect(screen.queryByText("Stall escalated")).not.toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("does not claim the score chose the tier on a redacted override row", () => {
|
||||
// `signals` is gone under redaction; the cause alone must suppress the band.
|
||||
render(<RoutingDecisionCard decision={{ ...heuristic, cause: "reasoning_override", signals: undefined }} />);
|
||||
|
|
|
|||
|
|
@ -25,6 +25,9 @@ export interface RoutingDecision {
|
|||
escalation_keyword?: string;
|
||||
classifier_model?: string;
|
||||
escalated?: boolean;
|
||||
stall_escalated?: boolean;
|
||||
stall_escalation_reason?: string;
|
||||
stall_escalation_original_tier?: string;
|
||||
tier_boundaries?: RoutingDecisionTierBoundaries;
|
||||
reasoning_override_min_score?: number;
|
||||
}
|
||||
|
|
@ -141,6 +144,23 @@ function describeEscalation(escalated: boolean, keyword: string | undefined): st
|
|||
return keyword ? `Requested via "${keyword}"; already at the highest tier` : "Requested; already at the highest tier";
|
||||
}
|
||||
|
||||
const STALL_REASON_LABELS: Record<string, string> = {
|
||||
repeated_tool_call: "the same tool call repeating",
|
||||
repeated_tool_error: "the same tool call erroring",
|
||||
};
|
||||
|
||||
/**
|
||||
* A stall can be detected on a request already at the highest configured tier, which has
|
||||
* nowhere to go. That row still has to say the detector fired, so an operator reading a bill
|
||||
* can tell a real bump from a no-op. Only called when a stall was detected.
|
||||
*/
|
||||
function describeStallEscalation(escalated: boolean, reason: string | undefined, originalTier: string | undefined) {
|
||||
const named = reason ? STALL_REASON_LABELS[reason] ?? reason : undefined;
|
||||
const because = named ? `, ${named}` : "";
|
||||
if (!escalated) return `Detected${because}; already at the highest tier`;
|
||||
return originalTier ? `Yes, from ${originalTier}${because}` : `Yes${because}`;
|
||||
}
|
||||
|
||||
function Row({ label, children }: { label: string; children: React.ReactNode }) {
|
||||
return (
|
||||
<div className="flex gap-3 py-1 text-sm">
|
||||
|
|
@ -170,6 +190,9 @@ export function RoutingDecisionCard({
|
|||
signals,
|
||||
escalated,
|
||||
escalation_keyword: escalationKeyword,
|
||||
stall_escalated: stallEscalated,
|
||||
stall_escalation_reason: stallReason,
|
||||
stall_escalation_original_tier: stallOriginalTier,
|
||||
tier_boundaries: tierBoundaries,
|
||||
} = decision;
|
||||
|
||||
|
|
@ -220,6 +243,12 @@ export function RoutingDecisionCard({
|
|||
|
||||
{escalated !== undefined && <Row label="Escalated">{describeEscalation(escalated, escalationKeyword)}</Row>}
|
||||
|
||||
{stallEscalated !== undefined && (
|
||||
<Row label="Stall escalated">
|
||||
{describeStallEscalation(stallEscalated, stallReason, stallOriginalTier)}
|
||||
</Row>
|
||||
)}
|
||||
|
||||
{signals && signals.length > 0 && (
|
||||
<Row label="Signals">
|
||||
<span className="flex flex-wrap gap-1">
|
||||
|
|
|
|||
9
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
9
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
|
|
@ -36452,6 +36452,15 @@ export interface components {
|
|||
score?: number;
|
||||
/** Signals */
|
||||
signals?: string[];
|
||||
/** Stall Escalated */
|
||||
stall_escalated?: boolean;
|
||||
/** Stall Escalation Original Tier */
|
||||
stall_escalation_original_tier?: string;
|
||||
/**
|
||||
* Stall Escalation Reason
|
||||
* @enum {string}
|
||||
*/
|
||||
stall_escalation_reason?: "repeated_tool_call" | "repeated_tool_error";
|
||||
/** Tier */
|
||||
tier?: string;
|
||||
tier_boundaries?: components["schemas"]["StandardLoggingRoutingDecisionTierBoundaries"];
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue