fix(router): classify new asks before reminder-only tails

This commit is contained in:
moe-berri 2026-09-10 12:24:42 -07:00
parent 1697684b68
commit 181b3fd94a
3 changed files with 121 additions and 9 deletions

View file

@ -452,6 +452,8 @@ The proxy records the incoming user agent in request metadata. SDK callers can s
The Codex `Message Type: NEW_TASK` wrapper and its delegated-task payload remain available for classification. Cleanup applies to the current ask and quoted prior turns; the routed request retains its original content
In `classification_mode: user_turn`, complete text-only reminder tails leave the preceding fresh ask eligible for classification. Assistant turns and tool results still mark continuations, including tool results carried alongside reminder text
`reminder_markers` replaces these defaults with your harness's own delimiters. Many harnesses use a different envelope per agent type, so list every pair you emit:
```yaml

View file

@ -599,6 +599,18 @@ def _last_human_ask_index(
)
def _is_reminder_only_turn(message: Mapping[str, object], marker_pairs: tuple[tuple[str, str], ...]) -> bool:
if message.get("role") != "user":
return False
content: Final = message.get("content")
if not isinstance(content, str) and not (
isinstance(content, list) and all(isinstance(part, Mapping) and part.get("type") == "text" for part in content)
):
return False
text: Final = _message_text(content)
return bool(text.strip()) and not _strip_reminder_blocks(text, marker_pairs)
def _newest_turn_is_human_ask(
messages: Sequence[Mapping[str, object]] | None,
marker_pairs: tuple[tuple[str, str], ...] = _DEFAULT_REMINDER_MARKERS,
@ -609,21 +621,23 @@ def _newest_turn_is_human_ask(
Anchored on `_last_human_ask_index` so every surface's plumbing reads as a continuation:
chat-completions tool turns are role=tool, Messages-surface tool_result turns flatten to empty
human text, and a hybrid turn carrying an ask alongside a tool_result still counts as an ask.
Compared against the newest non-system message rather than the raw tail, because Claude Code
appends a system-role reminder after the human turn; that trailing plumbing is neither an ask
nor loop traffic and must not turn a fresh ask into a continuation. An unreadable request (no
messages) is treated as a continuation: there is no ask to classify, which is the same reading
`_extract_current_ask_and_system_prompt` gives it downstream.
Trailing system messages and complete text-only reminders do not turn a fresh ask into a
continuation. Assistant turns and non-text content, including tool results alongside reminders,
still form continuation boundaries. An unreadable request has no ask to classify.
"""
if not messages:
return False
newest_non_system: Final = next(
(index for index in range(len(messages) - 1, -1, -1) if messages[index].get("role") != "system"),
newest_activity: Final = next(
(
index
for index in range(len(messages) - 1, -1, -1)
if messages[index].get("role") != "system" and not _is_reminder_only_turn(messages[index], marker_pairs)
),
None,
)
if newest_non_system is None:
if newest_activity is None:
return False
return _last_human_ask_index(messages, marker_pairs) == newest_non_system
return _last_human_ask_index(messages, marker_pairs) == newest_activity
def _iter_system_scope_texts(

View file

@ -7625,6 +7625,102 @@ _CODEX_ENVELOPES: Final = (
class TestContextAwareClassifier:
"""Test the new classifier context window and trajectory signals."""
@pytest.mark.parametrize(
"tail,expected",
(
([{"role": "user", "content": [{"type": "text", "text": _CODEX_ENVELOPES[0]}]}], True),
([{"role": "assistant", "content": _CODEX_ENVELOPES[0]}], False),
([{"role": "tool", "content": _CODEX_ENVELOPES[0]}], False),
([{"role": "user", "content": " "}], False),
(
[{"role": "user", "content": [_TOOL_RESULT, {"type": "text", "text": _CODEX_ENVELOPES[0]}]}],
False,
),
(
[{"role": "user", "content": [{"type": "image_url"}, {"type": "text", "text": _CODEX_ENVELOPES[0]}]}],
False,
),
),
)
def test_only_text_reminder_tails_are_ignored_for_new_asks(self, tail: list[dict[str, object]], expected: bool) -> None:
from litellm.router_strategy.complexity_router.complexity_router import (
_CODEX_REMINDER_MARKERS,
_newest_turn_is_human_ask,
)
assert _newest_turn_is_human_ask([_ASKED, *tail], _CODEX_REMINDER_MARKERS) is expected
assert _newest_turn_is_human_ask(tail, _CODEX_REMINDER_MARKERS) is False
@pytest.mark.asyncio
@pytest.mark.parametrize("new_ask", (_CODEX_NEW_TASK, "Now design cache invalidation"))
@pytest.mark.parametrize("responses_api", (False, True))
@pytest.mark.parametrize("session_affinity", (False, True))
async def test_codex_tail_preserves_new_ask_and_tool_continuation_boundaries(
self, new_ask: str, responses_api: bool, session_affinity: bool
) -> None:
completion: Final = AsyncMock(
side_effect=[_llm_response('{"tier":"SIMPLE"}'), _llm_response('{"tier":"COMPLEX"}')]
)
router: Final = ComplexityRouter(
model_name="router",
litellm_router_instance=MagicMock(acompletion=completion, cache=DualCache()),
complexity_router_config={
"tiers": {"SIMPLE": "simple-model", "COMPLEX": "task-model"},
"classifier_type": "llm",
"classifier_llm_config": {"model": "classifier-model"},
"classification_mode": "user_turn",
"session_affinity": session_affinity,
"escalation_keywords": [],
},
)
metadata: Final = {"user_agent": "codex-tui", "session_id": "codex-tail-session"}
first_messages: Final = [{"role": "user", "content": "Hello"}]
tail: Final = [{"role": "user", "content": envelope} for envelope in _CODEX_ENVELOPES]
new_messages: Final = [
*first_messages,
{"role": "assistant", "content": "Hello"},
{"role": "user", "content": new_ask},
*tail,
]
continuation: Final = [
*new_messages,
{"role": "assistant", "content": "Working on it"},
{
"role": "user",
"content": [
{"type": "tool_result", "tool_use_id": "read-cache", "content": "cache source"},
{"type": "text", "text": _CODEX_ENVELOPES[0]},
],
},
*tail,
]
results: Final = [
await router.async_pre_routing_hook(
model="router",
request_kwargs=(
{"input": messages, "litellm_metadata": {**metadata, "user_api_key_request_route": "/v1/responses"}}
if responses_api
else {"metadata": metadata}
),
messages=None if responses_api else messages,
input=messages if responses_api else None,
)
for messages in (first_messages, new_messages, continuation)
]
assert [result.model for result in results] == (
["simple-model", "simple-model", "simple-model"]
if session_affinity
else ["simple-model", "task-model", "task-model"]
)
assert completion.await_count == (1 if session_affinity else 2)
assert results[-1].routing_decision["cause"] == (
"session_affinity_pin" if session_affinity else "user_turn_continuation"
)
if not session_affinity:
assert completion.call_args.kwargs["messages"][1]["content"].endswith(f"Classify this message:\n{new_ask}")
assert results[1].messages == (None if responses_api else new_messages)
@pytest.mark.asyncio
@pytest.mark.parametrize("envelope", _CODEX_ENVELOPES)
@pytest.mark.parametrize("user_agent", (None, "curl/8.7.1", "codexify/1.0"))