mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-13 23:11:40 +00:00
fix(router): classify new asks before reminder-only tails
This commit is contained in:
parent
1697684b68
commit
181b3fd94a
3 changed files with 121 additions and 9 deletions
|
|
@ -452,6 +452,8 @@ The proxy records the incoming user agent in request metadata. SDK callers can s
|
|||
|
||||
The Codex `Message Type: NEW_TASK` wrapper and its delegated-task payload remain available for classification. Cleanup applies to the current ask and quoted prior turns; the routed request retains its original content
|
||||
|
||||
In `classification_mode: user_turn`, complete text-only reminder tails leave the preceding fresh ask eligible for classification. Assistant turns and tool results still mark continuations, including tool results carried alongside reminder text
|
||||
|
||||
`reminder_markers` replaces these defaults with your harness's own delimiters. Many harnesses use a different envelope per agent type, so list every pair you emit:
|
||||
|
||||
```yaml
|
||||
|
|
|
|||
|
|
@ -599,6 +599,18 @@ def _last_human_ask_index(
|
|||
)
|
||||
|
||||
|
||||
def _is_reminder_only_turn(message: Mapping[str, object], marker_pairs: tuple[tuple[str, str], ...]) -> bool:
|
||||
if message.get("role") != "user":
|
||||
return False
|
||||
content: Final = message.get("content")
|
||||
if not isinstance(content, str) and not (
|
||||
isinstance(content, list) and all(isinstance(part, Mapping) and part.get("type") == "text" for part in content)
|
||||
):
|
||||
return False
|
||||
text: Final = _message_text(content)
|
||||
return bool(text.strip()) and not _strip_reminder_blocks(text, marker_pairs)
|
||||
|
||||
|
||||
def _newest_turn_is_human_ask(
|
||||
messages: Sequence[Mapping[str, object]] | None,
|
||||
marker_pairs: tuple[tuple[str, str], ...] = _DEFAULT_REMINDER_MARKERS,
|
||||
|
|
@ -609,21 +621,23 @@ def _newest_turn_is_human_ask(
|
|||
Anchored on `_last_human_ask_index` so every surface's plumbing reads as a continuation:
|
||||
chat-completions tool turns are role=tool, Messages-surface tool_result turns flatten to empty
|
||||
human text, and a hybrid turn carrying an ask alongside a tool_result still counts as an ask.
|
||||
Compared against the newest non-system message rather than the raw tail, because Claude Code
|
||||
appends a system-role reminder after the human turn; that trailing plumbing is neither an ask
|
||||
nor loop traffic and must not turn a fresh ask into a continuation. An unreadable request (no
|
||||
messages) is treated as a continuation: there is no ask to classify, which is the same reading
|
||||
`_extract_current_ask_and_system_prompt` gives it downstream.
|
||||
Trailing system messages and complete text-only reminders do not turn a fresh ask into a
|
||||
continuation. Assistant turns and non-text content, including tool results alongside reminders,
|
||||
still form continuation boundaries. An unreadable request has no ask to classify.
|
||||
"""
|
||||
if not messages:
|
||||
return False
|
||||
newest_non_system: Final = next(
|
||||
(index for index in range(len(messages) - 1, -1, -1) if messages[index].get("role") != "system"),
|
||||
newest_activity: Final = next(
|
||||
(
|
||||
index
|
||||
for index in range(len(messages) - 1, -1, -1)
|
||||
if messages[index].get("role") != "system" and not _is_reminder_only_turn(messages[index], marker_pairs)
|
||||
),
|
||||
None,
|
||||
)
|
||||
if newest_non_system is None:
|
||||
if newest_activity is None:
|
||||
return False
|
||||
return _last_human_ask_index(messages, marker_pairs) == newest_non_system
|
||||
return _last_human_ask_index(messages, marker_pairs) == newest_activity
|
||||
|
||||
|
||||
def _iter_system_scope_texts(
|
||||
|
|
|
|||
|
|
@ -7625,6 +7625,102 @@ _CODEX_ENVELOPES: Final = (
|
|||
class TestContextAwareClassifier:
|
||||
"""Test the new classifier context window and trajectory signals."""
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"tail,expected",
|
||||
(
|
||||
([{"role": "user", "content": [{"type": "text", "text": _CODEX_ENVELOPES[0]}]}], True),
|
||||
([{"role": "assistant", "content": _CODEX_ENVELOPES[0]}], False),
|
||||
([{"role": "tool", "content": _CODEX_ENVELOPES[0]}], False),
|
||||
([{"role": "user", "content": " "}], False),
|
||||
(
|
||||
[{"role": "user", "content": [_TOOL_RESULT, {"type": "text", "text": _CODEX_ENVELOPES[0]}]}],
|
||||
False,
|
||||
),
|
||||
(
|
||||
[{"role": "user", "content": [{"type": "image_url"}, {"type": "text", "text": _CODEX_ENVELOPES[0]}]}],
|
||||
False,
|
||||
),
|
||||
),
|
||||
)
|
||||
def test_only_text_reminder_tails_are_ignored_for_new_asks(self, tail: list[dict[str, object]], expected: bool) -> None:
|
||||
from litellm.router_strategy.complexity_router.complexity_router import (
|
||||
_CODEX_REMINDER_MARKERS,
|
||||
_newest_turn_is_human_ask,
|
||||
)
|
||||
|
||||
assert _newest_turn_is_human_ask([_ASKED, *tail], _CODEX_REMINDER_MARKERS) is expected
|
||||
assert _newest_turn_is_human_ask(tail, _CODEX_REMINDER_MARKERS) is False
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("new_ask", (_CODEX_NEW_TASK, "Now design cache invalidation"))
|
||||
@pytest.mark.parametrize("responses_api", (False, True))
|
||||
@pytest.mark.parametrize("session_affinity", (False, True))
|
||||
async def test_codex_tail_preserves_new_ask_and_tool_continuation_boundaries(
|
||||
self, new_ask: str, responses_api: bool, session_affinity: bool
|
||||
) -> None:
|
||||
completion: Final = AsyncMock(
|
||||
side_effect=[_llm_response('{"tier":"SIMPLE"}'), _llm_response('{"tier":"COMPLEX"}')]
|
||||
)
|
||||
router: Final = ComplexityRouter(
|
||||
model_name="router",
|
||||
litellm_router_instance=MagicMock(acompletion=completion, cache=DualCache()),
|
||||
complexity_router_config={
|
||||
"tiers": {"SIMPLE": "simple-model", "COMPLEX": "task-model"},
|
||||
"classifier_type": "llm",
|
||||
"classifier_llm_config": {"model": "classifier-model"},
|
||||
"classification_mode": "user_turn",
|
||||
"session_affinity": session_affinity,
|
||||
"escalation_keywords": [],
|
||||
},
|
||||
)
|
||||
metadata: Final = {"user_agent": "codex-tui", "session_id": "codex-tail-session"}
|
||||
first_messages: Final = [{"role": "user", "content": "Hello"}]
|
||||
tail: Final = [{"role": "user", "content": envelope} for envelope in _CODEX_ENVELOPES]
|
||||
new_messages: Final = [
|
||||
*first_messages,
|
||||
{"role": "assistant", "content": "Hello"},
|
||||
{"role": "user", "content": new_ask},
|
||||
*tail,
|
||||
]
|
||||
continuation: Final = [
|
||||
*new_messages,
|
||||
{"role": "assistant", "content": "Working on it"},
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "tool_result", "tool_use_id": "read-cache", "content": "cache source"},
|
||||
{"type": "text", "text": _CODEX_ENVELOPES[0]},
|
||||
],
|
||||
},
|
||||
*tail,
|
||||
]
|
||||
results: Final = [
|
||||
await router.async_pre_routing_hook(
|
||||
model="router",
|
||||
request_kwargs=(
|
||||
{"input": messages, "litellm_metadata": {**metadata, "user_api_key_request_route": "/v1/responses"}}
|
||||
if responses_api
|
||||
else {"metadata": metadata}
|
||||
),
|
||||
messages=None if responses_api else messages,
|
||||
input=messages if responses_api else None,
|
||||
)
|
||||
for messages in (first_messages, new_messages, continuation)
|
||||
]
|
||||
|
||||
assert [result.model for result in results] == (
|
||||
["simple-model", "simple-model", "simple-model"]
|
||||
if session_affinity
|
||||
else ["simple-model", "task-model", "task-model"]
|
||||
)
|
||||
assert completion.await_count == (1 if session_affinity else 2)
|
||||
assert results[-1].routing_decision["cause"] == (
|
||||
"session_affinity_pin" if session_affinity else "user_turn_continuation"
|
||||
)
|
||||
if not session_affinity:
|
||||
assert completion.call_args.kwargs["messages"][1]["content"].endswith(f"Classify this message:\n{new_ask}")
|
||||
assert results[1].messages == (None if responses_api else new_messages)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("envelope", _CODEX_ENVELOPES)
|
||||
@pytest.mark.parametrize("user_agent", (None, "curl/8.7.1", "codexify/1.0"))
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue