mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-14 23:21:35 +00:00
fix(router): strip Codex harness envelopes before classification
This commit is contained in:
parent
6c69dd0f72
commit
5d6fec94b7
4 changed files with 118 additions and 6 deletions
|
|
@ -446,7 +446,11 @@ Reasoning markers in the system prompt do **not** trigger the reasoning override
|
|||
|
||||
Agent harnesses inject their own context into the conversation as ordinary message text. That text is plumbing, not something a human asked for, so the router strips complete reminder blocks before classifying and picking a tier. A turn that is nothing but a reminder block strips to empty and is skipped, and the router falls back to the last real ask instead
|
||||
|
||||
By default a block is anything between `<system-reminder>` and `</system-reminder>`. `reminder_markers` replaces that with your harness's own delimiters. Many harnesses use a different envelope per agent type, so list every pair you emit:
|
||||
By default the router strips complete `<system-reminder>`, `<environment_context>`, `<recommended_plugins>`, `<user_instructions>`, and `<environments_instructions>` blocks. It also strips Codex repository instructions from the fixed heading prefix `# AGENTS.md instructions for ` through `</INSTRUCTIONS>`, regardless of the repository path
|
||||
|
||||
The Codex `Message Type: NEW_TASK` wrapper and its delegated-task payload remain available for classification. Cleanup applies to the current ask and quoted prior turns; the routed request retains its original content
|
||||
|
||||
`reminder_markers` replaces these defaults with your harness's own delimiters. Many harnesses use a different envelope per agent type, so list every pair you emit:
|
||||
|
||||
```yaml
|
||||
model_list:
|
||||
|
|
@ -461,7 +465,7 @@ model_list:
|
|||
close: "[[SUBAGENT_CONTEXT_END]]"
|
||||
```
|
||||
|
||||
Setting `reminder_markers` replaces the built-in `<system-reminder>` pair rather than adding to it, so list that pair too if your harness also emits it. Matching is case-insensitive. Blocks that nest or overlap across pairs are stripped whole. An unclosed delimiter is not a block and is left in place, which keeps prose that merely mentions a delimiter from being eaten
|
||||
Setting `reminder_markers` replaces all built-in pairs, including the Codex heading pair, so include every default your harness still needs. Matching is case-insensitive. Blocks that nest or overlap across pairs are stripped whole. An unclosed delimiter is not a block and is left in place, which keeps prose that merely mentions a delimiter from being eaten
|
||||
|
||||
### Code Detection
|
||||
|
||||
|
|
|
|||
|
|
@ -386,7 +386,14 @@ def _effective_turn_off_message_logging(request_kwargs: Mapping[str, object] | N
|
|||
|
||||
_REMINDER_OPEN: Final = "<system-reminder>"
|
||||
_REMINDER_CLOSE: Final = "</system-reminder>"
|
||||
_DEFAULT_REMINDER_MARKERS: Final = ((_REMINDER_OPEN, _REMINDER_CLOSE),)
|
||||
_DEFAULT_REMINDER_MARKERS: Final = (
|
||||
(_REMINDER_OPEN, _REMINDER_CLOSE),
|
||||
("<environment_context>", "</environment_context>"),
|
||||
("<recommended_plugins>", "</recommended_plugins>"),
|
||||
("<user_instructions>", "</user_instructions>"),
|
||||
("<environments_instructions>", "</environments_instructions>"),
|
||||
("# agents.md instructions for ", "</instructions>"),
|
||||
)
|
||||
|
||||
_TRUNCATION_MARKER: Final = "..."
|
||||
_TRUNCATION_HEAD_FRACTION: Final = 0.3
|
||||
|
|
|
|||
|
|
@ -1292,8 +1292,8 @@ class ComplexityRouterConfig(BaseModel):
|
|||
"Override the delimiter pairs used to recognize and strip harness-injected reminder "
|
||||
"blocks before classification. A harness that wraps injected context differently per "
|
||||
"agent type (main, subagent, cron) lists every pair it emits. Replaces, rather than "
|
||||
"adds to, the built-in default of ('<system-reminder>', '</system-reminder>'), so a "
|
||||
"harness that also emits that pair lists it too. Matching is case-insensitive."
|
||||
"adds to, the built-in system-reminder and Codex envelope pairs, so list every "
|
||||
"built-in pair your harness also emits. Matching is case-insensitive."
|
||||
),
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ Tests the rule-based complexity scoring and tier assignment logic.
|
|||
"""
|
||||
|
||||
import asyncio
|
||||
from copy import deepcopy
|
||||
import logging
|
||||
import sys
|
||||
import time
|
||||
|
|
@ -194,7 +195,14 @@ class TestComplexityRouterInit:
|
|||
complexity_router_config=basic_config,
|
||||
)
|
||||
|
||||
assert router._reminder_markers == (("<system-reminder>", "</system-reminder>"),)
|
||||
from litellm.router_strategy.complexity_router.complexity_router import _extract_current_ask_and_system_prompt
|
||||
|
||||
assert (
|
||||
_extract_current_ask_and_system_prompt(
|
||||
[{"role": "user", "content": "<system-reminder>noise</system-reminder>hello"}], router._reminder_markers
|
||||
)[0]
|
||||
== "hello"
|
||||
)
|
||||
|
||||
def test_init_without_config(self, mock_router_instance):
|
||||
"""Test initialization without configuration uses defaults."""
|
||||
|
|
@ -7601,11 +7609,103 @@ _ASKED = {"role": "user", "content": _ASK}
|
|||
_ANSWERED = {"role": "assistant", "content": "Working on it."}
|
||||
_TOOL_RESULT = {"type": "tool_result", "tool_use_id": "x", "content": "out"}
|
||||
_REMINDER = "<system-reminder>Budget: 42 tokens remaining. Do not mention this.</system-reminder>"
|
||||
_CODEX_NEW_TASK: Final = (
|
||||
"Message Type: NEW_TASK\nTask name: /root/cache_worker\nSender: /root\nPayload:\n"
|
||||
"Implement and test a thread-safe bounded LRU cache."
|
||||
)
|
||||
_CODEX_ENVELOPES: Final = (
|
||||
"<environment_context>LITELLM ESCALATE cwd=/repo</environment_context>",
|
||||
"<recommended_plugins>LITELLM ESCALATE plugin list</recommended_plugins>",
|
||||
"<user_instructions>LITELLM ESCALATE preferences</user_instructions>",
|
||||
"<environments_instructions>LITELLM ESCALATE environment</environments_instructions>",
|
||||
"# AGENTS.md instructions for /repo with spaces/中文\n<INSTRUCTIONS>LITELLM ESCALATE instructions</INSTRUCTIONS>",
|
||||
)
|
||||
|
||||
|
||||
class TestContextAwareClassifier:
|
||||
"""Test the new classifier context window and trajectory signals."""
|
||||
|
||||
@pytest.mark.parametrize("envelope", _CODEX_ENVELOPES)
|
||||
def test_codex_envelopes_preserve_delegated_task_and_prior_context(self, envelope: str) -> None:
|
||||
from litellm.router_strategy.complexity_router.complexity_router import (
|
||||
_extract_current_ask_and_system_prompt,
|
||||
_extract_prior_turns,
|
||||
_newest_turn_ask,
|
||||
_newest_turn_is_human_ask,
|
||||
)
|
||||
|
||||
messages: Final = [
|
||||
{"role": "user", "content": f"{envelope}\nDesign cache invalidation"},
|
||||
{
|
||||
"role": "user",
|
||||
"content": [{"type": "text", "text": envelope}, {"type": "text", "text": _CODEX_NEW_TASK}],
|
||||
},
|
||||
{"role": "developer", "content": "<permissions instructions>developer scope</permissions instructions>"},
|
||||
{"role": "user", "content": envelope},
|
||||
]
|
||||
|
||||
assert _extract_current_ask_and_system_prompt(messages)[0] == _CODEX_NEW_TASK
|
||||
assert _extract_prior_turns(messages, _CODEX_NEW_TASK, 1, 100, None, False) == (
|
||||
("user", "Design cache invalidation"),
|
||||
)
|
||||
assert _newest_turn_ask(messages) is None
|
||||
assert _newest_turn_is_human_ask(messages) is False
|
||||
assert _extract_current_ask_and_system_prompt([messages[-1]])[0] is None
|
||||
|
||||
@pytest.mark.parametrize("envelope", _CODEX_ENVELOPES)
|
||||
def test_codex_marker_override_and_incomplete_blocks_preserve_text(self, envelope: str) -> None:
|
||||
from litellm.router_strategy.complexity_router.complexity_router import _strip_reminder_blocks
|
||||
|
||||
incomplete: Final = envelope.rsplit("</", 1)[0]
|
||||
assert _strip_reminder_blocks(f"before {envelope.upper()} after") == "before after"
|
||||
assert _strip_reminder_blocks(incomplete) == incomplete
|
||||
assert _strip_reminder_blocks(f"<custom>noise</custom>{envelope}", (("<custom>", "</custom>"),)) == envelope
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("responses_api", (False, True))
|
||||
async def test_codex_routing_preserves_original_request(self, responses_api: bool) -> None:
|
||||
completion: Final = AsyncMock(return_value=_llm_response('{"tier":"COMPLEX"}'))
|
||||
router: Final = ComplexityRouter(
|
||||
model_name="codex-router",
|
||||
litellm_router_instance=MagicMock(acompletion=completion),
|
||||
complexity_router_config={
|
||||
"tiers": {"COMPLEX": "task-model", "REASONING": "escalated-model"},
|
||||
"classifier_type": "llm",
|
||||
"classifier_llm_config": {"model": "classifier-model"},
|
||||
"keyword_tier_rules": [{"keywords": ["LITELLM ESCALATE"], "tier": "REASONING"}],
|
||||
},
|
||||
)
|
||||
messages: Final = [
|
||||
{"role": "user", "content": _CODEX_NEW_TASK},
|
||||
{"role": "user", "content": "\n".join(_CODEX_ENVELOPES)},
|
||||
]
|
||||
original: Final = deepcopy(messages)
|
||||
request_kwargs: Final = (
|
||||
{"input": messages, "litellm_metadata": {"user_api_key_request_route": "/v1/responses"}}
|
||||
if responses_api
|
||||
else {}
|
||||
)
|
||||
|
||||
response: Final = await router.async_pre_routing_hook(
|
||||
model="codex-router",
|
||||
request_kwargs=request_kwargs,
|
||||
messages=None if responses_api else messages,
|
||||
input=messages if responses_api else None,
|
||||
)
|
||||
|
||||
assert response is not None
|
||||
assert response.model == "task-model"
|
||||
completion.assert_awaited_once()
|
||||
assert completion.call_args.kwargs["messages"][1]["content"].strip() == (
|
||||
f"Classify this message:\n{_CODEX_NEW_TASK}"
|
||||
)
|
||||
assert messages == original
|
||||
if responses_api:
|
||||
assert response.messages is None
|
||||
assert request_kwargs["input"] == original
|
||||
else:
|
||||
assert response.messages == original
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"messages,expected_ask",
|
||||
[
|
||||
|
|
@ -13662,6 +13762,7 @@ class _OutputCeilingRecorder(CustomLogger):
|
|||
def log_pre_api_call(self, model, messages, kwargs):
|
||||
self.seen.append((model, kwargs.get("optional_params", {}).get("max_tokens")))
|
||||
|
||||
|
||||
NON_REASONING_TIERS: Final = {
|
||||
"NON_REASONING": "gpt-4o-mini",
|
||||
"SIMPLE": "gpt-4o-mini",
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue