From 5d6fec94b74371a7314a4900f3cf025a54f8bcff Mon Sep 17 00:00:00 2001 From: moe-berri Date: Thu, 10 Sep 2026 11:33:21 -0700 Subject: [PATCH 1/3] fix(router): strip Codex harness envelopes before classification --- .../complexity_router/README.md | 8 +- .../complexity_router/complexity_router.py | 9 +- .../complexity_router/config.py | 4 +- .../router_strategy/test_complexity_router.py | 103 +++++++++++++++++- 4 files changed, 118 insertions(+), 6 deletions(-) diff --git a/litellm/router_strategy/complexity_router/README.md b/litellm/router_strategy/complexity_router/README.md index d605b43e42a..3d2b33a2ecd 100644 --- a/litellm/router_strategy/complexity_router/README.md +++ b/litellm/router_strategy/complexity_router/README.md @@ -446,7 +446,11 @@ Reasoning markers in the system prompt do **not** trigger the reasoning override Agent harnesses inject their own context into the conversation as ordinary message text. That text is plumbing, not something a human asked for, so the router strips complete reminder blocks before classifying and picking a tier. A turn that is nothing but a reminder block strips to empty and is skipped, and the router falls back to the last real ask instead -By default a block is anything between `` and ``. `reminder_markers` replaces that with your harness's own delimiters. Many harnesses use a different envelope per agent type, so list every pair you emit: +By default the router strips complete ``, ``, ``, ``, and `` blocks. It also strips Codex repository instructions from the fixed heading prefix `# AGENTS.md instructions for ` through ``, regardless of the repository path + +The Codex `Message Type: NEW_TASK` wrapper and its delegated-task payload remain available for classification. Cleanup applies to the current ask and quoted prior turns; the routed request retains its original content + +`reminder_markers` replaces these defaults with your harness's own delimiters. Many harnesses use a different envelope per agent type, so list every pair you emit: ```yaml model_list: @@ -461,7 +465,7 @@ model_list: close: "[[SUBAGENT_CONTEXT_END]]" ``` -Setting `reminder_markers` replaces the built-in `` pair rather than adding to it, so list that pair too if your harness also emits it. Matching is case-insensitive. Blocks that nest or overlap across pairs are stripped whole. An unclosed delimiter is not a block and is left in place, which keeps prose that merely mentions a delimiter from being eaten +Setting `reminder_markers` replaces all built-in pairs, including the Codex heading pair, so include every default your harness still needs. Matching is case-insensitive. Blocks that nest or overlap across pairs are stripped whole. An unclosed delimiter is not a block and is left in place, which keeps prose that merely mentions a delimiter from being eaten ### Code Detection diff --git a/litellm/router_strategy/complexity_router/complexity_router.py b/litellm/router_strategy/complexity_router/complexity_router.py index faafcea404a..923aac85ee2 100644 --- a/litellm/router_strategy/complexity_router/complexity_router.py +++ b/litellm/router_strategy/complexity_router/complexity_router.py @@ -386,7 +386,14 @@ def _effective_turn_off_message_logging(request_kwargs: Mapping[str, object] | N _REMINDER_OPEN: Final = "" _REMINDER_CLOSE: Final = "" -_DEFAULT_REMINDER_MARKERS: Final = ((_REMINDER_OPEN, _REMINDER_CLOSE),) +_DEFAULT_REMINDER_MARKERS: Final = ( + (_REMINDER_OPEN, _REMINDER_CLOSE), + ("", ""), + ("", ""), + ("", ""), + ("", ""), + ("# agents.md instructions for ", ""), +) _TRUNCATION_MARKER: Final = "..." _TRUNCATION_HEAD_FRACTION: Final = 0.3 diff --git a/litellm/router_strategy/complexity_router/config.py b/litellm/router_strategy/complexity_router/config.py index d28924c69b2..6ab4aa85148 100644 --- a/litellm/router_strategy/complexity_router/config.py +++ b/litellm/router_strategy/complexity_router/config.py @@ -1292,8 +1292,8 @@ class ComplexityRouterConfig(BaseModel): "Override the delimiter pairs used to recognize and strip harness-injected reminder " "blocks before classification. A harness that wraps injected context differently per " "agent type (main, subagent, cron) lists every pair it emits. Replaces, rather than " - "adds to, the built-in default of ('', ''), so a " - "harness that also emits that pair lists it too. Matching is case-insensitive." + "adds to, the built-in system-reminder and Codex envelope pairs, so list every " + "built-in pair your harness also emits. Matching is case-insensitive." ), ) diff --git a/tests/test_litellm/router_strategy/test_complexity_router.py b/tests/test_litellm/router_strategy/test_complexity_router.py index 51103297c58..792963fde23 100644 --- a/tests/test_litellm/router_strategy/test_complexity_router.py +++ b/tests/test_litellm/router_strategy/test_complexity_router.py @@ -5,6 +5,7 @@ Tests the rule-based complexity scoring and tier assignment logic. """ import asyncio +from copy import deepcopy import logging import sys import time @@ -194,7 +195,14 @@ class TestComplexityRouterInit: complexity_router_config=basic_config, ) - assert router._reminder_markers == (("", ""),) + from litellm.router_strategy.complexity_router.complexity_router import _extract_current_ask_and_system_prompt + + assert ( + _extract_current_ask_and_system_prompt( + [{"role": "user", "content": "noisehello"}], router._reminder_markers + )[0] + == "hello" + ) def test_init_without_config(self, mock_router_instance): """Test initialization without configuration uses defaults.""" @@ -7601,11 +7609,103 @@ _ASKED = {"role": "user", "content": _ASK} _ANSWERED = {"role": "assistant", "content": "Working on it."} _TOOL_RESULT = {"type": "tool_result", "tool_use_id": "x", "content": "out"} _REMINDER = "Budget: 42 tokens remaining. Do not mention this." +_CODEX_NEW_TASK: Final = ( + "Message Type: NEW_TASK\nTask name: /root/cache_worker\nSender: /root\nPayload:\n" + "Implement and test a thread-safe bounded LRU cache." +) +_CODEX_ENVELOPES: Final = ( + "LITELLM ESCALATE cwd=/repo", + "LITELLM ESCALATE plugin list", + "LITELLM ESCALATE preferences", + "LITELLM ESCALATE environment", + "# AGENTS.md instructions for /repo with spaces/中文\nLITELLM ESCALATE instructions", +) class TestContextAwareClassifier: """Test the new classifier context window and trajectory signals.""" + @pytest.mark.parametrize("envelope", _CODEX_ENVELOPES) + def test_codex_envelopes_preserve_delegated_task_and_prior_context(self, envelope: str) -> None: + from litellm.router_strategy.complexity_router.complexity_router import ( + _extract_current_ask_and_system_prompt, + _extract_prior_turns, + _newest_turn_ask, + _newest_turn_is_human_ask, + ) + + messages: Final = [ + {"role": "user", "content": f"{envelope}\nDesign cache invalidation"}, + { + "role": "user", + "content": [{"type": "text", "text": envelope}, {"type": "text", "text": _CODEX_NEW_TASK}], + }, + {"role": "developer", "content": "developer scope"}, + {"role": "user", "content": envelope}, + ] + + assert _extract_current_ask_and_system_prompt(messages)[0] == _CODEX_NEW_TASK + assert _extract_prior_turns(messages, _CODEX_NEW_TASK, 1, 100, None, False) == ( + ("user", "Design cache invalidation"), + ) + assert _newest_turn_ask(messages) is None + assert _newest_turn_is_human_ask(messages) is False + assert _extract_current_ask_and_system_prompt([messages[-1]])[0] is None + + @pytest.mark.parametrize("envelope", _CODEX_ENVELOPES) + def test_codex_marker_override_and_incomplete_blocks_preserve_text(self, envelope: str) -> None: + from litellm.router_strategy.complexity_router.complexity_router import _strip_reminder_blocks + + incomplete: Final = envelope.rsplit("noise{envelope}", (("", ""),)) == envelope + + @pytest.mark.asyncio + @pytest.mark.parametrize("responses_api", (False, True)) + async def test_codex_routing_preserves_original_request(self, responses_api: bool) -> None: + completion: Final = AsyncMock(return_value=_llm_response('{"tier":"COMPLEX"}')) + router: Final = ComplexityRouter( + model_name="codex-router", + litellm_router_instance=MagicMock(acompletion=completion), + complexity_router_config={ + "tiers": {"COMPLEX": "task-model", "REASONING": "escalated-model"}, + "classifier_type": "llm", + "classifier_llm_config": {"model": "classifier-model"}, + "keyword_tier_rules": [{"keywords": ["LITELLM ESCALATE"], "tier": "REASONING"}], + }, + ) + messages: Final = [ + {"role": "user", "content": _CODEX_NEW_TASK}, + {"role": "user", "content": "\n".join(_CODEX_ENVELOPES)}, + ] + original: Final = deepcopy(messages) + request_kwargs: Final = ( + {"input": messages, "litellm_metadata": {"user_api_key_request_route": "/v1/responses"}} + if responses_api + else {} + ) + + response: Final = await router.async_pre_routing_hook( + model="codex-router", + request_kwargs=request_kwargs, + messages=None if responses_api else messages, + input=messages if responses_api else None, + ) + + assert response is not None + assert response.model == "task-model" + completion.assert_awaited_once() + assert completion.call_args.kwargs["messages"][1]["content"].strip() == ( + f"Classify this message:\n{_CODEX_NEW_TASK}" + ) + assert messages == original + if responses_api: + assert response.messages is None + assert request_kwargs["input"] == original + else: + assert response.messages == original + @pytest.mark.parametrize( "messages,expected_ask", [ @@ -13662,6 +13762,7 @@ class _OutputCeilingRecorder(CustomLogger): def log_pre_api_call(self, model, messages, kwargs): self.seen.append((model, kwargs.get("optional_params", {}).get("max_tokens"))) + NON_REASONING_TIERS: Final = { "NON_REASONING": "gpt-4o-mini", "SIMPLE": "gpt-4o-mini", From 1697684b685278ac68764a2f7278ffee9f05d2ff Mon Sep 17 00:00:00 2001 From: moe-berri Date: Thu, 10 Sep 2026 12:01:04 -0700 Subject: [PATCH 2/3] fix(router): scope Codex envelope defaults to Codex clients --- litellm/litellm_core_utils/core_helpers.py | 8 ++ litellm/proxy/litellm_pre_call_utils.py | 15 +-- .../complexity_router/README.md | 4 +- .../complexity_router/complexity_router.py | 40 +++++--- .../complexity_router/config.py | 5 +- .../router_strategy/test_complexity_router.py | 96 +++++++++++++++++-- ui/litellm-dashboard/src/lib/http/schema.d.ts | 2 +- 7 files changed, 127 insertions(+), 43 deletions(-) diff --git a/litellm/litellm_core_utils/core_helpers.py b/litellm/litellm_core_utils/core_helpers.py index eacc3e4860a..aa7d6ca1699 100644 --- a/litellm/litellm_core_utils/core_helpers.py +++ b/litellm/litellm_core_utils/core_helpers.py @@ -2,6 +2,7 @@ ## Helper utilities import copy import logging +import re from collections.abc import Iterable, Mapping from typing import TYPE_CHECKING, Any, Final, Literal @@ -21,6 +22,13 @@ else: Span = Any +_CODEX_CLIENT_PREFIX_RE: Final = re.compile(r"^codex[-_ /]", re.IGNORECASE) + + +def is_codex_user_agent(user_agent: str) -> bool: + return bool(_CODEX_CLIENT_PREFIX_RE.match(user_agent)) + + def safe_divide_seconds(seconds: float, denominator: float, default: float | None = None) -> float | None: """ Safely divide seconds by denominator, handling zero division. diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index da033dc2276..ad4687e95db 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -31,6 +31,7 @@ from litellm.constants import ( SESSION_ID_OMITTED_METADATA_KEY, X_LITELLM_DISABLE_CALLBACKS, ) +from litellm.litellm_core_utils.core_helpers import is_codex_user_agent from litellm.litellm_core_utils.credential_accessor import CredentialAccessor from litellm.litellm_core_utils.initialize_dynamic_callback_params import ( TRUSTED_CALLBACK_VARS_FIELD, @@ -83,10 +84,6 @@ _EXPLICIT_SESSION_HEADERS: Final = frozenset({"x-litellm-trace-id", "x-litellm-s # ``session-id``/``thread-id``; builds before the codex-api split sent # ``session_id``/``conversation_id``. Ordered session before thread. _CODEX_SESSION_ID_HEADERS: Final = ("session-id", "session_id", "thread-id", "conversation_id") -# Matches every first-party Codex originator: codex-tui, codex_cli_rs, codex_exec, -# codex_vscode, "Codex ...". A separator is required so an unrelated "codexfoo" client -# does not read as Codex. -_CODEX_CLIENT_PREFIX_RE: Final = re.compile(r"^codex[-_ /]", re.IGNORECASE) # Session-id values must be non-empty strings of alphanumerics, hyphens, or underscores # (covers UUIDs and most common session-id formats). _SESSION_ID_VALUE_RE: Final = re.compile(r"^[a-zA-Z0-9_\-]{8,}$") @@ -810,16 +807,6 @@ def apply_missing_session_id_policy( ) -def is_codex_user_agent(user_agent: str) -> bool: - """Codex builds its user agent as ``/ ...`` and ships - several first-party originators: ``codex-tui``, ``codex_cli_rs``, - ``codex_exec`` (exec mode), ``codex_vscode`` (IDE extension) and ``Codex ...`` - (see ``is_first_party_originator`` in codex-rs). They agree only on the - ``codex`` stem, and the TUI sends a bare ``codex-tui`` with no version at all, - so match the stem plus a separator rather than any one spelling.""" - return bool(_CODEX_CLIENT_PREFIX_RE.match(user_agent)) - - def should_auto_drop_params_for_agentic_cli(user_agent: str, data: dict, proxy_config: ProxyConfig) -> bool: """drop_params defaults to on for agentic CLIs so their client-specific params (e.g. Claude Code's thinking, Codex's service_tier) don't fail diff --git a/litellm/router_strategy/complexity_router/README.md b/litellm/router_strategy/complexity_router/README.md index 3d2b33a2ecd..be501db04f7 100644 --- a/litellm/router_strategy/complexity_router/README.md +++ b/litellm/router_strategy/complexity_router/README.md @@ -446,7 +446,9 @@ Reasoning markers in the system prompt do **not** trigger the reasoning override Agent harnesses inject their own context into the conversation as ordinary message text. That text is plumbing, not something a human asked for, so the router strips complete reminder blocks before classifying and picking a tier. A turn that is nothing but a reminder block strips to empty and is skipped, and the router falls back to the last real ask instead -By default the router strips complete ``, ``, ``, ``, and `` blocks. It also strips Codex repository instructions from the fixed heading prefix `# AGENTS.md instructions for ` through ``, regardless of the repository path +By default the router strips complete `` blocks. For requests with a Codex user agent, it also strips complete ``, ``, ``, and `` blocks, plus repository instructions from the fixed heading prefix `# AGENTS.md instructions for ` through ``, regardless of the repository path. Other clients keep those tags and their contents + +The proxy records the incoming user agent in request metadata. SDK callers can supply `metadata.user_agent` (or `litellm_metadata.user_agent` on Responses requests), or configure `reminder_markers` explicitly when their client identity is unavailable The Codex `Message Type: NEW_TASK` wrapper and its delegated-task payload remain available for classification. Cleanup applies to the current ask and quoted prior turns; the routed request retains its original content diff --git a/litellm/router_strategy/complexity_router/complexity_router.py b/litellm/router_strategy/complexity_router/complexity_router.py index 923aac85ee2..4c63452c1e9 100644 --- a/litellm/router_strategy/complexity_router/complexity_router.py +++ b/litellm/router_strategy/complexity_router/complexity_router.py @@ -39,6 +39,7 @@ from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.core_helpers import ( _get_parent_otel_span_from_kwargs, get_metadata_variable_name_from_kwargs, + is_codex_user_agent, ) from litellm.litellm_core_utils.internal_call_metadata import forwarded_internal_call_metadata from litellm.litellm_core_utils.prompt_templates.common_utils import ( @@ -386,8 +387,8 @@ def _effective_turn_off_message_logging(request_kwargs: Mapping[str, object] | N _REMINDER_OPEN: Final = "" _REMINDER_CLOSE: Final = "" -_DEFAULT_REMINDER_MARKERS: Final = ( - (_REMINDER_OPEN, _REMINDER_CLOSE), +_DEFAULT_REMINDER_MARKERS: Final = ((_REMINDER_OPEN, _REMINDER_CLOSE),) +_CODEX_REMINDER_MARKERS: Final = _DEFAULT_REMINDER_MARKERS + ( ("", ""), ("", ""), ("", ""), @@ -1951,6 +1952,7 @@ class ComplexityRouter(CustomLogger): raise ValueError("classifier_llm_config is not set") include_assistant: Final = self.config.classifier_context_include_assistant_turns + marker_pairs: Final = self._reminder_markers_for_request(request_kwargs or {}) context_enabled: Final = bool(messages) and self.config.classifier_context_window_size > 0 prior_turns: Final = ( _extract_prior_turns( @@ -1960,20 +1962,14 @@ class ComplexityRouter(CustomLogger): budget_chars=self.config.classifier_context_budget_chars, per_turn_chars=self.config.classifier_context_per_turn_chars, include_assistant=include_assistant, - marker_pairs=self._reminder_markers, + marker_pairs=marker_pairs, ) if context_enabled else () ) has_prior_conversation: Final = ( context_enabled - and len( - tuple( - islice( - _iter_context_turns_newest_first(messages or (), include_assistant, self._reminder_markers), 2 - ) - ) - ) + and len(tuple(islice(_iter_context_turns_newest_first(messages or (), include_assistant, marker_pairs), 2))) > 1 ) @@ -2434,7 +2430,7 @@ class ComplexityRouter(CustomLogger): body if isinstance(body, Mapping) else None, resolved_messages, tuple(self.config.plan_mode_patterns or ()), - self._reminder_markers, + self._reminder_markers_for_request(request_kwargs), ) def _matched_housekeeping_sentinel(self, newest_ask: str | None) -> str | None: @@ -3255,6 +3251,18 @@ class ComplexityRouter(CustomLogger): """ return _extract_current_ask_and_system_prompt(messages) + def _reminder_markers_for_request(self, request_kwargs: Mapping[str, object]) -> tuple[tuple[str, str], ...]: + if self.config.reminder_markers is not None: + return self._reminder_markers + if any( + is_codex_user_agent(user_agent) + for metadata_key in ("litellm_metadata", "metadata") + if isinstance(metadata := request_kwargs.get(metadata_key), Mapping) + if isinstance(user_agent := metadata.get("user_agent"), str) + ): + return _CODEX_REMINDER_MARKERS + return _DEFAULT_REMINDER_MARKERS + @staticmethod def _iter_metadata_dicts(request_kwargs: dict) -> list[dict]: """Metadata may land on `metadata` or `litellm_metadata` depending on the @@ -3356,6 +3364,7 @@ class ComplexityRouter(CustomLogger): # chat-completions messages, so it is real work on every non-chat surface, and # both the conversation shape and the classifier read the same list. resolved_messages: Final = self._resolve_messages(messages, request_kwargs) + marker_pairs: Final = self._reminder_markers_for_request(request_kwargs) conversation_continuing: Final = _conversation_is_continuing(resolved_messages) use_session_affinity: Final = self._uses_tier_pin @@ -3365,7 +3374,7 @@ class ComplexityRouter(CustomLogger): # In 'user_turn' mode a held pin is replayed only on continuation turns; a new human # ask falls through and re-classifies. session_affinity restores pin-first for asks too. pin_replay_allowed: Final = bool(self.config.session_affinity) or not _newest_turn_is_human_ask( - resolved_messages, self._reminder_markers + resolved_messages, marker_pairs ) if cache_key is not None and pin_replay_allowed: @@ -3376,7 +3385,7 @@ class ComplexityRouter(CustomLogger): pin_escalation_keyword: str | None = None if self.escalation_keywords: user_message: Final = ( - _newest_turn_ask(resolved_messages, self._reminder_markers) if resolved_messages else None + _newest_turn_ask(resolved_messages, marker_pairs) if resolved_messages else None ) if user_message is not None: pin_escalation_keyword = self._matched_escalation_keyword(user_message) @@ -3564,7 +3573,8 @@ class ComplexityRouter(CustomLogger): # Determine whether the original request used messages directly has_original_messages: Final = messages is not None and len(messages) > 0 - user_message, system_prompt = _extract_current_ask_and_system_prompt(resolved_messages, self._reminder_markers) + marker_pairs: Final = self._reminder_markers_for_request(request_kwargs) + user_message, system_prompt = _extract_current_ask_and_system_prompt(resolved_messages, marker_pairs) classifier_images: Final = self._classifier_image_parts(resolved_messages) if user_message is None and not classifier_images: @@ -3598,7 +3608,7 @@ class ComplexityRouter(CustomLogger): ) ask: Final = user_message or "" - newest_ask: Final = _newest_turn_ask(resolved_messages, self._reminder_markers) + newest_ask: Final = _newest_turn_ask(resolved_messages, marker_pairs) escalation_keyword: Final = self._matched_escalation_keyword(newest_ask) if newest_ask is not None else None # Resolved here rather than beside the classifier because the keyword-override path below # returns before any classification runs, and a forced tier gets stuck for the same reason diff --git a/litellm/router_strategy/complexity_router/config.py b/litellm/router_strategy/complexity_router/config.py index 6ab4aa85148..fb7003c887c 100644 --- a/litellm/router_strategy/complexity_router/config.py +++ b/litellm/router_strategy/complexity_router/config.py @@ -1292,8 +1292,9 @@ class ComplexityRouterConfig(BaseModel): "Override the delimiter pairs used to recognize and strip harness-injected reminder " "blocks before classification. A harness that wraps injected context differently per " "agent type (main, subagent, cron) lists every pair it emits. Replaces, rather than " - "adds to, the built-in system-reminder and Codex envelope pairs, so list every " - "built-in pair your harness also emits. Matching is case-insensitive." + "adds to, the built-in system-reminder pair and the Codex envelope pairs enabled " + "for Codex user agents, so list every built-in pair your harness also emits. " + "Matching is case-insensitive." ), ) diff --git a/tests/test_litellm/router_strategy/test_complexity_router.py b/tests/test_litellm/router_strategy/test_complexity_router.py index 792963fde23..44d7454e836 100644 --- a/tests/test_litellm/router_strategy/test_complexity_router.py +++ b/tests/test_litellm/router_strategy/test_complexity_router.py @@ -7625,9 +7625,38 @@ _CODEX_ENVELOPES: Final = ( class TestContextAwareClassifier: """Test the new classifier context window and trajectory signals.""" + @pytest.mark.asyncio + @pytest.mark.parametrize("envelope", _CODEX_ENVELOPES) + @pytest.mark.parametrize("user_agent", (None, "curl/8.7.1", "codexify/1.0")) + async def test_non_codex_requests_preserve_tagged_asks(self, envelope: str, user_agent: str | None) -> None: + completion: Final = AsyncMock(return_value=_llm_response('{"tier":"COMPLEX"}')) + router: Final = ComplexityRouter( + model_name="router", + litellm_router_instance=MagicMock(acompletion=completion), + complexity_router_config={ + "tiers": {"COMPLEX": "task-model"}, + "default_model": "fallback-model", + "classifier_type": "llm", + "classifier_llm_config": {"model": "classifier-model"}, + "escalation_keywords": [], + }, + ) + + response: Final = await router.async_pre_routing_hook( + model="router", + request_kwargs={"metadata": {"user_agent": user_agent}} if user_agent is not None else {}, + messages=[{"role": "user", "content": envelope}], + ) + + assert response is not None + assert response.model == "task-model" + completion.assert_awaited_once() + assert completion.call_args.kwargs["messages"][1]["content"].strip() == f"Classify this message:\n{envelope}" + @pytest.mark.parametrize("envelope", _CODEX_ENVELOPES) def test_codex_envelopes_preserve_delegated_task_and_prior_context(self, envelope: str) -> None: from litellm.router_strategy.complexity_router.complexity_router import ( + _CODEX_REMINDER_MARKERS, _extract_current_ask_and_system_prompt, _extract_prior_turns, _newest_turn_ask, @@ -7644,21 +7673,25 @@ class TestContextAwareClassifier: {"role": "user", "content": envelope}, ] - assert _extract_current_ask_and_system_prompt(messages)[0] == _CODEX_NEW_TASK - assert _extract_prior_turns(messages, _CODEX_NEW_TASK, 1, 100, None, False) == ( + assert _extract_current_ask_and_system_prompt(messages, _CODEX_REMINDER_MARKERS)[0] == _CODEX_NEW_TASK + assert _extract_prior_turns(messages, _CODEX_NEW_TASK, 1, 100, None, False, _CODEX_REMINDER_MARKERS) == ( ("user", "Design cache invalidation"), ) - assert _newest_turn_ask(messages) is None - assert _newest_turn_is_human_ask(messages) is False - assert _extract_current_ask_and_system_prompt([messages[-1]])[0] is None + assert _newest_turn_ask(messages, _CODEX_REMINDER_MARKERS) is None + assert _newest_turn_is_human_ask(messages, _CODEX_REMINDER_MARKERS) is False + assert _extract_current_ask_and_system_prompt([messages[-1]], _CODEX_REMINDER_MARKERS)[0] is None @pytest.mark.parametrize("envelope", _CODEX_ENVELOPES) def test_codex_marker_override_and_incomplete_blocks_preserve_text(self, envelope: str) -> None: - from litellm.router_strategy.complexity_router.complexity_router import _strip_reminder_blocks + from litellm.router_strategy.complexity_router.complexity_router import ( + _CODEX_REMINDER_MARKERS, + _strip_reminder_blocks, + ) incomplete: Final = envelope.rsplit("noise{envelope}", (("", ""),)) == envelope @pytest.mark.asyncio @@ -7681,9 +7714,12 @@ class TestContextAwareClassifier: ] original: Final = deepcopy(messages) request_kwargs: Final = ( - {"input": messages, "litellm_metadata": {"user_api_key_request_route": "/v1/responses"}} + { + "input": messages, + "litellm_metadata": {"user_api_key_request_route": "/v1/responses", "user_agent": "codex-tui"}, + } if responses_api - else {} + else {"metadata": {"user_agent": "codex-tui"}} ) response: Final = await router.async_pre_routing_hook( @@ -7706,6 +7742,46 @@ class TestContextAwareClassifier: else: assert response.messages == original + @pytest.mark.asyncio + @pytest.mark.parametrize("custom_markers", (False, True)) + async def test_codex_markers_are_request_scoped_and_respect_overrides(self, custom_markers: bool) -> None: + completion: Final = AsyncMock(return_value=_llm_response('{"tier":"COMPLEX"}')) + router: Final = ComplexityRouter( + model_name="router", + litellm_router_instance=MagicMock(acompletion=completion), + complexity_router_config={ + "tiers": {"COMPLEX": "task-model"}, + "classifier_type": "llm", + "classifier_llm_config": {"model": "classifier-model"}, + "classifier_context_window_size": 2, + "escalation_keywords": [], + **({"reminder_markers": [{"open": "", "close": ""}]} if custom_markers else {}), + }, + ) + envelope: Final = "\n".join(_CODEX_ENVELOPES) + prior: Final = f"{envelope}\nDesign cache invalidation" + messages: Final = [ + {"role": "user", "content": prior}, + {"role": "user", "content": _CODEX_NEW_TASK}, + {"role": "user", "content": envelope}, + ] + for user_agent in ("codex-tui", "curl/8.7.1", "codex_cli_rs/0.62.0"): + response: Final = await router.async_pre_routing_hook( + model="router", request_kwargs={"metadata": {"user_agent": user_agent}}, messages=messages + ) + assert response is not None + assert response.model == "task-model" + payload: Final = completion.call_args.kwargs["messages"][1]["content"] + if user_agent.startswith("codex") and not custom_markers: + assert payload.endswith(f"Classify this message:\n{_CODEX_NEW_TASK}") + assert "Design cache invalidation" in payload + assert "LITELLM ESCALATE" not in payload + else: + assert payload.endswith(f"Classify this message:\n{envelope}") + assert prior in payload + assert response.messages == messages + assert completion.await_count == 3 + @pytest.mark.parametrize( "messages,expected_ask", [ diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 5c4661ab357..ab2c1f27a59 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -35221,7 +35221,7 @@ export interface components { reasoning_override_min_score?: number | null; /** * Reminder Markers - * @description Override the delimiter pairs used to recognize and strip harness-injected reminder blocks before classification. A harness that wraps injected context differently per agent type (main, subagent, cron) lists every pair it emits. Replaces, rather than adds to, the built-in default of ('', ''), so a harness that also emits that pair lists it too. Matching is case-insensitive. + * @description Override the delimiter pairs used to recognize and strip harness-injected reminder blocks before classification. A harness that wraps injected context differently per agent type (main, subagent, cron) lists every pair it emits. Replaces, rather than adds to, the built-in system-reminder pair and the Codex envelope pairs enabled for Codex user agents, so list every built-in pair your harness also emits. Matching is case-insensitive. */ reminder_markers?: components["schemas"]["ReminderMarkerPair"][] | null; /** From 181b3fd94a17e54f0acbf42776bef36b5da4f2ed Mon Sep 17 00:00:00 2001 From: moe-berri Date: Thu, 10 Sep 2026 12:24:42 -0700 Subject: [PATCH 3/3] fix(router): classify new asks before reminder-only tails --- .../complexity_router/README.md | 2 + .../complexity_router/complexity_router.py | 32 +++++-- .../router_strategy/test_complexity_router.py | 96 +++++++++++++++++++ 3 files changed, 121 insertions(+), 9 deletions(-) diff --git a/litellm/router_strategy/complexity_router/README.md b/litellm/router_strategy/complexity_router/README.md index be501db04f7..f92e3569bbe 100644 --- a/litellm/router_strategy/complexity_router/README.md +++ b/litellm/router_strategy/complexity_router/README.md @@ -452,6 +452,8 @@ The proxy records the incoming user agent in request metadata. SDK callers can s The Codex `Message Type: NEW_TASK` wrapper and its delegated-task payload remain available for classification. Cleanup applies to the current ask and quoted prior turns; the routed request retains its original content +In `classification_mode: user_turn`, complete text-only reminder tails leave the preceding fresh ask eligible for classification. Assistant turns and tool results still mark continuations, including tool results carried alongside reminder text + `reminder_markers` replaces these defaults with your harness's own delimiters. Many harnesses use a different envelope per agent type, so list every pair you emit: ```yaml diff --git a/litellm/router_strategy/complexity_router/complexity_router.py b/litellm/router_strategy/complexity_router/complexity_router.py index 4c63452c1e9..a03b01ff1e0 100644 --- a/litellm/router_strategy/complexity_router/complexity_router.py +++ b/litellm/router_strategy/complexity_router/complexity_router.py @@ -599,6 +599,18 @@ def _last_human_ask_index( ) +def _is_reminder_only_turn(message: Mapping[str, object], marker_pairs: tuple[tuple[str, str], ...]) -> bool: + if message.get("role") != "user": + return False + content: Final = message.get("content") + if not isinstance(content, str) and not ( + isinstance(content, list) and all(isinstance(part, Mapping) and part.get("type") == "text" for part in content) + ): + return False + text: Final = _message_text(content) + return bool(text.strip()) and not _strip_reminder_blocks(text, marker_pairs) + + def _newest_turn_is_human_ask( messages: Sequence[Mapping[str, object]] | None, marker_pairs: tuple[tuple[str, str], ...] = _DEFAULT_REMINDER_MARKERS, @@ -609,21 +621,23 @@ def _newest_turn_is_human_ask( Anchored on `_last_human_ask_index` so every surface's plumbing reads as a continuation: chat-completions tool turns are role=tool, Messages-surface tool_result turns flatten to empty human text, and a hybrid turn carrying an ask alongside a tool_result still counts as an ask. - Compared against the newest non-system message rather than the raw tail, because Claude Code - appends a system-role reminder after the human turn; that trailing plumbing is neither an ask - nor loop traffic and must not turn a fresh ask into a continuation. An unreadable request (no - messages) is treated as a continuation: there is no ask to classify, which is the same reading - `_extract_current_ask_and_system_prompt` gives it downstream. + Trailing system messages and complete text-only reminders do not turn a fresh ask into a + continuation. Assistant turns and non-text content, including tool results alongside reminders, + still form continuation boundaries. An unreadable request has no ask to classify. """ if not messages: return False - newest_non_system: Final = next( - (index for index in range(len(messages) - 1, -1, -1) if messages[index].get("role") != "system"), + newest_activity: Final = next( + ( + index + for index in range(len(messages) - 1, -1, -1) + if messages[index].get("role") != "system" and not _is_reminder_only_turn(messages[index], marker_pairs) + ), None, ) - if newest_non_system is None: + if newest_activity is None: return False - return _last_human_ask_index(messages, marker_pairs) == newest_non_system + return _last_human_ask_index(messages, marker_pairs) == newest_activity def _iter_system_scope_texts( diff --git a/tests/test_litellm/router_strategy/test_complexity_router.py b/tests/test_litellm/router_strategy/test_complexity_router.py index 44d7454e836..750b4780c04 100644 --- a/tests/test_litellm/router_strategy/test_complexity_router.py +++ b/tests/test_litellm/router_strategy/test_complexity_router.py @@ -7625,6 +7625,102 @@ _CODEX_ENVELOPES: Final = ( class TestContextAwareClassifier: """Test the new classifier context window and trajectory signals.""" + @pytest.mark.parametrize( + "tail,expected", + ( + ([{"role": "user", "content": [{"type": "text", "text": _CODEX_ENVELOPES[0]}]}], True), + ([{"role": "assistant", "content": _CODEX_ENVELOPES[0]}], False), + ([{"role": "tool", "content": _CODEX_ENVELOPES[0]}], False), + ([{"role": "user", "content": " "}], False), + ( + [{"role": "user", "content": [_TOOL_RESULT, {"type": "text", "text": _CODEX_ENVELOPES[0]}]}], + False, + ), + ( + [{"role": "user", "content": [{"type": "image_url"}, {"type": "text", "text": _CODEX_ENVELOPES[0]}]}], + False, + ), + ), + ) + def test_only_text_reminder_tails_are_ignored_for_new_asks(self, tail: list[dict[str, object]], expected: bool) -> None: + from litellm.router_strategy.complexity_router.complexity_router import ( + _CODEX_REMINDER_MARKERS, + _newest_turn_is_human_ask, + ) + + assert _newest_turn_is_human_ask([_ASKED, *tail], _CODEX_REMINDER_MARKERS) is expected + assert _newest_turn_is_human_ask(tail, _CODEX_REMINDER_MARKERS) is False + + @pytest.mark.asyncio + @pytest.mark.parametrize("new_ask", (_CODEX_NEW_TASK, "Now design cache invalidation")) + @pytest.mark.parametrize("responses_api", (False, True)) + @pytest.mark.parametrize("session_affinity", (False, True)) + async def test_codex_tail_preserves_new_ask_and_tool_continuation_boundaries( + self, new_ask: str, responses_api: bool, session_affinity: bool + ) -> None: + completion: Final = AsyncMock( + side_effect=[_llm_response('{"tier":"SIMPLE"}'), _llm_response('{"tier":"COMPLEX"}')] + ) + router: Final = ComplexityRouter( + model_name="router", + litellm_router_instance=MagicMock(acompletion=completion, cache=DualCache()), + complexity_router_config={ + "tiers": {"SIMPLE": "simple-model", "COMPLEX": "task-model"}, + "classifier_type": "llm", + "classifier_llm_config": {"model": "classifier-model"}, + "classification_mode": "user_turn", + "session_affinity": session_affinity, + "escalation_keywords": [], + }, + ) + metadata: Final = {"user_agent": "codex-tui", "session_id": "codex-tail-session"} + first_messages: Final = [{"role": "user", "content": "Hello"}] + tail: Final = [{"role": "user", "content": envelope} for envelope in _CODEX_ENVELOPES] + new_messages: Final = [ + *first_messages, + {"role": "assistant", "content": "Hello"}, + {"role": "user", "content": new_ask}, + *tail, + ] + continuation: Final = [ + *new_messages, + {"role": "assistant", "content": "Working on it"}, + { + "role": "user", + "content": [ + {"type": "tool_result", "tool_use_id": "read-cache", "content": "cache source"}, + {"type": "text", "text": _CODEX_ENVELOPES[0]}, + ], + }, + *tail, + ] + results: Final = [ + await router.async_pre_routing_hook( + model="router", + request_kwargs=( + {"input": messages, "litellm_metadata": {**metadata, "user_api_key_request_route": "/v1/responses"}} + if responses_api + else {"metadata": metadata} + ), + messages=None if responses_api else messages, + input=messages if responses_api else None, + ) + for messages in (first_messages, new_messages, continuation) + ] + + assert [result.model for result in results] == ( + ["simple-model", "simple-model", "simple-model"] + if session_affinity + else ["simple-model", "task-model", "task-model"] + ) + assert completion.await_count == (1 if session_affinity else 2) + assert results[-1].routing_decision["cause"] == ( + "session_affinity_pin" if session_affinity else "user_turn_continuation" + ) + if not session_affinity: + assert completion.call_args.kwargs["messages"][1]["content"].endswith(f"Classify this message:\n{new_ask}") + assert results[1].messages == (None if responses_api else new_messages) + @pytest.mark.asyncio @pytest.mark.parametrize("envelope", _CODEX_ENVELOPES) @pytest.mark.parametrize("user_agent", (None, "curl/8.7.1", "codexify/1.0"))