diff --git a/litellm/router_strategy/complexity_router/complexity_router.py b/litellm/router_strategy/complexity_router/complexity_router.py index 3b8f22b41d7..d17ecf73ddd 100644 --- a/litellm/router_strategy/complexity_router/complexity_router.py +++ b/litellm/router_strategy/complexity_router/complexity_router.py @@ -1063,14 +1063,16 @@ def _extract_agent_progress( An agent loop learns what the ask really involves from its tools, so these are what the classifier needs on a continuation turn. Only what follows the newest human ask counts, since tool traffic - before it served an earlier ask. Newest first is what the budget keeps, so the latest result always - reaches the classifier. Harness reminders are stripped from tool output the same way they are from asks. + before it served an earlier ask; a follow-up approving that work ("yes, go ahead") is what + classifier_context_include_assistant_turns is for. The latest result always reaches the classifier. + Harness reminders are stripped from tool output the same way they are from asks. A loop that overflows the budget has each entry cut to a quarter of it first, since one large file read - would otherwise fill the block alone. Its first tool result is kept even then, ahead of the newest entries - the rest of the budget buys, because that is usually the read that told the agent what the task is, and a - long loop of later reads would otherwise push it out and leave the classifier with routine output alone. A - loop that fits is quoted whole. + would otherwise fill the block alone. The oldest entries then fill half the budget and the newest the + rest, because the reads a loop opens with usually told the agent what the task is, and a long loop of + later reads would otherwise push them out and leave the classifier with routine output alone. Half + rather than the first result alone, since that one can be a failed read that says nothing about the + task. A loop that fits is quoted whole. """ from litellm.litellm_core_utils.prompt_templates.factory import tool_exchange_texts @@ -1094,12 +1096,13 @@ def _extract_agent_progress( return tuple(reversed(clamped)) share: Final = budget_chars // _TOOL_ENTRY_SHARES_OF_BUDGET oldest_first: Final = tuple((kind, _truncate(text, share)) for kind, text in reversed(clamped)) - first_result: Final = next((index for index, (kind, _) in enumerate(oldest_first) if kind == "tool result"), None) - if first_result is None or share < _MIN_QUOTED_TURN_CHARS: + if share < _MIN_QUOTED_TURN_CHARS or sum(len(text) for _, text in oldest_first) <= budget_chars: return tuple(reversed(_turns_within_budget(tuple(reversed(oldest_first)), budget_chars))) - anchor: Final = oldest_first[first_result] - newer: Final = tuple(reversed(oldest_first[first_result + 1 :])) - return (anchor, *reversed(_turns_within_budget(newer, budget_chars - len(anchor[1])))) + opening: Final = _turns_within_budget(oldest_first, budget_chars // 2) + latest: Final = _turns_within_budget( + tuple(reversed(oldest_first[len(opening) :])), budget_chars - sum(len(text) for _, text in opening) + ) + return (*opening, *reversed(latest)) def _decision_is_pinnable(decision: StandardLoggingRoutingDecision | None) -> bool: diff --git a/litellm/router_strategy/complexity_router/config.py b/litellm/router_strategy/complexity_router/config.py index cafd1ac2b50..746710a6e18 100644 --- a/litellm/router_strategy/complexity_router/config.py +++ b/litellm/router_strategy/complexity_router/config.py @@ -1385,8 +1385,11 @@ class ComplexityRouterConfig(LiteLLMBaseModel): "always goes to the classifier model, since the local scorer reads only the ask. The block is bounded " "on its own by classifier_context_budget_chars and classifier_context_per_turn_chars, " "newest first, and once a loop overflows that budget each entry is cut to a quarter of it, so " - "one large file read cannot push out the rest, and the first tool result after the ask, usually " - "the read that described the task, is kept ahead of the newest ones. It is not bounded by " + "one large file read cannot push out the rest, and the oldest entries after the ask, usually " + "the reads that described the task, fill half of it ahead of the newest ones. Tool traffic " + "before the newest ask is left out, since it served an earlier ask, so a follow-up such as " + "'yes, go ahead' that approves that work needs classifier_context_include_assistant_turns " + "to be classified on it. It is not bounded by " "classifier_context_window_size, since one ask can run dozens " "of tool calls; setting the window size to 0 still turns all context off. Tool output " "such as file contents is sent to the classifier model, which may be a different " diff --git a/tests/unit/router_strategy/test_complexity_router.py b/tests/unit/router_strategy/test_complexity_router.py index 21161f2f086..977307bf04d 100644 --- a/tests/unit/router_strategy/test_complexity_router.py +++ b/tests/unit/router_strategy/test_complexity_router.py @@ -11734,12 +11734,35 @@ class TestAgentProgressReachesClassifier: assert "balance = 0" in client.states[-1] @pytest.mark.asyncio - @pytest.mark.parametrize("reads,read_chars", [(4, 3000), (6, 1500), (20, 400)]) - async def test_long_loop_keeps_the_read_that_described_the_task(self, reads: int, read_chars: int) -> None: + @pytest.mark.parametrize("failed_first_read", [False, True]) + @pytest.mark.parametrize("reads,read_chars", [(4, 3000), (6, 1500), (20, 400), (4, 20000)]) + async def test_long_loop_keeps_the_read_that_described_the_task( + self, reads: int, read_chars: int, failed_first_read: bool + ) -> None: """Every later read in a long loop is routine on its own, so once they overflow the budget the classifier - must still see the task file the loop opened with, or every_request drops the loop to the cheap tier.""" + must still see the task file the loop opened with, or every_request drops the loop to the cheap tier. That + holds when a failed read comes before the task file too.""" client: Final = _TaskAwareJevClient() router: Final = self._jev_router(client, classifier_context_include_tool_turns=True) + ask, *task_read = _tool_loop_continuation("chat_completions") + failed_read: Final = ( + ( + { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "call_0", + "type": "function", + "function": {"name": "read_file", "arguments": '{"path": "docs/TASK.md"}'}, + } + ], + }, + {"role": "tool", "tool_call_id": "call_0", "content": "Error: sandbox denied the read (exit code 1)"}, + ) + if failed_first_read + else () + ) later_reads: Final = tuple( turn for i in range(reads) @@ -11762,7 +11785,7 @@ class TestAgentProgressReachesClassifier: response: Final = await router.async_pre_routing_hook( model="test-router", request_kwargs={}, - messages=[*_tool_loop_continuation("chat_completions"), *later_reads], + messages=[ask, *failed_read, *task_read, *later_reads], ) assert response is not None