diff --git a/litellm/router_strategy/complexity_router/complexity_router.py b/litellm/router_strategy/complexity_router/complexity_router.py index cbaba69f696..4c07679c5ca 100644 --- a/litellm/router_strategy/complexity_router/complexity_router.py +++ b/litellm/router_strategy/complexity_router/complexity_router.py @@ -274,6 +274,7 @@ _REMINDER_CLOSE: Final = "" _DEFAULT_REMINDER_MARKERS: Final = ((_REMINDER_OPEN, _REMINDER_CLOSE),) _TRUNCATION_MARKER: Final = "..." +_TRUNCATION_HEAD_FRACTION: Final = 0.3 _CJK_CHARACTER: Final = re.compile("[぀-ヿㇰ-ㇿ㐀-䶿一-鿿豈-﫿ｦ-ﾝ\U00020000-\U0003ffff]") @@ -552,8 +553,21 @@ def _matched_plan_mode_sentinel( def _truncate(text: str, limit: int) -> str: - """Cap text at limit characters, marking it so the classifier can tell the turn was cut short.""" - return text if len(text) <= limit else f"{text[:limit]}{_TRUNCATION_MARKER}" + """Cap text at limit characters, keeping both ends and eliding the middle. + + A chat turn states its ask at the end, so cutting the tail keeps the preamble and discards the + request the turn exists to make: a turn opening with an incident report and closing with "rewrite + the retry path and prove it cannot livelock" reached the classifier as the incident report alone. + Keeping both ends costs nothing at the same budget and is what the truncation literature finds + best for classifying long text, head+tail measuring above both head-only and tail-only in Sun et + al. 2019. The marker sits at the cut, so the turn reads as having its middle removed rather than + as trailing off mid-thought. + """ + if len(text) <= limit: + return text + head_chars: Final = max(int(limit * _TRUNCATION_HEAD_FRACTION), 0) + tail_chars: Final = max(limit - head_chars, 0) + return f"{text[:head_chars]}{_TRUNCATION_MARKER}{text[len(text) - tail_chars :]}" def _iter_context_turns_newest_first( diff --git a/tests/test_litellm/router_strategy/test_complexity_router.py b/tests/test_litellm/router_strategy/test_complexity_router.py index cbaac4d89a7..d4333fde163 100644 --- a/tests/test_litellm/router_strategy/test_complexity_router.py +++ b/tests/test_litellm/router_strategy/test_complexity_router.py @@ -5988,6 +5988,77 @@ class TestContextAwareClassifier: assert _strip_reminder_blocks(text, pairs) == "<<>> why is my tag stripped?" + @pytest.mark.parametrize( + "text,limit,expected", + [ + pytest.param("short", 10, "short", id="under-the-limit-is-untouched"), + pytest.param("exact", 5, "exact", id="exactly-the-limit-is-untouched"), + pytest.param( + "Second request with more details and longer text", + 30, + "Second re...tails and longer text", + id="over-the-limit-keeps-both-ends", + ), + pytest.param("abcdefghij", 4, "a...hij", id="tiny-limit-still-splits"), + pytest.param("abcdefghij", 1, "...j", id="limit-too-small-for-a-head-keeps-the-tail"), + pytest.param("abcdefghij", 0, "...", id="zero-limit-quotes-nothing"), + pytest.param("日本語のテキストと最後の質問", 6, "日...最後の質問", id="cjk-slices-by-character"), + ], + ) + def test_truncate_keeps_the_end_of_an_over_long_turn(self, text, limit, expected): + """A cut turn keeps its tail, because that is where a chat turn puts its ask. + + Head-only truncation was the shipped behavior and it discarded exactly the part that carries + the difficulty. The degenerate limits are here because the budget hands this function whatever + space is left rather than a configured constant, so it must stay total: a limit too small to + hold a head degrades to tail-only rather than raising or slicing with a negative index. + """ + from litellm.router_strategy.complexity_router.complexity_router import _truncate + + assert _truncate(text, limit) == expected + + def test_truncate_holds_its_length_budget(self): + """Cutting to N spends N characters plus the marker, at every N including the degenerate ones. + + The marker is the cost of having cut at all, so it is charged uniformly rather than only once + the limit is large enough to hold a head; a caller sizing a cut against a remaining budget can + therefore price it as limit plus marker without special-casing the small end. + """ + from litellm.router_strategy.complexity_router.complexity_router import _TRUNCATION_MARKER, _truncate + + text = "x" * 500 + + assert all( + len(_truncate(text, limit)) == limit + len(_TRUNCATION_MARKER) for limit in (0, 1, 2, 4, 30, 200, 499) + ) + + def test_clipped_prior_turn_still_carries_the_ask_it_closes_on(self): + """The reported defect, at the level the classifier sees it. + + A prior turn that opens with an incident report and closes with the request routed to the + cheapest tier, because the 200-character cut kept the report and dropped the request. The + quoted turn must carry both ends. + """ + from litellm.router_strategy.complexity_router.complexity_router import _extract_prior_turns + + turn = ( + "We run a multi-region gateway and last night the eu-west pod returned 502s on the " + "streaming path only, for thirty minutes, while non-streaming stayed healthy the whole " + "window and the cooldown map was mid-failover. " + "Filler sentence to push past the cap. " * 4 + + "Now rewrite the streaming retry path and prove it cannot livelock." + ) + + quoted = _extract_prior_turns( + [{"role": "user", "content": turn}, {"role": "user", "content": "go ahead"}], + "go ahead", + 3, + 200, + False, + ) + + assert "multi-region gateway" in quoted[0][1] + assert "prove it cannot livelock" in quoted[0][1] + @pytest.mark.parametrize( "messages,current_ask,window,per_turn_chars,include_assistant,expected", [ @@ -6002,7 +6073,7 @@ class TestContextAwareClassifier: 2, 30, False, - (("user", "First request"), ("user", "Second request with more detai...")), + (("user", "First request"), ("user", "Second re...tails and longer text")), id="current-ask-excluded-and-long-turn-marked-as-clipped", ), pytest.param( @@ -6111,7 +6182,7 @@ class TestContextAwareClassifier: 1, 20, True, - (("assistant", "a very long plan tha..."),), + (("assistant", "a very...l past the cap"),), id="assistant-reply-is-clipped-at-per-turn-chars", ), pytest.param( @@ -6134,7 +6205,8 @@ class TestContextAwareClassifier: The current ask is excluded by matching it rather than by position, since `aclassify` takes `prompt` and `messages` separately and a caller may classify other than the newest turn. A turn - cut at per_turn_chars is marked so a clip does not read as an abandoned thought. + over per_turn_chars keeps both ends with its middle elided, so the ask it closes on survives the + cut and the marker does not read as an abandoned thought. With assistant turns enabled the window is the last N turns of the conversation rather than the last N asks, which is what makes a plan the assistant called complex visible under a bare "yes".