fix(headroom): keep compressing when the min_tokens estimate cannot count a row

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
yucheng 2026-09-20 00:04:25 +00:00
parent 6be6fea7cc
commit a56e5c835e
2 changed files with 29 additions and 6 deletions

View file

@ -108,18 +108,23 @@ async def _estimate_compressible_tokens(model: object, messages: list[dict[str,
Only rows whose flattened content is a plain string count: the service
passes list-of-parts rows through untouched, so their tokens can never
turn into savings. A disabled global counter reports 0 for everything,
so None keeps that case on the compression path.
turn into savings. None keeps the request on the compression path: a
disabled global counter reports 0 for everything, and a row shape the
counter rejects must not turn into a failed request.
"""
if litellm.disable_token_counter:
return None
string_rows: Final = [m for m in messages if isinstance(m.get("content"), str)]
if not string_rows:
return 0
return await offload_token_count(token_counter)(
model=model if isinstance(model, str) else "",
messages=string_rows,
)
try:
return await offload_token_count(token_counter)(
model=model if isinstance(model, str) else "",
messages=string_rows,
)
except ValueError as e:
verbose_proxy_logger.debug("Headroom: token estimate unavailable, compressing: %s", e)
return None
def _restore_content_shapes(

View file

@ -380,6 +380,24 @@ async def test_apply_guardrail_min_tokens_ignores_rows_the_service_cannot_compre
assert _recorded_guardrail_response(request_data)["compressible_tokens"] == text_only_tokens
@pytest.mark.asyncio
async def test_apply_guardrail_compresses_when_a_row_shape_cannot_be_counted():
odd_row = {"role": "user", "content": "Run it.", "tool_calls": "not-a-list"}
messages = [ORIGINAL_MESSAGES[0], odd_row, ORIGINAL_MESSAGES[2], ORIGINAL_MESSAGES[3]]
inputs = GenericGuardrailAPIInputs(texts=["Run it."], structured_messages=messages)
request_data = {"model": "gpt-4o"}
guardrail = _make_guardrail(min_tokens=1_000_000)
with patch.object(
guardrail.async_handler, "post", new_callable=AsyncMock, return_value=_make_compress_response([odd_row])
) as post:
result = await guardrail.apply_guardrail(inputs=inputs, request_data=request_data, input_type="request")
post.assert_awaited_once()
assert result["structured_messages"] is not None
assert "skipped" not in _recorded_guardrail_response(request_data)
@pytest.mark.asyncio
async def test_apply_guardrail_compresses_when_token_counter_disabled(monkeypatch: pytest.MonkeyPatch):
monkeypatch.setattr(litellm, "disable_token_counter", True)