fix(headroom): count only compressible text rows off the event loop for min_tokens

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
yucheng 2026-09-19 23:55:36 +00:00
parent 81e5d69109
commit 6be6fea7cc
2 changed files with 33 additions and 39 deletions

View file

@ -28,6 +28,7 @@ from litellm.litellm_core_utils.prompt_templates.factory import (
group_tool_exchanges,
has_tool_with_name,
)
from litellm.litellm_core_utils.token_counter import offload_token_count
from litellm.llms.custom_httpx.http_handler import (
get_async_httpx_client, # pyright: ignore[reportUnknownVariableType]
httpxSpecialProvider,
@ -102,24 +103,23 @@ def _flatten_messages_for_compression(messages: list[dict[str, object]]) -> list
return flattened
def _estimate_compressible_tokens(model: object, messages: list[dict[str, object]]) -> int | None:
async def _estimate_compressible_tokens(model: object, messages: list[dict[str, object]]) -> int | None:
"""Token estimate for the min_tokens gate, or None when no trustworthy count exists.
None keeps the request on the compression path: a disabled global counter
reports 0 for everything, and content parts the counter does not know
raise, and neither must turn into a skipped or failed request.
Only rows whose flattened content is a plain string count: the service
passes list-of-parts rows through untouched, so their tokens can never
turn into savings. A disabled global counter reports 0 for everything,
so None keeps that case on the compression path.
"""
if litellm.disable_token_counter:
return None
try:
return token_counter(
model=model if isinstance(model, str) else "",
messages=messages,
use_default_image_token_count=True,
)
except ValueError as e:
verbose_proxy_logger.debug("Headroom: token estimate unavailable, compressing: %s", e)
return None
string_rows: Final = [m for m in messages if isinstance(m.get("content"), str)]
if not string_rows:
return 0
return await offload_token_count(token_counter)(
model=model if isinstance(model, str) else "",
messages=string_rows,
)
def _restore_content_shapes(
@ -835,7 +835,7 @@ class HeadroomGuardrail(CustomGuardrail):
model: Final = self.headroom_model or request_data.get("model")
flattened: Final = _flatten_messages_for_compression(compressible)
compressible_tokens: Final = (
_estimate_compressible_tokens(model=model, messages=flattened) if self.min_tokens > 0 else None
await _estimate_compressible_tokens(model=model, messages=flattened) if self.min_tokens > 0 else None
)
from litellm.proxy.common_utils.callback_utils import (
add_guardrail_to_applied_guardrails_header,

View file

@ -32,12 +32,11 @@ import respx
from fastapi import HTTPException
import litellm
from litellm.proxy.guardrails.guardrail_hooks.headroom.headroom import (
DEFAULT_MIN_TOKENS,
HEADROOM_RETRIEVE_TOOL_NAME,
HeadroomGuardrail,
has_headroom_retrieve_tool,
HEADROOM_RETRIEVE_TOOL_NAME,
)
from litellm.proxy.spend_tracking.compression_savings import (
extract_compression_saved_tokens,
@ -356,34 +355,29 @@ async def test_apply_guardrail_min_tokens_never_fetches_image_urls(respx_mock: r
@pytest.mark.asyncio
async def test_apply_guardrail_compresses_when_content_part_is_uncountable():
audio_messages = [
ORIGINAL_MESSAGES[0],
{
"role": "user",
"content": [
{"type": "text", "text": "Transcribe."},
{"type": "input_audio", "input_audio": {"data": "AAAA", "format": "wav"}},
],
},
ORIGINAL_MESSAGES[2],
ORIGINAL_MESSAGES[3],
async def test_apply_guardrail_min_tokens_ignores_rows_the_service_cannot_compress():
image_part = {"type": "image_url", "image_url": {"url": "data:image/png;base64,AAAA", "detail": "high"}}
audio_part = {"type": "input_audio", "input_audio": {"data": "AAAA", "format": "wav"}}
media_history = [
{"role": "user", "content": [{"type": "text", "text": "Look."}, image_part]},
{"role": "user", "content": "Here is a picture."},
{"role": "user", "content": [{"type": "text", "text": "Listen."}, audio_part]},
{"role": "user", "content": "Here is a sound."},
]
inputs = GenericGuardrailAPIInputs(texts=["Transcribe."], structured_messages=audio_messages)
messages = [ORIGINAL_MESSAGES[0], *media_history * 4, ORIGINAL_MESSAGES[2], ORIGINAL_MESSAGES[3]]
inputs = GenericGuardrailAPIInputs(texts=["Look."], structured_messages=messages)
request_data = {"model": "gpt-4o"}
guardrail = _make_guardrail(min_tokens=1_000_000)
text_only_tokens = token_counter(
model="gpt-4o", messages=[m for m in messages[1:-2] if isinstance(m["content"], str)]
)
guardrail = _make_guardrail(min_tokens=text_only_tokens + 1)
with patch.object(
guardrail.async_handler,
"post",
new_callable=AsyncMock,
return_value=_make_compress_response([audio_messages[1]]),
) as post:
with patch.object(guardrail.async_handler, "post", new_callable=AsyncMock) as post:
result = await guardrail.apply_guardrail(inputs=inputs, request_data=request_data, input_type="request")
post.assert_awaited_once()
assert result["structured_messages"] is not None
assert "skipped" not in _recorded_guardrail_response(request_data)
post.assert_not_awaited()
assert result is inputs
assert _recorded_guardrail_response(request_data)["compressible_tokens"] == text_only_tokens
@pytest.mark.asyncio