From 6be6fea7ccf49af670e3b9480348afa6d83ec94c Mon Sep 17 00:00:00 2001 From: yucheng Date: Sat, 19 Sep 2026 23:55:36 +0000 Subject: [PATCH] fix(headroom): count only compressible text rows off the event loop for min_tokens Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../guardrail_hooks/headroom/headroom.py | 28 ++++++------ .../guardrail_hooks/test_headroom.py | 44 ++++++++----------- 2 files changed, 33 insertions(+), 39 deletions(-) diff --git a/litellm/proxy/guardrails/guardrail_hooks/headroom/headroom.py b/litellm/proxy/guardrails/guardrail_hooks/headroom/headroom.py index d68d32ada08..b1174ea71cb 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/headroom/headroom.py +++ b/litellm/proxy/guardrails/guardrail_hooks/headroom/headroom.py @@ -28,6 +28,7 @@ from litellm.litellm_core_utils.prompt_templates.factory import ( group_tool_exchanges, has_tool_with_name, ) +from litellm.litellm_core_utils.token_counter import offload_token_count from litellm.llms.custom_httpx.http_handler import ( get_async_httpx_client, # pyright: ignore[reportUnknownVariableType] httpxSpecialProvider, @@ -102,24 +103,23 @@ def _flatten_messages_for_compression(messages: list[dict[str, object]]) -> list return flattened -def _estimate_compressible_tokens(model: object, messages: list[dict[str, object]]) -> int | None: +async def _estimate_compressible_tokens(model: object, messages: list[dict[str, object]]) -> int | None: """Token estimate for the min_tokens gate, or None when no trustworthy count exists. - None keeps the request on the compression path: a disabled global counter - reports 0 for everything, and content parts the counter does not know - raise, and neither must turn into a skipped or failed request. + Only rows whose flattened content is a plain string count: the service + passes list-of-parts rows through untouched, so their tokens can never + turn into savings. A disabled global counter reports 0 for everything, + so None keeps that case on the compression path. """ if litellm.disable_token_counter: return None - try: - return token_counter( - model=model if isinstance(model, str) else "", - messages=messages, - use_default_image_token_count=True, - ) - except ValueError as e: - verbose_proxy_logger.debug("Headroom: token estimate unavailable, compressing: %s", e) - return None + string_rows: Final = [m for m in messages if isinstance(m.get("content"), str)] + if not string_rows: + return 0 + return await offload_token_count(token_counter)( + model=model if isinstance(model, str) else "", + messages=string_rows, + ) def _restore_content_shapes( @@ -835,7 +835,7 @@ class HeadroomGuardrail(CustomGuardrail): model: Final = self.headroom_model or request_data.get("model") flattened: Final = _flatten_messages_for_compression(compressible) compressible_tokens: Final = ( - _estimate_compressible_tokens(model=model, messages=flattened) if self.min_tokens > 0 else None + await _estimate_compressible_tokens(model=model, messages=flattened) if self.min_tokens > 0 else None ) from litellm.proxy.common_utils.callback_utils import ( add_guardrail_to_applied_guardrails_header, diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_headroom.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_headroom.py index 1410e3f5bc2..19626f2252b 100644 --- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_headroom.py +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_headroom.py @@ -32,12 +32,11 @@ import respx from fastapi import HTTPException import litellm - from litellm.proxy.guardrails.guardrail_hooks.headroom.headroom import ( DEFAULT_MIN_TOKENS, + HEADROOM_RETRIEVE_TOOL_NAME, HeadroomGuardrail, has_headroom_retrieve_tool, - HEADROOM_RETRIEVE_TOOL_NAME, ) from litellm.proxy.spend_tracking.compression_savings import ( extract_compression_saved_tokens, @@ -356,34 +355,29 @@ async def test_apply_guardrail_min_tokens_never_fetches_image_urls(respx_mock: r @pytest.mark.asyncio -async def test_apply_guardrail_compresses_when_content_part_is_uncountable(): - audio_messages = [ - ORIGINAL_MESSAGES[0], - { - "role": "user", - "content": [ - {"type": "text", "text": "Transcribe."}, - {"type": "input_audio", "input_audio": {"data": "AAAA", "format": "wav"}}, - ], - }, - ORIGINAL_MESSAGES[2], - ORIGINAL_MESSAGES[3], +async def test_apply_guardrail_min_tokens_ignores_rows_the_service_cannot_compress(): + image_part = {"type": "image_url", "image_url": {"url": "data:image/png;base64,AAAA", "detail": "high"}} + audio_part = {"type": "input_audio", "input_audio": {"data": "AAAA", "format": "wav"}} + media_history = [ + {"role": "user", "content": [{"type": "text", "text": "Look."}, image_part]}, + {"role": "user", "content": "Here is a picture."}, + {"role": "user", "content": [{"type": "text", "text": "Listen."}, audio_part]}, + {"role": "user", "content": "Here is a sound."}, ] - inputs = GenericGuardrailAPIInputs(texts=["Transcribe."], structured_messages=audio_messages) + messages = [ORIGINAL_MESSAGES[0], *media_history * 4, ORIGINAL_MESSAGES[2], ORIGINAL_MESSAGES[3]] + inputs = GenericGuardrailAPIInputs(texts=["Look."], structured_messages=messages) request_data = {"model": "gpt-4o"} - guardrail = _make_guardrail(min_tokens=1_000_000) + text_only_tokens = token_counter( + model="gpt-4o", messages=[m for m in messages[1:-2] if isinstance(m["content"], str)] + ) + guardrail = _make_guardrail(min_tokens=text_only_tokens + 1) - with patch.object( - guardrail.async_handler, - "post", - new_callable=AsyncMock, - return_value=_make_compress_response([audio_messages[1]]), - ) as post: + with patch.object(guardrail.async_handler, "post", new_callable=AsyncMock) as post: result = await guardrail.apply_guardrail(inputs=inputs, request_data=request_data, input_type="request") - post.assert_awaited_once() - assert result["structured_messages"] is not None - assert "skipped" not in _recorded_guardrail_response(request_data) + post.assert_not_awaited() + assert result is inputs + assert _recorded_guardrail_response(request_data)["compressible_tokens"] == text_only_tokens @pytest.mark.asyncio