mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-18 09:45:23 +00:00
fix(headroom): count only compressible text rows off the event loop for min_tokens
Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
parent
81e5d69109
commit
6be6fea7cc
2 changed files with 33 additions and 39 deletions
|
|
@ -28,6 +28,7 @@ from litellm.litellm_core_utils.prompt_templates.factory import (
|
|||
group_tool_exchanges,
|
||||
has_tool_with_name,
|
||||
)
|
||||
from litellm.litellm_core_utils.token_counter import offload_token_count
|
||||
from litellm.llms.custom_httpx.http_handler import (
|
||||
get_async_httpx_client, # pyright: ignore[reportUnknownVariableType]
|
||||
httpxSpecialProvider,
|
||||
|
|
@ -102,24 +103,23 @@ def _flatten_messages_for_compression(messages: list[dict[str, object]]) -> list
|
|||
return flattened
|
||||
|
||||
|
||||
def _estimate_compressible_tokens(model: object, messages: list[dict[str, object]]) -> int | None:
|
||||
async def _estimate_compressible_tokens(model: object, messages: list[dict[str, object]]) -> int | None:
|
||||
"""Token estimate for the min_tokens gate, or None when no trustworthy count exists.
|
||||
|
||||
None keeps the request on the compression path: a disabled global counter
|
||||
reports 0 for everything, and content parts the counter does not know
|
||||
raise, and neither must turn into a skipped or failed request.
|
||||
Only rows whose flattened content is a plain string count: the service
|
||||
passes list-of-parts rows through untouched, so their tokens can never
|
||||
turn into savings. A disabled global counter reports 0 for everything,
|
||||
so None keeps that case on the compression path.
|
||||
"""
|
||||
if litellm.disable_token_counter:
|
||||
return None
|
||||
try:
|
||||
return token_counter(
|
||||
model=model if isinstance(model, str) else "",
|
||||
messages=messages,
|
||||
use_default_image_token_count=True,
|
||||
)
|
||||
except ValueError as e:
|
||||
verbose_proxy_logger.debug("Headroom: token estimate unavailable, compressing: %s", e)
|
||||
return None
|
||||
string_rows: Final = [m for m in messages if isinstance(m.get("content"), str)]
|
||||
if not string_rows:
|
||||
return 0
|
||||
return await offload_token_count(token_counter)(
|
||||
model=model if isinstance(model, str) else "",
|
||||
messages=string_rows,
|
||||
)
|
||||
|
||||
|
||||
def _restore_content_shapes(
|
||||
|
|
@ -835,7 +835,7 @@ class HeadroomGuardrail(CustomGuardrail):
|
|||
model: Final = self.headroom_model or request_data.get("model")
|
||||
flattened: Final = _flatten_messages_for_compression(compressible)
|
||||
compressible_tokens: Final = (
|
||||
_estimate_compressible_tokens(model=model, messages=flattened) if self.min_tokens > 0 else None
|
||||
await _estimate_compressible_tokens(model=model, messages=flattened) if self.min_tokens > 0 else None
|
||||
)
|
||||
from litellm.proxy.common_utils.callback_utils import (
|
||||
add_guardrail_to_applied_guardrails_header,
|
||||
|
|
|
|||
|
|
@ -32,12 +32,11 @@ import respx
|
|||
from fastapi import HTTPException
|
||||
|
||||
import litellm
|
||||
|
||||
from litellm.proxy.guardrails.guardrail_hooks.headroom.headroom import (
|
||||
DEFAULT_MIN_TOKENS,
|
||||
HEADROOM_RETRIEVE_TOOL_NAME,
|
||||
HeadroomGuardrail,
|
||||
has_headroom_retrieve_tool,
|
||||
HEADROOM_RETRIEVE_TOOL_NAME,
|
||||
)
|
||||
from litellm.proxy.spend_tracking.compression_savings import (
|
||||
extract_compression_saved_tokens,
|
||||
|
|
@ -356,34 +355,29 @@ async def test_apply_guardrail_min_tokens_never_fetches_image_urls(respx_mock: r
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_apply_guardrail_compresses_when_content_part_is_uncountable():
|
||||
audio_messages = [
|
||||
ORIGINAL_MESSAGES[0],
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "Transcribe."},
|
||||
{"type": "input_audio", "input_audio": {"data": "AAAA", "format": "wav"}},
|
||||
],
|
||||
},
|
||||
ORIGINAL_MESSAGES[2],
|
||||
ORIGINAL_MESSAGES[3],
|
||||
async def test_apply_guardrail_min_tokens_ignores_rows_the_service_cannot_compress():
|
||||
image_part = {"type": "image_url", "image_url": {"url": "data:image/png;base64,AAAA", "detail": "high"}}
|
||||
audio_part = {"type": "input_audio", "input_audio": {"data": "AAAA", "format": "wav"}}
|
||||
media_history = [
|
||||
{"role": "user", "content": [{"type": "text", "text": "Look."}, image_part]},
|
||||
{"role": "user", "content": "Here is a picture."},
|
||||
{"role": "user", "content": [{"type": "text", "text": "Listen."}, audio_part]},
|
||||
{"role": "user", "content": "Here is a sound."},
|
||||
]
|
||||
inputs = GenericGuardrailAPIInputs(texts=["Transcribe."], structured_messages=audio_messages)
|
||||
messages = [ORIGINAL_MESSAGES[0], *media_history * 4, ORIGINAL_MESSAGES[2], ORIGINAL_MESSAGES[3]]
|
||||
inputs = GenericGuardrailAPIInputs(texts=["Look."], structured_messages=messages)
|
||||
request_data = {"model": "gpt-4o"}
|
||||
guardrail = _make_guardrail(min_tokens=1_000_000)
|
||||
text_only_tokens = token_counter(
|
||||
model="gpt-4o", messages=[m for m in messages[1:-2] if isinstance(m["content"], str)]
|
||||
)
|
||||
guardrail = _make_guardrail(min_tokens=text_only_tokens + 1)
|
||||
|
||||
with patch.object(
|
||||
guardrail.async_handler,
|
||||
"post",
|
||||
new_callable=AsyncMock,
|
||||
return_value=_make_compress_response([audio_messages[1]]),
|
||||
) as post:
|
||||
with patch.object(guardrail.async_handler, "post", new_callable=AsyncMock) as post:
|
||||
result = await guardrail.apply_guardrail(inputs=inputs, request_data=request_data, input_type="request")
|
||||
|
||||
post.assert_awaited_once()
|
||||
assert result["structured_messages"] is not None
|
||||
assert "skipped" not in _recorded_guardrail_response(request_data)
|
||||
post.assert_not_awaited()
|
||||
assert result is inputs
|
||||
assert _recorded_guardrail_response(request_data)["compressible_tokens"] == text_only_tokens
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue