mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-17 09:34:48 +00:00
fix(headroom): keep compressing when the min_tokens estimate cannot count a row
Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
parent
6be6fea7cc
commit
a56e5c835e
2 changed files with 29 additions and 6 deletions
|
|
@ -108,18 +108,23 @@ async def _estimate_compressible_tokens(model: object, messages: list[dict[str,
|
|||
|
||||
Only rows whose flattened content is a plain string count: the service
|
||||
passes list-of-parts rows through untouched, so their tokens can never
|
||||
turn into savings. A disabled global counter reports 0 for everything,
|
||||
so None keeps that case on the compression path.
|
||||
turn into savings. None keeps the request on the compression path: a
|
||||
disabled global counter reports 0 for everything, and a row shape the
|
||||
counter rejects must not turn into a failed request.
|
||||
"""
|
||||
if litellm.disable_token_counter:
|
||||
return None
|
||||
string_rows: Final = [m for m in messages if isinstance(m.get("content"), str)]
|
||||
if not string_rows:
|
||||
return 0
|
||||
return await offload_token_count(token_counter)(
|
||||
model=model if isinstance(model, str) else "",
|
||||
messages=string_rows,
|
||||
)
|
||||
try:
|
||||
return await offload_token_count(token_counter)(
|
||||
model=model if isinstance(model, str) else "",
|
||||
messages=string_rows,
|
||||
)
|
||||
except ValueError as e:
|
||||
verbose_proxy_logger.debug("Headroom: token estimate unavailable, compressing: %s", e)
|
||||
return None
|
||||
|
||||
|
||||
def _restore_content_shapes(
|
||||
|
|
|
|||
|
|
@ -380,6 +380,24 @@ async def test_apply_guardrail_min_tokens_ignores_rows_the_service_cannot_compre
|
|||
assert _recorded_guardrail_response(request_data)["compressible_tokens"] == text_only_tokens
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_apply_guardrail_compresses_when_a_row_shape_cannot_be_counted():
|
||||
odd_row = {"role": "user", "content": "Run it.", "tool_calls": "not-a-list"}
|
||||
messages = [ORIGINAL_MESSAGES[0], odd_row, ORIGINAL_MESSAGES[2], ORIGINAL_MESSAGES[3]]
|
||||
inputs = GenericGuardrailAPIInputs(texts=["Run it."], structured_messages=messages)
|
||||
request_data = {"model": "gpt-4o"}
|
||||
guardrail = _make_guardrail(min_tokens=1_000_000)
|
||||
|
||||
with patch.object(
|
||||
guardrail.async_handler, "post", new_callable=AsyncMock, return_value=_make_compress_response([odd_row])
|
||||
) as post:
|
||||
result = await guardrail.apply_guardrail(inputs=inputs, request_data=request_data, input_type="request")
|
||||
|
||||
post.assert_awaited_once()
|
||||
assert result["structured_messages"] is not None
|
||||
assert "skipped" not in _recorded_guardrail_response(request_data)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_apply_guardrail_compresses_when_token_counter_disabled(monkeypatch: pytest.MonkeyPatch):
|
||||
monkeypatch.setattr(litellm, "disable_token_counter", True)
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue