diff --git a/litellm/proxy/guardrails/guardrail_hooks/headroom/headroom.py b/litellm/proxy/guardrails/guardrail_hooks/headroom/headroom.py index b1174ea71cb..f58b2fcb868 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/headroom/headroom.py +++ b/litellm/proxy/guardrails/guardrail_hooks/headroom/headroom.py @@ -108,18 +108,23 @@ async def _estimate_compressible_tokens(model: object, messages: list[dict[str, Only rows whose flattened content is a plain string count: the service passes list-of-parts rows through untouched, so their tokens can never - turn into savings. A disabled global counter reports 0 for everything, - so None keeps that case on the compression path. + turn into savings. None keeps the request on the compression path: a + disabled global counter reports 0 for everything, and a row shape the + counter rejects must not turn into a failed request. """ if litellm.disable_token_counter: return None string_rows: Final = [m for m in messages if isinstance(m.get("content"), str)] if not string_rows: return 0 - return await offload_token_count(token_counter)( - model=model if isinstance(model, str) else "", - messages=string_rows, - ) + try: + return await offload_token_count(token_counter)( + model=model if isinstance(model, str) else "", + messages=string_rows, + ) + except ValueError as e: + verbose_proxy_logger.debug("Headroom: token estimate unavailable, compressing: %s", e) + return None def _restore_content_shapes( diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_headroom.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_headroom.py index 19626f2252b..bf3fd4bb6f4 100644 --- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_headroom.py +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_headroom.py @@ -380,6 +380,24 @@ async def test_apply_guardrail_min_tokens_ignores_rows_the_service_cannot_compre assert _recorded_guardrail_response(request_data)["compressible_tokens"] == text_only_tokens +@pytest.mark.asyncio +async def test_apply_guardrail_compresses_when_a_row_shape_cannot_be_counted(): + odd_row = {"role": "user", "content": "Run it.", "tool_calls": "not-a-list"} + messages = [ORIGINAL_MESSAGES[0], odd_row, ORIGINAL_MESSAGES[2], ORIGINAL_MESSAGES[3]] + inputs = GenericGuardrailAPIInputs(texts=["Run it."], structured_messages=messages) + request_data = {"model": "gpt-4o"} + guardrail = _make_guardrail(min_tokens=1_000_000) + + with patch.object( + guardrail.async_handler, "post", new_callable=AsyncMock, return_value=_make_compress_response([odd_row]) + ) as post: + result = await guardrail.apply_guardrail(inputs=inputs, request_data=request_data, input_type="request") + + post.assert_awaited_once() + assert result["structured_messages"] is not None + assert "skipped" not in _recorded_guardrail_response(request_data) + + @pytest.mark.asyncio async def test_apply_guardrail_compresses_when_token_counter_disabled(monkeypatch: pytest.MonkeyPatch): monkeypatch.setattr(litellm, "disable_token_counter", True)