fix(count_tokens): pass the system prompt into clearing so count_tokens matches inference

This commit is contained in:
mateo-berri 2026-10-29 10:58:41 -07:00
parent d8648dccd3
commit 9e23ea41d9
3 changed files with 51 additions and 7 deletions

View file

@ -88,7 +88,7 @@ async def _polyfilled(
model: str,
messages: AnthropicRequestMessages,
tools: Sequence[AllAnthropicToolsValues | Mapping[str, object]] | None,
system: str | None,
system: str | Sequence[Mapping[str, object]] | None,
edits: Sequence[Mapping[str, object]],
kwargs: Mapping[str, object],
) -> PolyfillResult | None:
@ -102,7 +102,7 @@ async def _polyfilled(
model=model,
messages=_BLOCK_LIST.validate_python(messages),
tools=_BLOCK_LIST.validate_python(tools) if tools else None,
system=system,
system=system if system is None or isinstance(system, str) else _BLOCK_LIST.validate_python(system),
context_management_spec={"edits": [dict(edit) for edit in edits]},
drop_invalid_edits=drop_invalid_edits,
)
@ -120,6 +120,7 @@ async def count_view_messages(
model: str,
messages: Sequence[Mapping[str, object]],
tools: Sequence[Mapping[str, object]] | None,
system: str | Sequence[Mapping[str, object]] | None,
context_management: CountContextManagement | None,
) -> Sequence[Mapping[str, object]]:
"""The messages /v1/messages sends the Responses API once the polyfill has run the edits it can't run natively."""
@ -130,7 +131,7 @@ async def count_view_messages(
model=model,
messages=_BLOCK_LIST.validate_python(messages),
tools=tools,
system=None,
system=system,
edits=edits,
kwargs={"drop_params": context_management.drop_invalid_edits},
)

View file

@ -8,7 +8,7 @@ from types import MappingProxyType
from typing import Any, Final
import openai
from pydantic import JsonValue
from pydantic import JsonValue, TypeAdapter, ValidationError
from typing_extensions import assert_never
from litellm._logging import verbose_logger
@ -41,6 +41,7 @@ from litellm.types.utils import CountContextManagement, LlmProviders, TokenCount
# Global handler instance - reuse across all token counting requests
openai_count_tokens_handler: Final = OpenAICountTokensHandler()
_SYSTEM: Final = TypeAdapter(str | list[dict[str, object]])
def _workload_identity_config(
@ -97,10 +98,18 @@ def _count_error(model_to_use: str, request_model: str, message: str, status_cod
)
def _clearing_system(system: object) -> str | Sequence[Mapping[str, object]] | None:
try:
return _SYSTEM.validate_python(system)
except ValidationError:
return None
async def _sent_messages(
model: str,
messages: Sequence[Mapping[str, object]],
tools: Sequence[Mapping[str, object]] | None,
system: object,
messages_format: TokenCountMessagesFormat,
context_management: CountContextManagement | None,
) -> Sequence[Mapping[str, object]] | AnthropicContextManagementError:
@ -110,7 +119,11 @@ async def _sent_messages(
case "anthropic":
try:
return await count_view_messages(
model=model, messages=messages, tools=tools, context_management=context_management
model=model,
messages=messages,
tools=tools,
system=_clearing_system(system),
context_management=context_management,
)
except AnthropicContextManagementError as error:
return error
@ -198,7 +211,9 @@ class OpenAITokenCounter(BaseTokenCounter):
rejected_before_provider_call=True,
)
sent_messages: Final = await _sent_messages(model_to_use, messages, tools, messages_format, context_management)
sent_messages: Final = await _sent_messages(
model_to_use, messages, tools, system, messages_format, context_management
)
if isinstance(sent_messages, AnthropicContextManagementError):
return TokenCountResponse(
total_tokens=0,

View file

@ -503,7 +503,7 @@ TOOL_PAIRS: Final[list[dict[str, object]]] = [
async def _count_with_edit(
monkeypatch: pytest.MonkeyPatch, edit: dict[str, JsonValue], drop_invalid_edits: bool
monkeypatch: pytest.MonkeyPatch, edit: dict[str, JsonValue], drop_invalid_edits: bool, system: object = None
) -> tuple[TokenCountResponse | None, respx.Route]:
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
with respx.mock:
@ -514,6 +514,7 @@ async def _count_with_edit(
contents=None,
deployment={"litellm_params": {"api_key": "sk-test"}},
request_model="openai/gpt-4o",
system=system,
messages_format="anthropic",
context_management=CountContextManagement(spec={"edits": [edit]}, drop_invalid_edits=drop_invalid_edits),
)
@ -547,6 +548,33 @@ async def test_anthropic_format_counts_the_history_the_clear_edit_leaves_for_inf
}
@pytest.mark.asyncio
@pytest.mark.parametrize(
"system",
["Follow the policy. " * 1500, [{"type": "text", "text": "Follow the policy. " * 1500}]],
ids=["string", "blocks"],
)
async def test_anthropic_format_input_tokens_trigger_counts_the_system_prompt_like_inference(
monkeypatch: pytest.MonkeyPatch, system: object
) -> None:
message_tokens: Final = litellm.token_counter(model="gpt-4o", messages=TOOL_PAIRS)
trigger_between_messages_and_system: Final = message_tokens + 1000
edit: Final[dict[str, JsonValue]] = {
**CLEAR_EDIT,
"trigger": {"type": "input_tokens", "value": trigger_between_messages_and_system},
}
_, without_system = await _count_with_edit(monkeypatch, edit, drop_invalid_edits=False)
_, with_system = await _count_with_edit(monkeypatch, edit, drop_invalid_edits=False, system=system)
assert _sent_outputs(without_system)["toolu_0"] == "log 0 " * 40
assert _sent_outputs(with_system) == {
"toolu_0": CLEARED_TOOL_RESULT_PLACEHOLDER,
"toolu_1": CLEARED_TOOL_RESULT_PLACEHOLDER,
"toolu_2": "log 2 " * 40,
}
@pytest.mark.asyncio
async def test_anthropic_format_malformed_clear_edit_answers_400_without_calling_openai(
monkeypatch: pytest.MonkeyPatch,