From 9e23ea41d9106d76f1fb79ba2c2588d8aaa4a901 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 29 Oct 2026 10:58:41 -0700 Subject: [PATCH] fix(count_tokens): pass the system prompt into clearing so count_tokens matches inference --- .../messages/responses_adapters/handler.py | 7 +++-- .../responses/count_tokens/token_counter.py | 21 +++++++++++-- .../count_tokens/test_token_counter.py | 30 ++++++++++++++++++- 3 files changed, 51 insertions(+), 7 deletions(-) diff --git a/litellm/llms/anthropic/messages/responses_adapters/handler.py b/litellm/llms/anthropic/messages/responses_adapters/handler.py index 4cf4aba4e0b..d2a16f0e5d3 100644 --- a/litellm/llms/anthropic/messages/responses_adapters/handler.py +++ b/litellm/llms/anthropic/messages/responses_adapters/handler.py @@ -88,7 +88,7 @@ async def _polyfilled( model: str, messages: AnthropicRequestMessages, tools: Sequence[AllAnthropicToolsValues | Mapping[str, object]] | None, - system: str | None, + system: str | Sequence[Mapping[str, object]] | None, edits: Sequence[Mapping[str, object]], kwargs: Mapping[str, object], ) -> PolyfillResult | None: @@ -102,7 +102,7 @@ async def _polyfilled( model=model, messages=_BLOCK_LIST.validate_python(messages), tools=_BLOCK_LIST.validate_python(tools) if tools else None, - system=system, + system=system if system is None or isinstance(system, str) else _BLOCK_LIST.validate_python(system), context_management_spec={"edits": [dict(edit) for edit in edits]}, drop_invalid_edits=drop_invalid_edits, ) @@ -120,6 +120,7 @@ async def count_view_messages( model: str, messages: Sequence[Mapping[str, object]], tools: Sequence[Mapping[str, object]] | None, + system: str | Sequence[Mapping[str, object]] | None, context_management: CountContextManagement | None, ) -> Sequence[Mapping[str, object]]: """The messages /v1/messages sends the Responses API once the polyfill has run the edits it can't run natively.""" @@ -130,7 +131,7 @@ async def count_view_messages( model=model, messages=_BLOCK_LIST.validate_python(messages), tools=tools, - system=None, + system=system, edits=edits, kwargs={"drop_params": context_management.drop_invalid_edits}, ) diff --git a/litellm/llms/openai/responses/count_tokens/token_counter.py b/litellm/llms/openai/responses/count_tokens/token_counter.py index 1b253808441..2c9bc442c0b 100644 --- a/litellm/llms/openai/responses/count_tokens/token_counter.py +++ b/litellm/llms/openai/responses/count_tokens/token_counter.py @@ -8,7 +8,7 @@ from types import MappingProxyType from typing import Any, Final import openai -from pydantic import JsonValue +from pydantic import JsonValue, TypeAdapter, ValidationError from typing_extensions import assert_never from litellm._logging import verbose_logger @@ -41,6 +41,7 @@ from litellm.types.utils import CountContextManagement, LlmProviders, TokenCount # Global handler instance - reuse across all token counting requests openai_count_tokens_handler: Final = OpenAICountTokensHandler() +_SYSTEM: Final = TypeAdapter(str | list[dict[str, object]]) def _workload_identity_config( @@ -97,10 +98,18 @@ def _count_error(model_to_use: str, request_model: str, message: str, status_cod ) +def _clearing_system(system: object) -> str | Sequence[Mapping[str, object]] | None: + try: + return _SYSTEM.validate_python(system) + except ValidationError: + return None + + async def _sent_messages( model: str, messages: Sequence[Mapping[str, object]], tools: Sequence[Mapping[str, object]] | None, + system: object, messages_format: TokenCountMessagesFormat, context_management: CountContextManagement | None, ) -> Sequence[Mapping[str, object]] | AnthropicContextManagementError: @@ -110,7 +119,11 @@ async def _sent_messages( case "anthropic": try: return await count_view_messages( - model=model, messages=messages, tools=tools, context_management=context_management + model=model, + messages=messages, + tools=tools, + system=_clearing_system(system), + context_management=context_management, ) except AnthropicContextManagementError as error: return error @@ -198,7 +211,9 @@ class OpenAITokenCounter(BaseTokenCounter): rejected_before_provider_call=True, ) - sent_messages: Final = await _sent_messages(model_to_use, messages, tools, messages_format, context_management) + sent_messages: Final = await _sent_messages( + model_to_use, messages, tools, system, messages_format, context_management + ) if isinstance(sent_messages, AnthropicContextManagementError): return TokenCountResponse( total_tokens=0, diff --git a/tests/unit/llms/openai/responses/count_tokens/test_token_counter.py b/tests/unit/llms/openai/responses/count_tokens/test_token_counter.py index 5fb1b9bd007..2b045148268 100644 --- a/tests/unit/llms/openai/responses/count_tokens/test_token_counter.py +++ b/tests/unit/llms/openai/responses/count_tokens/test_token_counter.py @@ -503,7 +503,7 @@ TOOL_PAIRS: Final[list[dict[str, object]]] = [ async def _count_with_edit( - monkeypatch: pytest.MonkeyPatch, edit: dict[str, JsonValue], drop_invalid_edits: bool + monkeypatch: pytest.MonkeyPatch, edit: dict[str, JsonValue], drop_invalid_edits: bool, system: object = None ) -> tuple[TokenCountResponse | None, respx.Route]: monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) with respx.mock: @@ -514,6 +514,7 @@ async def _count_with_edit( contents=None, deployment={"litellm_params": {"api_key": "sk-test"}}, request_model="openai/gpt-4o", + system=system, messages_format="anthropic", context_management=CountContextManagement(spec={"edits": [edit]}, drop_invalid_edits=drop_invalid_edits), ) @@ -547,6 +548,33 @@ async def test_anthropic_format_counts_the_history_the_clear_edit_leaves_for_inf } +@pytest.mark.asyncio +@pytest.mark.parametrize( + "system", + ["Follow the policy. " * 1500, [{"type": "text", "text": "Follow the policy. " * 1500}]], + ids=["string", "blocks"], +) +async def test_anthropic_format_input_tokens_trigger_counts_the_system_prompt_like_inference( + monkeypatch: pytest.MonkeyPatch, system: object +) -> None: + message_tokens: Final = litellm.token_counter(model="gpt-4o", messages=TOOL_PAIRS) + trigger_between_messages_and_system: Final = message_tokens + 1000 + edit: Final[dict[str, JsonValue]] = { + **CLEAR_EDIT, + "trigger": {"type": "input_tokens", "value": trigger_between_messages_and_system}, + } + + _, without_system = await _count_with_edit(monkeypatch, edit, drop_invalid_edits=False) + _, with_system = await _count_with_edit(monkeypatch, edit, drop_invalid_edits=False, system=system) + + assert _sent_outputs(without_system)["toolu_0"] == "log 0 " * 40 + assert _sent_outputs(with_system) == { + "toolu_0": CLEARED_TOOL_RESULT_PLACEHOLDER, + "toolu_1": CLEARED_TOOL_RESULT_PLACEHOLDER, + "toolu_2": "log 2 " * 40, + } + + @pytest.mark.asyncio async def test_anthropic_format_malformed_clear_edit_answers_400_without_calling_openai( monkeypatch: pytest.MonkeyPatch,