mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
fix(count_tokens): pass the system prompt into clearing so count_tokens matches inference
This commit is contained in:
parent
d8648dccd3
commit
9e23ea41d9
3 changed files with 51 additions and 7 deletions
|
|
@ -88,7 +88,7 @@ async def _polyfilled(
|
|||
model: str,
|
||||
messages: AnthropicRequestMessages,
|
||||
tools: Sequence[AllAnthropicToolsValues | Mapping[str, object]] | None,
|
||||
system: str | None,
|
||||
system: str | Sequence[Mapping[str, object]] | None,
|
||||
edits: Sequence[Mapping[str, object]],
|
||||
kwargs: Mapping[str, object],
|
||||
) -> PolyfillResult | None:
|
||||
|
|
@ -102,7 +102,7 @@ async def _polyfilled(
|
|||
model=model,
|
||||
messages=_BLOCK_LIST.validate_python(messages),
|
||||
tools=_BLOCK_LIST.validate_python(tools) if tools else None,
|
||||
system=system,
|
||||
system=system if system is None or isinstance(system, str) else _BLOCK_LIST.validate_python(system),
|
||||
context_management_spec={"edits": [dict(edit) for edit in edits]},
|
||||
drop_invalid_edits=drop_invalid_edits,
|
||||
)
|
||||
|
|
@ -120,6 +120,7 @@ async def count_view_messages(
|
|||
model: str,
|
||||
messages: Sequence[Mapping[str, object]],
|
||||
tools: Sequence[Mapping[str, object]] | None,
|
||||
system: str | Sequence[Mapping[str, object]] | None,
|
||||
context_management: CountContextManagement | None,
|
||||
) -> Sequence[Mapping[str, object]]:
|
||||
"""The messages /v1/messages sends the Responses API once the polyfill has run the edits it can't run natively."""
|
||||
|
|
@ -130,7 +131,7 @@ async def count_view_messages(
|
|||
model=model,
|
||||
messages=_BLOCK_LIST.validate_python(messages),
|
||||
tools=tools,
|
||||
system=None,
|
||||
system=system,
|
||||
edits=edits,
|
||||
kwargs={"drop_params": context_management.drop_invalid_edits},
|
||||
)
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@ from types import MappingProxyType
|
|||
from typing import Any, Final
|
||||
|
||||
import openai
|
||||
from pydantic import JsonValue
|
||||
from pydantic import JsonValue, TypeAdapter, ValidationError
|
||||
from typing_extensions import assert_never
|
||||
|
||||
from litellm._logging import verbose_logger
|
||||
|
|
@ -41,6 +41,7 @@ from litellm.types.utils import CountContextManagement, LlmProviders, TokenCount
|
|||
|
||||
# Global handler instance - reuse across all token counting requests
|
||||
openai_count_tokens_handler: Final = OpenAICountTokensHandler()
|
||||
_SYSTEM: Final = TypeAdapter(str | list[dict[str, object]])
|
||||
|
||||
|
||||
def _workload_identity_config(
|
||||
|
|
@ -97,10 +98,18 @@ def _count_error(model_to_use: str, request_model: str, message: str, status_cod
|
|||
)
|
||||
|
||||
|
||||
def _clearing_system(system: object) -> str | Sequence[Mapping[str, object]] | None:
|
||||
try:
|
||||
return _SYSTEM.validate_python(system)
|
||||
except ValidationError:
|
||||
return None
|
||||
|
||||
|
||||
async def _sent_messages(
|
||||
model: str,
|
||||
messages: Sequence[Mapping[str, object]],
|
||||
tools: Sequence[Mapping[str, object]] | None,
|
||||
system: object,
|
||||
messages_format: TokenCountMessagesFormat,
|
||||
context_management: CountContextManagement | None,
|
||||
) -> Sequence[Mapping[str, object]] | AnthropicContextManagementError:
|
||||
|
|
@ -110,7 +119,11 @@ async def _sent_messages(
|
|||
case "anthropic":
|
||||
try:
|
||||
return await count_view_messages(
|
||||
model=model, messages=messages, tools=tools, context_management=context_management
|
||||
model=model,
|
||||
messages=messages,
|
||||
tools=tools,
|
||||
system=_clearing_system(system),
|
||||
context_management=context_management,
|
||||
)
|
||||
except AnthropicContextManagementError as error:
|
||||
return error
|
||||
|
|
@ -198,7 +211,9 @@ class OpenAITokenCounter(BaseTokenCounter):
|
|||
rejected_before_provider_call=True,
|
||||
)
|
||||
|
||||
sent_messages: Final = await _sent_messages(model_to_use, messages, tools, messages_format, context_management)
|
||||
sent_messages: Final = await _sent_messages(
|
||||
model_to_use, messages, tools, system, messages_format, context_management
|
||||
)
|
||||
if isinstance(sent_messages, AnthropicContextManagementError):
|
||||
return TokenCountResponse(
|
||||
total_tokens=0,
|
||||
|
|
|
|||
|
|
@ -503,7 +503,7 @@ TOOL_PAIRS: Final[list[dict[str, object]]] = [
|
|||
|
||||
|
||||
async def _count_with_edit(
|
||||
monkeypatch: pytest.MonkeyPatch, edit: dict[str, JsonValue], drop_invalid_edits: bool
|
||||
monkeypatch: pytest.MonkeyPatch, edit: dict[str, JsonValue], drop_invalid_edits: bool, system: object = None
|
||||
) -> tuple[TokenCountResponse | None, respx.Route]:
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
with respx.mock:
|
||||
|
|
@ -514,6 +514,7 @@ async def _count_with_edit(
|
|||
contents=None,
|
||||
deployment={"litellm_params": {"api_key": "sk-test"}},
|
||||
request_model="openai/gpt-4o",
|
||||
system=system,
|
||||
messages_format="anthropic",
|
||||
context_management=CountContextManagement(spec={"edits": [edit]}, drop_invalid_edits=drop_invalid_edits),
|
||||
)
|
||||
|
|
@ -547,6 +548,33 @@ async def test_anthropic_format_counts_the_history_the_clear_edit_leaves_for_inf
|
|||
}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"system",
|
||||
["Follow the policy. " * 1500, [{"type": "text", "text": "Follow the policy. " * 1500}]],
|
||||
ids=["string", "blocks"],
|
||||
)
|
||||
async def test_anthropic_format_input_tokens_trigger_counts_the_system_prompt_like_inference(
|
||||
monkeypatch: pytest.MonkeyPatch, system: object
|
||||
) -> None:
|
||||
message_tokens: Final = litellm.token_counter(model="gpt-4o", messages=TOOL_PAIRS)
|
||||
trigger_between_messages_and_system: Final = message_tokens + 1000
|
||||
edit: Final[dict[str, JsonValue]] = {
|
||||
**CLEAR_EDIT,
|
||||
"trigger": {"type": "input_tokens", "value": trigger_between_messages_and_system},
|
||||
}
|
||||
|
||||
_, without_system = await _count_with_edit(monkeypatch, edit, drop_invalid_edits=False)
|
||||
_, with_system = await _count_with_edit(monkeypatch, edit, drop_invalid_edits=False, system=system)
|
||||
|
||||
assert _sent_outputs(without_system)["toolu_0"] == "log 0 " * 40
|
||||
assert _sent_outputs(with_system) == {
|
||||
"toolu_0": CLEARED_TOOL_RESULT_PLACEHOLDER,
|
||||
"toolu_1": CLEARED_TOOL_RESULT_PLACEHOLDER,
|
||||
"toolu_2": "log 2 " * 40,
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_anthropic_format_malformed_clear_edit_answers_400_without_calling_openai(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue