diff --git a/litellm/llms/anthropic/messages/context_management/editors/clear_tool_uses.py b/litellm/llms/anthropic/messages/context_management/editors/clear_tool_uses.py index e1dfff45138..79e0d8319f4 100644 --- a/litellm/llms/anthropic/messages/context_management/editors/clear_tool_uses.py +++ b/litellm/llms/anthropic/messages/context_management/editors/clear_tool_uses.py @@ -31,7 +31,7 @@ from ..constants import ( ) from ..errors import AnthropicContextManagementError from ..placeholders import build_cleared_tool_result_content -from .compact import system_to_text +from .compact import count_system_tokens if TYPE_CHECKING: from litellm.types.llms.openai import ChatCompletionToolParam @@ -199,14 +199,6 @@ def _select_in_batches(candidates: Sequence[str], tokens: Mapping[str, int], cle return tuple(candidates[:batch_end]) -def _with_system( - messages: Sequence[Mapping[str, object]], system: str | Sequence[Mapping[str, object]] | None -) -> Sequence[Mapping[str, object]]: - """The messages to count: like the API, an ``input_tokens`` trigger counts the system prompt too.""" - system_text: Final = system_to_text(system) - return ({"role": "system", "content": system_text}, *messages) if system_text else messages - - def _trigger_met( trigger: dict[str, object], model: str, @@ -215,7 +207,7 @@ def _trigger_met( system: str | Sequence[Mapping[str, object]] | None, counted_tool_uses: int, ) -> tuple[bool, int | None]: - """Return (trigger_met, input_tokens if counted for reuse).""" + """Return (trigger_met, message and tool tokens for reuse); like the API, input_tokens adds the system prompt.""" trigger_type: Final = trigger.get("type", "input_tokens") threshold = trigger.get("value") @@ -226,14 +218,15 @@ def _trigger_met( if not isinstance(threshold, int): threshold = DEFAULT_INPUT_TOKENS_TRIGGER - current_tokens: Final = litellm.token_counter( + message_tokens: Final = litellm.token_counter( model=model, messages=messages, tools=cast("list[ChatCompletionToolParam] | None", tools), ) + current_tokens: Final = message_tokens + count_system_tokens(model, system) verbose_logger.debug("context_management polyfill: current_tokens: %s", current_tokens) verbose_logger.debug("context_management polyfill: threshold: %s", threshold) - return current_tokens > threshold, current_tokens + return current_tokens > threshold, message_tokens def _resolve_keep_count(keep: dict[str, object]) -> int: diff --git a/litellm/llms/anthropic/messages/context_management/editors/compact.py b/litellm/llms/anthropic/messages/context_management/editors/compact.py index 753766708b6..9a07caee0b1 100644 --- a/litellm/llms/anthropic/messages/context_management/editors/compact.py +++ b/litellm/llms/anthropic/messages/context_management/editors/compact.py @@ -320,13 +320,15 @@ def _count_effective_tokens( content: Final = compaction_block.get("content") or "" if content: total += litellm.token_counter(model=model, text=content) - system_text: Final = system_to_text(system) - if system_text: - total += litellm.token_counter(model=model, text=system_text) - return total + return total + count_system_tokens(model, system) -def system_to_text( +def count_system_tokens(model: str, system: str | Sequence[Mapping[str, object]] | None) -> int: + system_text: Final = _system_to_text(system) + return litellm.token_counter(model=model, text=system_text) if system_text else 0 + + +def _system_to_text( system: str | Sequence[Mapping[str, object]] | None, ) -> str: """Flatten an Anthropic-style ``system`` value into a single string for diff --git a/tests/unit/llms/anthropic/messages/context_management/test_clear_tool_uses.py b/tests/unit/llms/anthropic/messages/context_management/test_clear_tool_uses.py index 9d0d9233c6b..3084cba36be 100644 --- a/tests/unit/llms/anthropic/messages/context_management/test_clear_tool_uses.py +++ b/tests/unit/llms/anthropic/messages/context_management/test_clear_tool_uses.py @@ -709,10 +709,18 @@ def test_input_tokens_trigger_counts_the_system_prompt(system: str | list[dict[s _, with_system = apply_clear_tool_uses_20250919( model=MODEL, messages=deepcopy(messages), tools=None, system=system, edit_spec=edit_spec ) + _, met_without_system = apply_clear_tool_uses_20250919( + model=MODEL, + messages=deepcopy(messages), + tools=None, + system=None, + edit_spec={**edit_spec, "trigger": {"type": "input_tokens", "value": 0}}, + ) assert without_system is None assert with_system is not None + assert met_without_system is not None assert with_system["cleared_tool_uses"] == 2 - assert with_system["cleared_input_tokens"] < history_tokens + assert with_system["cleared_input_tokens"] == met_without_system["cleared_input_tokens"] > 0 @pytest.mark.parametrize(("clear_tool_inputs", "triggers"), [(False, False), (True, True)])