From dd243bdccba7d2ebe2efb0c634c5c1109c9cf718 Mon Sep 17 00:00:00 2001 From: Alon Sokolovski Date: Tue, 20 Oct 2026 10:11:59 +0300 Subject: [PATCH] feat(context_management)!: honor clear_at_least, exclude_tools and clear_tool_inputs in the clear_tool_uses polyfill (#47123) * feat(context_management): honor clear_at_least, exclude_tools and clear_tool_inputs in the clear_tool_uses polyfill The clear_tool_uses_20250919 polyfill honored only trigger and keep and ignored the other three fields with a warning. It now follows the Anthropic API, as measured against it: - keep counts clearable tool uses only; uses of exclude_tools are never cleared and count toward neither keep nor clear_at_least. - clear_at_least clears oldest first in batches of at least that many tokens and leaves an incomplete last batch, so the cleared prefix only grows between requests and each batch costs one prompt-cache miss. - clear_tool_inputs (true or a list of tool names) also empties the input of the cleared tool uses. Invalid values are ignored with a _invalid warning on the applied edit. * refactor(context_management): build the clear_tool_uses selection without mutable state * test(context_management): cover a history made only of excluded tools * fix(context_management): leave tiny tool results out of clear_tool_uses like the API The API neither counts nor clears a tool use whose result is 1 to 46 bytes of text, whatever the tool or error flag; empty results still count. Probed against the API with crafted histories. Test data under 46 bytes is lengthened so those tests keep exercising clearing * fix(context_management): count tiny tool results when clear_tool_inputs covers their tool The API leaves a use whose result is 1 to 46 bytes out of clear_tool_uses only while its input is kept. When clear_tool_inputs is true or names the use's tool, the API counts it toward keep and clears it like any other use. * fix(context_management): count cleared inputs and images toward clear_at_least, answer malformed fields with a 400 * fix(context_management): validate clear_tool_uses edits before any editor runs, drop only the malformed edit under drop_params * fix(context_management): drop only the malformed clear_tool_uses fields under drop_params A malformed clear_at_least, exclude_tools or clear_tool_inputs no longer drops the whole edit under drop_params. The edit still runs with its well-formed fields, so the clearing that worked before is kept. Also types the new tests and swaps the hand-rolled spy router for an AsyncMock * test(context_management): pin the 46 byte cutoff in UTF-8 bytes and a batch closing at exactly clear_at_least * fix(context_management): leave a non-string tool_use id for the provider, size lone surrogates, and read drop_params strings * fix(context_management): read a drop_params string the same way on count_tokens and both /v1/messages paths count_tokens still checked drop_params with is True, so a body drop_params of "true" answered 400 there while /v1/messages dropped the malformed field. Both routes and the sync path now use drops_unsupported_params. --------- Co-authored-by: AlonSoko <25619863+AlonSoko@users.noreply.github.com> Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- .../pass_through/adapters/handler.py | 22 +- .../context_management/dispatcher.py | 25 +- .../context_management/editors/__init__.py | 4 +- .../editors/clear_tool_uses.py | 287 +++++++-- litellm/proxy/proxy_server.py | 3 +- .../test_handler_context_management.py | 14 +- .../test_clear_tool_uses.py | 561 +++++++++++++++++- .../context_management/test_compact.py | 16 +- .../context_management/test_dispatcher.py | 4 +- tests/unit/proxy/test_proxy_token_counter.py | 9 +- 10 files changed, 835 insertions(+), 110 deletions(-) diff --git a/litellm/llms/anthropic/pass_through/adapters/handler.py b/litellm/llms/anthropic/pass_through/adapters/handler.py index cca00a8383b..01e781485a0 100644 --- a/litellm/llms/anthropic/pass_through/adapters/handler.py +++ b/litellm/llms/anthropic/pass_through/adapters/handler.py @@ -13,7 +13,7 @@ from typing_extensions import TypedDict import litellm from litellm._logging import verbose_logger from litellm.litellm_core_utils.asyncify import run_async_function -from litellm.litellm_core_utils.core_helpers import normalize_drop_params +from litellm.litellm_core_utils.core_helpers import drops_unsupported_params from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObject from litellm.llms.anthropic.pass_through.adapters.transformation import ( AnthropicAdapter, @@ -331,8 +331,10 @@ async def _run_polyfill_if_enabled( does not disable a valid spec because context_management is a supported param). Raises ``AnthropicContextManagementError`` so the /v1/messages endpoint can emit an Anthropic-format 400, unless ``drop_invalid_edits`` - (``drop_params``) drops the invalid spec. All other exceptions are - best-effort swallowed (matches v0 behavior). + (``drop_params``) drops what is invalid: the malformed fields of a + ``clear_tool_uses`` edit alone, or the whole spec for an invalid ``compact`` + trigger. All other + exceptions are best-effort swallowed (matches v0 behavior). """ if not context_management_spec: return None @@ -353,6 +355,7 @@ async def _run_polyfill_if_enabled( sub_call_logging_kwargs=sub_call_logging_kwargs, parent_trace_id=parent_trace_id, count_only=count_only, + drop_invalid_edits=drop_invalid_edits, ) except AnthropicContextManagementError: if drop_invalid_edits: @@ -649,7 +652,7 @@ class LiteLLMMessagesToCompletionTransformationHandler: ) = ANTHROPIC_ADAPTER.translate_completion_input_params_with_tool_mapping( request_data, custom_llm_provider=custom_llm_provider if isinstance(custom_llm_provider, str) else None, - drop_params=litellm.drop_params is True or normalize_drop_params(extra_kwargs.get("drop_params")) is True, + drop_params=drops_unsupported_params(extra_kwargs.get("drop_params")), ) if openai_request is None: @@ -736,14 +739,12 @@ class LiteLLMMessagesToCompletionTransformationHandler: """Handle non-Anthropic models asynchronously using the adapter""" context_management: Final = kwargs.pop("context_management", None) additional_drop_params: Final[list[str] | None] = kwargs.get("additional_drop_params", None) - drop_invalid_edits: Final = litellm.drop_params is True or ( - "drop_params" in kwargs and kwargs["drop_params"] is True - ) litellm_router: Final[Router | None] = _router_or_proxy_fallback(kwargs.pop("litellm_router", None)) proxy_litellm_metadata, user_api_key_auth = _extract_proxy_litellm_metadata(kwargs) request_kwargs: Final = _REQUEST_KWARGS.validate_python(kwargs) reject_malformed_tool_result_text(messages, model, request_kwargs) + drop_invalid_edits: Final = drops_unsupported_params(request_kwargs.get("drop_params")) sub_call_logging_kwargs: Final = proxy_sub_call_logging_kwargs(request_kwargs) sub_call_trace_id: Final = proxy_sub_call_trace_id(request_kwargs) @@ -866,7 +867,8 @@ class LiteLLMMessagesToCompletionTransformationHandler: **kwargs, ) - reject_malformed_tool_result_text(messages, model, _REQUEST_KWARGS.validate_python(kwargs)) + sync_request_kwargs: Final = _REQUEST_KWARGS.validate_python(kwargs) + reject_malformed_tool_result_text(messages, model, sync_request_kwargs) # Run the context_management polyfill on the sync path too so that # ``litellm.messages.create()`` callers don't silently lose edits like @@ -875,9 +877,7 @@ class LiteLLMMessagesToCompletionTransformationHandler: # bridge to it via ``run_async_function``. context_management: Final = kwargs.pop("context_management", None) additional_drop_params: Final[list[str] | None] = kwargs.get("additional_drop_params", None) - drop_invalid_edits: Final = litellm.drop_params is True or ( - "drop_params" in kwargs and kwargs["drop_params"] is True - ) + drop_invalid_edits: Final = drops_unsupported_params(sync_request_kwargs.get("drop_params")) # Deliberately do NOT auto-attach the proxy ``llm_router`` here: # ``run_async_function`` spawns a new event loop in a worker thread # to bridge to the async dispatcher, but the proxy router's httpx diff --git a/litellm/llms/anthropic/pass_through/context_management/dispatcher.py b/litellm/llms/anthropic/pass_through/context_management/dispatcher.py index 3e1bab09d5a..bce653df6d1 100644 --- a/litellm/llms/anthropic/pass_through/context_management/dispatcher.py +++ b/litellm/llms/anthropic/pass_through/context_management/dispatcher.py @@ -9,7 +9,12 @@ from litellm.litellm_core_utils.asyncify import asyncify from litellm.types.llms.anthropic import AppliedEdit from .constants import CLEAR_TOOL_USES_EDIT_TYPE, COMPACT_EDIT_TYPE -from .editors import apply_clear_tool_uses_20250919, apply_compact_20260112 +from .editors import ( + CheckedEdit, + apply_clear_tool_uses_20250919, + apply_compact_20260112, + check_clear_tool_uses_spec, +) from .result import PolyfillResult if TYPE_CHECKING: @@ -73,6 +78,12 @@ def _wrap_editor_return( ) +def _checked_edit(index: int, edit_spec: dict[str, object]) -> CheckedEdit: + if edit_spec.get("type") != CLEAR_TOOL_USES_EDIT_TYPE: + return CheckedEdit(edit=edit_spec, error=None) + return check_clear_tool_uses_spec(edit_spec, index) + + async def apply_context_management( *, model: str, @@ -86,9 +97,14 @@ async def apply_context_management( sub_call_logging_kwargs: "ProxySubCallLoggingKwargs | None" = None, parent_trace_id: str | None = None, count_only: bool = False, + drop_invalid_edits: bool = False, ) -> PolyfillResult: """Run edits in order; return a single ``PolyfillResult``. + Every ``clear_tool_uses_20250919`` edit is validated before any editor runs, so a malformed one raises its 400 + before a ``compact_20260112`` summary call is paid for, or runs without its malformed fields under + ``drop_invalid_edits``. + The dispatcher is async so async editors (``compact_20260112``) can ``await`` the configured summarization model. Sync editors run in a worker thread so their token counts stay off the event loop; @@ -97,6 +113,11 @@ async def apply_context_management( edits: Final = normalize_spec(context_management_spec) if not edits: return PolyfillResult(messages=messages, system=system, applied_edits=[]) + checked: Final = tuple(_checked_edit(index, edit_spec) for index, edit_spec in enumerate(edits)) + first_error: Final = next((check.error for check in checked if check.error is not None), None) + if first_error is not None and not drop_invalid_edits: + raise first_error + well_formed_edits: Final = tuple(check.edit for check in checked) current_messages = messages current_system = system @@ -104,7 +125,7 @@ async def apply_context_management( aggregated_compaction_block = None aggregated_iterations_usage = None - for edit_spec in edits: + for edit_spec in well_formed_edits: edit_type = edit_spec.get("type") editor = _EDITOR_REGISTRY.get(edit_type) if isinstance(edit_type, str) else None if editor is None: diff --git a/litellm/llms/anthropic/pass_through/context_management/editors/__init__.py b/litellm/llms/anthropic/pass_through/context_management/editors/__init__.py index 3e933a9880a..5fc8c553d4f 100644 --- a/litellm/llms/anthropic/pass_through/context_management/editors/__init__.py +++ b/litellm/llms/anthropic/pass_through/context_management/editors/__init__.py @@ -1,4 +1,4 @@ -from .clear_tool_uses import apply_clear_tool_uses_20250919 +from .clear_tool_uses import CheckedEdit, apply_clear_tool_uses_20250919, check_clear_tool_uses_spec from .compact import apply_compact_20260112 -__all__ = ["apply_clear_tool_uses_20250919", "apply_compact_20260112"] +__all__ = ["CheckedEdit", "apply_clear_tool_uses_20250919", "apply_compact_20260112", "check_clear_tool_uses_spec"] diff --git a/litellm/llms/anthropic/pass_through/context_management/editors/clear_tool_uses.py b/litellm/llms/anthropic/pass_through/context_management/editors/clear_tool_uses.py index 00ecb315bf1..935ce77ef87 100644 --- a/litellm/llms/anthropic/pass_through/context_management/editors/clear_tool_uses.py +++ b/litellm/llms/anthropic/pass_through/context_management/editors/clear_tool_uses.py @@ -1,7 +1,23 @@ -"""``clear_tool_uses_20250919`` polyfill (v0: ``trigger`` and ``keep`` only).""" +"""``clear_tool_uses_20250919`` polyfill, matching the Anthropic API's selection. -from typing import Any, Final, cast +``keep`` counts clearable tool uses only. Uses of ``exclude_tools`` are never cleared and never count toward a +``tool_uses`` trigger, and neither are uses whose result is 1 to 46 bytes of text, which the API leaves out of its +count, unless ``clear_tool_inputs`` covers their tool: then their input can still be cleared, and the API counts and +clears them like any other. Clearing goes oldest first in batches that free at least ``clear_at_least`` tokens, +counting the result and any input it clears, and leaves an incomplete last batch, so the cleared prefix only grows +between requests and each batch costs one prompt-cache miss. ``check_clear_tool_uses_spec`` answers a malformed field +the way the API's 400 does, so the dispatcher can reject the edit, or drop only its malformed fields, before any editor +runs. +""" +from collections.abc import Mapping, Sequence, Set +from dataclasses import dataclass +from functools import reduce +from itertools import chain +from types import MappingProxyType +from typing import Annotated, Any, Final, Literal, TypeVar, cast + +from pydantic import BaseModel, ConfigDict, Field, StrictInt, TypeAdapter, ValidationError from typing_extensions import ReadOnly, TypedDict import litellm @@ -13,8 +29,39 @@ from ..constants import ( DEFAULT_INPUT_TOKENS_TRIGGER, DEFAULT_KEEP_TOOL_USES, ) +from ..errors import AnthropicContextManagementError from ..placeholders import build_cleared_tool_result_content +T = TypeVar("T") +# api.anthropic.com, measured 2026-10-19: text results of 1 to 46 bytes are never cleared or counted toward keep +_TINY_RESULT_BYTES: Final = 46 +ToolNames = Literal["all"] | frozenset[str] + + +class _ClearAtLeast(BaseModel): + model_config = ConfigDict(frozen=True) + + type: Literal["input_tokens"] + value: Annotated[StrictInt, Field(ge=0)] + + +class _ClearToolUsesFields(BaseModel): + model_config = ConfigDict(frozen=True) + + clear_at_least: _ClearAtLeast | None = None + exclude_tools: tuple[str, ...] | None = None + clear_tool_inputs: bool | tuple[str, ...] | None = None + + +_BLOCKS: Final = TypeAdapter(tuple[Mapping[str, object], ...]) +# api.anthropic.com, measured 2026-10-19: its wording for the two shapes pydantic words differently +_API_MESSAGES: Final = MappingProxyType( + { + "tuple_type": "Input should be a valid array", + "model_type": "Input does not match the expected shape.", + } +) + class ClearToolUsesEditSpec(TypedDict, total=False): """The ``clear_tool_uses_20250919`` entry of a ``context_management`` spec.""" @@ -27,36 +74,125 @@ class ClearToolUsesEditSpec(TypedDict, total=False): clear_tool_inputs: ReadOnly[object] -def _count_tool_uses(messages: list[dict[str, object]]) -> int: - """Return the number of tool_use content blocks across all messages. - - Only counts blocks with a string ``id`` to stay consistent with - :func:`_collect_tool_use_ids_in_order`, which is the source of truth for - which blocks are clearable. - """ - count = 0 - for msg in messages: - content = msg.get("content") - if isinstance(content, list): - for block in content: - if isinstance(block, dict) and block.get("type") == "tool_use": - if isinstance(block.get("id"), str): - count += 1 - return count +def _parse(adapter: TypeAdapter[T], value: object) -> T | None: + try: + return adapter.validate_python(value) + except ValidationError: + return None -def _collect_tool_use_ids_in_order(messages: list[dict[str, object]]) -> list[str]: - """Return tool_use ids in the chronological order they appear in messages.""" - ids: Final[list[str]] = [] - for msg in messages: - content = msg.get("content") - if isinstance(content, list): - for block in content: - if isinstance(block, dict) and block.get("type") == "tool_use": - block_id = block.get("id") - if isinstance(block_id, str): - ids.append(block_id) - return ids +@dataclass(frozen=True, slots=True) +class CheckedEdit: + edit: dict[str, object] + error: AnthropicContextManagementError | None + + +def check_clear_tool_uses_spec(edit_spec: dict[str, object], index: int) -> CheckedEdit: + """``edits[index]`` without its malformed fields, with the API's 400 for the first one, or None when there is none.""" + try: + _ClearToolUsesFields.model_validate(edit_spec) + except ValidationError as error: + details: Final = error.errors(include_url=False) + malformed: Final = frozenset(detail["loc"][0] for detail in details) + first: Final = details[0] + location: Final = ".".join(map(str, ("edits", index, CLEAR_TOOL_USES_EDIT_TYPE, *first["loc"]))) + return CheckedEdit( + edit={key: value for key, value in edit_spec.items() if key not in malformed}, + error=AnthropicContextManagementError( + status_code=400, + message=f"context_management.{location}: {_API_MESSAGES.get(first['type'], first['msg'])}", + ), + ) + return CheckedEdit(edit=edit_spec, error=None) + + +def _input_tools(clear_tool_inputs: bool | tuple[str, ...] | None) -> ToolNames: + return "all" if clear_tool_inputs is True else frozenset(clear_tool_inputs or ()) + + +def _all_blocks(messages: Sequence[Mapping[str, object]]) -> tuple[Mapping[str, object], ...]: + return tuple(chain.from_iterable(_parse(_BLOCKS, message.get("content")) or () for message in messages)) + + +def _tool_use_ref(block: Mapping[str, object]) -> tuple[str, str | None] | None: + block_id: Final = block.get("id") + name: Final = block.get("name") + if block.get("type") != "tool_use" or not isinstance(block_id, str): + return None + return block_id, name if isinstance(name, str) else None + + +def _tool_uses_in_order(blocks: Sequence[Mapping[str, object]]) -> tuple[tuple[str, str | None], ...]: + refs: Final = (_tool_use_ref(block) for block in blocks) + return tuple(ref for ref in refs if ref is not None) + + +def _block_id(block: Mapping[str, object], block_type: str, id_key: str) -> str | None: + block_id: Final = block.get(id_key) + return block_id if block.get("type") == block_type and isinstance(block_id, str) else None + + +def _blocks_by_id( + blocks: Sequence[Mapping[str, object]], block_type: str, id_key: str +) -> Mapping[str, Mapping[str, object]]: + pairs: Final = ((_block_id(block, block_type, id_key), block) for block in blocks) + return {block_id: block for block_id, block in pairs if block_id is not None} + + +def _utf8_size(text: str) -> int: + return len(text.encode("utf-8", "surrogatepass")) + + +def _text_bytes(content: object) -> int | None: + """Return the UTF-8 size of a text-only result, or None when it holds anything but text.""" + if isinstance(content, str): + return _utf8_size(content) + parts: Final = _parse(_BLOCKS, content) or () + texts: Final = tuple(part.get("text") for part in parts) + if any(part.get("type") != "text" or not isinstance(text, str) for part, text in zip(parts, texts)): + return None + return sum(_utf8_size(text) for text in texts if isinstance(text, str)) + + +def _is_tiny(content: object) -> bool: + size: Final = _text_bytes(content) + return size is not None and 0 < size <= _TINY_RESULT_BYTES + + +def _tiny_result_ids(results: Mapping[str, Mapping[str, object]]) -> frozenset[str]: + """Return ids whose result is 1 to 46 bytes of text; the API neither counts nor clears those uses.""" + return frozenset(tool_use_id for tool_use_id, block in results.items() if _is_tiny(block.get("content"))) + + +def _clears_input(name: str | None, input_tools: ToolNames) -> bool: + return input_tools == "all" or (name is not None and name in input_tools) + + +def _cleared_result(block: Mapping[str, object]) -> dict[str, object]: + return {**block, "content": build_cleared_tool_result_content(block.get("content"))} + + +def _freed_tokens(model: str, result: Mapping[str, object], use: Mapping[str, object], clears_input: bool) -> int: + """Tokens that clearing this use frees: its result, and its input when ``clear_tool_inputs`` covers it.""" + before: Final = [result, use] if clears_input else [result] + after: Final = [_cleared_result(result), {**use, "input": {}}] if clears_input else [_cleared_result(result)] + counts: Final = tuple( + litellm.token_counter(model=model, messages=[{"role": "user", "content": blocks}]) for blocks in (before, after) + ) + return max(counts[0] - counts[1], 0) + + +def _select_in_batches(candidates: Sequence[str], tokens: Mapping[str, int], clear_at_least: int) -> tuple[str, ...]: + """Oldest first, close a batch once it holds ``clear_at_least`` tokens; drop the incomplete last batch.""" + + def step(state: tuple[int, int], item: tuple[int, str]) -> tuple[int, int]: + batch_end, batch_tokens = state + index, tool_use_id = item + total: Final = batch_tokens + tokens[tool_use_id] + return (index + 1, 0) if total >= clear_at_least else (batch_end, total) + + batch_end, _ = reduce(step, enumerate(candidates), (0, 0)) + return tuple(candidates[:batch_end]) def _trigger_met( @@ -64,6 +200,7 @@ def _trigger_met( model: str, messages: list[dict[str, object]], tools: list[dict[str, object]] | None, + counted_tool_uses: int, ) -> tuple[bool, int | None]: """Return (trigger_met, input_tokens if counted for reuse).""" trigger_type: Final = trigger.get("type", "input_tokens") @@ -72,7 +209,7 @@ def _trigger_met( if trigger_type == "tool_uses": if not isinstance(threshold, int): return False, None - return _count_tool_uses(messages) > threshold, None + return counted_tool_uses > threshold, None if not isinstance(threshold, int): threshold = DEFAULT_INPUT_TOKENS_TRIGGER @@ -132,11 +269,7 @@ def _clear_tool_results( and block.get("type") == "tool_result" and block.get("tool_use_id") in ids_to_clear ): - new_block = { - **block, - "content": build_cleared_tool_result_content(block.get("content")), - } - new_blocks.append(new_block) + new_blocks.append(_cleared_result(block)) mutated = True cleared += 1 else: @@ -150,6 +283,23 @@ def _clear_tool_results( return new_messages, cleared +def _without_input(block: Mapping[str, object], ids_to_clear: Set[str], tool_names: ToolNames) -> Mapping[str, object]: + match _tool_use_ref(block): + case (block_id, name) if block_id in ids_to_clear and _clears_input(name, tool_names): + return {**block, "input": dict[str, object]()} + case _: + return block + + +def _message_without_inputs( + message: dict[str, object], ids_to_clear: Set[str], tool_names: ToolNames +) -> dict[str, object]: # mutable-ok: the editor contract hands messages back as list[dict] + blocks: Final = _parse(_BLOCKS, message.get("content")) + if blocks is None: + return message + return {**message, "content": [_without_input(block, ids_to_clear, tool_names) for block in blocks]} + + def apply_clear_tool_uses_20250919( *, model: str, @@ -158,15 +308,11 @@ def apply_clear_tool_uses_20250919( system: str | list[dict[str, object]] | None, edit_spec: ClearToolUsesEditSpec, ) -> tuple[list[dict[str, object]], AppliedEdit | None]: - """Apply clear_tool_uses; return (messages, AppliedEdit or None).""" - ignored_knobs = [knob for knob in ("clear_at_least", "exclude_tools", "clear_tool_inputs") if knob in edit_spec] - for ignored_knob in ignored_knobs: - verbose_logger.warning( - "context_management polyfill: ignoring '%s' on %s " - "(supported only on Anthropic-family forwarding path in v0)", - ignored_knob, - CLEAR_TOOL_USES_EDIT_TYPE, - ) + """Apply clear_tool_uses to an edit ``check_clear_tool_uses_spec`` returned; return (messages, AppliedEdit or None).""" + fields: Final = _ClearToolUsesFields.model_validate(edit_spec) + clear_at_least: Final = 0 if fields.clear_at_least is None else fields.clear_at_least.value + exclude_tools: Final = frozenset(fields.exclude_tools or ()) + input_tools: Final = _input_tools(fields.clear_tool_inputs) trigger: Final[dict[str, object]] = edit_spec.get("trigger") or { "type": "input_tokens", @@ -177,37 +323,56 @@ def apply_clear_tool_uses_20250919( "value": DEFAULT_KEEP_TOOL_USES, } - met, tokens_before = _trigger_met(trigger, model, messages, tools) + blocks: Final = _all_blocks(messages) + results: Final = _blocks_by_id(blocks, "tool_result", "tool_use_id") + uses: Final = _blocks_by_id(blocks, "tool_use", "id") + tiny: Final = _tiny_result_ids(results) + clearable: Final = tuple( + (tool_use_id, name) + for tool_use_id, name in _tool_uses_in_order(blocks) + if name not in exclude_tools and (tool_use_id not in tiny or _clears_input(name, input_tools)) + ) + met, counted_tokens = _trigger_met(trigger, model, messages, tools, len(clearable)) if not met: return messages, None keep_count: Final = _resolve_keep_count(keep) - tool_use_ids: Final = _collect_tool_use_ids_in_order(messages) - if len(tool_use_ids) <= keep_count: + if len(clearable) <= keep_count: return messages, None - ids_to_clear: Final = set(tool_use_ids[: len(tool_use_ids) - keep_count]) - - # Never clear the latest completed tool_result (reply context). last_completed_id: Final = _last_completed_tool_use_id(messages) - if last_completed_id is not None: - ids_to_clear.discard(last_completed_id) + candidates: Final = tuple( + (tool_use_id, name) + for tool_use_id, name in clearable[: len(clearable) - keep_count] + if tool_use_id in results and tool_use_id != last_completed_id + ) + candidate_ids: Final = tuple(tool_use_id for tool_use_id, _ in candidates) + tokens: Final = { + tool_use_id: _freed_tokens(model, results[tool_use_id], uses[tool_use_id], _clears_input(name, input_tools)) + for tool_use_id, name in (candidates if clear_at_least > 0 else ()) + } + ids_to_clear: Final = frozenset( + _select_in_batches(candidate_ids, tokens, clear_at_least) if clear_at_least > 0 else candidate_ids + ) - edited, cleared_count = _clear_tool_results(messages, ids_to_clear) + edited, cleared_count = _clear_tool_results(messages, set(ids_to_clear)) verbose_logger.debug("context_management polyfill: edited: %s", edited) if cleared_count == 0: return messages, None + final_messages: Final = ( + [_message_without_inputs(message, ids_to_clear, input_tools) for message in edited] if input_tools else edited + ) - if tokens_before is None: - tokens_before = litellm.token_counter(model=model, messages=messages, tools=cast(Any, tools)) - tokens_after: Final = litellm.token_counter(model=model, messages=edited, tools=cast(Any, tools)) + tokens_before: Final = ( + counted_tokens + if counted_tokens is not None + else litellm.token_counter(model=model, messages=messages, tools=cast(Any, tools)) + ) + tokens_after: Final = litellm.token_counter(model=model, messages=final_messages, tools=cast(Any, tools)) cleared_input_tokens: Final = max(tokens_before - tokens_after, 0) - applied: Final[AppliedEdit] = { + return final_messages, { "type": CLEAR_TOOL_USES_EDIT_TYPE, "cleared_tool_uses": cleared_count, "cleared_input_tokens": cleared_input_tokens, } - if ignored_knobs: - applied["warnings"] = [f"{knob}_ignored" for knob in ignored_knobs] - return edited, applied diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index c17e0c67d68..1840586a08e 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -360,6 +360,7 @@ from litellm.litellm_core_utils.core_helpers import ( get_litellm_metadata_from_kwargs, get_parent_otel_span_from_kwargs, iter_items, + normalize_drop_params, ) from litellm.litellm_core_utils.credential_accessor import CredentialAccessor from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj @@ -15208,7 +15209,7 @@ def _count_context_management( return CountContextManagement( spec=context_management, drop_invalid_edits=litellm.drop_params is True - or any(params.get("drop_params") is True for params in sent_params), + or any(normalize_drop_params(params.get("drop_params")) is True for params in sent_params), ) diff --git a/tests/unit/llms/anthropic/pass_through/adapters/test_handler_context_management.py b/tests/unit/llms/anthropic/pass_through/adapters/test_handler_context_management.py index f906530c8d9..9b5477e3206 100644 --- a/tests/unit/llms/anthropic/pass_through/adapters/test_handler_context_management.py +++ b/tests/unit/llms/anthropic/pass_through/adapters/test_handler_context_management.py @@ -83,9 +83,16 @@ async def _send(router: Router, request_options: Mapping[str, object], backend: ({"drop_params": True}, {}, {}, False), ({}, {"drop_params": True}, {}, False), ({}, {}, {"drop_params": True}, False), + ({}, {}, {"drop_params": "true"}, False), ({}, {}, {}, True), ), - ids=("deployment-drop-params", "router-default-drop-params", "request-drop-params", "litellm-drop-params"), + ids=( + "deployment-drop-params", + "router-default-drop-params", + "request-drop-params", + "request-drop-params-string", + "litellm-drop-params", + ), ) async def test_messages_sends_the_unedited_body_when_drop_params_drops_an_invalid_edit( deployment_params: Mapping[str, object], @@ -113,7 +120,8 @@ async def test_messages_rejects_an_invalid_edit_without_drop_params() -> None: assert backend.calls == [] -def test_sync_messages_sends_the_unedited_body_when_drop_params_drops_an_invalid_edit() -> None: +@pytest.mark.parametrize("drop_params", (True, "true"), ids=("drop-params", "drop-params-string")) +def test_sync_messages_sends_the_unedited_body_when_drop_params_drops_an_invalid_edit(drop_params: bool | str) -> None: backend: Final = _SyncBackend() with patch.object(litellm, "drop_params", False), patch("litellm.completion", new=backend): response: Final = LiteLLMMessagesToCompletionTransformationHandler.anthropic_messages_handler( @@ -122,7 +130,7 @@ def test_sync_messages_sends_the_unedited_body_when_drop_params_drops_an_invalid model="gemini/gemini-2.5-flash", api_key="test-key", context_management=_INVALID_COMPACT_EDIT, - drop_params=True, + drop_params=drop_params, ) assert isinstance(response, dict) diff --git a/tests/unit/llms/anthropic/pass_through/context_management/test_clear_tool_uses.py b/tests/unit/llms/anthropic/pass_through/context_management/test_clear_tool_uses.py index ee50bdff461..2ffa73dda21 100644 --- a/tests/unit/llms/anthropic/pass_through/context_management/test_clear_tool_uses.py +++ b/tests/unit/llms/anthropic/pass_through/context_management/test_clear_tool_uses.py @@ -3,18 +3,32 @@ Unit tests for the in-gateway `clear_tool_uses_20250919` polyfill editor. """ from copy import deepcopy +from itertools import chain +from typing import Final +from unittest.mock import AsyncMock, MagicMock +import pytest + +import litellm +from litellm.llms.anthropic.pass_through.adapters.handler import _run_polyfill_if_enabled +from litellm.llms.anthropic.pass_through.context_management import ( + AnthropicContextManagementError, + PolyfillResult, + apply_context_management, +) from litellm.llms.anthropic.pass_through.context_management.constants import ( CLEARED_TOOL_RESULT_PLACEHOLDER, + COMPACT_SUMMARY_MODEL_SETTING_KEY, ) from litellm.llms.anthropic.pass_through.context_management.editors.clear_tool_uses import ( apply_clear_tool_uses_20250919, ) +from litellm.types.llms.anthropic import AppliedEdit MODEL = "xai/grok-4" -def _make_pair(tool_use_id: str, result_text: str, location: str = "Mumbai"): +def _make_pair(tool_use_id: str, result_text: str, location: str = "Mumbai", name: str = "get_weather"): """Return an (assistant, user) message pair with one tool_use + tool_result.""" assistant_msg = { "role": "assistant", @@ -22,7 +36,7 @@ def _make_pair(tool_use_id: str, result_text: str, location: str = "Mumbai"): { "type": "tool_use", "id": tool_use_id, - "name": "get_weather", + "name": name, "input": {"location": location}, } ], @@ -203,10 +217,71 @@ def test_cleared_input_tokens_is_nonnegative(): assert applied["cleared_input_tokens"] >= 0 -def test_ignored_knobs_do_not_alter_behavior(): - """clear_at_least / exclude_tools / clear_tool_inputs are accepted but ignored in v0.""" +def _blocks_of(messages: list[dict[str, object]], block_type: str) -> tuple[dict[str, object], ...]: + contents: Final = (message["content"] for message in messages) + blocks: Final = chain.from_iterable(content for content in contents if isinstance(content, list)) + return tuple(block for block in blocks if isinstance(block, dict) and block["type"] == block_type) + + +def _result_contents(messages: list[dict[str, object]]) -> dict[object, object]: + return {block["tool_use_id"]: block["content"] for block in _blocks_of(messages, "tool_result")} + + +def _cleared_ids(messages: list[dict[str, object]]) -> set[object]: + placeholders: Final = (CLEARED_TOOL_RESULT_PLACEHOLDER, [{"type": "text", "text": CLEARED_TOOL_RESULT_PLACEHOLDER}]) + return {tool_use_id for tool_use_id, content in _result_contents(messages).items() if content in placeholders} + + +def _result_tokens(text: str) -> int: + return litellm.token_counter(model=MODEL, text=text) + + +def test_clear_at_least_clears_in_batches_and_leaves_an_incomplete_batch(): + """Oldest first, a batch closes once it reaches clear_at_least; the rest waits.""" + messages = _make_history(n_pairs=8) + one_result = _result_tokens(_result_contents(messages)["toolu_00"]) + edit_spec = { + "type": "clear_tool_uses_20250919", + "trigger": {"type": "tool_uses", "value": 0}, + "keep": {"type": "tool_uses", "value": 1}, + # Three results fill a batch; two don't. + "clear_at_least": {"type": "input_tokens", "value": 2 * one_result + 1}, + } + new_messages, applied = apply_clear_tool_uses_20250919( + model=MODEL, messages=messages, tools=None, system=None, edit_spec=edit_spec + ) + # 7 eligible: two batches of 3; the 7th waits for its batch to fill. + assert applied is not None + assert applied["cleared_tool_uses"] == 6 + assert _cleared_ids(new_messages) == {f"toolu_{i:02d}" for i in range(6)} + + +def test_clear_at_least_keeps_the_cleared_prefix_stable_between_requests(): + """A new tool use doesn't move the cleared prefix until the next batch fills (prompt-cache friendly).""" + history = _make_history(n_pairs=10) + one_result = _result_tokens(_result_contents(history)["toolu_00"]) + edit_spec = { + "type": "clear_tool_uses_20250919", + "trigger": {"type": "tool_uses", "value": 0}, + "keep": {"type": "tool_uses", "value": 1}, + "clear_at_least": {"type": "input_tokens", "value": 2 * one_result + 1}, + } + cleared: Final = tuple( + _cleared_ids( + apply_clear_tool_uses_20250919( + model=MODEL, messages=history[: 1 + 2 * n_pairs], tools=None, system=None, edit_spec=edit_spec + )[0] + ) + for n_pairs in (8, 9, 10) + ) + assert cleared[0] == cleared[1] == {f"toolu_{i:02d}" for i in range(6)} + assert cleared[2] == {f"toolu_{i:02d}" for i in range(9)} + + +def test_clear_at_least_out_of_reach_leaves_messages_unchanged(): messages = _make_history(n_pairs=3) - _, applied = apply_clear_tool_uses_20250919( + original = deepcopy(messages) + new_messages, applied = apply_clear_tool_uses_20250919( model=MODEL, messages=messages, tools=None, @@ -216,28 +291,46 @@ def test_ignored_knobs_do_not_alter_behavior(): "trigger": {"type": "tool_uses", "value": 0}, "keep": {"type": "tool_uses", "value": 1}, "clear_at_least": {"type": "input_tokens", "value": 999_999_999}, - "exclude_tools": ["get_weather"], - "clear_tool_inputs": True, }, ) - # Despite clear_at_least being huge, polyfill still applies (knob ignored). - # Despite clear_tool_inputs=True, inputs are NOT cleared (knob ignored). - assert applied is not None - assert applied["cleared_tool_uses"] == 2 - # Ignored knobs surface as warnings on the AppliedEdit so operators can - # see what was dropped (the v0 polyfill silently dropping them at debug - # log level made misconfiguration invisible from the response). - assert set(applied.get("warnings", [])) == { - "clear_at_least_ignored", - "exclude_tools_ignored", - "clear_tool_inputs_ignored", + assert applied is None + assert new_messages == original + + +def test_a_batch_closes_when_it_frees_exactly_clear_at_least(): + messages = _make_history(n_pairs=6) + clear_one_use = { + "type": "clear_tool_uses_20250919", + "trigger": {"type": "tool_uses", "value": 0}, + "keep": {"type": "tool_uses", "value": 5}, } + _, one = apply_clear_tool_uses_20250919( + model=MODEL, messages=messages, tools=None, system=None, edit_spec=clear_one_use + ) + assert one is not None + assert one["cleared_tool_uses"] == 1 + new_messages, applied = apply_clear_tool_uses_20250919( + model=MODEL, + messages=messages, + tools=None, + system=None, + edit_spec={ + **clear_one_use, + "keep": {"type": "tool_uses", "value": 1}, + "clear_at_least": {"type": "input_tokens", "value": 2 * one["cleared_input_tokens"]}, + }, + ) + assert _cleared_ids(new_messages) == {f"toolu_{i:02d}" for i in range(4)} + assert applied is not None + assert applied["cleared_tool_uses"] == 4 -def test_no_ignored_knobs_omits_warnings_field(): - """When the caller doesn't pass any unsupported knobs, no ``warnings`` are added.""" - messages = _make_history(n_pairs=3) - _, applied = apply_clear_tool_uses_20250919( +def test_exclude_tools_are_never_cleared_and_do_not_count_toward_keep(): + """keep counts clearable uses only: with keep=1 the newest get_weather result stays, not the newest search.""" + messages = [{"role": "user", "content": "Plan a trip."}] + for i, name in enumerate(["get_weather", "search", "get_weather", "get_weather", "search"]): + messages.extend(_make_pair(f"toolu_{i:02d}", f"Result {i}: " + "x" * 200, name=name)) + new_messages, applied = apply_clear_tool_uses_20250919( model=MODEL, messages=messages, tools=None, @@ -246,10 +339,428 @@ def test_no_ignored_knobs_omits_warnings_field(): "type": "clear_tool_uses_20250919", "trigger": {"type": "tool_uses", "value": 0}, "keep": {"type": "tool_uses", "value": 1}, + "exclude_tools": ["search"], }, ) assert applied is not None - assert "warnings" not in applied + assert _cleared_ids(new_messages) == {"toolu_00", "toolu_02"} + assert applied["cleared_tool_uses"] == 2 + + +def test_history_of_only_excluded_tools_is_left_unchanged(): + messages = [{"role": "user", "content": "Plan a trip."}] + for i in range(4): + messages.extend(_make_pair(f"toolu_{i:02d}", f"Result {i}: " + "x" * 200, name="search")) + original = deepcopy(messages) + new_messages, applied = apply_clear_tool_uses_20250919( + model=MODEL, + messages=messages, + tools=None, + system=None, + edit_spec={ + "type": "clear_tool_uses_20250919", + "trigger": {"type": "tool_uses", "value": 0}, + "keep": {"type": "tool_uses", "value": 1}, + "exclude_tools": ["search"], + }, + ) + assert applied is None + assert new_messages == original + + +def _clear_with_keep_two(messages: list[dict[str, object]]) -> tuple[list[dict[str, object]], AppliedEdit | None]: + return apply_clear_tool_uses_20250919( + model=MODEL, + messages=messages, + tools=None, + system=None, + edit_spec={ + "type": "clear_tool_uses_20250919", + "trigger": {"type": "tool_uses", "value": 0}, + "keep": {"type": "tool_uses", "value": 2}, + }, + ) + + +def test_tiny_results_are_never_cleared_and_do_not_count_toward_keep(): + messages = _make_history(6) + messages[2]["content"][0]["content"] = "(Bash completed with no output)" + messages[12]["content"][0]["content"] = [{"type": "text", "text": "\u9ce5" * 15}] + new_messages, applied = _clear_with_keep_two(messages) + assert _cleared_ids(new_messages) == {"toolu_01", "toolu_02"} + assert applied is not None + assert applied["cleared_tool_uses"] == 2 + + +def _clear_with_keep_two_and_inputs( + messages: list[dict[str, object]], clear_tool_inputs: bool | list[str] +) -> tuple[list[dict[str, object]], AppliedEdit | None]: + return apply_clear_tool_uses_20250919( + model=MODEL, + messages=messages, + tools=None, + system=None, + edit_spec={ + "type": "clear_tool_uses_20250919", + "trigger": {"type": "tool_uses", "value": 0}, + "keep": {"type": "tool_uses", "value": 2}, + "clear_tool_inputs": clear_tool_inputs, + }, + ) + + +def test_tiny_results_count_and_clear_when_their_tool_inputs_are_cleared(): + for clear_tool_inputs in (True, ["get_weather"]): + messages = _make_history(6) + messages[2]["content"][0]["content"] = "(Bash completed with no output)" + messages[12]["content"][0]["content"] = "ok" + new_messages, applied = _clear_with_keep_two_and_inputs(messages, clear_tool_inputs) + assert _cleared_ids(new_messages) == {"toolu_00", "toolu_01", "toolu_02", "toolu_03"} + assert applied is not None + assert applied["cleared_tool_uses"] == 4 + + +def test_tiny_results_stay_out_when_clear_tool_inputs_names_other_tools(): + messages = _make_history(6) + messages[2]["content"][0]["content"] = "(Bash completed with no output)" + messages[12]["content"][0]["content"] = "ok" + new_messages, applied = _clear_with_keep_two_and_inputs(messages, ["read_file"]) + assert _cleared_ids(new_messages) == {"toolu_01", "toolu_02"} + assert applied is not None + assert applied["cleared_tool_uses"] == 2 + + +def test_results_over_46_bytes_and_empty_results_still_count(): + # The 46 byte cutoff was measured on api.anthropic.com on 2026-10-19. + messages = _make_history(5) + messages[2]["content"][0]["content"] = "x" * 47 + messages[4]["content"][0]["content"] = "" + messages[6]["content"][0]["content"] = [ + {"type": "text", "text": "ok"}, + {"type": "image", "source": {"type": "base64", "media_type": "image/png", "data": "iVBORw0KGgo="}}, + ] + new_messages, applied = _clear_with_keep_two(messages) + assert _cleared_ids(new_messages) == {"toolu_00", "toolu_01", "toolu_02"} + assert new_messages[6]["content"][0]["content"] == [{"type": "text", "text": CLEARED_TOOL_RESULT_PLACEHOLDER}] + assert applied is not None + assert applied["cleared_tool_uses"] == 3 + + +def test_the_46_byte_cutoff_includes_46_and_counts_utf8_bytes(): + # The 46 byte cutoff was measured on api.anthropic.com on 2026-10-19. + messages = _make_history(5) + messages[2]["content"][0]["content"] = "x" * 46 + messages[4]["content"][0]["content"] = "鳥" * 16 + new_messages, applied = _clear_with_keep_two(messages) + assert _cleared_ids(new_messages) == {"toolu_01", "toolu_02"} + assert applied is not None + assert applied["cleared_tool_uses"] == 2 + + +def test_a_result_holding_a_lone_surrogate_is_sized_like_any_other_text(): + messages = _make_history(5) + messages[2]["content"][0]["content"] = "\ud800" + new_messages, applied = _clear_with_keep_two(messages) + assert _cleared_ids(new_messages) == {"toolu_01", "toolu_02"} + assert applied is not None + + +def _tool_inputs(messages: list[dict[str, object]]) -> dict[object, object]: + return {block["id"]: block["input"] for block in _blocks_of(messages, "tool_use")} + + +def test_clear_tool_inputs_true_empties_the_inputs_of_cleared_uses_only(): + messages = _make_history(n_pairs=3) + new_messages, applied = apply_clear_tool_uses_20250919( + model=MODEL, + messages=messages, + tools=None, + system=None, + edit_spec={ + "type": "clear_tool_uses_20250919", + "trigger": {"type": "tool_uses", "value": 0}, + "keep": {"type": "tool_uses", "value": 1}, + "clear_tool_inputs": True, + }, + ) + assert applied is not None + assert _tool_inputs(new_messages) == {"toolu_00": {}, "toolu_01": {}, "toolu_02": {"location": "City2"}} + # The caller's messages are not mutated. + assert _tool_inputs(messages)["toolu_00"] == {"location": "City0"} + + +def test_clear_tool_inputs_list_clears_only_the_named_tools(): + messages = [{"role": "user", "content": "Plan a trip."}] + for i, name in enumerate(["get_weather", "search", "get_weather"]): + messages.extend(_make_pair(f"toolu_{i:02d}", f"Result {i}: " + "x" * 200, location=f"City{i}", name=name)) + new_messages, applied = apply_clear_tool_uses_20250919( + model=MODEL, + messages=messages, + tools=None, + system=None, + edit_spec={ + "type": "clear_tool_uses_20250919", + "trigger": {"type": "tool_uses", "value": 0}, + "keep": {"type": "tool_uses", "value": 1}, + "clear_tool_inputs": ["search"], + }, + ) + assert applied is not None + assert _cleared_ids(new_messages) == {"toolu_00", "toolu_01"} + assert _tool_inputs(new_messages) == { + "toolu_00": {"location": "City0"}, + "toolu_01": {}, + "toolu_02": {"location": "City2"}, + } + + +def test_a_tool_use_with_a_non_string_id_is_left_for_the_provider_to_reject(): + messages = _make_history(n_pairs=4) + messages[1]["content"][0]["id"] = ["toolu_00"] + new_messages, applied = apply_clear_tool_uses_20250919( + model=MODEL, + messages=messages, + tools=None, + system=None, + edit_spec={ + "type": "clear_tool_uses_20250919", + "trigger": {"type": "tool_uses", "value": 0}, + "keep": {"type": "tool_uses", "value": 1}, + "clear_tool_inputs": True, + }, + ) + assert applied is not None + assert new_messages[1] == messages[1] + assert _cleared_ids(new_messages) == {"toolu_01", "toolu_02"} + assert (new_messages[3]["content"][0]["input"], new_messages[5]["content"][0]["input"]) == ({}, {}) + + +async def _malformed_spec_error(edits: list[dict[str, object]]) -> AnthropicContextManagementError: + with pytest.raises(AnthropicContextManagementError) as raised: + await apply_context_management( + model=MODEL, + messages=_make_history(n_pairs=3), + tools=None, + system=None, + context_management_spec={"edits": edits}, + ) + return raised.value + + +@pytest.mark.parametrize( + ("field", "value"), + [ + ("clear_at_least", {"type": "tool_uses", "value": 5}), + ("clear_at_least", {"value": 5}), + ("clear_at_least", {"type": "input_tokens", "value": -1}), + ("clear_at_least", {"type": "input_tokens", "value": 1.5}), + ("clear_at_least", {"type": "input_tokens", "value": "1200"}), + ("clear_at_least", {"type": "input_tokens", "value": True}), + ("exclude_tools", "get_weather"), + ("exclude_tools", [1]), + ("clear_tool_inputs", "get_weather"), + ], +) +async def test_malformed_fields_are_a_400_even_below_the_trigger(field: str, value: object) -> None: + error = await _malformed_spec_error( + [{"type": "clear_tool_uses_20250919", "trigger": {"type": "tool_uses", "value": 99}, field: value}] + ) + assert error.status_code == 400 + assert error.message.startswith(f"context_management.edits.0.clear_tool_uses_20250919.{field}") + + +@pytest.mark.parametrize( + ("field", "value", "message"), + [ + ("exclude_tools", "memory", "exclude_tools: Input should be a valid array"), + ("clear_at_least", 1200, "clear_at_least: Input does not match the expected shape."), + ("clear_at_least", {"type": "input_tokens", "value": "1200"}, "value: Input should be a valid integer"), + ("clear_tool_inputs", 5, "clear_tool_inputs.bool: Input should be a valid boolean, unable to interpret input"), + ], +) +async def test_malformed_field_400_names_the_edit_and_field_as_the_api_does( + field: str, value: object, message: str +) -> None: + # api.anthropic.com, measured 2026-10-19, answers these four with these messages for the edit at index 1. + error = await _malformed_spec_error( + [{"type": "clear_thinking_20251015"}, {"type": "clear_tool_uses_20250919", field: value}] + ) + assert error.message.startswith("context_management.edits.1.clear_tool_uses_20250919.") + assert error.message.endswith(message) + + +_VALID_EDIT: Final = { + "type": "clear_tool_uses_20250919", + "trigger": {"type": "tool_uses", "value": 0}, + "keep": {"type": "tool_uses", "value": 1}, +} +_MALFORMED_EDIT: Final = {**_VALID_EDIT, "exclude_tools": "get_weather"} + + +@pytest.mark.parametrize( + ("field", "value"), + [("exclude_tools", "get_weather"), ("clear_at_least", {"type": "input_tokens", "value": "1200"})], +) +async def test_drop_params_drops_only_the_malformed_field(field: str, value: object) -> None: + async def run(drop_invalid_edits: bool) -> PolyfillResult | None: + return await _run_polyfill_if_enabled( + model=MODEL, + messages=_make_history(n_pairs=3), + tools=None, + system=None, + context_management_spec={"edits": [{**_VALID_EDIT, field: value, "clear_tool_inputs": True}]}, + litellm_metadata={}, + additional_drop_params=None, + llm_router=None, + drop_invalid_edits=drop_invalid_edits, + ) + + with pytest.raises( + AnthropicContextManagementError, match=rf"^context_management\.edits\.0\.clear_tool_uses_20250919\.{field}" + ): + await run(drop_invalid_edits=False) + result: Final = await run(drop_invalid_edits=True) + assert result is not None + assert _cleared_ids(result.messages) == {"toolu_00", "toolu_01"} + assert _tool_inputs(result.messages) == {"toolu_00": {}, "toolu_01": {}, "toolu_02": {"location": "City2"}} + + +async def test_malformed_edit_is_rejected_before_a_compact_summary_is_paid_for( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from litellm.proxy.proxy_server import general_settings + + monkeypatch.setitem(general_settings, COMPACT_SUMMARY_MODEL_SETTING_KEY, "summary-model") + router: Final = MagicMock() + router.acompletion = AsyncMock(side_effect=RuntimeError("summary model unavailable")) + with pytest.raises(AnthropicContextManagementError): + await apply_context_management( + model=MODEL, + messages=[{"role": "user", "content": "hello " * 60_000}], + tools=None, + system=None, + context_management_spec={ + "edits": [ + {"type": "compact_20260112", "trigger": {"type": "input_tokens", "value": 50_000}}, + _MALFORMED_EDIT, + ] + }, + llm_router=router, + ) + router.acompletion.assert_not_awaited() + + +@pytest.mark.parametrize(("clear_tool_inputs", "cleared"), [("yes", True), ("true", True), (1, True), ("no", False)]) +def test_clear_tool_inputs_reads_boolean_strings_and_numbers_like_the_api( + clear_tool_inputs: object, cleared: bool +) -> None: + new_messages, applied = apply_clear_tool_uses_20250919( + model=MODEL, + messages=_make_history(n_pairs=3), + tools=None, + system=None, + edit_spec={ + "type": "clear_tool_uses_20250919", + "trigger": {"type": "tool_uses", "value": 0}, + "keep": {"type": "tool_uses", "value": 1}, + "clear_tool_inputs": clear_tool_inputs, + }, + ) + assert applied is not None + assert _tool_inputs(new_messages)["toolu_00"] == ({} if cleared else {"location": "City0"}) + + +@pytest.mark.parametrize(("trigger_value", "triggers"), [(2, False), (1, True)]) +def test_tool_uses_trigger_does_not_count_excluded_tools(trigger_value: int, triggers: bool) -> None: + messages = [{"role": "user", "content": "Plan a trip."}] + for i, name in enumerate(["search", "get_weather", "search", "get_weather"]): + messages.extend(_make_pair(f"toolu_{i:02d}", f"Result {i}: " + "x" * 200, name=name)) + _, applied = apply_clear_tool_uses_20250919( + model=MODEL, + messages=messages, + tools=None, + system=None, + edit_spec={ + "type": "clear_tool_uses_20250919", + "trigger": {"type": "tool_uses", "value": trigger_value}, + "keep": {"type": "tool_uses", "value": 0}, + "exclude_tools": ["search"], + }, + ) + assert (applied is not None) == triggers + + +@pytest.mark.parametrize(("clear_tool_inputs", "triggers"), [(False, False), (True, True)]) +def test_tool_uses_trigger_counts_tiny_results_only_when_their_inputs_are_cleared( + clear_tool_inputs: bool, triggers: bool +) -> None: + messages = _make_history(n_pairs=3) + messages[2]["content"][0]["content"] = "ok" + _, applied = apply_clear_tool_uses_20250919( + model=MODEL, + messages=messages, + tools=None, + system=None, + edit_spec={ + "type": "clear_tool_uses_20250919", + "trigger": {"type": "tool_uses", "value": 2}, + "keep": {"type": "tool_uses", "value": 1}, + "clear_tool_inputs": clear_tool_inputs, + }, + ) + assert (applied is not None) == triggers + + +def test_clear_at_least_counts_the_tool_inputs_it_clears(): + messages = [{"role": "user", "content": "Write the files."}] + for i in range(4): + assistant_msg, user_msg = _make_pair(f"toolu_{i:02d}", "ok", name="write_file") + assistant_msg["content"][0]["input"] = {"path": f"file{i}.txt", "content": "lorem ipsum " * 300} + messages.extend((assistant_msg, user_msg)) + one_input = litellm.token_counter(model=MODEL, text="lorem ipsum " * 300) + new_messages, applied = apply_clear_tool_uses_20250919( + model=MODEL, + messages=messages, + tools=None, + system=None, + edit_spec={ + "type": "clear_tool_uses_20250919", + "trigger": {"type": "tool_uses", "value": 0}, + "keep": {"type": "tool_uses", "value": 1}, + "clear_at_least": {"type": "input_tokens", "value": 3 * one_input // 2}, + "clear_tool_inputs": True, + }, + ) + assert applied is not None + assert _cleared_ids(new_messages) == {"toolu_00", "toolu_01"} + assert [_tool_inputs(new_messages)[f"toolu_{i:02d}"] == {} for i in range(4)] == [True, True, False, False] + assert applied["cleared_input_tokens"] >= 3 * one_input // 2 + + +_ONE_PIXEL_PNG: Final = ( + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNkYPhfDwAChwGA60e6kgAAAABJRU5ErkJggg==" +) + + +def test_clear_at_least_counts_image_results(): + messages = _make_history(n_pairs=4) + image: Final = {"type": "image", "source": {"type": "base64", "media_type": "image/png", "data": _ONE_PIXEL_PNG}} + for user_msg in messages[2::2]: + user_msg["content"][0]["content"] = [image] + new_messages, applied = apply_clear_tool_uses_20250919( + model=MODEL, + messages=messages, + tools=None, + system=None, + edit_spec={ + "type": "clear_tool_uses_20250919", + "trigger": {"type": "tool_uses", "value": 0}, + "keep": {"type": "tool_uses", "value": 1}, + "clear_at_least": {"type": "input_tokens", "value": 1}, + }, + ) + assert applied is not None + assert _cleared_ids(new_messages) == {"toolu_00", "toolu_01", "toolu_02"} def test_tool_result_list_content_shape_preserved(): @@ -266,7 +777,7 @@ def test_tool_result_list_content_shape_preserved(): { "type": "tool_result", "tool_use_id": "toolu_a", - "content": [{"type": "text", "text": "huge result"}], + "content": [{"type": "text", "text": "huge result: " + "x" * 200}], } ], }, @@ -280,7 +791,7 @@ def test_tool_result_list_content_shape_preserved(): { "type": "tool_result", "tool_use_id": "toolu_b", - "content": [{"type": "text", "text": "keep me"}], + "content": [{"type": "text", "text": "keep me: " + "y" * 200}], } ], }, diff --git a/tests/unit/llms/anthropic/pass_through/context_management/test_compact.py b/tests/unit/llms/anthropic/pass_through/context_management/test_compact.py index 5e55c0ade32..9b3084f0e7c 100644 --- a/tests/unit/llms/anthropic/pass_through/context_management/test_compact.py +++ b/tests/unit/llms/anthropic/pass_through/context_management/test_compact.py @@ -2406,7 +2406,13 @@ def _tool_use_messages() -> List[Dict[str, Any]]: }, { "role": "user", - "content": [{"type": "tool_result", "tool_use_id": "toolu_01", "content": "sunny in SF"}], + "content": [ + { + "type": "tool_result", + "tool_use_id": "toolu_01", + "content": "sunny in SF, 18 C, light wind from the west, no rain today", + } + ], }, { "role": "assistant", @@ -2414,7 +2420,13 @@ def _tool_use_messages() -> List[Dict[str, Any]]: }, { "role": "user", - "content": [{"type": "tool_result", "tool_use_id": "toolu_02", "content": "rainy in NY"}], + "content": [ + { + "type": "tool_result", + "tool_use_id": "toolu_02", + "content": "rainy in NY, 11 C, strong wind from the east, rain all day", + } + ], }, {"role": "user", "content": "now summarize both"}, ] diff --git a/tests/unit/llms/anthropic/pass_through/context_management/test_dispatcher.py b/tests/unit/llms/anthropic/pass_through/context_management/test_dispatcher.py index e28d2ac5e65..66957456c95 100644 --- a/tests/unit/llms/anthropic/pass_through/context_management/test_dispatcher.py +++ b/tests/unit/llms/anthropic/pass_through/context_management/test_dispatcher.py @@ -22,7 +22,7 @@ def _history_with_two_tool_pairs(): { "type": "tool_result", "tool_use_id": "t1", - "content": "first result", + "content": "first result: the forecast for the first city, sunny and mild", } ], }, @@ -36,7 +36,7 @@ def _history_with_two_tool_pairs(): { "type": "tool_result", "tool_use_id": "t2", - "content": "second result", + "content": "second result: the forecast for the second city, rainy and cold", } ], }, diff --git a/tests/unit/proxy/test_proxy_token_counter.py b/tests/unit/proxy/test_proxy_token_counter.py index f1b8d3fd591..714d8bb77d7 100644 --- a/tests/unit/proxy/test_proxy_token_counter.py +++ b/tests/unit/proxy/test_proxy_token_counter.py @@ -3236,9 +3236,16 @@ def _gemini_router(drop_params: bool = False) -> Router: ({"drop_params": True}, {}, {}, False), ({}, {"drop_params": True}, {}, False), ({}, {}, {"drop_params": True}, False), + ({}, {}, {"drop_params": "true"}, False), ({}, {}, {}, True), ), - ids=("deployment-drop-params", "router-default-drop-params", "request-drop-params", "litellm-drop-params"), + ids=( + "deployment-drop-params", + "router-default-drop-params", + "request-drop-params", + "request-drop-params-string", + "litellm-drop-params", + ), ) async def test_gemini_messages_count_tokens_counts_the_unedited_body_when_drop_params_drops_an_invalid_edit( deployment_params: dict[str, JsonValue],