diff --git a/litellm/llms/anthropic/pass_through/adapters/handler.py b/litellm/llms/anthropic/pass_through/adapters/handler.py index cca00a8383b..01e781485a0 100644 --- a/litellm/llms/anthropic/pass_through/adapters/handler.py +++ b/litellm/llms/anthropic/pass_through/adapters/handler.py @@ -13,7 +13,7 @@ from typing_extensions import TypedDict import litellm from litellm._logging import verbose_logger from litellm.litellm_core_utils.asyncify import run_async_function -from litellm.litellm_core_utils.core_helpers import normalize_drop_params +from litellm.litellm_core_utils.core_helpers import drops_unsupported_params from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObject from litellm.llms.anthropic.pass_through.adapters.transformation import ( AnthropicAdapter, @@ -331,8 +331,10 @@ async def _run_polyfill_if_enabled( does not disable a valid spec because context_management is a supported param). Raises ``AnthropicContextManagementError`` so the /v1/messages endpoint can emit an Anthropic-format 400, unless ``drop_invalid_edits`` - (``drop_params``) drops the invalid spec. All other exceptions are - best-effort swallowed (matches v0 behavior). + (``drop_params``) drops what is invalid: the malformed fields of a + ``clear_tool_uses`` edit alone, or the whole spec for an invalid ``compact`` + trigger. All other + exceptions are best-effort swallowed (matches v0 behavior). """ if not context_management_spec: return None @@ -353,6 +355,7 @@ async def _run_polyfill_if_enabled( sub_call_logging_kwargs=sub_call_logging_kwargs, parent_trace_id=parent_trace_id, count_only=count_only, + drop_invalid_edits=drop_invalid_edits, ) except AnthropicContextManagementError: if drop_invalid_edits: @@ -649,7 +652,7 @@ class LiteLLMMessagesToCompletionTransformationHandler: ) = ANTHROPIC_ADAPTER.translate_completion_input_params_with_tool_mapping( request_data, custom_llm_provider=custom_llm_provider if isinstance(custom_llm_provider, str) else None, - drop_params=litellm.drop_params is True or normalize_drop_params(extra_kwargs.get("drop_params")) is True, + drop_params=drops_unsupported_params(extra_kwargs.get("drop_params")), ) if openai_request is None: @@ -736,14 +739,12 @@ class LiteLLMMessagesToCompletionTransformationHandler: """Handle non-Anthropic models asynchronously using the adapter""" context_management: Final = kwargs.pop("context_management", None) additional_drop_params: Final[list[str] | None] = kwargs.get("additional_drop_params", None) - drop_invalid_edits: Final = litellm.drop_params is True or ( - "drop_params" in kwargs and kwargs["drop_params"] is True - ) litellm_router: Final[Router | None] = _router_or_proxy_fallback(kwargs.pop("litellm_router", None)) proxy_litellm_metadata, user_api_key_auth = _extract_proxy_litellm_metadata(kwargs) request_kwargs: Final = _REQUEST_KWARGS.validate_python(kwargs) reject_malformed_tool_result_text(messages, model, request_kwargs) + drop_invalid_edits: Final = drops_unsupported_params(request_kwargs.get("drop_params")) sub_call_logging_kwargs: Final = proxy_sub_call_logging_kwargs(request_kwargs) sub_call_trace_id: Final = proxy_sub_call_trace_id(request_kwargs) @@ -866,7 +867,8 @@ class LiteLLMMessagesToCompletionTransformationHandler: **kwargs, ) - reject_malformed_tool_result_text(messages, model, _REQUEST_KWARGS.validate_python(kwargs)) + sync_request_kwargs: Final = _REQUEST_KWARGS.validate_python(kwargs) + reject_malformed_tool_result_text(messages, model, sync_request_kwargs) # Run the context_management polyfill on the sync path too so that # ``litellm.messages.create()`` callers don't silently lose edits like @@ -875,9 +877,7 @@ class LiteLLMMessagesToCompletionTransformationHandler: # bridge to it via ``run_async_function``. context_management: Final = kwargs.pop("context_management", None) additional_drop_params: Final[list[str] | None] = kwargs.get("additional_drop_params", None) - drop_invalid_edits: Final = litellm.drop_params is True or ( - "drop_params" in kwargs and kwargs["drop_params"] is True - ) + drop_invalid_edits: Final = drops_unsupported_params(sync_request_kwargs.get("drop_params")) # Deliberately do NOT auto-attach the proxy ``llm_router`` here: # ``run_async_function`` spawns a new event loop in a worker thread # to bridge to the async dispatcher, but the proxy router's httpx diff --git a/litellm/llms/anthropic/pass_through/context_management/dispatcher.py b/litellm/llms/anthropic/pass_through/context_management/dispatcher.py index 3e1bab09d5a..bce653df6d1 100644 --- a/litellm/llms/anthropic/pass_through/context_management/dispatcher.py +++ b/litellm/llms/anthropic/pass_through/context_management/dispatcher.py @@ -9,7 +9,12 @@ from litellm.litellm_core_utils.asyncify import asyncify from litellm.types.llms.anthropic import AppliedEdit from .constants import CLEAR_TOOL_USES_EDIT_TYPE, COMPACT_EDIT_TYPE -from .editors import apply_clear_tool_uses_20250919, apply_compact_20260112 +from .editors import ( + CheckedEdit, + apply_clear_tool_uses_20250919, + apply_compact_20260112, + check_clear_tool_uses_spec, +) from .result import PolyfillResult if TYPE_CHECKING: @@ -73,6 +78,12 @@ def _wrap_editor_return( ) +def _checked_edit(index: int, edit_spec: dict[str, object]) -> CheckedEdit: + if edit_spec.get("type") != CLEAR_TOOL_USES_EDIT_TYPE: + return CheckedEdit(edit=edit_spec, error=None) + return check_clear_tool_uses_spec(edit_spec, index) + + async def apply_context_management( *, model: str, @@ -86,9 +97,14 @@ async def apply_context_management( sub_call_logging_kwargs: "ProxySubCallLoggingKwargs | None" = None, parent_trace_id: str | None = None, count_only: bool = False, + drop_invalid_edits: bool = False, ) -> PolyfillResult: """Run edits in order; return a single ``PolyfillResult``. + Every ``clear_tool_uses_20250919`` edit is validated before any editor runs, so a malformed one raises its 400 + before a ``compact_20260112`` summary call is paid for, or runs without its malformed fields under + ``drop_invalid_edits``. + The dispatcher is async so async editors (``compact_20260112``) can ``await`` the configured summarization model. Sync editors run in a worker thread so their token counts stay off the event loop; @@ -97,6 +113,11 @@ async def apply_context_management( edits: Final = normalize_spec(context_management_spec) if not edits: return PolyfillResult(messages=messages, system=system, applied_edits=[]) + checked: Final = tuple(_checked_edit(index, edit_spec) for index, edit_spec in enumerate(edits)) + first_error: Final = next((check.error for check in checked if check.error is not None), None) + if first_error is not None and not drop_invalid_edits: + raise first_error + well_formed_edits: Final = tuple(check.edit for check in checked) current_messages = messages current_system = system @@ -104,7 +125,7 @@ async def apply_context_management( aggregated_compaction_block = None aggregated_iterations_usage = None - for edit_spec in edits: + for edit_spec in well_formed_edits: edit_type = edit_spec.get("type") editor = _EDITOR_REGISTRY.get(edit_type) if isinstance(edit_type, str) else None if editor is None: diff --git a/litellm/llms/anthropic/pass_through/context_management/editors/__init__.py b/litellm/llms/anthropic/pass_through/context_management/editors/__init__.py index 3e933a9880a..5fc8c553d4f 100644 --- a/litellm/llms/anthropic/pass_through/context_management/editors/__init__.py +++ b/litellm/llms/anthropic/pass_through/context_management/editors/__init__.py @@ -1,4 +1,4 @@ -from .clear_tool_uses import apply_clear_tool_uses_20250919 +from .clear_tool_uses import CheckedEdit, apply_clear_tool_uses_20250919, check_clear_tool_uses_spec from .compact import apply_compact_20260112 -__all__ = ["apply_clear_tool_uses_20250919", "apply_compact_20260112"] +__all__ = ["CheckedEdit", "apply_clear_tool_uses_20250919", "apply_compact_20260112", "check_clear_tool_uses_spec"] diff --git a/litellm/llms/anthropic/pass_through/context_management/editors/clear_tool_uses.py b/litellm/llms/anthropic/pass_through/context_management/editors/clear_tool_uses.py index 00ecb315bf1..935ce77ef87 100644 --- a/litellm/llms/anthropic/pass_through/context_management/editors/clear_tool_uses.py +++ b/litellm/llms/anthropic/pass_through/context_management/editors/clear_tool_uses.py @@ -1,7 +1,23 @@ -"""``clear_tool_uses_20250919`` polyfill (v0: ``trigger`` and ``keep`` only).""" +"""``clear_tool_uses_20250919`` polyfill, matching the Anthropic API's selection. -from typing import Any, Final, cast +``keep`` counts clearable tool uses only. Uses of ``exclude_tools`` are never cleared and never count toward a +``tool_uses`` trigger, and neither are uses whose result is 1 to 46 bytes of text, which the API leaves out of its +count, unless ``clear_tool_inputs`` covers their tool: then their input can still be cleared, and the API counts and +clears them like any other. Clearing goes oldest first in batches that free at least ``clear_at_least`` tokens, +counting the result and any input it clears, and leaves an incomplete last batch, so the cleared prefix only grows +between requests and each batch costs one prompt-cache miss. ``check_clear_tool_uses_spec`` answers a malformed field +the way the API's 400 does, so the dispatcher can reject the edit, or drop only its malformed fields, before any editor +runs. +""" +from collections.abc import Mapping, Sequence, Set +from dataclasses import dataclass +from functools import reduce +from itertools import chain +from types import MappingProxyType +from typing import Annotated, Any, Final, Literal, TypeVar, cast + +from pydantic import BaseModel, ConfigDict, Field, StrictInt, TypeAdapter, ValidationError from typing_extensions import ReadOnly, TypedDict import litellm @@ -13,8 +29,39 @@ from ..constants import ( DEFAULT_INPUT_TOKENS_TRIGGER, DEFAULT_KEEP_TOOL_USES, ) +from ..errors import AnthropicContextManagementError from ..placeholders import build_cleared_tool_result_content +T = TypeVar("T") +# api.anthropic.com, measured 2026-10-19: text results of 1 to 46 bytes are never cleared or counted toward keep +_TINY_RESULT_BYTES: Final = 46 +ToolNames = Literal["all"] | frozenset[str] + + +class _ClearAtLeast(BaseModel): + model_config = ConfigDict(frozen=True) + + type: Literal["input_tokens"] + value: Annotated[StrictInt, Field(ge=0)] + + +class _ClearToolUsesFields(BaseModel): + model_config = ConfigDict(frozen=True) + + clear_at_least: _ClearAtLeast | None = None + exclude_tools: tuple[str, ...] | None = None + clear_tool_inputs: bool | tuple[str, ...] | None = None + + +_BLOCKS: Final = TypeAdapter(tuple[Mapping[str, object], ...]) +# api.anthropic.com, measured 2026-10-19: its wording for the two shapes pydantic words differently +_API_MESSAGES: Final = MappingProxyType( + { + "tuple_type": "Input should be a valid array", + "model_type": "Input does not match the expected shape.", + } +) + class ClearToolUsesEditSpec(TypedDict, total=False): """The ``clear_tool_uses_20250919`` entry of a ``context_management`` spec.""" @@ -27,36 +74,125 @@ class ClearToolUsesEditSpec(TypedDict, total=False): clear_tool_inputs: ReadOnly[object] -def _count_tool_uses(messages: list[dict[str, object]]) -> int: - """Return the number of tool_use content blocks across all messages. - - Only counts blocks with a string ``id`` to stay consistent with - :func:`_collect_tool_use_ids_in_order`, which is the source of truth for - which blocks are clearable. - """ - count = 0 - for msg in messages: - content = msg.get("content") - if isinstance(content, list): - for block in content: - if isinstance(block, dict) and block.get("type") == "tool_use": - if isinstance(block.get("id"), str): - count += 1 - return count +def _parse(adapter: TypeAdapter[T], value: object) -> T | None: + try: + return adapter.validate_python(value) + except ValidationError: + return None -def _collect_tool_use_ids_in_order(messages: list[dict[str, object]]) -> list[str]: - """Return tool_use ids in the chronological order they appear in messages.""" - ids: Final[list[str]] = [] - for msg in messages: - content = msg.get("content") - if isinstance(content, list): - for block in content: - if isinstance(block, dict) and block.get("type") == "tool_use": - block_id = block.get("id") - if isinstance(block_id, str): - ids.append(block_id) - return ids +@dataclass(frozen=True, slots=True) +class CheckedEdit: + edit: dict[str, object] + error: AnthropicContextManagementError | None + + +def check_clear_tool_uses_spec(edit_spec: dict[str, object], index: int) -> CheckedEdit: + """``edits[index]`` without its malformed fields, with the API's 400 for the first one, or None when there is none.""" + try: + _ClearToolUsesFields.model_validate(edit_spec) + except ValidationError as error: + details: Final = error.errors(include_url=False) + malformed: Final = frozenset(detail["loc"][0] for detail in details) + first: Final = details[0] + location: Final = ".".join(map(str, ("edits", index, CLEAR_TOOL_USES_EDIT_TYPE, *first["loc"]))) + return CheckedEdit( + edit={key: value for key, value in edit_spec.items() if key not in malformed}, + error=AnthropicContextManagementError( + status_code=400, + message=f"context_management.{location}: {_API_MESSAGES.get(first['type'], first['msg'])}", + ), + ) + return CheckedEdit(edit=edit_spec, error=None) + + +def _input_tools(clear_tool_inputs: bool | tuple[str, ...] | None) -> ToolNames: + return "all" if clear_tool_inputs is True else frozenset(clear_tool_inputs or ()) + + +def _all_blocks(messages: Sequence[Mapping[str, object]]) -> tuple[Mapping[str, object], ...]: + return tuple(chain.from_iterable(_parse(_BLOCKS, message.get("content")) or () for message in messages)) + + +def _tool_use_ref(block: Mapping[str, object]) -> tuple[str, str | None] | None: + block_id: Final = block.get("id") + name: Final = block.get("name") + if block.get("type") != "tool_use" or not isinstance(block_id, str): + return None + return block_id, name if isinstance(name, str) else None + + +def _tool_uses_in_order(blocks: Sequence[Mapping[str, object]]) -> tuple[tuple[str, str | None], ...]: + refs: Final = (_tool_use_ref(block) for block in blocks) + return tuple(ref for ref in refs if ref is not None) + + +def _block_id(block: Mapping[str, object], block_type: str, id_key: str) -> str | None: + block_id: Final = block.get(id_key) + return block_id if block.get("type") == block_type and isinstance(block_id, str) else None + + +def _blocks_by_id( + blocks: Sequence[Mapping[str, object]], block_type: str, id_key: str +) -> Mapping[str, Mapping[str, object]]: + pairs: Final = ((_block_id(block, block_type, id_key), block) for block in blocks) + return {block_id: block for block_id, block in pairs if block_id is not None} + + +def _utf8_size(text: str) -> int: + return len(text.encode("utf-8", "surrogatepass")) + + +def _text_bytes(content: object) -> int | None: + """Return the UTF-8 size of a text-only result, or None when it holds anything but text.""" + if isinstance(content, str): + return _utf8_size(content) + parts: Final = _parse(_BLOCKS, content) or () + texts: Final = tuple(part.get("text") for part in parts) + if any(part.get("type") != "text" or not isinstance(text, str) for part, text in zip(parts, texts)): + return None + return sum(_utf8_size(text) for text in texts if isinstance(text, str)) + + +def _is_tiny(content: object) -> bool: + size: Final = _text_bytes(content) + return size is not None and 0 < size <= _TINY_RESULT_BYTES + + +def _tiny_result_ids(results: Mapping[str, Mapping[str, object]]) -> frozenset[str]: + """Return ids whose result is 1 to 46 bytes of text; the API neither counts nor clears those uses.""" + return frozenset(tool_use_id for tool_use_id, block in results.items() if _is_tiny(block.get("content"))) + + +def _clears_input(name: str | None, input_tools: ToolNames) -> bool: + return input_tools == "all" or (name is not None and name in input_tools) + + +def _cleared_result(block: Mapping[str, object]) -> dict[str, object]: + return {**block, "content": build_cleared_tool_result_content(block.get("content"))} + + +def _freed_tokens(model: str, result: Mapping[str, object], use: Mapping[str, object], clears_input: bool) -> int: + """Tokens that clearing this use frees: its result, and its input when ``clear_tool_inputs`` covers it.""" + before: Final = [result, use] if clears_input else [result] + after: Final = [_cleared_result(result), {**use, "input": {}}] if clears_input else [_cleared_result(result)] + counts: Final = tuple( + litellm.token_counter(model=model, messages=[{"role": "user", "content": blocks}]) for blocks in (before, after) + ) + return max(counts[0] - counts[1], 0) + + +def _select_in_batches(candidates: Sequence[str], tokens: Mapping[str, int], clear_at_least: int) -> tuple[str, ...]: + """Oldest first, close a batch once it holds ``clear_at_least`` tokens; drop the incomplete last batch.""" + + def step(state: tuple[int, int], item: tuple[int, str]) -> tuple[int, int]: + batch_end, batch_tokens = state + index, tool_use_id = item + total: Final = batch_tokens + tokens[tool_use_id] + return (index + 1, 0) if total >= clear_at_least else (batch_end, total) + + batch_end, _ = reduce(step, enumerate(candidates), (0, 0)) + return tuple(candidates[:batch_end]) def _trigger_met( @@ -64,6 +200,7 @@ def _trigger_met( model: str, messages: list[dict[str, object]], tools: list[dict[str, object]] | None, + counted_tool_uses: int, ) -> tuple[bool, int | None]: """Return (trigger_met, input_tokens if counted for reuse).""" trigger_type: Final = trigger.get("type", "input_tokens") @@ -72,7 +209,7 @@ def _trigger_met( if trigger_type == "tool_uses": if not isinstance(threshold, int): return False, None - return _count_tool_uses(messages) > threshold, None + return counted_tool_uses > threshold, None if not isinstance(threshold, int): threshold = DEFAULT_INPUT_TOKENS_TRIGGER @@ -132,11 +269,7 @@ def _clear_tool_results( and block.get("type") == "tool_result" and block.get("tool_use_id") in ids_to_clear ): - new_block = { - **block, - "content": build_cleared_tool_result_content(block.get("content")), - } - new_blocks.append(new_block) + new_blocks.append(_cleared_result(block)) mutated = True cleared += 1 else: @@ -150,6 +283,23 @@ def _clear_tool_results( return new_messages, cleared +def _without_input(block: Mapping[str, object], ids_to_clear: Set[str], tool_names: ToolNames) -> Mapping[str, object]: + match _tool_use_ref(block): + case (block_id, name) if block_id in ids_to_clear and _clears_input(name, tool_names): + return {**block, "input": dict[str, object]()} + case _: + return block + + +def _message_without_inputs( + message: dict[str, object], ids_to_clear: Set[str], tool_names: ToolNames +) -> dict[str, object]: # mutable-ok: the editor contract hands messages back as list[dict] + blocks: Final = _parse(_BLOCKS, message.get("content")) + if blocks is None: + return message + return {**message, "content": [_without_input(block, ids_to_clear, tool_names) for block in blocks]} + + def apply_clear_tool_uses_20250919( *, model: str, @@ -158,15 +308,11 @@ def apply_clear_tool_uses_20250919( system: str | list[dict[str, object]] | None, edit_spec: ClearToolUsesEditSpec, ) -> tuple[list[dict[str, object]], AppliedEdit | None]: - """Apply clear_tool_uses; return (messages, AppliedEdit or None).""" - ignored_knobs = [knob for knob in ("clear_at_least", "exclude_tools", "clear_tool_inputs") if knob in edit_spec] - for ignored_knob in ignored_knobs: - verbose_logger.warning( - "context_management polyfill: ignoring '%s' on %s " - "(supported only on Anthropic-family forwarding path in v0)", - ignored_knob, - CLEAR_TOOL_USES_EDIT_TYPE, - ) + """Apply clear_tool_uses to an edit ``check_clear_tool_uses_spec`` returned; return (messages, AppliedEdit or None).""" + fields: Final = _ClearToolUsesFields.model_validate(edit_spec) + clear_at_least: Final = 0 if fields.clear_at_least is None else fields.clear_at_least.value + exclude_tools: Final = frozenset(fields.exclude_tools or ()) + input_tools: Final = _input_tools(fields.clear_tool_inputs) trigger: Final[dict[str, object]] = edit_spec.get("trigger") or { "type": "input_tokens", @@ -177,37 +323,56 @@ def apply_clear_tool_uses_20250919( "value": DEFAULT_KEEP_TOOL_USES, } - met, tokens_before = _trigger_met(trigger, model, messages, tools) + blocks: Final = _all_blocks(messages) + results: Final = _blocks_by_id(blocks, "tool_result", "tool_use_id") + uses: Final = _blocks_by_id(blocks, "tool_use", "id") + tiny: Final = _tiny_result_ids(results) + clearable: Final = tuple( + (tool_use_id, name) + for tool_use_id, name in _tool_uses_in_order(blocks) + if name not in exclude_tools and (tool_use_id not in tiny or _clears_input(name, input_tools)) + ) + met, counted_tokens = _trigger_met(trigger, model, messages, tools, len(clearable)) if not met: return messages, None keep_count: Final = _resolve_keep_count(keep) - tool_use_ids: Final = _collect_tool_use_ids_in_order(messages) - if len(tool_use_ids) <= keep_count: + if len(clearable) <= keep_count: return messages, None - ids_to_clear: Final = set(tool_use_ids[: len(tool_use_ids) - keep_count]) - - # Never clear the latest completed tool_result (reply context). last_completed_id: Final = _last_completed_tool_use_id(messages) - if last_completed_id is not None: - ids_to_clear.discard(last_completed_id) + candidates: Final = tuple( + (tool_use_id, name) + for tool_use_id, name in clearable[: len(clearable) - keep_count] + if tool_use_id in results and tool_use_id != last_completed_id + ) + candidate_ids: Final = tuple(tool_use_id for tool_use_id, _ in candidates) + tokens: Final = { + tool_use_id: _freed_tokens(model, results[tool_use_id], uses[tool_use_id], _clears_input(name, input_tools)) + for tool_use_id, name in (candidates if clear_at_least > 0 else ()) + } + ids_to_clear: Final = frozenset( + _select_in_batches(candidate_ids, tokens, clear_at_least) if clear_at_least > 0 else candidate_ids + ) - edited, cleared_count = _clear_tool_results(messages, ids_to_clear) + edited, cleared_count = _clear_tool_results(messages, set(ids_to_clear)) verbose_logger.debug("context_management polyfill: edited: %s", edited) if cleared_count == 0: return messages, None + final_messages: Final = ( + [_message_without_inputs(message, ids_to_clear, input_tools) for message in edited] if input_tools else edited + ) - if tokens_before is None: - tokens_before = litellm.token_counter(model=model, messages=messages, tools=cast(Any, tools)) - tokens_after: Final = litellm.token_counter(model=model, messages=edited, tools=cast(Any, tools)) + tokens_before: Final = ( + counted_tokens + if counted_tokens is not None + else litellm.token_counter(model=model, messages=messages, tools=cast(Any, tools)) + ) + tokens_after: Final = litellm.token_counter(model=model, messages=final_messages, tools=cast(Any, tools)) cleared_input_tokens: Final = max(tokens_before - tokens_after, 0) - applied: Final[AppliedEdit] = { + return final_messages, { "type": CLEAR_TOOL_USES_EDIT_TYPE, "cleared_tool_uses": cleared_count, "cleared_input_tokens": cleared_input_tokens, } - if ignored_knobs: - applied["warnings"] = [f"{knob}_ignored" for knob in ignored_knobs] - return edited, applied diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index c17e0c67d68..1840586a08e 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -360,6 +360,7 @@ from litellm.litellm_core_utils.core_helpers import ( get_litellm_metadata_from_kwargs, get_parent_otel_span_from_kwargs, iter_items, + normalize_drop_params, ) from litellm.litellm_core_utils.credential_accessor import CredentialAccessor from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj @@ -15208,7 +15209,7 @@ def _count_context_management( return CountContextManagement( spec=context_management, drop_invalid_edits=litellm.drop_params is True - or any(params.get("drop_params") is True for params in sent_params), + or any(normalize_drop_params(params.get("drop_params")) is True for params in sent_params), ) diff --git a/tests/unit/llms/anthropic/pass_through/adapters/test_handler_context_management.py b/tests/unit/llms/anthropic/pass_through/adapters/test_handler_context_management.py index f906530c8d9..9b5477e3206 100644 --- a/tests/unit/llms/anthropic/pass_through/adapters/test_handler_context_management.py +++ b/tests/unit/llms/anthropic/pass_through/adapters/test_handler_context_management.py @@ -83,9 +83,16 @@ async def _send(router: Router, request_options: Mapping[str, object], backend: ({"drop_params": True}, {}, {}, False), ({}, {"drop_params": True}, {}, False), ({}, {}, {"drop_params": True}, False), + ({}, {}, {"drop_params": "true"}, False), ({}, {}, {}, True), ), - ids=("deployment-drop-params", "router-default-drop-params", "request-drop-params", "litellm-drop-params"), + ids=( + "deployment-drop-params", + "router-default-drop-params", + "request-drop-params", + "request-drop-params-string", + "litellm-drop-params", + ), ) async def test_messages_sends_the_unedited_body_when_drop_params_drops_an_invalid_edit( deployment_params: Mapping[str, object], @@ -113,7 +120,8 @@ async def test_messages_rejects_an_invalid_edit_without_drop_params() -> None: assert backend.calls == [] -def test_sync_messages_sends_the_unedited_body_when_drop_params_drops_an_invalid_edit() -> None: +@pytest.mark.parametrize("drop_params", (True, "true"), ids=("drop-params", "drop-params-string")) +def test_sync_messages_sends_the_unedited_body_when_drop_params_drops_an_invalid_edit(drop_params: bool | str) -> None: backend: Final = _SyncBackend() with patch.object(litellm, "drop_params", False), patch("litellm.completion", new=backend): response: Final = LiteLLMMessagesToCompletionTransformationHandler.anthropic_messages_handler( @@ -122,7 +130,7 @@ def test_sync_messages_sends_the_unedited_body_when_drop_params_drops_an_invalid model="gemini/gemini-2.5-flash", api_key="test-key", context_management=_INVALID_COMPACT_EDIT, - drop_params=True, + drop_params=drop_params, ) assert isinstance(response, dict) diff --git a/tests/unit/llms/anthropic/pass_through/context_management/test_clear_tool_uses.py b/tests/unit/llms/anthropic/pass_through/context_management/test_clear_tool_uses.py index ee50bdff461..2ffa73dda21 100644 --- a/tests/unit/llms/anthropic/pass_through/context_management/test_clear_tool_uses.py +++ b/tests/unit/llms/anthropic/pass_through/context_management/test_clear_tool_uses.py @@ -3,18 +3,32 @@ Unit tests for the in-gateway `clear_tool_uses_20250919` polyfill editor. """ from copy import deepcopy +from itertools import chain +from typing import Final +from unittest.mock import AsyncMock, MagicMock +import pytest + +import litellm +from litellm.llms.anthropic.pass_through.adapters.handler import _run_polyfill_if_enabled +from litellm.llms.anthropic.pass_through.context_management import ( + AnthropicContextManagementError, + PolyfillResult, + apply_context_management, +) from litellm.llms.anthropic.pass_through.context_management.constants import ( CLEARED_TOOL_RESULT_PLACEHOLDER, + COMPACT_SUMMARY_MODEL_SETTING_KEY, ) from litellm.llms.anthropic.pass_through.context_management.editors.clear_tool_uses import ( apply_clear_tool_uses_20250919, ) +from litellm.types.llms.anthropic import AppliedEdit MODEL = "xai/grok-4" -def _make_pair(tool_use_id: str, result_text: str, location: str = "Mumbai"): +def _make_pair(tool_use_id: str, result_text: str, location: str = "Mumbai", name: str = "get_weather"): """Return an (assistant, user) message pair with one tool_use + tool_result.""" assistant_msg = { "role": "assistant", @@ -22,7 +36,7 @@ def _make_pair(tool_use_id: str, result_text: str, location: str = "Mumbai"): { "type": "tool_use", "id": tool_use_id, - "name": "get_weather", + "name": name, "input": {"location": location}, } ], @@ -203,10 +217,71 @@ def test_cleared_input_tokens_is_nonnegative(): assert applied["cleared_input_tokens"] >= 0 -def test_ignored_knobs_do_not_alter_behavior(): - """clear_at_least / exclude_tools / clear_tool_inputs are accepted but ignored in v0.""" +def _blocks_of(messages: list[dict[str, object]], block_type: str) -> tuple[dict[str, object], ...]: + contents: Final = (message["content"] for message in messages) + blocks: Final = chain.from_iterable(content for content in contents if isinstance(content, list)) + return tuple(block for block in blocks if isinstance(block, dict) and block["type"] == block_type) + + +def _result_contents(messages: list[dict[str, object]]) -> dict[object, object]: + return {block["tool_use_id"]: block["content"] for block in _blocks_of(messages, "tool_result")} + + +def _cleared_ids(messages: list[dict[str, object]]) -> set[object]: + placeholders: Final = (CLEARED_TOOL_RESULT_PLACEHOLDER, [{"type": "text", "text": CLEARED_TOOL_RESULT_PLACEHOLDER}]) + return {tool_use_id for tool_use_id, content in _result_contents(messages).items() if content in placeholders} + + +def _result_tokens(text: str) -> int: + return litellm.token_counter(model=MODEL, text=text) + + +def test_clear_at_least_clears_in_batches_and_leaves_an_incomplete_batch(): + """Oldest first, a batch closes once it reaches clear_at_least; the rest waits.""" + messages = _make_history(n_pairs=8) + one_result = _result_tokens(_result_contents(messages)["toolu_00"]) + edit_spec = { + "type": "clear_tool_uses_20250919", + "trigger": {"type": "tool_uses", "value": 0}, + "keep": {"type": "tool_uses", "value": 1}, + # Three results fill a batch; two don't. + "clear_at_least": {"type": "input_tokens", "value": 2 * one_result + 1}, + } + new_messages, applied = apply_clear_tool_uses_20250919( + model=MODEL, messages=messages, tools=None, system=None, edit_spec=edit_spec + ) + # 7 eligible: two batches of 3; the 7th waits for its batch to fill. + assert applied is not None + assert applied["cleared_tool_uses"] == 6 + assert _cleared_ids(new_messages) == {f"toolu_{i:02d}" for i in range(6)} + + +def test_clear_at_least_keeps_the_cleared_prefix_stable_between_requests(): + """A new tool use doesn't move the cleared prefix until the next batch fills (prompt-cache friendly).""" + history = _make_history(n_pairs=10) + one_result = _result_tokens(_result_contents(history)["toolu_00"]) + edit_spec = { + "type": "clear_tool_uses_20250919", + "trigger": {"type": "tool_uses", "value": 0}, + "keep": {"type": "tool_uses", "value": 1}, + "clear_at_least": {"type": "input_tokens", "value": 2 * one_result + 1}, + } + cleared: Final = tuple( + _cleared_ids( + apply_clear_tool_uses_20250919( + model=MODEL, messages=history[: 1 + 2 * n_pairs], tools=None, system=None, edit_spec=edit_spec + )[0] + ) + for n_pairs in (8, 9, 10) + ) + assert cleared[0] == cleared[1] == {f"toolu_{i:02d}" for i in range(6)} + assert cleared[2] == {f"toolu_{i:02d}" for i in range(9)} + + +def test_clear_at_least_out_of_reach_leaves_messages_unchanged(): messages = _make_history(n_pairs=3) - _, applied = apply_clear_tool_uses_20250919( + original = deepcopy(messages) + new_messages, applied = apply_clear_tool_uses_20250919( model=MODEL, messages=messages, tools=None, @@ -216,28 +291,46 @@ def test_ignored_knobs_do_not_alter_behavior(): "trigger": {"type": "tool_uses", "value": 0}, "keep": {"type": "tool_uses", "value": 1}, "clear_at_least": {"type": "input_tokens", "value": 999_999_999}, - "exclude_tools": ["get_weather"], - "clear_tool_inputs": True, }, ) - # Despite clear_at_least being huge, polyfill still applies (knob ignored). - # Despite clear_tool_inputs=True, inputs are NOT cleared (knob ignored). - assert applied is not None - assert applied["cleared_tool_uses"] == 2 - # Ignored knobs surface as warnings on the AppliedEdit so operators can - # see what was dropped (the v0 polyfill silently dropping them at debug - # log level made misconfiguration invisible from the response). - assert set(applied.get("warnings", [])) == { - "clear_at_least_ignored", - "exclude_tools_ignored", - "clear_tool_inputs_ignored", + assert applied is None + assert new_messages == original + + +def test_a_batch_closes_when_it_frees_exactly_clear_at_least(): + messages = _make_history(n_pairs=6) + clear_one_use = { + "type": "clear_tool_uses_20250919", + "trigger": {"type": "tool_uses", "value": 0}, + "keep": {"type": "tool_uses", "value": 5}, } + _, one = apply_clear_tool_uses_20250919( + model=MODEL, messages=messages, tools=None, system=None, edit_spec=clear_one_use + ) + assert one is not None + assert one["cleared_tool_uses"] == 1 + new_messages, applied = apply_clear_tool_uses_20250919( + model=MODEL, + messages=messages, + tools=None, + system=None, + edit_spec={ + **clear_one_use, + "keep": {"type": "tool_uses", "value": 1}, + "clear_at_least": {"type": "input_tokens", "value": 2 * one["cleared_input_tokens"]}, + }, + ) + assert _cleared_ids(new_messages) == {f"toolu_{i:02d}" for i in range(4)} + assert applied is not None + assert applied["cleared_tool_uses"] == 4 -def test_no_ignored_knobs_omits_warnings_field(): - """When the caller doesn't pass any unsupported knobs, no ``warnings`` are added.""" - messages = _make_history(n_pairs=3) - _, applied = apply_clear_tool_uses_20250919( +def test_exclude_tools_are_never_cleared_and_do_not_count_toward_keep(): + """keep counts clearable uses only: with keep=1 the newest get_weather result stays, not the newest search.""" + messages = [{"role": "user", "content": "Plan a trip."}] + for i, name in enumerate(["get_weather", "search", "get_weather", "get_weather", "search"]): + messages.extend(_make_pair(f"toolu_{i:02d}", f"Result {i}: " + "x" * 200, name=name)) + new_messages, applied = apply_clear_tool_uses_20250919( model=MODEL, messages=messages, tools=None, @@ -246,10 +339,428 @@ def test_no_ignored_knobs_omits_warnings_field(): "type": "clear_tool_uses_20250919", "trigger": {"type": "tool_uses", "value": 0}, "keep": {"type": "tool_uses", "value": 1}, + "exclude_tools": ["search"], }, ) assert applied is not None - assert "warnings" not in applied + assert _cleared_ids(new_messages) == {"toolu_00", "toolu_02"} + assert applied["cleared_tool_uses"] == 2 + + +def test_history_of_only_excluded_tools_is_left_unchanged(): + messages = [{"role": "user", "content": "Plan a trip."}] + for i in range(4): + messages.extend(_make_pair(f"toolu_{i:02d}", f"Result {i}: " + "x" * 200, name="search")) + original = deepcopy(messages) + new_messages, applied = apply_clear_tool_uses_20250919( + model=MODEL, + messages=messages, + tools=None, + system=None, + edit_spec={ + "type": "clear_tool_uses_20250919", + "trigger": {"type": "tool_uses", "value": 0}, + "keep": {"type": "tool_uses", "value": 1}, + "exclude_tools": ["search"], + }, + ) + assert applied is None + assert new_messages == original + + +def _clear_with_keep_two(messages: list[dict[str, object]]) -> tuple[list[dict[str, object]], AppliedEdit | None]: + return apply_clear_tool_uses_20250919( + model=MODEL, + messages=messages, + tools=None, + system=None, + edit_spec={ + "type": "clear_tool_uses_20250919", + "trigger": {"type": "tool_uses", "value": 0}, + "keep": {"type": "tool_uses", "value": 2}, + }, + ) + + +def test_tiny_results_are_never_cleared_and_do_not_count_toward_keep(): + messages = _make_history(6) + messages[2]["content"][0]["content"] = "(Bash completed with no output)" + messages[12]["content"][0]["content"] = [{"type": "text", "text": "\u9ce5" * 15}] + new_messages, applied = _clear_with_keep_two(messages) + assert _cleared_ids(new_messages) == {"toolu_01", "toolu_02"} + assert applied is not None + assert applied["cleared_tool_uses"] == 2 + + +def _clear_with_keep_two_and_inputs( + messages: list[dict[str, object]], clear_tool_inputs: bool | list[str] +) -> tuple[list[dict[str, object]], AppliedEdit | None]: + return apply_clear_tool_uses_20250919( + model=MODEL, + messages=messages, + tools=None, + system=None, + edit_spec={ + "type": "clear_tool_uses_20250919", + "trigger": {"type": "tool_uses", "value": 0}, + "keep": {"type": "tool_uses", "value": 2}, + "clear_tool_inputs": clear_tool_inputs, + }, + ) + + +def test_tiny_results_count_and_clear_when_their_tool_inputs_are_cleared(): + for clear_tool_inputs in (True, ["get_weather"]): + messages = _make_history(6) + messages[2]["content"][0]["content"] = "(Bash completed with no output)" + messages[12]["content"][0]["content"] = "ok" + new_messages, applied = _clear_with_keep_two_and_inputs(messages, clear_tool_inputs) + assert _cleared_ids(new_messages) == {"toolu_00", "toolu_01", "toolu_02", "toolu_03"} + assert applied is not None + assert applied["cleared_tool_uses"] == 4 + + +def test_tiny_results_stay_out_when_clear_tool_inputs_names_other_tools(): + messages = _make_history(6) + messages[2]["content"][0]["content"] = "(Bash completed with no output)" + messages[12]["content"][0]["content"] = "ok" + new_messages, applied = _clear_with_keep_two_and_inputs(messages, ["read_file"]) + assert _cleared_ids(new_messages) == {"toolu_01", "toolu_02"} + assert applied is not None + assert applied["cleared_tool_uses"] == 2 + + +def test_results_over_46_bytes_and_empty_results_still_count(): + # The 46 byte cutoff was measured on api.anthropic.com on 2026-10-19. + messages = _make_history(5) + messages[2]["content"][0]["content"] = "x" * 47 + messages[4]["content"][0]["content"] = "" + messages[6]["content"][0]["content"] = [ + {"type": "text", "text": "ok"}, + {"type": "image", "source": {"type": "base64", "media_type": "image/png", "data": "iVBORw0KGgo="}}, + ] + new_messages, applied = _clear_with_keep_two(messages) + assert _cleared_ids(new_messages) == {"toolu_00", "toolu_01", "toolu_02"} + assert new_messages[6]["content"][0]["content"] == [{"type": "text", "text": CLEARED_TOOL_RESULT_PLACEHOLDER}] + assert applied is not None + assert applied["cleared_tool_uses"] == 3 + + +def test_the_46_byte_cutoff_includes_46_and_counts_utf8_bytes(): + # The 46 byte cutoff was measured on api.anthropic.com on 2026-10-19. + messages = _make_history(5) + messages[2]["content"][0]["content"] = "x" * 46 + messages[4]["content"][0]["content"] = "鳥" * 16 + new_messages, applied = _clear_with_keep_two(messages) + assert _cleared_ids(new_messages) == {"toolu_01", "toolu_02"} + assert applied is not None + assert applied["cleared_tool_uses"] == 2 + + +def test_a_result_holding_a_lone_surrogate_is_sized_like_any_other_text(): + messages = _make_history(5) + messages[2]["content"][0]["content"] = "\ud800" + new_messages, applied = _clear_with_keep_two(messages) + assert _cleared_ids(new_messages) == {"toolu_01", "toolu_02"} + assert applied is not None + + +def _tool_inputs(messages: list[dict[str, object]]) -> dict[object, object]: + return {block["id"]: block["input"] for block in _blocks_of(messages, "tool_use")} + + +def test_clear_tool_inputs_true_empties_the_inputs_of_cleared_uses_only(): + messages = _make_history(n_pairs=3) + new_messages, applied = apply_clear_tool_uses_20250919( + model=MODEL, + messages=messages, + tools=None, + system=None, + edit_spec={ + "type": "clear_tool_uses_20250919", + "trigger": {"type": "tool_uses", "value": 0}, + "keep": {"type": "tool_uses", "value": 1}, + "clear_tool_inputs": True, + }, + ) + assert applied is not None + assert _tool_inputs(new_messages) == {"toolu_00": {}, "toolu_01": {}, "toolu_02": {"location": "City2"}} + # The caller's messages are not mutated. + assert _tool_inputs(messages)["toolu_00"] == {"location": "City0"} + + +def test_clear_tool_inputs_list_clears_only_the_named_tools(): + messages = [{"role": "user", "content": "Plan a trip."}] + for i, name in enumerate(["get_weather", "search", "get_weather"]): + messages.extend(_make_pair(f"toolu_{i:02d}", f"Result {i}: " + "x" * 200, location=f"City{i}", name=name)) + new_messages, applied = apply_clear_tool_uses_20250919( + model=MODEL, + messages=messages, + tools=None, + system=None, + edit_spec={ + "type": "clear_tool_uses_20250919", + "trigger": {"type": "tool_uses", "value": 0}, + "keep": {"type": "tool_uses", "value": 1}, + "clear_tool_inputs": ["search"], + }, + ) + assert applied is not None + assert _cleared_ids(new_messages) == {"toolu_00", "toolu_01"} + assert _tool_inputs(new_messages) == { + "toolu_00": {"location": "City0"}, + "toolu_01": {}, + "toolu_02": {"location": "City2"}, + } + + +def test_a_tool_use_with_a_non_string_id_is_left_for_the_provider_to_reject(): + messages = _make_history(n_pairs=4) + messages[1]["content"][0]["id"] = ["toolu_00"] + new_messages, applied = apply_clear_tool_uses_20250919( + model=MODEL, + messages=messages, + tools=None, + system=None, + edit_spec={ + "type": "clear_tool_uses_20250919", + "trigger": {"type": "tool_uses", "value": 0}, + "keep": {"type": "tool_uses", "value": 1}, + "clear_tool_inputs": True, + }, + ) + assert applied is not None + assert new_messages[1] == messages[1] + assert _cleared_ids(new_messages) == {"toolu_01", "toolu_02"} + assert (new_messages[3]["content"][0]["input"], new_messages[5]["content"][0]["input"]) == ({}, {}) + + +async def _malformed_spec_error(edits: list[dict[str, object]]) -> AnthropicContextManagementError: + with pytest.raises(AnthropicContextManagementError) as raised: + await apply_context_management( + model=MODEL, + messages=_make_history(n_pairs=3), + tools=None, + system=None, + context_management_spec={"edits": edits}, + ) + return raised.value + + +@pytest.mark.parametrize( + ("field", "value"), + [ + ("clear_at_least", {"type": "tool_uses", "value": 5}), + ("clear_at_least", {"value": 5}), + ("clear_at_least", {"type": "input_tokens", "value": -1}), + ("clear_at_least", {"type": "input_tokens", "value": 1.5}), + ("clear_at_least", {"type": "input_tokens", "value": "1200"}), + ("clear_at_least", {"type": "input_tokens", "value": True}), + ("exclude_tools", "get_weather"), + ("exclude_tools", [1]), + ("clear_tool_inputs", "get_weather"), + ], +) +async def test_malformed_fields_are_a_400_even_below_the_trigger(field: str, value: object) -> None: + error = await _malformed_spec_error( + [{"type": "clear_tool_uses_20250919", "trigger": {"type": "tool_uses", "value": 99}, field: value}] + ) + assert error.status_code == 400 + assert error.message.startswith(f"context_management.edits.0.clear_tool_uses_20250919.{field}") + + +@pytest.mark.parametrize( + ("field", "value", "message"), + [ + ("exclude_tools", "memory", "exclude_tools: Input should be a valid array"), + ("clear_at_least", 1200, "clear_at_least: Input does not match the expected shape."), + ("clear_at_least", {"type": "input_tokens", "value": "1200"}, "value: Input should be a valid integer"), + ("clear_tool_inputs", 5, "clear_tool_inputs.bool: Input should be a valid boolean, unable to interpret input"), + ], +) +async def test_malformed_field_400_names_the_edit_and_field_as_the_api_does( + field: str, value: object, message: str +) -> None: + # api.anthropic.com, measured 2026-10-19, answers these four with these messages for the edit at index 1. + error = await _malformed_spec_error( + [{"type": "clear_thinking_20251015"}, {"type": "clear_tool_uses_20250919", field: value}] + ) + assert error.message.startswith("context_management.edits.1.clear_tool_uses_20250919.") + assert error.message.endswith(message) + + +_VALID_EDIT: Final = { + "type": "clear_tool_uses_20250919", + "trigger": {"type": "tool_uses", "value": 0}, + "keep": {"type": "tool_uses", "value": 1}, +} +_MALFORMED_EDIT: Final = {**_VALID_EDIT, "exclude_tools": "get_weather"} + + +@pytest.mark.parametrize( + ("field", "value"), + [("exclude_tools", "get_weather"), ("clear_at_least", {"type": "input_tokens", "value": "1200"})], +) +async def test_drop_params_drops_only_the_malformed_field(field: str, value: object) -> None: + async def run(drop_invalid_edits: bool) -> PolyfillResult | None: + return await _run_polyfill_if_enabled( + model=MODEL, + messages=_make_history(n_pairs=3), + tools=None, + system=None, + context_management_spec={"edits": [{**_VALID_EDIT, field: value, "clear_tool_inputs": True}]}, + litellm_metadata={}, + additional_drop_params=None, + llm_router=None, + drop_invalid_edits=drop_invalid_edits, + ) + + with pytest.raises( + AnthropicContextManagementError, match=rf"^context_management\.edits\.0\.clear_tool_uses_20250919\.{field}" + ): + await run(drop_invalid_edits=False) + result: Final = await run(drop_invalid_edits=True) + assert result is not None + assert _cleared_ids(result.messages) == {"toolu_00", "toolu_01"} + assert _tool_inputs(result.messages) == {"toolu_00": {}, "toolu_01": {}, "toolu_02": {"location": "City2"}} + + +async def test_malformed_edit_is_rejected_before_a_compact_summary_is_paid_for( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from litellm.proxy.proxy_server import general_settings + + monkeypatch.setitem(general_settings, COMPACT_SUMMARY_MODEL_SETTING_KEY, "summary-model") + router: Final = MagicMock() + router.acompletion = AsyncMock(side_effect=RuntimeError("summary model unavailable")) + with pytest.raises(AnthropicContextManagementError): + await apply_context_management( + model=MODEL, + messages=[{"role": "user", "content": "hello " * 60_000}], + tools=None, + system=None, + context_management_spec={ + "edits": [ + {"type": "compact_20260112", "trigger": {"type": "input_tokens", "value": 50_000}}, + _MALFORMED_EDIT, + ] + }, + llm_router=router, + ) + router.acompletion.assert_not_awaited() + + +@pytest.mark.parametrize(("clear_tool_inputs", "cleared"), [("yes", True), ("true", True), (1, True), ("no", False)]) +def test_clear_tool_inputs_reads_boolean_strings_and_numbers_like_the_api( + clear_tool_inputs: object, cleared: bool +) -> None: + new_messages, applied = apply_clear_tool_uses_20250919( + model=MODEL, + messages=_make_history(n_pairs=3), + tools=None, + system=None, + edit_spec={ + "type": "clear_tool_uses_20250919", + "trigger": {"type": "tool_uses", "value": 0}, + "keep": {"type": "tool_uses", "value": 1}, + "clear_tool_inputs": clear_tool_inputs, + }, + ) + assert applied is not None + assert _tool_inputs(new_messages)["toolu_00"] == ({} if cleared else {"location": "City0"}) + + +@pytest.mark.parametrize(("trigger_value", "triggers"), [(2, False), (1, True)]) +def test_tool_uses_trigger_does_not_count_excluded_tools(trigger_value: int, triggers: bool) -> None: + messages = [{"role": "user", "content": "Plan a trip."}] + for i, name in enumerate(["search", "get_weather", "search", "get_weather"]): + messages.extend(_make_pair(f"toolu_{i:02d}", f"Result {i}: " + "x" * 200, name=name)) + _, applied = apply_clear_tool_uses_20250919( + model=MODEL, + messages=messages, + tools=None, + system=None, + edit_spec={ + "type": "clear_tool_uses_20250919", + "trigger": {"type": "tool_uses", "value": trigger_value}, + "keep": {"type": "tool_uses", "value": 0}, + "exclude_tools": ["search"], + }, + ) + assert (applied is not None) == triggers + + +@pytest.mark.parametrize(("clear_tool_inputs", "triggers"), [(False, False), (True, True)]) +def test_tool_uses_trigger_counts_tiny_results_only_when_their_inputs_are_cleared( + clear_tool_inputs: bool, triggers: bool +) -> None: + messages = _make_history(n_pairs=3) + messages[2]["content"][0]["content"] = "ok" + _, applied = apply_clear_tool_uses_20250919( + model=MODEL, + messages=messages, + tools=None, + system=None, + edit_spec={ + "type": "clear_tool_uses_20250919", + "trigger": {"type": "tool_uses", "value": 2}, + "keep": {"type": "tool_uses", "value": 1}, + "clear_tool_inputs": clear_tool_inputs, + }, + ) + assert (applied is not None) == triggers + + +def test_clear_at_least_counts_the_tool_inputs_it_clears(): + messages = [{"role": "user", "content": "Write the files."}] + for i in range(4): + assistant_msg, user_msg = _make_pair(f"toolu_{i:02d}", "ok", name="write_file") + assistant_msg["content"][0]["input"] = {"path": f"file{i}.txt", "content": "lorem ipsum " * 300} + messages.extend((assistant_msg, user_msg)) + one_input = litellm.token_counter(model=MODEL, text="lorem ipsum " * 300) + new_messages, applied = apply_clear_tool_uses_20250919( + model=MODEL, + messages=messages, + tools=None, + system=None, + edit_spec={ + "type": "clear_tool_uses_20250919", + "trigger": {"type": "tool_uses", "value": 0}, + "keep": {"type": "tool_uses", "value": 1}, + "clear_at_least": {"type": "input_tokens", "value": 3 * one_input // 2}, + "clear_tool_inputs": True, + }, + ) + assert applied is not None + assert _cleared_ids(new_messages) == {"toolu_00", "toolu_01"} + assert [_tool_inputs(new_messages)[f"toolu_{i:02d}"] == {} for i in range(4)] == [True, True, False, False] + assert applied["cleared_input_tokens"] >= 3 * one_input // 2 + + +_ONE_PIXEL_PNG: Final = ( + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNkYPhfDwAChwGA60e6kgAAAABJRU5ErkJggg==" +) + + +def test_clear_at_least_counts_image_results(): + messages = _make_history(n_pairs=4) + image: Final = {"type": "image", "source": {"type": "base64", "media_type": "image/png", "data": _ONE_PIXEL_PNG}} + for user_msg in messages[2::2]: + user_msg["content"][0]["content"] = [image] + new_messages, applied = apply_clear_tool_uses_20250919( + model=MODEL, + messages=messages, + tools=None, + system=None, + edit_spec={ + "type": "clear_tool_uses_20250919", + "trigger": {"type": "tool_uses", "value": 0}, + "keep": {"type": "tool_uses", "value": 1}, + "clear_at_least": {"type": "input_tokens", "value": 1}, + }, + ) + assert applied is not None + assert _cleared_ids(new_messages) == {"toolu_00", "toolu_01", "toolu_02"} def test_tool_result_list_content_shape_preserved(): @@ -266,7 +777,7 @@ def test_tool_result_list_content_shape_preserved(): { "type": "tool_result", "tool_use_id": "toolu_a", - "content": [{"type": "text", "text": "huge result"}], + "content": [{"type": "text", "text": "huge result: " + "x" * 200}], } ], }, @@ -280,7 +791,7 @@ def test_tool_result_list_content_shape_preserved(): { "type": "tool_result", "tool_use_id": "toolu_b", - "content": [{"type": "text", "text": "keep me"}], + "content": [{"type": "text", "text": "keep me: " + "y" * 200}], } ], }, diff --git a/tests/unit/llms/anthropic/pass_through/context_management/test_compact.py b/tests/unit/llms/anthropic/pass_through/context_management/test_compact.py index 5e55c0ade32..9b3084f0e7c 100644 --- a/tests/unit/llms/anthropic/pass_through/context_management/test_compact.py +++ b/tests/unit/llms/anthropic/pass_through/context_management/test_compact.py @@ -2406,7 +2406,13 @@ def _tool_use_messages() -> List[Dict[str, Any]]: }, { "role": "user", - "content": [{"type": "tool_result", "tool_use_id": "toolu_01", "content": "sunny in SF"}], + "content": [ + { + "type": "tool_result", + "tool_use_id": "toolu_01", + "content": "sunny in SF, 18 C, light wind from the west, no rain today", + } + ], }, { "role": "assistant", @@ -2414,7 +2420,13 @@ def _tool_use_messages() -> List[Dict[str, Any]]: }, { "role": "user", - "content": [{"type": "tool_result", "tool_use_id": "toolu_02", "content": "rainy in NY"}], + "content": [ + { + "type": "tool_result", + "tool_use_id": "toolu_02", + "content": "rainy in NY, 11 C, strong wind from the east, rain all day", + } + ], }, {"role": "user", "content": "now summarize both"}, ] diff --git a/tests/unit/llms/anthropic/pass_through/context_management/test_dispatcher.py b/tests/unit/llms/anthropic/pass_through/context_management/test_dispatcher.py index e28d2ac5e65..66957456c95 100644 --- a/tests/unit/llms/anthropic/pass_through/context_management/test_dispatcher.py +++ b/tests/unit/llms/anthropic/pass_through/context_management/test_dispatcher.py @@ -22,7 +22,7 @@ def _history_with_two_tool_pairs(): { "type": "tool_result", "tool_use_id": "t1", - "content": "first result", + "content": "first result: the forecast for the first city, sunny and mild", } ], }, @@ -36,7 +36,7 @@ def _history_with_two_tool_pairs(): { "type": "tool_result", "tool_use_id": "t2", - "content": "second result", + "content": "second result: the forecast for the second city, rainy and cold", } ], }, diff --git a/tests/unit/proxy/test_proxy_token_counter.py b/tests/unit/proxy/test_proxy_token_counter.py index f1b8d3fd591..714d8bb77d7 100644 --- a/tests/unit/proxy/test_proxy_token_counter.py +++ b/tests/unit/proxy/test_proxy_token_counter.py @@ -3236,9 +3236,16 @@ def _gemini_router(drop_params: bool = False) -> Router: ({"drop_params": True}, {}, {}, False), ({}, {"drop_params": True}, {}, False), ({}, {}, {"drop_params": True}, False), + ({}, {}, {"drop_params": "true"}, False), ({}, {}, {}, True), ), - ids=("deployment-drop-params", "router-default-drop-params", "request-drop-params", "litellm-drop-params"), + ids=( + "deployment-drop-params", + "router-default-drop-params", + "request-drop-params", + "request-drop-params-string", + "litellm-drop-params", + ), ) async def test_gemini_messages_count_tokens_counts_the_unedited_body_when_drop_params_drops_an_invalid_edit( deployment_params: dict[str, JsonValue],