From 2b4cb92c8b0e470b1fd8a4ae0d91c8193353abde Mon Sep 17 00:00:00 2001 From: Alon Sokolovski Date: Fri, 30 Oct 2026 02:42:11 +0200 Subject: [PATCH] fix(context_management)!: run clear_tool_uses in the gateway on the Responses API path (#47861) * fix(context_management): run clear_tool_uses in the gateway on the Responses API path /v1/messages to openai/ models goes through the Responses API, which translated only compact_20260112 (to its native compaction) and dropped every other edit without a warning, so clear_tool_uses_20250919 never cleared anything. Edits without a Responses API form now run in the same polyfill as the chat completions path, before the call, and their applied_edits come back on the response and the final message_delta. compact stays native. * fix(context_management): check the sync polyfill result instead of casting it The strict ruff gate (TID251) bans typing.cast. * test(context_management): cover the Responses path's polyfill error handling A polyfill crash is a 500 before the call, a rejected edit under drop_params is skipped, and the sync stream reports its edits. normalize_spec never raises for a dict spec, so its guard is gone. * fix(context_management): count unreadable blocks and check tool_result text before clearing clear_tool_uses now maps blocks the token counter cannot read (server tool, compaction) to JSON text, so an input_tokens trigger no longer 500s on that history. The Responses API path checks malformed tool_result text on the original messages before clearing, as the chat completions path does. Also removes comments that repeated the code and makes the test fixtures immutable. * fix(context_management): count only blocks the model receives and keep list specs unsent clear_tool_uses now leaves blocks token_counter cannot read, like server tool results and compaction, out of its count instead of counting them as JSON text. No adapter sends them to the model, so an input_tokens trigger no longer fires on tokens the model never sees. On the Responses API path, an OpenAI-style list context_management stays unsent, as it was before, instead of reaching OpenAI's native compaction, where a threshold under 1000 is a 400. On the chat completions path, a clear edit listed before compact_20260112 now runs on the turns after the latest compaction block, as Anthropic does, instead of counting and clearing turns the compaction block drops. The crash test that replaced apply_context_management with a mock is removed. * fix(context_management): apply clear edits in OpenAI count_tokens and skip nested off-spec blocks when counting * fix(context_management): run edits on the history drop_params sends, so count_tokens and /v1/messages clear alike * fix(context_management): count clear_at_least freed tokens over sent blocks only * fix(context_management): count tools with JSON Schema shorthands and reject a non-object trigger or keep A tool whose schema has an array without items, a boolean subschema, or a search_result block without a title made the token counter raise, so a clear edit answered 500. A string trigger or keep also answered 500, even under drop_params. Both now count or get the API's 400, and drop_params drops the malformed field. * fix(token_counter): count a malformed tool schema keyword as absent and move shorthand framed-estimate cases * fix(count_tokens): pass the system prompt into clearing so count_tokens matches inference * fix(context_management): keep a malformed system prompt from failing clear_tool_uses on the Responses API A number, an object, or a list holding non-object items as system made the polyfill answer 500. The polyfill now keeps a list's object items and treats any other shape as no system, on both /v1/messages and count_tokens, so the two paths still count the same text --------- Co-authored-by: AlonSoko <25619863+AlonSoko@users.noreply.github.com> Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- litellm/litellm_core_utils/token_counter.py | 25 +- .../anthropic/messages/adapters/handler.py | 3 +- .../messages/context_management/dispatcher.py | 5 +- .../editors/clear_tool_uses.py | 75 +++- .../context_management/editors/compact.py | 8 +- .../messages/responses_adapters/handler.py | 153 ++++++++- .../responses_adapters/streaming_iterator.py | 10 +- .../responses/count_tokens/token_counter.py | 45 ++- .../litellm_core_utils/test_token_counter.py | 51 +++ .../test_clear_tool_uses.py | 130 ++++++- .../context_management/test_compact.py | 6 +- .../context_management/test_dispatcher.py | 55 +++ .../test_responses_adapters_handler.py | 325 +++++++++++++++++- .../count_tokens/test_token_counter.py | 120 ++++++- .../vertex_ai/count_tokens/test_context.py | 2 +- tests/unit/proxy/test_proxy_token_counter.py | 54 ++- 16 files changed, 993 insertions(+), 74 deletions(-) diff --git a/litellm/litellm_core_utils/token_counter.py b/litellm/litellm_core_utils/token_counter.py index 591d77bdb16..5ad4e130e62 100644 --- a/litellm/litellm_core_utils/token_counter.py +++ b/litellm/litellm_core_utils/token_counter.py @@ -1224,10 +1224,6 @@ def _validate_anthropic_content(content: Mapping[str, object]) -> type: if expected_cls is None: raise ValueError(f"Unknown Anthropic content type: '{content_type}'") - missing: Final = [k for k in getattr(expected_cls, "__required_keys__", set()) if k not in content] - if missing: - raise ValueError(f"Missing required fields in {content_type} block: {', '.join(missing)}") - return expected_cls @@ -1552,12 +1548,13 @@ def _format_single_function_definition(function: Mapping[str, object]) -> tuple[ def _format_object_parameters(parameters, indent): properties: Final = parameters.get("properties") - if not properties: + if not isinstance(properties, Mapping) or not properties: return "" - required_params: Final = parameters.get("required", []) + required: Final = parameters.get("required") + required_params: Final = tuple(required) if isinstance(required, list) else () lines: Final = [] for key, props in properties.items(): - description = props.get("description") + description = props.get("description") if isinstance(props, Mapping) else None if description: lines.append(f"// {description}") question = "?" @@ -1568,19 +1565,21 @@ def _format_object_parameters(parameters, indent): def _format_type(props, indent): + if not isinstance(props, Mapping): + return "any" type: Final = props.get("type") + enum: Final = props.get("enum") if type == "string": - if "enum" in props: - return " | ".join([f'"{item}"' for item in props["enum"]]) + if isinstance(enum, list): + return " | ".join([f'"{item}"' for item in enum]) return "string" elif type == "array": - # items is required, OpenAI throws an error if it's missing - return f"{_format_type(props['items'], indent)}[]" + return f"{_format_type(props.get('items'), indent)}[]" elif type == "object": return f"{{\n{_format_object_parameters(props, indent + 2)}\n}}" elif type in ["integer", "number"]: - if "enum" in props: - return " | ".join([f'"{item}"' for item in props["enum"]]) + if isinstance(enum, list): + return " | ".join([f'"{item}"' for item in enum]) return "number" elif type == "boolean": return "boolean" diff --git a/litellm/llms/anthropic/messages/adapters/handler.py b/litellm/llms/anthropic/messages/adapters/handler.py index 71ccfbaf915..704bec9056f 100644 --- a/litellm/llms/anthropic/messages/adapters/handler.py +++ b/litellm/llms/anthropic/messages/adapters/handler.py @@ -160,6 +160,7 @@ async def prepare_context_managed_request( """Apply client compaction history, then optional context_management polyfill.""" from litellm.llms.anthropic.messages.context_management.editors.compact import ( apply_client_compaction_block_history, + slice_around_compaction_block, ) # Skip the client-history pre-processing when a ``compact_20260112`` @@ -175,7 +176,7 @@ async def prepare_context_managed_request( if polyfill_will_run: history_result: PolyfillResult | None = None - working_messages: _AnthropicMessages = messages + working_messages: _AnthropicMessages = list(slice_around_compaction_block(messages)[0]) working_system: _AnthropicSystem = system else: history_result = apply_client_compaction_block_history( diff --git a/litellm/llms/anthropic/messages/context_management/dispatcher.py b/litellm/llms/anthropic/messages/context_management/dispatcher.py index 1557cf206ac..2b713646d50 100644 --- a/litellm/llms/anthropic/messages/context_management/dispatcher.py +++ b/litellm/llms/anthropic/messages/context_management/dispatcher.py @@ -8,6 +8,7 @@ from litellm._logging import verbose_logger from litellm.litellm_core_utils.asyncify import asyncify from litellm.types.llms.anthropic import AppliedEdit +from ..convertible_blocks import drop_non_string_tool_result_texts from .constants import CLEAR_TOOL_USES_EDIT_TYPE, COMPACT_EDIT_TYPE from .editors import ( CheckedEdit, @@ -103,7 +104,7 @@ async def apply_context_management( Every ``clear_tool_uses_20250919`` edit is validated before any editor runs, so a malformed one raises its 400 before a ``compact_20260112`` summary call is paid for, or runs without its malformed fields under - ``drop_invalid_edits``. + ``drop_invalid_edits``, which also drops non-string ``tool_result`` text first, as the provider never receives it. The dispatcher is async so async editors (``compact_20260112``) can ``await`` the configured summarization model. Sync editors run in a @@ -119,7 +120,7 @@ async def apply_context_management( raise first_error well_formed_edits: Final = tuple(check.edit for check in checked) - current_messages = messages + current_messages = list(drop_non_string_tool_result_texts(messages, drop_invalid_edits)) current_system = system aggregated_applied: Final[list[AppliedEdit]] = [] aggregated_compaction_block = None diff --git a/litellm/llms/anthropic/messages/context_management/editors/clear_tool_uses.py b/litellm/llms/anthropic/messages/context_management/editors/clear_tool_uses.py index c4f7966f429..fabc112d443 100644 --- a/litellm/llms/anthropic/messages/context_management/editors/clear_tool_uses.py +++ b/litellm/llms/anthropic/messages/context_management/editors/clear_tool_uses.py @@ -39,6 +39,21 @@ if TYPE_CHECKING: T = TypeVar("T") # api.anthropic.com, measured 2026-10-19: text results of 1 to 46 bytes are never cleared or counted toward keep _TINY_RESULT_BYTES: Final = 46 +_TOKEN_COUNTER_BLOCK_TYPES: Final = frozenset( + { + "text", + "image_url", + "image", + "document", + "file", + "tool_use", + "tool_result", + "thinking", + "redacted_thinking", + "tool_reference", + "search_result", + } +) ToolNames = Literal["all"] | frozenset[str] @@ -52,16 +67,19 @@ class _ClearAtLeast(BaseModel): class _ClearToolUsesFields(BaseModel): model_config = ConfigDict(frozen=True) + trigger: Mapping[str, object] | None = None + keep: Mapping[str, object] | None = None clear_at_least: _ClearAtLeast | None = None exclude_tools: tuple[str, ...] | None = None clear_tool_inputs: bool | tuple[str, ...] | None = None _BLOCKS: Final = TypeAdapter(tuple[Mapping[str, object], ...]) -# api.anthropic.com, measured 2026-10-19: its wording for the two shapes pydantic words differently +# api.anthropic.com, measured 2026-10-19 and 2026-10-28: its wording for the shapes pydantic words differently _API_MESSAGES: Final = MappingProxyType( { "tuple_type": "Input should be a valid array", + "dict_type": "Input should be an object", "model_type": "Input does not match the expected shape.", } ) @@ -178,10 +196,12 @@ def _cleared_result(block: Mapping[str, object]) -> dict[str, object]: def _freed_tokens(model: str, result: Mapping[str, object], use: Mapping[str, object], clears_input: bool) -> int: """Tokens that clearing this use frees: its result, and its input when ``clear_tool_inputs`` covers it.""" - before: Final = [result, use] if clears_input else [result] - after: Final = [_cleared_result(result), {**use, "input": {}}] if clears_input else [_cleared_result(result)] + before: Final[Sequence[Mapping[str, object]]] = (result, use) if clears_input else (result,) + after: Final[Sequence[Mapping[str, object]]] = ( + (_cleared_result(result), {**use, "input": {}}) if clears_input else (_cleared_result(result),) + ) counts: Final = tuple( - litellm.token_counter(model=model, messages=[{"role": "user", "content": blocks}]) for blocks in (before, after) + _count_tokens(model, [{"role": "user", "content": blocks}], None) for blocks in (before, after) ) return max(counts[0] - counts[1], 0) @@ -199,6 +219,35 @@ def _select_in_batches(candidates: Sequence[str], tokens: Mapping[str, int], cle return tuple(candidates[:batch_end]) +def _without_uncountable_blocks(block: Mapping[str, object]) -> Mapping[str, object]: + nested: Final = _parse(_BLOCKS, block.get("content")) + if nested is None: + return block + return {**block, "content": [inner for inner in nested if inner.get("type") in _TOKEN_COUNTER_BLOCK_TYPES]} + + +def _countable_message(message: Mapping[str, object]) -> Mapping[str, object]: + blocks: Final = _parse(_BLOCKS, message.get("content")) + if blocks is None: + return message + return { + **message, + "content": [ + _without_uncountable_blocks(block) for block in blocks if block.get("type") in _TOKEN_COUNTER_BLOCK_TYPES + ], + } + + +def _count_tokens( + model: str, messages: Sequence[Mapping[str, object]], tools: Sequence[Mapping[str, object]] | None +) -> int: + return litellm.token_counter( + model=model, + messages=[_countable_message(message) for message in messages], + tools=cast("list[ChatCompletionToolParam] | None", tools), + ) + + def _trigger_met( trigger: dict[str, object], model: str, @@ -218,11 +267,7 @@ def _trigger_met( if not isinstance(threshold, int): threshold = DEFAULT_INPUT_TOKENS_TRIGGER - message_tokens: Final = litellm.token_counter( - model=model, - messages=messages, - tools=cast("list[ChatCompletionToolParam] | None", tools), - ) + message_tokens: Final = _count_tokens(model, messages, tools) if message_tokens > threshold: return True, message_tokens current_tokens: Final = message_tokens + count_system_tokens(model, system) @@ -375,17 +420,9 @@ def apply_clear_tool_uses_20250919( [_message_without_inputs(message, ids_to_clear, input_tools) for message in edited] if input_tools else edited ) - tokens_before: Final = ( - counted_tokens - if counted_tokens is not None - else litellm.token_counter( - model=model, messages=kept, tools=cast("list[ChatCompletionToolParam] | None", tools) - ) - ) + tokens_before: Final = counted_tokens if counted_tokens is not None else _count_tokens(model, kept, tools) kept_after, _ = history_after_client_compaction(final_messages, system) - tokens_after: Final = litellm.token_counter( - model=model, messages=kept_after, tools=cast("list[ChatCompletionToolParam] | None", tools) - ) + tokens_after: Final = _count_tokens(model, kept_after, tools) cleared_input_tokens: Final = max(tokens_before - tokens_after, 0) return final_messages, { diff --git a/litellm/llms/anthropic/messages/context_management/editors/compact.py b/litellm/llms/anthropic/messages/context_management/editors/compact.py index 0bf8cac7344..d1cfcc2f2bc 100644 --- a/litellm/llms/anthropic/messages/context_management/editors/compact.py +++ b/litellm/llms/anthropic/messages/context_management/editors/compact.py @@ -137,7 +137,7 @@ def _find_latest_compaction_index( return None, None -def _slice_around_compaction_block( +def slice_around_compaction_block( messages: Sequence[_MsgT], ) -> tuple[Sequence[_MsgT | dict[str, object]], dict[str, object] | None]: """Apply Anthropic's "drop everything before the compaction block" rule. @@ -194,7 +194,7 @@ def history_after_client_compaction( system: str | Sequence[Mapping[str, object]] | None, ) -> tuple[list[dict[str, object]], str | Sequence[Mapping[str, object]] | None]: """Return the history the API keeps after the latest compaction block, without it, and the system with its summary.""" - sliced, block = _slice_around_compaction_block(messages) + sliced, block = slice_around_compaction_block(messages) if block is None: return list(messages), system tail: Final = _strip_compaction_blocks(sliced) @@ -615,7 +615,7 @@ def apply_client_compaction_block_history( tail is forwarded unchanged (with compaction blocks stripped) so recent turns the summary does not cover are preserved. """ - effective_messages, prior_compaction_block = _slice_around_compaction_block(messages) + effective_messages, prior_compaction_block = slice_around_compaction_block(messages) if prior_compaction_block is None: return None @@ -686,7 +686,7 @@ async def apply_compact_20260112( # opt-in gate below so that even when summarization is disabled we still # strip Anthropic-only ``compaction`` blocks from messages going to # non-Anthropic backends (which would reject them). - effective_messages, prior_compaction_block = _slice_around_compaction_block(messages) + effective_messages, prior_compaction_block = slice_around_compaction_block(messages) prior_summary_text: Final = prior_compaction_block.get("content") if prior_compaction_block else None augmented_system: str | list[dict[str, object]] | None = system if isinstance(prior_summary_text, str) and prior_summary_text: diff --git a/litellm/llms/anthropic/messages/responses_adapters/handler.py b/litellm/llms/anthropic/messages/responses_adapters/handler.py index 9b22170c9db..0eef3fd87c0 100644 --- a/litellm/llms/anthropic/messages/responses_adapters/handler.py +++ b/litellm/llms/anthropic/messages/responses_adapters/handler.py @@ -10,11 +10,15 @@ from typing import Final, TypeAlias from pydantic import TypeAdapter, ValidationError import litellm +from litellm._logging import verbose_logger from litellm.integrations.anthropic_cache_control_hook import ( AnthropicCacheControlHook, supports_openai_prompt_cache_breakpoint, ) -from litellm.litellm_core_utils.core_helpers import drops_unsupported_params +from litellm.litellm_core_utils.asyncify import ( + run_async_function, # pyright: ignore[reportUnknownVariableType] # untyped helper +) +from litellm.litellm_core_utils.core_helpers import as_str_mapping, drops_unsupported_params from litellm.litellm_core_utils.prompt_templates.common_utils import ( cache_control_as_prompt_cache_breakpoint, prompt_cache_breakpoint_for_wire, @@ -28,14 +32,19 @@ from litellm.types.llms.anthropic import ( AnthropicMessagesRequest, AnthropicOutputConfig, AnthropicOutputSchema, + ContextManagementResponse, ) from litellm.types.llms.anthropic_messages.anthropic_response import ( AnthropicMessagesResponse, ) from litellm.types.llms.openai import ResponsesAPIResponse +from litellm.types.utils import CountContextManagement from litellm.utils import ProviderConfigManager -from ..adapters.handler import request_safeguards_evaluator +from ..adapters.handler import context_management_explicitly_dropped, request_safeguards_evaluator +from ..context_management import AnthropicContextManagementError, PolyfillResult, apply_context_management +from ..context_management.constants import COMPACT_EDIT_TYPE +from ..context_management.dispatcher import normalize_spec from ..convertible_blocks import reject_malformed_tool_result_text from ..safeguards import with_safeguard_results from ..utils import litellm_logging_obj_from_kwargs, local_model_name @@ -45,10 +54,111 @@ from .transformation import LiteLLMAnthropicToResponsesAPIAdapter AnthropicRequestMessages: TypeAlias = list[AllAnthropicMessageValues] | list[dict[str, object]] BlockRewrite: TypeAlias = Callable[[Mapping[str, object]], Mapping[str, object]] _BLOCK_LIST: Final = TypeAdapter(list[dict[str, object]]) +_SYSTEM_ITEMS: Final = TypeAdapter(list[object]) +_SPEC: Final = TypeAdapter(dict[str, object]) +_SplitEdits: TypeAlias = tuple[dict[str, object] | None, Sequence[Mapping[str, object]]] +AnthropicContextManagementSpec: TypeAlias = dict[str, object] | list[dict[str, object]] | None _ADAPTER: Final = LiteLLMAnthropicToResponsesAPIAdapter() +def _split_context_management( + spec: AnthropicContextManagementSpec, + kwargs: Mapping[str, object], +) -> _SplitEdits: + """Return (the edits the Responses API runs natively, the edits the gateway polyfill runs). + + The Responses API compacts natively (``compact_20260112`` becomes its ``compaction``); any other + edit, such as ``clear_tool_uses_20250919``, has no Responses API form and runs in the polyfill, as + it does on the chat completions path. An OpenAI-style list spec is not Anthropic's shape and stays + unsent, as it was before the polyfill ran here. + """ + if not isinstance(spec, dict): + return None, () + if not spec or context_management_explicitly_dropped(kwargs.get("additional_drop_params")): + return spec, () + edits: Final = normalize_spec(spec) or [] + native: Final = [edit for edit in edits if edit.get("type") == COMPACT_EDIT_TYPE] + return ({"edits": native} if native else None), tuple( + edit for edit in edits if edit.get("type") != COMPACT_EDIT_TYPE + ) + + +def _system_blocks(system: object) -> str | Sequence[Mapping[str, object]] | None: + if system is None or isinstance(system, str): + return system + try: + items: Final = _SYSTEM_ITEMS.validate_python(system) + except ValidationError: + return None + return tuple(block for block in map(as_str_mapping, items) if block is not None) + + +async def _polyfilled( + *, + model: str, + messages: AnthropicRequestMessages, + tools: Sequence[AllAnthropicToolsValues | Mapping[str, object]] | None, + system: object, + edits: Sequence[Mapping[str, object]], + kwargs: Mapping[str, object], +) -> PolyfillResult | None: + """Run the edits the Responses API can't run natively; raise like the chat completions path does.""" + if not edits: + return None + reject_malformed_tool_result_text(messages, model, kwargs) + drop_invalid_edits: Final = drops_unsupported_params(kwargs.get("drop_params")) + blocks: Final = _system_blocks(system) + try: + return await apply_context_management( + model=model, + messages=_BLOCK_LIST.validate_python(messages), + tools=_BLOCK_LIST.validate_python(tools) if tools else None, + system=blocks if blocks is None or isinstance(blocks, str) else _BLOCK_LIST.validate_python(blocks), + context_management_spec={"edits": [dict(edit) for edit in edits]}, + drop_invalid_edits=drop_invalid_edits, + ) + except AnthropicContextManagementError: + raise + except Exception as e: + verbose_logger.exception("context_management polyfill failed on the Responses API path: %s", e) + raise AnthropicContextManagementError( + status_code=500, message=f"context_management polyfill failed: {e}" + ) from e + + +async def count_view_messages( + *, + model: str, + messages: Sequence[Mapping[str, object]], + tools: Sequence[Mapping[str, object]] | None, + system: object, + context_management: CountContextManagement | None, +) -> Sequence[Mapping[str, object]]: + """The messages /v1/messages sends the Responses API once the polyfill has run the edits it can't run natively.""" + if context_management is None or not isinstance(context_management.spec, dict): + return messages + _, edits = _split_context_management(_SPEC.validate_python(context_management.spec), {}) + result: Final = await _polyfilled( + model=model, + messages=_BLOCK_LIST.validate_python(messages), + tools=tools, + system=system, + edits=edits, + kwargs={"drop_params": context_management.drop_invalid_edits}, + ) + return messages if result is None else result.messages + + +def _with_applied_edits( + response: AnthropicMessagesResponse, polyfill_result: PolyfillResult | None +) -> AnthropicMessagesResponse: + applied_edits: Final = polyfill_result.applied_edits_for_response() if polyfill_result else None + if not applied_edits: + return response + return {**response, "context_management": ContextManagementResponse(applied_edits=list(applied_edits))} + + def _record_bridged_response(responses_kwargs: Mapping[str, object], result: ResponsesAPIResponse) -> None: logging_obj: Final = litellm_logging_obj_from_kwargs(responses_kwargs) if logging_obj is not None: @@ -281,7 +391,7 @@ class LiteLLMMessagesToResponsesAPIHandler: max_tokens: int, messages: AnthropicRequestMessages, model: str, - context_management: dict[str, object] | None = None, + context_management: AnthropicContextManagementSpec = None, metadata: dict[str, object] | None = None, output_config: AnthropicOutputConfig | None = None, stop_sequences: list[str] | None = None, @@ -296,11 +406,15 @@ class LiteLLMMessagesToResponsesAPIHandler: output_format: AnthropicOutputSchema | None = None, **kwargs: object, ) -> AnthropicMessagesResponse | AsyncIterator[bytes]: + native_context_management, polyfilled_edits = _split_context_management(context_management, kwargs) + polyfill_result: Final = await _polyfilled( + model=model, messages=messages, tools=tools, system=system, edits=polyfilled_edits, kwargs=kwargs + ) responses_kwargs: Final = _build_responses_kwargs( max_tokens=max_tokens, - messages=messages, + messages=polyfill_result.messages if polyfill_result is not None else messages, model=model, - context_management=context_management, + context_management=native_context_management, metadata=metadata, output_config=output_config, stop_sequences=stop_sequences, @@ -325,13 +439,16 @@ class LiteLLMMessagesToResponsesAPIHandler: model=local_model_name(model, kwargs.get("custom_llm_provider")), litellm_logging_obj=litellm_logging_obj_from_kwargs(responses_kwargs), safeguards_evaluator=safeguards_evaluator, + applied_edits=polyfill_result.applied_edits_for_response() if polyfill_result else None, ) return wrapper.async_anthropic_sse_wrapper() if not isinstance(result, ResponsesAPIResponse): raise ValueError(f"Expected ResponsesAPIResponse, got {type(result)}") - served: Final = await with_safeguard_results(_ADAPTER.translate_response(result), safeguards_evaluator) + served: Final = await with_safeguard_results( + _with_applied_edits(_ADAPTER.translate_response(result), polyfill_result), safeguards_evaluator + ) _record_bridged_response(responses_kwargs, result) return served @@ -340,7 +457,7 @@ class LiteLLMMessagesToResponsesAPIHandler: max_tokens: int, messages: AnthropicRequestMessages, model: str, - context_management: dict[str, object] | None = None, + context_management: AnthropicContextManagementSpec = None, metadata: dict[str, object] | None = None, output_config: AnthropicOutputConfig | None = None, stop_sequences: list[str] | None = None, @@ -382,11 +499,26 @@ class LiteLLMMessagesToResponsesAPIHandler: ) # Sync path + native_context_management, polyfilled_edits = _split_context_management(context_management, kwargs) + ran: Final[object] = ( # pyright: ignore[reportUnknownVariableType] # run_async_function is untyped + run_async_function( + _polyfilled, + model=model, + messages=messages, + tools=tools, + system=system, + edits=polyfilled_edits, + kwargs=kwargs, + ) + if polyfilled_edits + else None + ) + polyfill_result: Final = ran if isinstance(ran, PolyfillResult) else None responses_kwargs: Final = _build_responses_kwargs( max_tokens=max_tokens, - messages=messages, + messages=polyfill_result.messages if polyfill_result is not None else messages, model=model, - context_management=context_management, + context_management=native_context_management, metadata=metadata, output_config=output_config, stop_sequences=stop_sequences, @@ -409,10 +541,11 @@ class LiteLLMMessagesToResponsesAPIHandler: responses_stream=result, model=local_model_name(model, kwargs.get("custom_llm_provider")), litellm_logging_obj=litellm_logging_obj_from_kwargs(responses_kwargs), + applied_edits=polyfill_result.applied_edits_for_response() if polyfill_result else None, ) return wrapper.async_anthropic_sse_wrapper() if not isinstance(result, ResponsesAPIResponse): raise ValueError(f"Expected ResponsesAPIResponse, got {type(result)}") - return _ADAPTER.translate_response(result) + return _with_applied_edits(_ADAPTER.translate_response(result), polyfill_result) diff --git a/litellm/llms/anthropic/messages/responses_adapters/streaming_iterator.py b/litellm/llms/anthropic/messages/responses_adapters/streaming_iterator.py index 44f233f2254..dae96ba4ce1 100644 --- a/litellm/llms/anthropic/messages/responses_adapters/streaming_iterator.py +++ b/litellm/llms/anthropic/messages/responses_adapters/streaming_iterator.py @@ -2,7 +2,7 @@ ## Translates OpenAI call to Anthropic `/v1/messages` format import asyncio from collections import deque -from collections.abc import AsyncGenerator, Awaitable, Callable, Iterator, Mapping +from collections.abc import AsyncGenerator, Awaitable, Callable, Iterator, Mapping, Sequence from typing import TYPE_CHECKING, Any, Final import anyio @@ -27,6 +27,7 @@ from litellm.responses.streaming_iterator import ( partial_responses_incomplete_event_on_disconnect, stream_error_status_and_message, ) +from litellm.types.llms.anthropic import AppliedEdit, ContextManagementResponse from litellm.types.llms.anthropic_messages.anthropic_response import AnthropicUsage from litellm.types.llms.base import LiteLLMBaseModel @@ -129,8 +130,10 @@ class AnthropicResponsesStreamWrapper: model: str, litellm_logging_obj: "LiteLLMLoggingObject | None" = None, safeguards_evaluator: "SafeguardsEvaluator | None" = None, + applied_edits: Sequence[AppliedEdit] | None = None, ) -> None: self.responses_stream = responses_stream + self._applied_edits: Final = tuple(applied_edits or ()) self.model = model self.litellm_logging_obj = litellm_logging_obj self.safeguards_evaluator: Final = safeguards_evaluator @@ -445,6 +448,11 @@ class AnthropicResponsesStreamWrapper: "type": "message_delta", "delta": message_delta_payload, "usage": dict(anthropic_usage), + **( + {"context_management": ContextManagementResponse(applied_edits=list(self._applied_edits))} + if self._applied_edits + else {} + ), } ) self._chunk_queue.append({"type": "message_stop"}) diff --git a/litellm/llms/openai/responses/count_tokens/token_counter.py b/litellm/llms/openai/responses/count_tokens/token_counter.py index 6ae61e15953..87a75dae95e 100644 --- a/litellm/llms/openai/responses/count_tokens/token_counter.py +++ b/litellm/llms/openai/responses/count_tokens/token_counter.py @@ -15,7 +15,9 @@ from litellm._logging import verbose_logger from litellm.litellm_core_utils.asyncify import asyncify from litellm.litellm_core_utils.core_helpers import as_str_mapping from litellm.litellm_core_utils.tool_fields import DropPolicy +from litellm.llms.anthropic.messages.context_management import AnthropicContextManagementError from litellm.llms.anthropic.messages.convertible_blocks import deployment_drops_params, unconvertible_blocks_error +from litellm.llms.anthropic.messages.responses_adapters.handler import count_view_messages from litellm.llms.anthropic.messages.responses_adapters.transformation import ( LiteLLMAnthropicToResponsesAPIAdapter, ) @@ -95,6 +97,32 @@ def _count_error(model_to_use: str, request_model: str, message: str, status_cod ) +async def _sent_messages( + model: str, + messages: Sequence[Mapping[str, object]], + tools: Sequence[Mapping[str, object]] | None, + system: object, + messages_format: TokenCountMessagesFormat, + context_management: CountContextManagement | None, +) -> Sequence[Mapping[str, object]] | AnthropicContextManagementError: + match messages_format: + case "openai": + return messages + case "anthropic": + try: + return await count_view_messages( + model=model, + messages=messages, + tools=tools, + system=system, + context_management=context_management, + ) + except AnthropicContextManagementError as error: + return error + case _: + assert_never(messages_format) + + class OpenAITokenCounter(BaseTokenCounter): """Token counter implementation for OpenAI provider using the Responses API.""" @@ -175,13 +203,28 @@ class OpenAITokenCounter(BaseTokenCounter): rejected_before_provider_call=True, ) + sent_messages: Final = await _sent_messages( + model_to_use, messages, tools, system, messages_format, context_management + ) + if isinstance(sent_messages, AnthropicContextManagementError): + return TokenCountResponse( + total_tokens=0, + request_model=request_model, + model_used=model_to_use, + tokenizer_type="openai_api", + error=True, + error_message=sent_messages.message, + status_code=sent_messages.status_code, + rejected_before_provider_call=True, + ) + resolved: Final = _count_credential(deployment, request_model or model_to_use) if isinstance(resolved, OpenAIError): return _count_error(model_to_use, request_model, resolved.message, resolved.status_code) input_items, instructions, counted_tools, text = self._responses_request_parts( model_to_use=model_to_use, - messages=messages, + messages=sent_messages, tools=tools, system=system, messages_format=messages_format, diff --git a/tests/unit/litellm_core_utils/test_token_counter.py b/tests/unit/litellm_core_utils/test_token_counter.py index d97970753f8..f03c7421899 100644 --- a/tests/unit/litellm_core_utils/test_token_counter.py +++ b/tests/unit/litellm_core_utils/test_token_counter.py @@ -523,6 +523,43 @@ def test_token_counter_skips_non_mapping_tools(): assert mixed_tokens == valid_tokens +@pytest.mark.parametrize( + ("shorthand", "explicit"), + [({"type": "array"}, {"type": "array", "items": {}}), (True, {})], + ids=["array_without_items", "boolean_subschema"], +) +def test_token_counter_counts_json_schema_shorthands_like_their_explicit_form(shorthand: object, explicit: object) -> None: + assert _count_with_property_schema(shorthand) == _count_with_property_schema(explicit) + + +@pytest.mark.parametrize( + ("malformed", "without_keyword"), + [ + ( + {"type": "object", "properties": {"id": {"type": "string"}}, "required": True}, + {"type": "object", "properties": {"id": {"type": "string"}}}, + ), + ({"type": "object", "properties": [{"id": {"type": "string"}}]}, {"type": "object"}), + ({"type": "string", "enum": None}, {"type": "string"}), + ({"type": "integer", "enum": 3}, {"type": "integer"}), + ], + ids=["required_true", "properties_list", "string_enum_null", "integer_enum_number"], +) +def test_token_counter_counts_a_malformed_schema_keyword_as_absent(malformed: object, without_keyword: object) -> None: + assert _count_with_property_schema(malformed) == _count_with_property_schema(without_keyword) + + +def _count_with_property_schema(property_schema: object) -> int: + tool: Final = { + "type": "function", + "function": { + "name": "read_files", + "parameters": {"type": "object", "properties": {"paths": property_schema}}, + }, + } + return token_counter_new(model="gpt-4o", messages=[{"role": "user", "content": "Read the config files"}], tools=[tool]) + + _WEATHER_FUNCTION: Final = { "name": "get_weather", "description": "Get the current weather for a city", @@ -2132,6 +2169,20 @@ def test_token_counter_counts_anthropic_search_result_text() -> None: ) +def test_token_counter_counts_a_search_result_without_a_title() -> None: + from litellm.litellm_core_utils.token_counter import _count_content_list + + search_result: Final = { + "type": "search_result", + "source": "https://example.invalid/notes", + "content": [{"type": "text", "text": "The code word is 7f3a."}], + } + + assert _count_content_list(len, [search_result], False, None) == len( + "https://example.invalid/notesThe code word is 7f3a." + ) + + @pytest.mark.parametrize("nested", (False, True)) def test_unknown_content_type_remains_a_value_error_with_an_optional_fallback(nested: bool) -> None: from litellm.litellm_core_utils.token_counter import _count_content_list diff --git a/tests/unit/llms/anthropic/messages/context_management/test_clear_tool_uses.py b/tests/unit/llms/anthropic/messages/context_management/test_clear_tool_uses.py index f03ffb25b2b..4cbf4cfa5f6 100644 --- a/tests/unit/llms/anthropic/messages/context_management/test_clear_tool_uses.py +++ b/tests/unit/llms/anthropic/messages/context_management/test_clear_tool_uses.py @@ -217,6 +217,106 @@ def test_cleared_input_tokens_is_nonnegative(): assert applied["cleared_input_tokens"] >= 0 +SERVER_TOOL_TURN: Final = { + "role": "assistant", + "content": [ + {"type": "server_tool_use", "id": "srvtoolu_01", "name": "web_search", "input": {"query": "Mumbai rain"}}, + { + "type": "web_search_tool_result", + "tool_use_id": "srvtoolu_01", + "content": [{"type": "web_search_result", "url": "https://example.com", "title": "rain " * 300}], + }, + ], +} + + +def _clear_above( + messages: list[dict[str, object]], threshold: int +) -> tuple[list[dict[str, object]], AppliedEdit | None]: + return apply_clear_tool_uses_20250919( + model=MODEL, + messages=messages, + tools=None, + system=None, + edit_spec={ + "type": "clear_tool_uses_20250919", + "trigger": {"type": "input_tokens", "value": threshold}, + "keep": {"type": "tool_uses", "value": 1}, + }, + ) + + +def test_an_input_tokens_trigger_skips_blocks_no_adapter_sends() -> None: + history: Final = _make_history(n_pairs=3) + sent_tokens: Final = litellm.token_counter(model=MODEL, messages=history) + turn: Final = SERVER_TOOL_TURN + messages: Final = [history[0], turn, *history[1:]] + + untouched, not_applied = _clear_above(messages, sent_tokens + 100) + cleared, applied = _clear_above(messages, sent_tokens - 100) + + assert not_applied is None + assert untouched == messages + assert applied is not None + assert applied["cleared_tool_uses"] == 2 + assert _cleared_ids(cleared) == {"toolu_00", "toolu_01"} + assert cleared[1] == turn + + +def _first_result_with(blocks: list[dict[str, object]]) -> dict[str, object]: + return {"role": "user", "content": [{"type": "tool_result", "tool_use_id": "toolu_00", "content": blocks}]} + + +def test_an_input_tokens_trigger_skips_blocks_no_adapter_sends_inside_a_tool_result() -> None: + history: Final = _make_history(n_pairs=3) + result_text: Final = {"type": "text", "text": "Result 0: " + "x" * 200} + search_result: Final = {"type": "web_search_result", "url": "https://example.com", "title": "rain " * 300} + sent_tokens: Final = litellm.token_counter( + model=MODEL, messages=[*history[:2], _first_result_with([result_text]), *history[3:]] + ) + messages: Final = [*history[:2], _first_result_with([result_text, search_result]), *history[3:]] + + untouched, not_applied = _clear_above(messages, sent_tokens + 100) + cleared, applied = _clear_above(messages, sent_tokens - 100) + + assert not_applied is None + assert untouched == messages + assert applied is not None + assert _cleared_ids(cleared) == {"toolu_00", "toolu_01"} + + +def _clear_at_least_one_token( + messages: list[dict[str, object]], +) -> tuple[list[dict[str, object]], AppliedEdit | None]: + return apply_clear_tool_uses_20250919( + model=MODEL, + messages=messages, + tools=None, + system=None, + edit_spec={ + "type": "clear_tool_uses_20250919", + "trigger": {"type": "tool_uses", "value": 0}, + "keep": {"type": "tool_uses", "value": 1}, + "clear_at_least": {"type": "input_tokens", "value": 1}, + }, + ) + + +def test_clear_at_least_frees_only_blocks_an_adapter_sends_inside_a_tool_result() -> None: + history: Final = _make_history(n_pairs=3) + result_text: Final = {"type": "text", "text": "Result 0: " + "x" * 200} + search_result: Final = {"type": "web_search_result", "url": "https://example.com", "title": "rain " * 300} + + _, sent_only = _clear_at_least_one_token([*history[:2], _first_result_with([result_text]), *history[3:]]) + cleared, applied = _clear_at_least_one_token( + [*history[:2], _first_result_with([result_text, search_result]), *history[3:]] + ) + + assert applied is not None + assert applied == sent_only + assert _cleared_ids(cleared) == {"toolu_00", "toolu_01"} + + def _blocks_of(messages: list[dict[str, object]], block_type: str) -> tuple[dict[str, object], ...]: contents: Final = (message["content"] for message in messages) blocks: Final = chain.from_iterable(content for content in contents if isinstance(content, list)) @@ -559,6 +659,9 @@ async def _malformed_spec_error(edits: list[dict[str, object]]) -> AnthropicCont ("exclude_tools", "get_weather"), ("exclude_tools", [1]), ("clear_tool_inputs", "get_weather"), + ("trigger", "soon"), + ("trigger", [1]), + ("keep", 5), ], ) async def test_malformed_fields_are_a_400_even_below_the_trigger(field: str, value: object) -> None: @@ -576,12 +679,15 @@ async def test_malformed_fields_are_a_400_even_below_the_trigger(field: str, val ("clear_at_least", 1200, "clear_at_least: Input does not match the expected shape."), ("clear_at_least", {"type": "input_tokens", "value": "1200"}, "value: Input should be a valid integer"), ("clear_tool_inputs", 5, "clear_tool_inputs.bool: Input should be a valid boolean, unable to interpret input"), + ("trigger", "soon", "trigger: Input should be an object"), + ("keep", 5, "keep: Input should be an object"), ], ) async def test_malformed_field_400_names_the_edit_and_field_as_the_api_does( field: str, value: object, message: str ) -> None: - # api.anthropic.com, measured 2026-10-19, answers these four with these messages for the edit at index 1. + # api.anthropic.com answers these with these messages for the edit at index 1 (trigger and keep measured + # 2026-10-28, the rest 2026-10-19). error = await _malformed_spec_error( [{"type": "clear_thinking_20251015"}, {"type": "clear_tool_uses_20250919", field: value}] ) @@ -625,6 +731,28 @@ async def test_drop_params_drops_only_the_malformed_field(field: str, value: obj assert _tool_inputs(result.messages) == {"toolu_00": {}, "toolu_01": {}, "toolu_02": {"location": "City2"}} +@pytest.mark.parametrize(("field", "value"), [("trigger", "soon"), ("keep", 5)]) +async def test_drop_params_treats_a_malformed_trigger_or_keep_as_never_sent(field: str, value: object) -> None: + async def run(edit: dict[str, object], drop_invalid_edits: bool) -> PolyfillResult | None: + return await _run_polyfill_if_enabled( + model=MODEL, + messages=_make_history(n_pairs=5), + tools=None, + system=None, + context_management_spec={"edits": [edit]}, + litellm_metadata={}, + additional_drop_params=None, + llm_router=None, + drop_invalid_edits=drop_invalid_edits, + ) + + malformed: Final = {**_VALID_EDIT, field: value} + with pytest.raises(AnthropicContextManagementError, match=rf"\.{field}: Input should be an object$"): + await run(malformed, drop_invalid_edits=False) + never_sent: Final = {key: spec for key, spec in _VALID_EDIT.items() if key != field} + assert await run(malformed, drop_invalid_edits=True) == await run(never_sent, drop_invalid_edits=True) + + async def test_malformed_edit_is_rejected_before_a_compact_summary_is_paid_for( monkeypatch: pytest.MonkeyPatch, ) -> None: diff --git a/tests/unit/llms/anthropic/messages/context_management/test_compact.py b/tests/unit/llms/anthropic/messages/context_management/test_compact.py index dff931a6074..32899a8e3c8 100644 --- a/tests/unit/llms/anthropic/messages/context_management/test_compact.py +++ b/tests/unit/llms/anthropic/messages/context_management/test_compact.py @@ -29,7 +29,7 @@ from litellm.llms.anthropic.messages.context_management.editors.compact import ( _augment_system_with_summary, _extract_summary_text, _select_last_user_question, - _slice_around_compaction_block, + slice_around_compaction_block, _strip_compaction_blocks, apply_client_compaction_block_history, apply_compact_20260112, @@ -153,7 +153,7 @@ def test_applied_edits_for_response_includes_compact_when_block_present(): def test_slice_around_compaction_block_found(): messages = _messages_with_compaction("my summary") - sliced, block = _slice_around_compaction_block(messages) + sliced, block = slice_around_compaction_block(messages) assert block is not None assert block["type"] == "compaction" assert block["content"] == "my summary" @@ -164,7 +164,7 @@ def test_slice_around_compaction_block_found(): def test_slice_around_compaction_block_not_found(): messages = _simple_messages() - sliced, block = _slice_around_compaction_block(messages) + sliced, block = slice_around_compaction_block(messages) assert block is None assert sliced is messages # same object, no copy diff --git a/tests/unit/llms/anthropic/messages/context_management/test_dispatcher.py b/tests/unit/llms/anthropic/messages/context_management/test_dispatcher.py index fabb05c2e61..e55af602013 100644 --- a/tests/unit/llms/anthropic/messages/context_management/test_dispatcher.py +++ b/tests/unit/llms/anthropic/messages/context_management/test_dispatcher.py @@ -5,6 +5,7 @@ Unit tests for the context_management polyfill dispatcher. from litellm.llms.anthropic.messages.context_management import ( apply_context_management, ) +from litellm.llms.anthropic.messages.convertible_blocks import drop_non_string_tool_result_texts MODEL = "xai/grok-4" @@ -160,3 +161,57 @@ async def test_sync_editor_counts_tokens_off_the_event_loop(): assert result.messages == messages assert_loop_stayed_free(took, lags) + + +def _history_with_null_text_beside_a_short_first_result(): + return [ + {"role": "user", "content": "Hi"}, + { + "role": "assistant", + "content": [{"type": "tool_use", "id": "t0", "name": "f", "input": {}}], + }, + { + "role": "user", + "content": [ + { + "type": "tool_result", + "tool_use_id": "t0", + "content": [{"type": "text", "text": "ok"}, {"type": "text", "text": None}], + } + ], + }, + *_history_with_two_tool_pairs()[1:], + ] + + +async def test_drop_invalid_edits_clears_as_if_malformed_tool_result_text_was_never_sent(): + spec = { + "edits": [ + { + "type": "clear_tool_uses_20250919", + "trigger": {"type": "tool_uses", "value": 2}, + "keep": {"type": "tool_uses", "value": 1}, + } + ] + } + history = _history_with_null_text_beside_a_short_first_result() + + from_raw = await apply_context_management( + model=MODEL, + messages=history, + tools=None, + system=None, + context_management_spec=spec, + drop_invalid_edits=True, + ) + from_sent = await apply_context_management( + model=MODEL, + messages=list(drop_non_string_tool_result_texts(history, True)), + tools=None, + system=None, + context_management_spec=spec, + drop_invalid_edits=True, + ) + + assert from_raw.applied_edits == from_sent.applied_edits + assert from_raw.messages == from_sent.messages diff --git a/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_handler.py b/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_handler.py index bba5413fbdf..382917fc756 100644 --- a/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_handler.py +++ b/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_handler.py @@ -2,7 +2,8 @@ import datetime import json import os import sys -from collections.abc import Mapping +from collections.abc import AsyncIterator, Mapping +from itertools import chain from typing import Final, cast from unittest.mock import AsyncMock, patch @@ -761,3 +762,325 @@ async def test_recomputed_cost_of_a_served_bridge_response_prices_the_reasoning_ assert logging_obj.cost_breakdown is not None assert logging_obj.cost_breakdown["output_cost"] == pytest.approx(expected_output) assert logging_obj.cost_breakdown["reasoning_cost"] == pytest.approx(expected_reasoning) + + +TEXT_RESPONSE: Final = { + "id": "resp_cleared", + "object": "response", + "created_at": 1, + "status": "completed", + "model": "gpt-5.6-luna", + "output": [ + { + "type": "message", + "id": "msg_1", + "role": "assistant", + "status": "completed", + "content": [{"type": "output_text", "text": "done", "annotations": []}], + } + ], + "parallel_tool_calls": True, + "tool_choice": "auto", + "tools": [], + "usage": {"input_tokens": 3, "output_tokens": 4, "total_tokens": 7}, +} +CLEAR_EDIT: Final = { + "type": "clear_tool_uses_20250919", + "trigger": {"type": "tool_uses", "value": 1}, + "keep": {"type": "tool_uses", "value": 1}, +} +COMPACT_EDIT: Final = {"type": "compact_20260112", "trigger": {"type": "input_tokens", "value": 150000}} + + +def _read(i: int) -> tuple[dict[str, object], dict[str, object]]: + return ( + { + "role": "assistant", + "content": [{"type": "tool_use", "id": f"toolu_{i}", "name": "Bash", "input": {"command": f"cat {i}"}}], + }, + { + "role": "user", + "content": [{"type": "tool_result", "tool_use_id": f"toolu_{i}", "content": f"log {i} " * 40}], + }, + ) + + +def _three_reads() -> list[dict[str, object]]: + return [{"role": "user", "content": "Read the logs."}, *chain.from_iterable(_read(i) for i in range(3))] + + +async def _cleared_bridge( + respx_mock: respx.MockRouter, + monkeypatch: pytest.MonkeyPatch, + *, + stream: bool, + edits: list[dict[str, object]], + drop_params: bool = False, + system: object = None, +) -> tuple[object, Mapping[str, object]]: + monkeypatch.setenv("OPENAI_API_KEY", "sk-clear-test") + monkeypatch.setenv("DISABLE_AIOHTTP_TRANSPORT", "True") + litellm.in_memory_llm_clients_cache.flush_cache() + upstream: Final = respx_mock.post("https://api.openai.com/v1/responses") + if stream: + upstream.respond(status_code=200, headers={"Content-Type": "text/event-stream"}, content=RESPONSES_SSE_BODY) + else: + upstream.respond(status_code=200, json=TEXT_RESPONSE) + response: Final = await LiteLLMMessagesToResponsesAPIHandler.async_anthropic_messages_handler( + max_tokens=1024, + messages=_three_reads(), + model="openai/gpt-5.6-luna", + tools=[dict(BASH_TOOL)], + stream=stream, + custom_llm_provider="openai", + context_management={"edits": edits}, + drop_params=drop_params, + system=system, + ) + body: Final = json.loads(upstream.calls.last.request.content) + if not stream: + return response, body + return [json.loads(chunk.decode().split("data: ", 1)[1]) async for chunk in response], body + + +@pytest.mark.asyncio +async def test_clear_tool_uses_clears_old_results_before_the_responses_api_call( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + from litellm.llms.anthropic.messages.context_management.constants import CLEARED_TOOL_RESULT_PLACEHOLDER + + served, body = await _cleared_bridge(respx_mock, monkeypatch, stream=False, edits=[dict(CLEAR_EDIT)]) + + outputs: Final = { + item["call_id"]: item["output"] for item in body["input"] if item.get("type") == "function_call_output" + } + assert [CLEARED_TOOL_RESULT_PLACEHOLDER in json.dumps(outputs[f"toolu_{i}"]) for i in range(3)] == [ + True, + True, + False, + ] + assert "context_management" not in body + assert isinstance(served, dict) + assert served["context_management"]["applied_edits"][0]["cleared_tool_uses"] == 2 + + +@pytest.mark.asyncio +async def test_streamed_clear_tool_uses_reports_its_edits_on_the_final_message_delta( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + events, _ = await _cleared_bridge(respx_mock, monkeypatch, stream=True, edits=[dict(CLEAR_EDIT)]) + + assert isinstance(events, list) + message_delta: Final = next(event for event in events if event["type"] == "message_delta") + assert message_delta["context_management"]["applied_edits"][0]["cleared_tool_uses"] == 2 + + +@pytest.mark.asyncio +async def test_compact_stays_native_while_clear_tool_uses_runs_in_the_gateway( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + served, body = await _cleared_bridge( + respx_mock, monkeypatch, stream=False, edits=[dict(CLEAR_EDIT), dict(COMPACT_EDIT)] + ) + + assert body["context_management"] == [{"type": "compaction", "compact_threshold": 150000}] + assert isinstance(served, dict) + assert [edit["type"] for edit in served["context_management"]["applied_edits"]] == ["clear_tool_uses_20250919"] + + +@pytest.mark.asyncio +async def test_a_malformed_clear_tool_uses_edit_is_a_400_before_the_responses_api_call( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + from litellm.llms.anthropic.messages.context_management import AnthropicContextManagementError + + upstream: Final = respx_mock.post("https://api.openai.com/v1/responses").respond( + status_code=200, json=TEXT_RESPONSE + ) + monkeypatch.setenv("OPENAI_API_KEY", "sk-clear-test") + with pytest.raises(AnthropicContextManagementError) as raised: + await LiteLLMMessagesToResponsesAPIHandler.async_anthropic_messages_handler( + max_tokens=1024, + messages=_three_reads(), + model="openai/gpt-5.6-luna", + custom_llm_provider="openai", + context_management={"edits": [{**CLEAR_EDIT, "exclude_tools": "Bash"}]}, + ) + assert raised.value.status_code == 400 + assert not upstream.called + + +@pytest.mark.asyncio +async def test_a_malformed_text_in_a_result_clearing_would_drop_is_still_a_400( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + upstream: Final = respx_mock.post("https://api.openai.com/v1/responses").respond( + status_code=200, json=TEXT_RESPONSE + ) + monkeypatch.setenv("OPENAI_API_KEY", "sk-clear-test") + reads: Final = _three_reads() + malformed_oldest_result: Final = { + "role": "user", + "content": [{"type": "tool_result", "tool_use_id": "toolu_0", "content": [{"type": "text", "text": None}]}], + } + with pytest.raises(litellm.BadRequestError) as rejected: + await LiteLLMMessagesToResponsesAPIHandler.async_anthropic_messages_handler( + max_tokens=1024, + messages=[*reads[:2], malformed_oldest_result, *reads[3:]], + model="openai/gpt-5.6-luna", + tools=[dict(BASH_TOOL)], + custom_llm_provider="openai", + context_management={"edits": [dict(CLEAR_EDIT)]}, + ) + assert rejected.value.status_code == 400 + assert "tool_result toolu_0 content.0.text: Input should be a valid string" in rejected.value.message + assert not upstream.called + + +def test_the_sync_path_clears_old_results_too(respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch) -> None: + from litellm.llms.anthropic.messages.context_management.constants import CLEARED_TOOL_RESULT_PLACEHOLDER + + monkeypatch.setenv("OPENAI_API_KEY", "sk-clear-test") + monkeypatch.setenv("DISABLE_AIOHTTP_TRANSPORT", "True") + litellm.in_memory_llm_clients_cache.flush_cache() + upstream: Final = respx_mock.post("https://api.openai.com/v1/responses").respond( + status_code=200, json=TEXT_RESPONSE + ) + + served: Final = LiteLLMMessagesToResponsesAPIHandler.anthropic_messages_handler( + max_tokens=1024, + messages=_three_reads(), + model="openai/gpt-5.6-luna", + tools=[dict(BASH_TOOL)], + custom_llm_provider="openai", + context_management={"edits": [dict(CLEAR_EDIT)]}, + ) + + body: Final = json.loads(upstream.calls.last.request.content) + outputs: Final = { + item["call_id"]: item["output"] for item in body["input"] if item.get("type") == "function_call_output" + } + assert [CLEARED_TOOL_RESULT_PLACEHOLDER in json.dumps(outputs[f"toolu_{i}"]) for i in range(3)] == [ + True, + True, + False, + ] + assert isinstance(served, dict) + assert served["context_management"]["applied_edits"][0]["cleared_tool_uses"] == 2 + + +@pytest.mark.asyncio +async def test_an_openai_style_context_management_list_is_still_not_sent_to_the_responses_api( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + upstream: Final = respx_mock.post("https://api.openai.com/v1/responses").respond( + status_code=200, json=TEXT_RESPONSE + ) + monkeypatch.setenv("OPENAI_API_KEY", "sk-clear-test") + monkeypatch.setenv("DISABLE_AIOHTTP_TRANSPORT", "True") + litellm.in_memory_llm_clients_cache.flush_cache() + + served: Final = await LiteLLMMessagesToResponsesAPIHandler.async_anthropic_messages_handler( + max_tokens=1024, + messages=_three_reads(), + model="openai/gpt-5.6-luna", + custom_llm_provider="openai", + context_management=[{"type": "compaction", "compact_threshold": 500}], + ) + + assert "context_management" not in json.loads(upstream.calls.last.request.content) + assert isinstance(served, dict) + assert "context_management" not in served + + +@pytest.mark.asyncio +async def test_under_drop_params_a_malformed_edit_still_clears_without_its_bad_field( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + from litellm.llms.anthropic.messages.context_management.constants import CLEARED_TOOL_RESULT_PLACEHOLDER + + served, body = await _cleared_bridge( + respx_mock, monkeypatch, stream=False, edits=[{**CLEAR_EDIT, "exclude_tools": "Bash"}], drop_params=True + ) + + outputs: Final = { + item["call_id"]: item["output"] for item in body["input"] if item.get("type") == "function_call_output" + } + assert [CLEARED_TOOL_RESULT_PLACEHOLDER in json.dumps(outputs[f"toolu_{i}"]) for i in range(3)] == [ + True, + True, + False, + ] + assert isinstance(served, dict) + assert served["context_management"]["applied_edits"][0]["cleared_tool_uses"] == 2 + + +def _cleared_outputs(body: Mapping[str, object]) -> list[bool]: + from litellm.llms.anthropic.messages.context_management.constants import CLEARED_TOOL_RESULT_PLACEHOLDER + + outputs: Final = { + item["call_id"]: item["output"] for item in body["input"] if item.get("type") == "function_call_output" + } + return [CLEARED_TOOL_RESULT_PLACEHOLDER in json.dumps(outputs[f"toolu_{i}"]) for i in range(3)] + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "system", + [7, {"type": "text", "text": "Be brief."}, ["Be brief."], ["Be brief.", {"type": "text", "text": "Be kind."}]], + ids=["number", "object", "strings", "mixed_list"], +) +@pytest.mark.parametrize("stream", [False, True], ids=["non_stream", "stream"]) +async def test_clear_tool_uses_still_clears_when_the_system_prompt_is_not_text_or_a_block_list( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, system: object, stream: bool +) -> None: + _, body = await _cleared_bridge(respx_mock, monkeypatch, stream=stream, edits=[dict(CLEAR_EDIT)], system=system) + + assert _cleared_outputs(body) == [True, True, False] + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("system", "expect_cleared"), + [ + (["Stray.", {"type": "text", "text": "Follow the policy. " * 1500}], [True, True, False]), + (["Follow the policy. " * 1500], [False, False, False]), + ], + ids=["text_block_counts", "stray_string_does_not"], +) +async def test_input_tokens_trigger_counts_the_text_blocks_of_a_system_list_with_stray_items( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, system: object, expect_cleared: list[bool] +) -> None: + trigger_between_messages_and_system: Final = litellm.token_counter(model="gpt-4o", messages=_three_reads()) + 1000 + edit: Final = {**CLEAR_EDIT, "trigger": {"type": "input_tokens", "value": trigger_between_messages_and_system}} + + _, body = await _cleared_bridge(respx_mock, monkeypatch, stream=False, edits=[edit], system=system) + + assert _cleared_outputs(body) == expect_cleared + + +@pytest.mark.asyncio +async def test_the_sync_path_reports_its_edits_on_the_streamed_message_delta( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv("OPENAI_API_KEY", "sk-clear-test") + monkeypatch.setenv("DISABLE_AIOHTTP_TRANSPORT", "True") + litellm.in_memory_llm_clients_cache.flush_cache() + respx_mock.post("https://api.openai.com/v1/responses").respond( + status_code=200, headers={"Content-Type": "text/event-stream"}, content=RESPONSES_SSE_BODY + ) + + sse: Final = LiteLLMMessagesToResponsesAPIHandler.anthropic_messages_handler( + max_tokens=1024, + messages=_three_reads(), + model="openai/gpt-5.6-luna", + tools=[dict(BASH_TOOL)], + stream=True, + custom_llm_provider="openai", + context_management={"edits": [dict(CLEAR_EDIT)]}, + ) + events: Final = [ + json.loads(chunk.decode().split("data: ", 1)[1]) async for chunk in cast(AsyncIterator[bytes], sse) + ] + message_delta: Final = next(event for event in events if event["type"] == "message_delta") + assert message_delta["context_management"]["applied_edits"][0]["cleared_tool_uses"] == 2 diff --git a/tests/unit/llms/openai/responses/count_tokens/test_token_counter.py b/tests/unit/llms/openai/responses/count_tokens/test_token_counter.py index 17c1b35e211..29c25c5fdbf 100644 --- a/tests/unit/llms/openai/responses/count_tokens/test_token_counter.py +++ b/tests/unit/llms/openai/responses/count_tokens/test_token_counter.py @@ -2,6 +2,7 @@ import copy import json import operator from collections.abc import Mapping +from itertools import chain from typing import Final import httpx @@ -11,9 +12,10 @@ from pydantic import JsonValue, ValidationError import litellm from litellm.litellm_core_utils.tool_fields import DropPolicy +from litellm.llms.anthropic.messages.context_management.constants import CLEARED_TOOL_RESULT_PLACEHOLDER from litellm.llms.openai.responses.count_tokens.token_counter import OpenAITokenCounter from litellm.responses.utils import ResponsesAPIRequestUtils -from litellm.types.utils import TokenCountMessagesFormat, TokenCountResponse +from litellm.types.utils import CountContextManagement, TokenCountMessagesFormat, TokenCountResponse COUNT_URL: Final = "https://api.openai.com/v1/responses/input_tokens" @@ -484,6 +486,122 @@ async def test_anthropic_format_tool_result_text_block_without_text_still_counts assert {"type": "function_call_output", "call_id": "toolu_log", "output": "build ok\n\ndone"} in sent["input"] +def _tool_pair(index: int) -> tuple[dict[str, object], dict[str, object]]: + return ( + {"role": "assistant", "content": [{"type": "tool_use", "id": f"toolu_{index}", "name": "log", "input": {}}]}, + { + "role": "user", + "content": [{"type": "tool_result", "tool_use_id": f"toolu_{index}", "content": f"log {index} " * 40}], + }, + ) + + +TOOL_PAIRS: Final[list[dict[str, object]]] = [ + {"role": "user", "content": "Compare the logs."}, + *chain.from_iterable(_tool_pair(index) for index in range(3)), +] + + +async def _count_with_edit( + monkeypatch: pytest.MonkeyPatch, edit: dict[str, JsonValue], drop_invalid_edits: bool, system: object = None +) -> tuple[TokenCountResponse | None, respx.Route]: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + with respx.mock: + route: Final = respx.post(COUNT_URL).mock(return_value=httpx.Response(200, json={"input_tokens": 9})) + result: Final = await OpenAITokenCounter().count_tokens( + model_to_use="gpt-4o", + messages=copy.deepcopy(TOOL_PAIRS), + contents=None, + deployment={"litellm_params": {"api_key": "sk-test"}}, + request_model="openai/gpt-4o", + system=system, + messages_format="anthropic", + context_management=CountContextManagement(spec={"edits": [edit]}, drop_invalid_edits=drop_invalid_edits), + ) + return result, route + + +def _sent_outputs(route: respx.Route) -> dict[str, str]: + sent: Final = json.loads(route.calls.last.request.content) + return {item["call_id"]: item["output"] for item in sent["input"] if item["type"] == "function_call_output"} + + +CLEAR_EDIT: Final[dict[str, JsonValue]] = { + "type": "clear_tool_uses_20250919", + "trigger": {"type": "tool_uses", "value": 1}, + "keep": {"type": "tool_uses", "value": 1}, +} + + +@pytest.mark.asyncio +async def test_anthropic_format_counts_the_history_the_clear_edit_leaves_for_inference( + monkeypatch: pytest.MonkeyPatch, +) -> None: + result, route = await _count_with_edit(monkeypatch, CLEAR_EDIT, drop_invalid_edits=False) + + assert result is not None + assert (result.error, result.total_tokens) == (False, 9) + assert _sent_outputs(route) == { + "toolu_0": CLEARED_TOOL_RESULT_PLACEHOLDER, + "toolu_1": CLEARED_TOOL_RESULT_PLACEHOLDER, + "toolu_2": "log 2 " * 40, + } + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "system", + [ + "Follow the policy. " * 1500, + [{"type": "text", "text": "Follow the policy. " * 1500}], + ["Stray.", {"type": "text", "text": "Follow the policy. " * 1500}], + ], + ids=["string", "blocks", "blocks_beside_a_stray_string"], +) +async def test_anthropic_format_input_tokens_trigger_counts_the_system_prompt_like_inference( + monkeypatch: pytest.MonkeyPatch, system: object +) -> None: + message_tokens: Final = litellm.token_counter(model="gpt-4o", messages=TOOL_PAIRS) + trigger_between_messages_and_system: Final = message_tokens + 1000 + edit: Final[dict[str, JsonValue]] = { + **CLEAR_EDIT, + "trigger": {"type": "input_tokens", "value": trigger_between_messages_and_system}, + } + + _, without_system = await _count_with_edit(monkeypatch, edit, drop_invalid_edits=False) + _, with_system = await _count_with_edit(monkeypatch, edit, drop_invalid_edits=False, system=system) + + assert _sent_outputs(without_system)["toolu_0"] == "log 0 " * 40 + assert _sent_outputs(with_system) == { + "toolu_0": CLEARED_TOOL_RESULT_PLACEHOLDER, + "toolu_1": CLEARED_TOOL_RESULT_PLACEHOLDER, + "toolu_2": "log 2 " * 40, + } + + +@pytest.mark.asyncio +async def test_anthropic_format_malformed_clear_edit_answers_400_without_calling_openai( + monkeypatch: pytest.MonkeyPatch, +) -> None: + result, route = await _count_with_edit(monkeypatch, {**CLEAR_EDIT, "exclude_tools": "log"}, drop_invalid_edits=False) + + assert route.called is False + assert result is not None + assert (result.error, result.status_code) == (True, 400) + assert "exclude_tools" in str(result.error_message) + + +@pytest.mark.asyncio +async def test_anthropic_format_drop_params_counts_the_clear_without_its_malformed_field( + monkeypatch: pytest.MonkeyPatch, +) -> None: + result, route = await _count_with_edit(monkeypatch, {**CLEAR_EDIT, "exclude_tools": "log"}, drop_invalid_edits=True) + + assert result is not None + assert result.error is False + assert _sent_outputs(route)["toolu_0"] == CLEARED_TOOL_RESULT_PLACEHOLDER + + @pytest.mark.asyncio async def test_count_uses_the_global_openai_key_chat_would_use(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) diff --git a/tests/unit/llms/vertex_ai/count_tokens/test_context.py b/tests/unit/llms/vertex_ai/count_tokens/test_context.py index b2a0b735b0f..38eb87debfd 100644 --- a/tests/unit/llms/vertex_ai/count_tokens/test_context.py +++ b/tests/unit/llms/vertex_ai/count_tokens/test_context.py @@ -649,7 +649,7 @@ async def test_anthropic_view_counts_the_unedited_body_without_an_error_log_when assert not any(record.levelno >= logging.ERROR for record in caplog.records) -async def test_anthropic_view_counts_the_turns_after_a_compaction_block_when_an_edit_before_compact_fails() -> None: +async def test_anthropic_view_clears_only_the_turns_after_a_compaction_block_when_clear_runs_before_compact() -> None: clear_before_compact: Final[JsonValue] = { "edits": [ { diff --git a/tests/unit/proxy/test_proxy_token_counter.py b/tests/unit/proxy/test_proxy_token_counter.py index 43c017468ff..c182cd1ec75 100644 --- a/tests/unit/proxy/test_proxy_token_counter.py +++ b/tests/unit/proxy/test_proxy_token_counter.py @@ -2064,22 +2064,6 @@ async def test_framing_keeps_the_400_for_blocks_the_provider_counter_cannot_conv }, id="web_search_blocks", ), - pytest.param( - { - "messages": [{"role": "user", "content": "What is new?"}], - "tools": [{"name": "search", "input_schema": {"type": "object", "properties": {"query": True}}}], - }, - id="boolean_subschema_tool", - ), - pytest.param( - { - "messages": [{"role": "user", "content": "What is new?"}], - "tools": [ - {"name": "search", "input_schema": {"type": "object", "properties": {"tags": {"type": "array"}}}} - ], - }, - id="array_schema_without_items", - ), pytest.param( {"messages": [], "contents": [{"role": "user", "parts": [{"text": "What is new?"}]}]}, id="google_contents", @@ -2138,6 +2122,44 @@ async def test_framed_estimate_that_cannot_read_the_body_answers_the_provider_er assert readable.count_source == "local_estimate" +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("shorthand", "explicit"), + [ + pytest.param({"type": "array"}, {"type": "array", "items": {}}, id="array_schema_without_items"), + pytest.param(True, {}, id="boolean_subschema"), + ], +) +async def test_framed_estimate_counts_a_json_schema_shorthand_like_its_explicit_form( + monkeypatch: pytest.MonkeyPatch, shorthand: object, explicit: dict[str, object] +): + counter: Final = _ScriptedCounter({"us.moonshotai.kimi-k3": _provider_reply("kimi", status_code=400)}) + monkeypatch.setattr( + litellm.proxy.proxy_server, + "llm_router", + Router(model_list=[{"model_name": "kimi", "litellm_params": {"model": "bedrock/us.moonshotai.kimi-k3"}}]), + ) + monkeypatch.setattr( + litellm.proxy.proxy_server, "_get_provider_token_counter", lambda deployment, model: (counter, model, "bedrock") + ) + + async def count(schema: object) -> TokenCountResponse: + return await token_counter( + request=TokenCountRequest( + model="kimi", + messages=[{"role": "user", "content": "What is new?"}], + tools=[{"name": "search", "input_schema": {"type": "object", "properties": {"tags": schema}}}], + ), + call_endpoint=True, + ) + + from_shorthand: Final = await count(shorthand) + from_explicit: Final = await count(explicit) + + assert from_shorthand.count_source == "local_estimate" + assert from_shorthand.total_tokens == from_explicit.total_tokens + + @pytest.mark.asyncio @pytest.mark.parametrize( "contents",