scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-11-11 08:38:24 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 280

Merge b089560973 into 086daf4653

Browse source
This commit is contained in:
Alon Sokolovski 2026-10-28 18:17:46 +00:00 • committed by GitHub
commit 6def7cd9ad
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
13 changed files with 725 additions and 38 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
litellm/llms/anthropic/messages/adapters/handler.py
View file

@ -160,6 +160,7 @@ async def prepare_context_managed_request(
"""Apply client compaction history, then optional context_management polyfill."""
from litellm.llms.anthropic.messages.context_management.editors.compact import (
apply_client_compaction_block_history,
slice_around_compaction_block,
)
# Skip the client-history pre-processing when a ``compact_20260112``
@ -175,7 +176,7 @@ async def prepare_context_managed_request(
if polyfill_will_run:
history_result: PolyfillResult | None = None
working_messages: _AnthropicMessages = messages
working_messages: _AnthropicMessages = list(slice_around_compaction_block(messages)[0])
working_system: _AnthropicSystem = system
else:
history_result = apply_client_compaction_block_history(

5
litellm/llms/anthropic/messages/context_management/dispatcher.py
View file

@ -8,6 +8,7 @@ from litellm._logging import verbose_logger
from litellm.litellm_core_utils.asyncify import asyncify
from litellm.types.llms.anthropic import AppliedEdit
from ..convertible_blocks import drop_non_string_tool_result_texts
from .constants import CLEAR_TOOL_USES_EDIT_TYPE, COMPACT_EDIT_TYPE
from .editors import (
CheckedEdit,
@ -103,7 +104,7 @@ async def apply_context_management(
Every ``clear_tool_uses_20250919`` edit is validated before any editor runs, so a malformed one raises its 400
before a ``compact_20260112`` summary call is paid for, or runs without its malformed fields under
``drop_invalid_edits``.
``drop_invalid_edits``, which also drops non-string ``tool_result`` text first, as the provider never receives it.
The dispatcher is async so async editors (``compact_20260112``) can
``await`` the configured summarization model. Sync editors run in a
@ -119,7 +120,7 @@ async def apply_context_management(
raise first_error
well_formed_edits: Final = tuple(check.edit for check in checked)
current_messages = messages
current_messages = list(drop_non_string_tool_result_texts(messages, drop_invalid_edits))
current_system = system
aggregated_applied: Final[list[AppliedEdit]] = []
aggregated_compaction_block = None

62
litellm/llms/anthropic/messages/context_management/editors/clear_tool_uses.py
View file

@ -38,6 +38,21 @@ if TYPE_CHECKING:
T = TypeVar("T")
# api.anthropic.com, measured 2026-10-19: text results of 1 to 46 bytes are never cleared or counted toward keep
_TINY_RESULT_BYTES: Final = 46
_TOKEN_COUNTER_BLOCK_TYPES: Final = frozenset(
{
"text",
"image_url",
"image",
"document",
"file",
"tool_use",
"tool_result",
"thinking",
"redacted_thinking",
"tool_reference",
"search_result",
}
)
ToolNames = Literal["all"] | frozenset[str]
@ -198,6 +213,35 @@ def _select_in_batches(candidates: Sequence[str], tokens: Mapping[str, int], cle
return tuple(candidates[:batch_end])
def _without_uncountable_blocks(block: Mapping[str, object]) -> Mapping[str, object]:
nested: Final = _parse(_BLOCKS, block.get("content"))
if nested is None:
return block
return {**block, "content": [inner for inner in nested if inner.get("type") in _TOKEN_COUNTER_BLOCK_TYPES]}
def _countable_message(message: Mapping[str, object]) -> Mapping[str, object]:
blocks: Final = _parse(_BLOCKS, message.get("content"))
if blocks is None:
return message
return {
**message,
"content": [
_without_uncountable_blocks(block) for block in blocks if block.get("type") in _TOKEN_COUNTER_BLOCK_TYPES
],
}
def _count_tokens(
model: str, messages: Sequence[Mapping[str, object]], tools: Sequence[Mapping[str, object]] | None
) -> int:
return litellm.token_counter(
model=model,
messages=[_countable_message(message) for message in messages],
tools=cast("list[ChatCompletionToolParam] | None", tools),
)
def _trigger_met(
trigger: dict[str, object],
model: str,
@ -216,11 +260,7 @@ def _trigger_met(
if not isinstance(threshold, int):
threshold = DEFAULT_INPUT_TOKENS_TRIGGER
current_tokens: Final = litellm.token_counter(
model=model,
messages=messages,
tools=cast("list[ChatCompletionToolParam] | None", tools),
)
current_tokens: Final = _count_tokens(model, messages, tools)
verbose_logger.debug("context_management polyfill: current_tokens: %s", current_tokens)
verbose_logger.debug("context_management polyfill: threshold: %s", threshold)
return current_tokens > threshold, current_tokens
@ -366,16 +406,8 @@ def apply_clear_tool_uses_20250919(
[_message_without_inputs(message, ids_to_clear, input_tools) for message in edited] if input_tools else edited
)
tokens_before: Final = (
counted_tokens
if counted_tokens is not None
else litellm.token_counter(
model=model, messages=messages, tools=cast("list[ChatCompletionToolParam] | None", tools)
)
)
tokens_after: Final = litellm.token_counter(
model=model, messages=final_messages, tools=cast("list[ChatCompletionToolParam] | None", tools)
)
tokens_before: Final = counted_tokens if counted_tokens is not None else _count_tokens(model, messages, tools)
tokens_after: Final = _count_tokens(model, final_messages, tools)
cleared_input_tokens: Final = max(tokens_before - tokens_after, 0)
return final_messages, {

6
litellm/llms/anthropic/messages/context_management/editors/compact.py
View file

@ -137,7 +137,7 @@ def _find_latest_compaction_index(
return None, None
def _slice_around_compaction_block(
def slice_around_compaction_block(
messages: Sequence[_MsgT],
) -> tuple[Sequence[_MsgT | dict[str, object]], dict[str, object] | None]:
"""Apply Anthropic's "drop everything before the compaction block" rule.
@ -598,7 +598,7 @@ def apply_client_compaction_block_history(
tail is forwarded unchanged (with compaction blocks stripped) so recent
turns the summary does not cover are preserved.
"""
effective_messages, prior_compaction_block = _slice_around_compaction_block(messages)
effective_messages, prior_compaction_block = slice_around_compaction_block(messages)
if prior_compaction_block is None:
return None
@ -669,7 +669,7 @@ async def apply_compact_20260112(
# opt-in gate below so that even when summarization is disabled we still
# strip Anthropic-only ``compaction`` blocks from messages going to
# non-Anthropic backends (which would reject them).
effective_messages, prior_compaction_block = _slice_around_compaction_block(messages)
effective_messages, prior_compaction_block = slice_around_compaction_block(messages)
prior_summary_text: Final = prior_compaction_block.get("content") if prior_compaction_block else None
augmented_system: str | list[dict[str, object]] | None = system
if isinstance(prior_summary_text, str) and prior_summary_text:

138
litellm/llms/anthropic/messages/responses_adapters/handler.py
View file

@ -10,10 +10,14 @@ from typing import Final, TypeAlias
from pydantic import TypeAdapter, ValidationError
import litellm
from litellm._logging import verbose_logger
from litellm.integrations.anthropic_cache_control_hook import (
AnthropicCacheControlHook,
supports_openai_prompt_cache_breakpoint,
)
from litellm.litellm_core_utils.asyncify import (
run_async_function, # pyright: ignore[reportUnknownVariableType] # untyped helper
)
from litellm.litellm_core_utils.core_helpers import drops_unsupported_params
from litellm.litellm_core_utils.prompt_templates.common_utils import cache_control_as_prompt_cache_breakpoint
from litellm.responses.utils import auto_reasoning_summary
@ -24,14 +28,19 @@ from litellm.types.llms.anthropic import (
AnthropicMessagesRequest,
AnthropicOutputConfig,
AnthropicOutputSchema,
ContextManagementResponse,
)
from litellm.types.llms.anthropic_messages.anthropic_response import (
AnthropicMessagesResponse,
)
from litellm.types.llms.openai import ResponsesAPIResponse
from litellm.types.utils import CountContextManagement
from litellm.utils import ProviderConfigManager
from ..adapters.handler import request_safeguards_evaluator
from ..adapters.handler import context_management_explicitly_dropped, request_safeguards_evaluator
from ..context_management import AnthropicContextManagementError, PolyfillResult, apply_context_management
from ..context_management.constants import COMPACT_EDIT_TYPE
from ..context_management.dispatcher import normalize_spec
from ..convertible_blocks import reject_malformed_tool_result_text
from ..safeguards import with_safeguard_results
from ..utils import litellm_logging_obj_from_kwargs, local_model_name
@ -41,10 +50,98 @@ from .transformation import LiteLLMAnthropicToResponsesAPIAdapter
AnthropicRequestMessages: TypeAlias = list[AllAnthropicMessageValues] | list[dict[str, object]]
BlockRewrite: TypeAlias = Callable[[Mapping[str, object]], Mapping[str, object]]
_BLOCK_LIST: Final = TypeAdapter(list[dict[str, object]])
_SPEC: Final = TypeAdapter(dict[str, object])
_SplitEdits: TypeAlias = tuple[dict[str, object] | None, Sequence[Mapping[str, object]]]
AnthropicContextManagementSpec: TypeAlias = dict[str, object] | list[dict[str, object]] | None
_ADAPTER: Final = LiteLLMAnthropicToResponsesAPIAdapter()
def _split_context_management(
spec: AnthropicContextManagementSpec,
kwargs: Mapping[str, object],
) -> _SplitEdits:
"""Return (the edits the Responses API runs natively, the edits the gateway polyfill runs).
The Responses API compacts natively (``compact_20260112`` becomes its ``compaction``); any other
edit, such as ``clear_tool_uses_20250919``, has no Responses API form and runs in the polyfill, as
it does on the chat completions path. An OpenAI-style list spec is not Anthropic's shape and stays
unsent, as it was before the polyfill ran here.
"""
if not isinstance(spec, dict):
return None, ()
if not spec or context_management_explicitly_dropped(kwargs.get("additional_drop_params")):
return spec, ()
edits: Final = normalize_spec(spec) or []
native: Final = [edit for edit in edits if edit.get("type") == COMPACT_EDIT_TYPE]
return ({"edits": native} if native else None), tuple(
edit for edit in edits if edit.get("type") != COMPACT_EDIT_TYPE
)
async def _polyfilled(
*,
model: str,
messages: AnthropicRequestMessages,
tools: Sequence[AllAnthropicToolsValues | Mapping[str, object]] | None,
system: str | None,
edits: Sequence[Mapping[str, object]],
kwargs: Mapping[str, object],
) -> PolyfillResult | None:
"""Run the edits the Responses API can't run natively; raise like the chat completions path does."""
if not edits:
return None
reject_malformed_tool_result_text(messages, model, kwargs)
drop_invalid_edits: Final = drops_unsupported_params(kwargs.get("drop_params"))
try:
return await apply_context_management(
model=model,
messages=_BLOCK_LIST.validate_python(messages),
tools=_BLOCK_LIST.validate_python(tools) if tools else None,
system=system,
context_management_spec={"edits": [dict(edit) for edit in edits]},
drop_invalid_edits=drop_invalid_edits,
)
except AnthropicContextManagementError:
raise
except Exception as e:
verbose_logger.exception("context_management polyfill failed on the Responses API path: %s", e)
raise AnthropicContextManagementError(
status_code=500, message=f"context_management polyfill failed: {e}"
) from e
async def count_view_messages(
*,
model: str,
messages: Sequence[Mapping[str, object]],
tools: Sequence[Mapping[str, object]] | None,
context_management: CountContextManagement | None,
) -> Sequence[Mapping[str, object]]:
"""The messages /v1/messages sends the Responses API once the polyfill has run the edits it can't run natively."""
if context_management is None or not isinstance(context_management.spec, dict):
return messages
_, edits = _split_context_management(_SPEC.validate_python(context_management.spec), {})
result: Final = await _polyfilled(
model=model,
messages=_BLOCK_LIST.validate_python(messages),
tools=tools,
system=None,
edits=edits,
kwargs={"drop_params": context_management.drop_invalid_edits},
)
return messages if result is None else result.messages
def _with_applied_edits(
response: AnthropicMessagesResponse, polyfill_result: PolyfillResult | None
) -> AnthropicMessagesResponse:
applied_edits: Final = polyfill_result.applied_edits_for_response() if polyfill_result else None
if not applied_edits:
return response
return {**response, "context_management": ContextManagementResponse(applied_edits=list(applied_edits))}
def _record_bridged_response(responses_kwargs: Mapping[str, object], result: ResponsesAPIResponse) -> None:
logging_obj: Final = litellm_logging_obj_from_kwargs(responses_kwargs)
if logging_obj is not None:
@ -263,7 +360,7 @@ class LiteLLMMessagesToResponsesAPIHandler:
max_tokens: int,
messages: AnthropicRequestMessages,
model: str,
context_management: dict[str, object] | None = None,
context_management: AnthropicContextManagementSpec = None,
metadata: dict[str, object] | None = None,
output_config: AnthropicOutputConfig | None = None,
stop_sequences: list[str] | None = None,
@ -278,11 +375,15 @@ class LiteLLMMessagesToResponsesAPIHandler:
output_format: AnthropicOutputSchema | None = None,
**kwargs: object,
) -> AnthropicMessagesResponse | AsyncIterator[bytes]:
native_context_management, polyfilled_edits = _split_context_management(context_management, kwargs)
polyfill_result: Final = await _polyfilled(
model=model, messages=messages, tools=tools, system=system, edits=polyfilled_edits, kwargs=kwargs
)
responses_kwargs: Final = _build_responses_kwargs(
max_tokens=max_tokens,
messages=messages,
messages=polyfill_result.messages if polyfill_result is not None else messages,
model=model,
context_management=context_management,
context_management=native_context_management,
metadata=metadata,
output_config=output_config,
stop_sequences=stop_sequences,
@ -307,13 +408,16 @@ class LiteLLMMessagesToResponsesAPIHandler:
model=local_model_name(model, kwargs.get("custom_llm_provider")),
litellm_logging_obj=litellm_logging_obj_from_kwargs(responses_kwargs),
safeguards_evaluator=safeguards_evaluator,
applied_edits=polyfill_result.applied_edits_for_response() if polyfill_result else None,
)
return wrapper.async_anthropic_sse_wrapper()
if not isinstance(result, ResponsesAPIResponse):
raise ValueError(f"Expected ResponsesAPIResponse, got {type(result)}")
served: Final = await with_safeguard_results(_ADAPTER.translate_response(result), safeguards_evaluator)
served: Final = await with_safeguard_results(
_with_applied_edits(_ADAPTER.translate_response(result), polyfill_result), safeguards_evaluator
)
_record_bridged_response(responses_kwargs, result)
return served
@ -322,7 +426,7 @@ class LiteLLMMessagesToResponsesAPIHandler:
max_tokens: int,
messages: AnthropicRequestMessages,
model: str,
context_management: dict[str, object] | None = None,
context_management: AnthropicContextManagementSpec = None,
metadata: dict[str, object] | None = None,
output_config: AnthropicOutputConfig | None = None,
stop_sequences: list[str] | None = None,
@ -364,11 +468,26 @@ class LiteLLMMessagesToResponsesAPIHandler:
)
# Sync path
native_context_management, polyfilled_edits = _split_context_management(context_management, kwargs)
ran: Final[object] = ( # pyright: ignore[reportUnknownVariableType] # run_async_function is untyped
run_async_function(
_polyfilled,
model=model,
messages=messages,
tools=tools,
system=system,
edits=polyfilled_edits,
kwargs=kwargs,
)
if polyfilled_edits
else None
)
polyfill_result: Final = ran if isinstance(ran, PolyfillResult) else None
responses_kwargs: Final = _build_responses_kwargs(
max_tokens=max_tokens,
messages=messages,
messages=polyfill_result.messages if polyfill_result is not None else messages,
model=model,
context_management=context_management,
context_management=native_context_management,
metadata=metadata,
output_config=output_config,
stop_sequences=stop_sequences,
@ -391,10 +510,11 @@ class LiteLLMMessagesToResponsesAPIHandler:
responses_stream=result,
model=local_model_name(model, kwargs.get("custom_llm_provider")),
litellm_logging_obj=litellm_logging_obj_from_kwargs(responses_kwargs),
applied_edits=polyfill_result.applied_edits_for_response() if polyfill_result else None,
)
return wrapper.async_anthropic_sse_wrapper()
if not isinstance(result, ResponsesAPIResponse):
raise ValueError(f"Expected ResponsesAPIResponse, got {type(result)}")
return _ADAPTER.translate_response(result)
return _with_applied_edits(_ADAPTER.translate_response(result), polyfill_result)

10
litellm/llms/anthropic/messages/responses_adapters/streaming_iterator.py
View file

@ -2,7 +2,7 @@
## Translates OpenAI call to Anthropic `/v1/messages` format
import asyncio
from collections import deque
from collections.abc import AsyncGenerator, Iterator, Mapping
from collections.abc import AsyncGenerator, Iterator, Mapping, Sequence
from typing import TYPE_CHECKING, Any, Final
from pydantic import ConfigDict, field_validator
@ -25,6 +25,7 @@ from litellm.responses.streaming_iterator import (
partial_responses_incomplete_event_on_disconnect,
stream_error_status_and_message,
)
from litellm.types.llms.anthropic import AppliedEdit, ContextManagementResponse
from litellm.types.llms.anthropic_messages.anthropic_response import AnthropicUsage
from litellm.types.llms.base import LiteLLMBaseModel
@ -127,8 +128,10 @@ class AnthropicResponsesStreamWrapper:
model: str,
litellm_logging_obj: "LiteLLMLoggingObject | None" = None,
safeguards_evaluator: "SafeguardsEvaluator | None" = None,
applied_edits: Sequence[AppliedEdit] | None = None,
) -> None:
self.responses_stream = responses_stream
self._applied_edits: Final = tuple(applied_edits or ())
self.model = model
self.safeguards_evaluator: Final = safeguards_evaluator
self._message_id: str = f"msg_{uuid.uuid4()}"
@ -433,6 +436,11 @@ class AnthropicResponsesStreamWrapper:
"type": "message_delta",
"delta": message_delta_payload,
"usage": dict(anthropic_usage),
**(
{"context_management": ContextManagementResponse(applied_edits=list(self._applied_edits))}
if self._applied_edits
else {}
),
}
)
self._chunk_queue.append({"type": "message_stop"})

38
litellm/llms/openai/responses/count_tokens/token_counter.py
View file

@ -15,7 +15,9 @@ from litellm._logging import verbose_logger
from litellm.litellm_core_utils.asyncify import asyncify
from litellm.litellm_core_utils.core_helpers import as_str_mapping
from litellm.litellm_core_utils.tool_fields import DropPolicy
from litellm.llms.anthropic.messages.context_management import AnthropicContextManagementError
from litellm.llms.anthropic.messages.convertible_blocks import deployment_drops_params, unconvertible_blocks_error
from litellm.llms.anthropic.messages.responses_adapters.handler import count_view_messages
from litellm.llms.anthropic.messages.responses_adapters.transformation import (
LiteLLMAnthropicToResponsesAPIAdapter,
)
@ -95,6 +97,27 @@ def _count_error(model_to_use: str, request_model: str, message: str, status_cod
)
async def _sent_messages(
model: str,
messages: Sequence[Mapping[str, object]],
tools: Sequence[Mapping[str, object]] | None,
messages_format: TokenCountMessagesFormat,
context_management: CountContextManagement | None,
) -> Sequence[Mapping[str, object]] | AnthropicContextManagementError:
match messages_format:
case "openai":
return messages
case "anthropic":
try:
return await count_view_messages(
model=model, messages=messages, tools=tools, context_management=context_management
)
except AnthropicContextManagementError as error:
return error
case _:
assert_never(messages_format)
class OpenAITokenCounter(BaseTokenCounter):
"""Token counter implementation for OpenAI provider using the Responses API."""
@ -175,13 +198,26 @@ class OpenAITokenCounter(BaseTokenCounter):
rejected_before_provider_call=True,
)
sent_messages: Final = await _sent_messages(model_to_use, messages, tools, messages_format, context_management)
if isinstance(sent_messages, AnthropicContextManagementError):
return TokenCountResponse(
total_tokens=0,
request_model=request_model,
model_used=model_to_use,
tokenizer_type="openai_api",
error=True,
error_message=sent_messages.message,
status_code=sent_messages.status_code,
rejected_before_provider_call=True,
)
resolved: Final = _count_credential(deployment, request_model or model_to_use)
if isinstance(resolved, OpenAIError):
return _count_error(model_to_use, request_model, resolved.message, resolved.status_code)
input_items, instructions, counted_tools, text = self._responses_request_parts(
model_to_use=model_to_use,
messages=messages,
messages=sent_messages,
tools=tools,
system=system,
messages_format=messages_format,

71
tests/unit/llms/anthropic/messages/context_management/test_clear_tool_uses.py
View file

@ -217,6 +217,77 @@ def test_cleared_input_tokens_is_nonnegative():
assert applied["cleared_input_tokens"] >= 0
SERVER_TOOL_TURN: Final = {
"role": "assistant",
"content": [
{"type": "server_tool_use", "id": "srvtoolu_01", "name": "web_search", "input": {"query": "Mumbai rain"}},
{
"type": "web_search_tool_result",
"tool_use_id": "srvtoolu_01",
"content": [{"type": "web_search_result", "url": "https://example.com", "title": "rain " * 300}],
},
],
}
COMPACTION_TURN: Final = {"role": "assistant", "content": [{"type": "compaction", "content": "summary " * 300}]}
def _clear_above(
messages: list[dict[str, object]], threshold: int
) -> tuple[list[dict[str, object]], AppliedEdit | None]:
return apply_clear_tool_uses_20250919(
model=MODEL,
messages=messages,
tools=None,
system=None,
edit_spec={
"type": "clear_tool_uses_20250919",
"trigger": {"type": "input_tokens", "value": threshold},
"keep": {"type": "tool_uses", "value": 1},
},
)
@pytest.mark.parametrize(
"turn", [pytest.param(SERVER_TOOL_TURN, id="server_tool"), pytest.param(COMPACTION_TURN, id="compaction")]
)
def test_an_input_tokens_trigger_skips_blocks_no_adapter_sends(turn: dict[str, object]) -> None:
history: Final = _make_history(n_pairs=3)
sent_tokens: Final = litellm.token_counter(model=MODEL, messages=history)
messages: Final = [history[0], turn, *history[1:]]
untouched, not_applied = _clear_above(messages, sent_tokens + 100)
cleared, applied = _clear_above(messages, sent_tokens - 100)
assert not_applied is None
assert untouched == messages
assert applied is not None
assert applied["cleared_tool_uses"] == 2
assert _cleared_ids(cleared) == {"toolu_00", "toolu_01"}
assert cleared[1] == turn
def _first_result_with(blocks: list[dict[str, object]]) -> dict[str, object]:
return {"role": "user", "content": [{"type": "tool_result", "tool_use_id": "toolu_00", "content": blocks}]}
def test_an_input_tokens_trigger_skips_blocks_no_adapter_sends_inside_a_tool_result() -> None:
history: Final = _make_history(n_pairs=3)
result_text: Final = {"type": "text", "text": "Result 0: " + "x" * 200}
search_result: Final = {"type": "web_search_result", "url": "https://example.com", "title": "rain " * 300}
sent_tokens: Final = litellm.token_counter(
model=MODEL, messages=[*history[:2], _first_result_with([result_text]), *history[3:]]
)
messages: Final = [*history[:2], _first_result_with([result_text, search_result]), *history[3:]]
untouched, not_applied = _clear_above(messages, sent_tokens + 100)
cleared, applied = _clear_above(messages, sent_tokens - 100)
assert not_applied is None
assert untouched == messages
assert applied is not None
assert _cleared_ids(cleared) == {"toolu_00", "toolu_01"}
def _blocks_of(messages: list[dict[str, object]], block_type: str) -> tuple[dict[str, object], ...]:
contents: Final = (message["content"] for message in messages)
blocks: Final = chain.from_iterable(content for content in contents if isinstance(content, list))

6
tests/unit/llms/anthropic/messages/context_management/test_compact.py
View file

@ -29,7 +29,7 @@ from litellm.llms.anthropic.messages.context_management.editors.compact import (
_augment_system_with_summary,
_extract_summary_text,
_select_last_user_question,
_slice_around_compaction_block,
slice_around_compaction_block,
_strip_compaction_blocks,
apply_client_compaction_block_history,
apply_compact_20260112,
@ -153,7 +153,7 @@ def test_applied_edits_for_response_includes_compact_when_block_present():
def test_slice_around_compaction_block_found():
messages = _messages_with_compaction("my summary")
sliced, block = _slice_around_compaction_block(messages)
sliced, block = slice_around_compaction_block(messages)
assert block is not None
assert block["type"] == "compaction"
assert block["content"] == "my summary"
@ -164,7 +164,7 @@ def test_slice_around_compaction_block_found():
def test_slice_around_compaction_block_not_found():
messages = _simple_messages()
sliced, block = _slice_around_compaction_block(messages)
sliced, block = slice_around_compaction_block(messages)
assert block is None
assert sliced is messages # same object, no copy

55
tests/unit/llms/anthropic/messages/context_management/test_dispatcher.py
View file

@ -5,6 +5,7 @@ Unit tests for the context_management polyfill dispatcher.
from litellm.llms.anthropic.messages.context_management import (
apply_context_management,
)
from litellm.llms.anthropic.messages.convertible_blocks import drop_non_string_tool_result_texts
MODEL = "xai/grok-4"
@ -159,3 +160,57 @@ async def test_sync_editor_counts_tokens_off_the_event_loop():
assert result.messages == messages
assert_loop_stayed_free(took, lags)
def _history_with_null_text_beside_a_short_first_result():
return [
{"role": "user", "content": "Hi"},
{
"role": "assistant",
"content": [{"type": "tool_use", "id": "t0", "name": "f", "input": {}}],
},
{
"role": "user",
"content": [
{
"type": "tool_result",
"tool_use_id": "t0",
"content": [{"type": "text", "text": "ok"}, {"type": "text", "text": None}],
}
],
},
*_history_with_two_tool_pairs()[1:],
]
async def test_drop_invalid_edits_clears_as_if_malformed_tool_result_text_was_never_sent():
spec = {
"edits": [
{
"type": "clear_tool_uses_20250919",
"trigger": {"type": "tool_uses", "value": 2},
"keep": {"type": "tool_uses", "value": 1},
}
]
}
history = _history_with_null_text_beside_a_short_first_result()
from_raw = await apply_context_management(
model=MODEL,
messages=history,
tools=None,
system=None,
context_management_spec=spec,
drop_invalid_edits=True,
)
from_sent = await apply_context_management(
model=MODEL,
messages=list(drop_non_string_tool_result_texts(history, True)),
tools=None,
system=None,
context_management_spec=spec,
drop_invalid_edits=True,
)
assert from_raw.applied_edits == from_sent.applied_edits
assert from_raw.messages == from_sent.messages

279
tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_handler.py
View file

@ -2,7 +2,8 @@ import datetime
import json
import os
import sys
from collections.abc import Mapping
from collections.abc import AsyncIterator, Mapping
from itertools import chain
from typing import Final, cast
from unittest.mock import AsyncMock, patch
@ -739,3 +740,279 @@ async def test_recomputed_cost_of_a_served_bridge_response_prices_the_reasoning_
assert logging_obj.cost_breakdown is not None
assert logging_obj.cost_breakdown["output_cost"] == pytest.approx(expected_output)
assert logging_obj.cost_breakdown["reasoning_cost"] == pytest.approx(expected_reasoning)
TEXT_RESPONSE: Final = {
"id": "resp_cleared",
"object": "response",
"created_at": 1,
"status": "completed",
"model": "gpt-5.6-luna",
"output": [
{
"type": "message",
"id": "msg_1",
"role": "assistant",
"status": "completed",
"content": [{"type": "output_text", "text": "done", "annotations": []}],
}
],
"parallel_tool_calls": True,
"tool_choice": "auto",
"tools": [],
"usage": {"input_tokens": 3, "output_tokens": 4, "total_tokens": 7},
}
CLEAR_EDIT: Final = {
"type": "clear_tool_uses_20250919",
"trigger": {"type": "tool_uses", "value": 1},
"keep": {"type": "tool_uses", "value": 1},
}
COMPACT_EDIT: Final = {"type": "compact_20260112", "trigger": {"type": "input_tokens", "value": 150000}}
def _read(i: int) -> tuple[dict[str, object], dict[str, object]]:
return (
{
"role": "assistant",
"content": [{"type": "tool_use", "id": f"toolu_{i}", "name": "Bash", "input": {"command": f"cat {i}"}}],
},
{
"role": "user",
"content": [{"type": "tool_result", "tool_use_id": f"toolu_{i}", "content": f"log {i} " * 40}],
},
)
def _three_reads() -> list[dict[str, object]]:
return [{"role": "user", "content": "Read the logs."}, *chain.from_iterable(_read(i) for i in range(3))]
async def _cleared_bridge(
respx_mock: respx.MockRouter,
monkeypatch: pytest.MonkeyPatch,
*,
stream: bool,
edits: list[dict[str, object]],
drop_params: bool = False,
) -> tuple[object, Mapping[str, object]]:
monkeypatch.setenv("OPENAI_API_KEY", "sk-clear-test")
monkeypatch.setenv("DISABLE_AIOHTTP_TRANSPORT", "True")
litellm.in_memory_llm_clients_cache.flush_cache()
upstream: Final = respx_mock.post("https://api.openai.com/v1/responses")
if stream:
upstream.respond(status_code=200, headers={"Content-Type": "text/event-stream"}, content=RESPONSES_SSE_BODY)
else:
upstream.respond(status_code=200, json=TEXT_RESPONSE)
response: Final = await LiteLLMMessagesToResponsesAPIHandler.async_anthropic_messages_handler(
max_tokens=1024,
messages=_three_reads(),
model="openai/gpt-5.6-luna",
tools=[dict(BASH_TOOL)],
stream=stream,
custom_llm_provider="openai",
context_management={"edits": edits},
drop_params=drop_params,
)
body: Final = json.loads(upstream.calls.last.request.content)
if not stream:
return response, body
return [json.loads(chunk.decode().split("data: ", 1)[1]) async for chunk in response], body
@pytest.mark.asyncio
async def test_clear_tool_uses_clears_old_results_before_the_responses_api_call(
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
) -> None:
from litellm.llms.anthropic.messages.context_management.constants import CLEARED_TOOL_RESULT_PLACEHOLDER
served, body = await _cleared_bridge(respx_mock, monkeypatch, stream=False, edits=[dict(CLEAR_EDIT)])
outputs: Final = {
item["call_id"]: item["output"] for item in body["input"] if item.get("type") == "function_call_output"
}
assert [CLEARED_TOOL_RESULT_PLACEHOLDER in json.dumps(outputs[f"toolu_{i}"]) for i in range(3)] == [
True,
True,
False,
]
assert "context_management" not in body
assert isinstance(served, dict)
assert served["context_management"]["applied_edits"][0]["cleared_tool_uses"] == 2
@pytest.mark.asyncio
async def test_streamed_clear_tool_uses_reports_its_edits_on_the_final_message_delta(
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
) -> None:
events, _ = await _cleared_bridge(respx_mock, monkeypatch, stream=True, edits=[dict(CLEAR_EDIT)])
assert isinstance(events, list)
message_delta: Final = next(event for event in events if event["type"] == "message_delta")
assert message_delta["context_management"]["applied_edits"][0]["cleared_tool_uses"] == 2
@pytest.mark.asyncio
async def test_compact_stays_native_while_clear_tool_uses_runs_in_the_gateway(
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
) -> None:
served, body = await _cleared_bridge(
respx_mock, monkeypatch, stream=False, edits=[dict(CLEAR_EDIT), dict(COMPACT_EDIT)]
)
assert body["context_management"] == [{"type": "compaction", "compact_threshold": 150000}]
assert isinstance(served, dict)
assert [edit["type"] for edit in served["context_management"]["applied_edits"]] == ["clear_tool_uses_20250919"]
@pytest.mark.asyncio
async def test_a_malformed_clear_tool_uses_edit_is_a_400_before_the_responses_api_call(
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
) -> None:
from litellm.llms.anthropic.messages.context_management import AnthropicContextManagementError
upstream: Final = respx_mock.post("https://api.openai.com/v1/responses").respond(
status_code=200, json=TEXT_RESPONSE
)
monkeypatch.setenv("OPENAI_API_KEY", "sk-clear-test")
with pytest.raises(AnthropicContextManagementError) as raised:
await LiteLLMMessagesToResponsesAPIHandler.async_anthropic_messages_handler(
max_tokens=1024,
messages=_three_reads(),
model="openai/gpt-5.6-luna",
custom_llm_provider="openai",
context_management={"edits": [{**CLEAR_EDIT, "exclude_tools": "Bash"}]},
)
assert raised.value.status_code == 400
assert not upstream.called
@pytest.mark.asyncio
async def test_a_malformed_text_in_a_result_clearing_would_drop_is_still_a_400(
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
) -> None:
upstream: Final = respx_mock.post("https://api.openai.com/v1/responses").respond(
status_code=200, json=TEXT_RESPONSE
)
monkeypatch.setenv("OPENAI_API_KEY", "sk-clear-test")
reads: Final = _three_reads()
malformed_oldest_result: Final = {
"role": "user",
"content": [{"type": "tool_result", "tool_use_id": "toolu_0", "content": [{"type": "text", "text": None}]}],
}
with pytest.raises(litellm.BadRequestError) as rejected:
await LiteLLMMessagesToResponsesAPIHandler.async_anthropic_messages_handler(
max_tokens=1024,
messages=[*reads[:2], malformed_oldest_result, *reads[3:]],
model="openai/gpt-5.6-luna",
tools=[dict(BASH_TOOL)],
custom_llm_provider="openai",
context_management={"edits": [dict(CLEAR_EDIT)]},
)
assert rejected.value.status_code == 400
assert "tool_result toolu_0 content.0.text: Input should be a valid string" in rejected.value.message
assert not upstream.called
def test_the_sync_path_clears_old_results_too(respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch) -> None:
from litellm.llms.anthropic.messages.context_management.constants import CLEARED_TOOL_RESULT_PLACEHOLDER
monkeypatch.setenv("OPENAI_API_KEY", "sk-clear-test")
monkeypatch.setenv("DISABLE_AIOHTTP_TRANSPORT", "True")
litellm.in_memory_llm_clients_cache.flush_cache()
upstream: Final = respx_mock.post("https://api.openai.com/v1/responses").respond(
status_code=200, json=TEXT_RESPONSE
)
served: Final = LiteLLMMessagesToResponsesAPIHandler.anthropic_messages_handler(
max_tokens=1024,
messages=_three_reads(),
model="openai/gpt-5.6-luna",
tools=[dict(BASH_TOOL)],
custom_llm_provider="openai",
context_management={"edits": [dict(CLEAR_EDIT)]},
)
body: Final = json.loads(upstream.calls.last.request.content)
outputs: Final = {
item["call_id"]: item["output"] for item in body["input"] if item.get("type") == "function_call_output"
}
assert [CLEARED_TOOL_RESULT_PLACEHOLDER in json.dumps(outputs[f"toolu_{i}"]) for i in range(3)] == [
True,
True,
False,
]
assert isinstance(served, dict)
assert served["context_management"]["applied_edits"][0]["cleared_tool_uses"] == 2
@pytest.mark.asyncio
async def test_an_openai_style_context_management_list_is_still_not_sent_to_the_responses_api(
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
) -> None:
upstream: Final = respx_mock.post("https://api.openai.com/v1/responses").respond(
status_code=200, json=TEXT_RESPONSE
)
monkeypatch.setenv("OPENAI_API_KEY", "sk-clear-test")
monkeypatch.setenv("DISABLE_AIOHTTP_TRANSPORT", "True")
litellm.in_memory_llm_clients_cache.flush_cache()
served: Final = await LiteLLMMessagesToResponsesAPIHandler.async_anthropic_messages_handler(
max_tokens=1024,
messages=_three_reads(),
model="openai/gpt-5.6-luna",
custom_llm_provider="openai",
context_management=[{"type": "compaction", "compact_threshold": 500}],
)
assert "context_management" not in json.loads(upstream.calls.last.request.content)
assert isinstance(served, dict)
assert "context_management" not in served
@pytest.mark.asyncio
async def test_under_drop_params_a_malformed_edit_still_clears_without_its_bad_field(
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
) -> None:
from litellm.llms.anthropic.messages.context_management.constants import CLEARED_TOOL_RESULT_PLACEHOLDER
served, body = await _cleared_bridge(
respx_mock, monkeypatch, stream=False, edits=[{**CLEAR_EDIT, "exclude_tools": "Bash"}], drop_params=True
)
outputs: Final = {
item["call_id"]: item["output"] for item in body["input"] if item.get("type") == "function_call_output"
}
assert [CLEARED_TOOL_RESULT_PLACEHOLDER in json.dumps(outputs[f"toolu_{i}"]) for i in range(3)] == [
True,
True,
False,
]
assert isinstance(served, dict)
assert served["context_management"]["applied_edits"][0]["cleared_tool_uses"] == 2
@pytest.mark.asyncio
async def test_the_sync_path_reports_its_edits_on_the_streamed_message_delta(
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.setenv("OPENAI_API_KEY", "sk-clear-test")
monkeypatch.setenv("DISABLE_AIOHTTP_TRANSPORT", "True")
litellm.in_memory_llm_clients_cache.flush_cache()
respx_mock.post("https://api.openai.com/v1/responses").respond(
status_code=200, headers={"Content-Type": "text/event-stream"}, content=RESPONSES_SSE_BODY
)
sse: Final = LiteLLMMessagesToResponsesAPIHandler.anthropic_messages_handler(
max_tokens=1024,
messages=_three_reads(),
model="openai/gpt-5.6-luna",
tools=[dict(BASH_TOOL)],
stream=True,
custom_llm_provider="openai",
context_management={"edits": [dict(CLEAR_EDIT)]},
)
events: Final = [
json.loads(chunk.decode().split("data: ", 1)[1]) async for chunk in cast(AsyncIterator[bytes], sse)
]
message_delta: Final = next(event for event in events if event["type"] == "message_delta")
assert message_delta["context_management"]["applied_edits"][0]["cleared_tool_uses"] == 2

88
tests/unit/llms/openai/responses/count_tokens/test_token_counter.py
View file

@ -2,6 +2,7 @@ import copy
import json
import operator
from collections.abc import Mapping
from itertools import chain
from typing import Final
import httpx
@ -11,9 +12,10 @@ from pydantic import JsonValue, ValidationError
import litellm
from litellm.litellm_core_utils.tool_fields import DropPolicy
from litellm.llms.anthropic.messages.context_management.constants import CLEARED_TOOL_RESULT_PLACEHOLDER
from litellm.llms.openai.responses.count_tokens.token_counter import OpenAITokenCounter
from litellm.responses.utils import ResponsesAPIRequestUtils
from litellm.types.utils import TokenCountMessagesFormat, TokenCountResponse
from litellm.types.utils import CountContextManagement, TokenCountMessagesFormat, TokenCountResponse
COUNT_URL: Final = "https://api.openai.com/v1/responses/input_tokens"
@ -484,6 +486,90 @@ async def test_anthropic_format_tool_result_text_block_without_text_still_counts
assert {"type": "function_call_output", "call_id": "toolu_log", "output": "build ok\n\ndone"} in sent["input"]
def _tool_pair(index: int) -> tuple[dict[str, object], dict[str, object]]:
return (
{"role": "assistant", "content": [{"type": "tool_use", "id": f"toolu_{index}", "name": "log", "input": {}}]},
{
"role": "user",
"content": [{"type": "tool_result", "tool_use_id": f"toolu_{index}", "content": f"log {index} " * 40}],
},
)
TOOL_PAIRS: Final[list[dict[str, object]]] = [
{"role": "user", "content": "Compare the logs."},
*chain.from_iterable(_tool_pair(index) for index in range(3)),
]
async def _count_with_edit(
monkeypatch: pytest.MonkeyPatch, edit: dict[str, JsonValue], drop_invalid_edits: bool
) -> tuple[TokenCountResponse | None, respx.Route]:
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
with respx.mock:
route: Final = respx.post(COUNT_URL).mock(return_value=httpx.Response(200, json={"input_tokens": 9}))
result: Final = await OpenAITokenCounter().count_tokens(
model_to_use="gpt-4o",
messages=copy.deepcopy(TOOL_PAIRS),
contents=None,
deployment={"litellm_params": {"api_key": "sk-test"}},
request_model="openai/gpt-4o",
messages_format="anthropic",
context_management=CountContextManagement(spec={"edits": [edit]}, drop_invalid_edits=drop_invalid_edits),
)
return result, route
def _sent_outputs(route: respx.Route) -> dict[str, str]:
sent: Final = json.loads(route.calls.last.request.content)
return {item["call_id"]: item["output"] for item in sent["input"] if item["type"] == "function_call_output"}
CLEAR_EDIT: Final[dict[str, JsonValue]] = {
"type": "clear_tool_uses_20250919",
"trigger": {"type": "tool_uses", "value": 1},
"keep": {"type": "tool_uses", "value": 1},
}
@pytest.mark.asyncio
async def test_anthropic_format_counts_the_history_the_clear_edit_leaves_for_inference(
monkeypatch: pytest.MonkeyPatch,
) -> None:
result, route = await _count_with_edit(monkeypatch, CLEAR_EDIT, drop_invalid_edits=False)
assert result is not None
assert (result.error, result.total_tokens) == (False, 9)
assert _sent_outputs(route) == {
"toolu_0": CLEARED_TOOL_RESULT_PLACEHOLDER,
"toolu_1": CLEARED_TOOL_RESULT_PLACEHOLDER,
"toolu_2": "log 2 " * 40,
}
@pytest.mark.asyncio
async def test_anthropic_format_malformed_clear_edit_answers_400_without_calling_openai(
monkeypatch: pytest.MonkeyPatch,
) -> None:
result, route = await _count_with_edit(monkeypatch, {**CLEAR_EDIT, "exclude_tools": "log"}, drop_invalid_edits=False)
assert route.called is False
assert result is not None
assert (result.error, result.status_code) == (True, 400)
assert "exclude_tools" in str(result.error_message)
@pytest.mark.asyncio
async def test_anthropic_format_drop_params_counts_the_clear_without_its_malformed_field(
monkeypatch: pytest.MonkeyPatch,
) -> None:
result, route = await _count_with_edit(monkeypatch, {**CLEAR_EDIT, "exclude_tools": "log"}, drop_invalid_edits=True)
assert result is not None
assert result.error is False
assert _sent_outputs(route)["toolu_0"] == CLEARED_TOOL_RESULT_PLACEHOLDER
@pytest.mark.asyncio
async def test_count_uses_the_global_openai_key_chat_would_use(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)

2
tests/unit/llms/vertex_ai/count_tokens/test_context.py
View file

@ -649,7 +649,7 @@ async def test_anthropic_view_counts_the_unedited_body_without_an_error_log_when
assert not any(record.levelno >= logging.ERROR for record in caplog.records)
async def test_anthropic_view_counts_the_turns_after_a_compaction_block_when_an_edit_before_compact_fails() -> None:
async def test_anthropic_view_clears_only_the_turns_after_a_compaction_block_when_clear_runs_before_compact() -> None:
clear_before_compact: Final[JsonValue] = {
"edits": [
{