mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-01 07:03:02 +00:00
fix(context_management)!: run clear_tool_uses in the gateway on the Responses API path (#47861)
* fix(context_management): run clear_tool_uses in the gateway on the Responses API path /v1/messages to openai/ models goes through the Responses API, which translated only compact_20260112 (to its native compaction) and dropped every other edit without a warning, so clear_tool_uses_20250919 never cleared anything. Edits without a Responses API form now run in the same polyfill as the chat completions path, before the call, and their applied_edits come back on the response and the final message_delta. compact stays native. * fix(context_management): check the sync polyfill result instead of casting it The strict ruff gate (TID251) bans typing.cast. * test(context_management): cover the Responses path's polyfill error handling A polyfill crash is a 500 before the call, a rejected edit under drop_params is skipped, and the sync stream reports its edits. normalize_spec never raises for a dict spec, so its guard is gone. * fix(context_management): count unreadable blocks and check tool_result text before clearing clear_tool_uses now maps blocks the token counter cannot read (server tool, compaction) to JSON text, so an input_tokens trigger no longer 500s on that history. The Responses API path checks malformed tool_result text on the original messages before clearing, as the chat completions path does. Also removes comments that repeated the code and makes the test fixtures immutable. * fix(context_management): count only blocks the model receives and keep list specs unsent clear_tool_uses now leaves blocks token_counter cannot read, like server tool results and compaction, out of its count instead of counting them as JSON text. No adapter sends them to the model, so an input_tokens trigger no longer fires on tokens the model never sees. On the Responses API path, an OpenAI-style list context_management stays unsent, as it was before, instead of reaching OpenAI's native compaction, where a threshold under 1000 is a 400. On the chat completions path, a clear edit listed before compact_20260112 now runs on the turns after the latest compaction block, as Anthropic does, instead of counting and clearing turns the compaction block drops. The crash test that replaced apply_context_management with a mock is removed. * fix(context_management): apply clear edits in OpenAI count_tokens and skip nested off-spec blocks when counting * fix(context_management): run edits on the history drop_params sends, so count_tokens and /v1/messages clear alike * fix(context_management): count clear_at_least freed tokens over sent blocks only * fix(context_management): count tools with JSON Schema shorthands and reject a non-object trigger or keep A tool whose schema has an array without items, a boolean subschema, or a search_result block without a title made the token counter raise, so a clear edit answered 500. A string trigger or keep also answered 500, even under drop_params. Both now count or get the API's 400, and drop_params drops the malformed field. * fix(token_counter): count a malformed tool schema keyword as absent and move shorthand framed-estimate cases * fix(count_tokens): pass the system prompt into clearing so count_tokens matches inference * fix(context_management): keep a malformed system prompt from failing clear_tool_uses on the Responses API A number, an object, or a list holding non-object items as system made the polyfill answer 500. The polyfill now keeps a list's object items and treats any other shape as no system, on both /v1/messages and count_tokens, so the two paths still count the same text --------- Co-authored-by: AlonSoko <25619863+AlonSoko@users.noreply.github.com> Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
This commit is contained in:
Alon Sokolovski
GitHub
parent
ea51e77dc5
commit
2b4cb92c8b
16 changed files with 993 additions and 74 deletions
|
|
@ -1224,10 +1224,6 @@ def _validate_anthropic_content(content: Mapping[str, object]) -> type:
|
|||
if expected_cls is None:
|
||||
raise ValueError(f"Unknown Anthropic content type: '{content_type}'")
|
||||
|
||||
missing: Final = [k for k in getattr(expected_cls, "__required_keys__", set()) if k not in content]
|
||||
if missing:
|
||||
raise ValueError(f"Missing required fields in {content_type} block: {', '.join(missing)}")
|
||||
|
||||
return expected_cls
|
||||
|
||||
|
||||
|
|
@ -1552,12 +1548,13 @@ def _format_single_function_definition(function: Mapping[str, object]) -> tuple[
|
|||
|
||||
def _format_object_parameters(parameters, indent):
|
||||
properties: Final = parameters.get("properties")
|
||||
if not properties:
|
||||
if not isinstance(properties, Mapping) or not properties:
|
||||
return ""
|
||||
required_params: Final = parameters.get("required", [])
|
||||
required: Final = parameters.get("required")
|
||||
required_params: Final = tuple(required) if isinstance(required, list) else ()
|
||||
lines: Final = []
|
||||
for key, props in properties.items():
|
||||
description = props.get("description")
|
||||
description = props.get("description") if isinstance(props, Mapping) else None
|
||||
if description:
|
||||
lines.append(f"// {description}")
|
||||
question = "?"
|
||||
|
|
@ -1568,19 +1565,21 @@ def _format_object_parameters(parameters, indent):
|
|||
|
||||
|
||||
def _format_type(props, indent):
|
||||
if not isinstance(props, Mapping):
|
||||
return "any"
|
||||
type: Final = props.get("type")
|
||||
enum: Final = props.get("enum")
|
||||
if type == "string":
|
||||
if "enum" in props:
|
||||
return " | ".join([f'"{item}"' for item in props["enum"]])
|
||||
if isinstance(enum, list):
|
||||
return " | ".join([f'"{item}"' for item in enum])
|
||||
return "string"
|
||||
elif type == "array":
|
||||
# items is required, OpenAI throws an error if it's missing
|
||||
return f"{_format_type(props['items'], indent)}[]"
|
||||
return f"{_format_type(props.get('items'), indent)}[]"
|
||||
elif type == "object":
|
||||
return f"{{\n{_format_object_parameters(props, indent + 2)}\n}}"
|
||||
elif type in ["integer", "number"]:
|
||||
if "enum" in props:
|
||||
return " | ".join([f'"{item}"' for item in props["enum"]])
|
||||
if isinstance(enum, list):
|
||||
return " | ".join([f'"{item}"' for item in enum])
|
||||
return "number"
|
||||
elif type == "boolean":
|
||||
return "boolean"
|
||||
|
|
|
|||
|
|
@ -160,6 +160,7 @@ async def prepare_context_managed_request(
|
|||
"""Apply client compaction history, then optional context_management polyfill."""
|
||||
from litellm.llms.anthropic.messages.context_management.editors.compact import (
|
||||
apply_client_compaction_block_history,
|
||||
slice_around_compaction_block,
|
||||
)
|
||||
|
||||
# Skip the client-history pre-processing when a ``compact_20260112``
|
||||
|
|
@ -175,7 +176,7 @@ async def prepare_context_managed_request(
|
|||
|
||||
if polyfill_will_run:
|
||||
history_result: PolyfillResult | None = None
|
||||
working_messages: _AnthropicMessages = messages
|
||||
working_messages: _AnthropicMessages = list(slice_around_compaction_block(messages)[0])
|
||||
working_system: _AnthropicSystem = system
|
||||
else:
|
||||
history_result = apply_client_compaction_block_history(
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@ from litellm._logging import verbose_logger
|
|||
from litellm.litellm_core_utils.asyncify import asyncify
|
||||
from litellm.types.llms.anthropic import AppliedEdit
|
||||
|
||||
from ..convertible_blocks import drop_non_string_tool_result_texts
|
||||
from .constants import CLEAR_TOOL_USES_EDIT_TYPE, COMPACT_EDIT_TYPE
|
||||
from .editors import (
|
||||
CheckedEdit,
|
||||
|
|
@ -103,7 +104,7 @@ async def apply_context_management(
|
|||
|
||||
Every ``clear_tool_uses_20250919`` edit is validated before any editor runs, so a malformed one raises its 400
|
||||
before a ``compact_20260112`` summary call is paid for, or runs without its malformed fields under
|
||||
``drop_invalid_edits``.
|
||||
``drop_invalid_edits``, which also drops non-string ``tool_result`` text first, as the provider never receives it.
|
||||
|
||||
The dispatcher is async so async editors (``compact_20260112``) can
|
||||
``await`` the configured summarization model. Sync editors run in a
|
||||
|
|
@ -119,7 +120,7 @@ async def apply_context_management(
|
|||
raise first_error
|
||||
well_formed_edits: Final = tuple(check.edit for check in checked)
|
||||
|
||||
current_messages = messages
|
||||
current_messages = list(drop_non_string_tool_result_texts(messages, drop_invalid_edits))
|
||||
current_system = system
|
||||
aggregated_applied: Final[list[AppliedEdit]] = []
|
||||
aggregated_compaction_block = None
|
||||
|
|
|
|||
|
|
@ -39,6 +39,21 @@ if TYPE_CHECKING:
|
|||
T = TypeVar("T")
|
||||
# api.anthropic.com, measured 2026-10-19: text results of 1 to 46 bytes are never cleared or counted toward keep
|
||||
_TINY_RESULT_BYTES: Final = 46
|
||||
_TOKEN_COUNTER_BLOCK_TYPES: Final = frozenset(
|
||||
{
|
||||
"text",
|
||||
"image_url",
|
||||
"image",
|
||||
"document",
|
||||
"file",
|
||||
"tool_use",
|
||||
"tool_result",
|
||||
"thinking",
|
||||
"redacted_thinking",
|
||||
"tool_reference",
|
||||
"search_result",
|
||||
}
|
||||
)
|
||||
ToolNames = Literal["all"] | frozenset[str]
|
||||
|
||||
|
||||
|
|
@ -52,16 +67,19 @@ class _ClearAtLeast(BaseModel):
|
|||
class _ClearToolUsesFields(BaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
|
||||
trigger: Mapping[str, object] | None = None
|
||||
keep: Mapping[str, object] | None = None
|
||||
clear_at_least: _ClearAtLeast | None = None
|
||||
exclude_tools: tuple[str, ...] | None = None
|
||||
clear_tool_inputs: bool | tuple[str, ...] | None = None
|
||||
|
||||
|
||||
_BLOCKS: Final = TypeAdapter(tuple[Mapping[str, object], ...])
|
||||
# api.anthropic.com, measured 2026-10-19: its wording for the two shapes pydantic words differently
|
||||
# api.anthropic.com, measured 2026-10-19 and 2026-10-28: its wording for the shapes pydantic words differently
|
||||
_API_MESSAGES: Final = MappingProxyType(
|
||||
{
|
||||
"tuple_type": "Input should be a valid array",
|
||||
"dict_type": "Input should be an object",
|
||||
"model_type": "Input does not match the expected shape.",
|
||||
}
|
||||
)
|
||||
|
|
@ -178,10 +196,12 @@ def _cleared_result(block: Mapping[str, object]) -> dict[str, object]:
|
|||
|
||||
def _freed_tokens(model: str, result: Mapping[str, object], use: Mapping[str, object], clears_input: bool) -> int:
|
||||
"""Tokens that clearing this use frees: its result, and its input when ``clear_tool_inputs`` covers it."""
|
||||
before: Final = [result, use] if clears_input else [result]
|
||||
after: Final = [_cleared_result(result), {**use, "input": {}}] if clears_input else [_cleared_result(result)]
|
||||
before: Final[Sequence[Mapping[str, object]]] = (result, use) if clears_input else (result,)
|
||||
after: Final[Sequence[Mapping[str, object]]] = (
|
||||
(_cleared_result(result), {**use, "input": {}}) if clears_input else (_cleared_result(result),)
|
||||
)
|
||||
counts: Final = tuple(
|
||||
litellm.token_counter(model=model, messages=[{"role": "user", "content": blocks}]) for blocks in (before, after)
|
||||
_count_tokens(model, [{"role": "user", "content": blocks}], None) for blocks in (before, after)
|
||||
)
|
||||
return max(counts[0] - counts[1], 0)
|
||||
|
||||
|
|
@ -199,6 +219,35 @@ def _select_in_batches(candidates: Sequence[str], tokens: Mapping[str, int], cle
|
|||
return tuple(candidates[:batch_end])
|
||||
|
||||
|
||||
def _without_uncountable_blocks(block: Mapping[str, object]) -> Mapping[str, object]:
|
||||
nested: Final = _parse(_BLOCKS, block.get("content"))
|
||||
if nested is None:
|
||||
return block
|
||||
return {**block, "content": [inner for inner in nested if inner.get("type") in _TOKEN_COUNTER_BLOCK_TYPES]}
|
||||
|
||||
|
||||
def _countable_message(message: Mapping[str, object]) -> Mapping[str, object]:
|
||||
blocks: Final = _parse(_BLOCKS, message.get("content"))
|
||||
if blocks is None:
|
||||
return message
|
||||
return {
|
||||
**message,
|
||||
"content": [
|
||||
_without_uncountable_blocks(block) for block in blocks if block.get("type") in _TOKEN_COUNTER_BLOCK_TYPES
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def _count_tokens(
|
||||
model: str, messages: Sequence[Mapping[str, object]], tools: Sequence[Mapping[str, object]] | None
|
||||
) -> int:
|
||||
return litellm.token_counter(
|
||||
model=model,
|
||||
messages=[_countable_message(message) for message in messages],
|
||||
tools=cast("list[ChatCompletionToolParam] | None", tools),
|
||||
)
|
||||
|
||||
|
||||
def _trigger_met(
|
||||
trigger: dict[str, object],
|
||||
model: str,
|
||||
|
|
@ -218,11 +267,7 @@ def _trigger_met(
|
|||
|
||||
if not isinstance(threshold, int):
|
||||
threshold = DEFAULT_INPUT_TOKENS_TRIGGER
|
||||
message_tokens: Final = litellm.token_counter(
|
||||
model=model,
|
||||
messages=messages,
|
||||
tools=cast("list[ChatCompletionToolParam] | None", tools),
|
||||
)
|
||||
message_tokens: Final = _count_tokens(model, messages, tools)
|
||||
if message_tokens > threshold:
|
||||
return True, message_tokens
|
||||
current_tokens: Final = message_tokens + count_system_tokens(model, system)
|
||||
|
|
@ -375,17 +420,9 @@ def apply_clear_tool_uses_20250919(
|
|||
[_message_without_inputs(message, ids_to_clear, input_tools) for message in edited] if input_tools else edited
|
||||
)
|
||||
|
||||
tokens_before: Final = (
|
||||
counted_tokens
|
||||
if counted_tokens is not None
|
||||
else litellm.token_counter(
|
||||
model=model, messages=kept, tools=cast("list[ChatCompletionToolParam] | None", tools)
|
||||
)
|
||||
)
|
||||
tokens_before: Final = counted_tokens if counted_tokens is not None else _count_tokens(model, kept, tools)
|
||||
kept_after, _ = history_after_client_compaction(final_messages, system)
|
||||
tokens_after: Final = litellm.token_counter(
|
||||
model=model, messages=kept_after, tools=cast("list[ChatCompletionToolParam] | None", tools)
|
||||
)
|
||||
tokens_after: Final = _count_tokens(model, kept_after, tools)
|
||||
cleared_input_tokens: Final = max(tokens_before - tokens_after, 0)
|
||||
|
||||
return final_messages, {
|
||||
|
|
|
|||
|
|
@ -137,7 +137,7 @@ def _find_latest_compaction_index(
|
|||
return None, None
|
||||
|
||||
|
||||
def _slice_around_compaction_block(
|
||||
def slice_around_compaction_block(
|
||||
messages: Sequence[_MsgT],
|
||||
) -> tuple[Sequence[_MsgT | dict[str, object]], dict[str, object] | None]:
|
||||
"""Apply Anthropic's "drop everything before the compaction block" rule.
|
||||
|
|
@ -194,7 +194,7 @@ def history_after_client_compaction(
|
|||
system: str | Sequence[Mapping[str, object]] | None,
|
||||
) -> tuple[list[dict[str, object]], str | Sequence[Mapping[str, object]] | None]:
|
||||
"""Return the history the API keeps after the latest compaction block, without it, and the system with its summary."""
|
||||
sliced, block = _slice_around_compaction_block(messages)
|
||||
sliced, block = slice_around_compaction_block(messages)
|
||||
if block is None:
|
||||
return list(messages), system
|
||||
tail: Final = _strip_compaction_blocks(sliced)
|
||||
|
|
@ -615,7 +615,7 @@ def apply_client_compaction_block_history(
|
|||
tail is forwarded unchanged (with compaction blocks stripped) so recent
|
||||
turns the summary does not cover are preserved.
|
||||
"""
|
||||
effective_messages, prior_compaction_block = _slice_around_compaction_block(messages)
|
||||
effective_messages, prior_compaction_block = slice_around_compaction_block(messages)
|
||||
if prior_compaction_block is None:
|
||||
return None
|
||||
|
||||
|
|
@ -686,7 +686,7 @@ async def apply_compact_20260112(
|
|||
# opt-in gate below so that even when summarization is disabled we still
|
||||
# strip Anthropic-only ``compaction`` blocks from messages going to
|
||||
# non-Anthropic backends (which would reject them).
|
||||
effective_messages, prior_compaction_block = _slice_around_compaction_block(messages)
|
||||
effective_messages, prior_compaction_block = slice_around_compaction_block(messages)
|
||||
prior_summary_text: Final = prior_compaction_block.get("content") if prior_compaction_block else None
|
||||
augmented_system: str | list[dict[str, object]] | None = system
|
||||
if isinstance(prior_summary_text, str) and prior_summary_text:
|
||||
|
|
|
|||
|
|
@ -10,11 +10,15 @@ from typing import Final, TypeAlias
|
|||
from pydantic import TypeAdapter, ValidationError
|
||||
|
||||
import litellm
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.integrations.anthropic_cache_control_hook import (
|
||||
AnthropicCacheControlHook,
|
||||
supports_openai_prompt_cache_breakpoint,
|
||||
)
|
||||
from litellm.litellm_core_utils.core_helpers import drops_unsupported_params
|
||||
from litellm.litellm_core_utils.asyncify import (
|
||||
run_async_function, # pyright: ignore[reportUnknownVariableType] # untyped helper
|
||||
)
|
||||
from litellm.litellm_core_utils.core_helpers import as_str_mapping, drops_unsupported_params
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
||||
cache_control_as_prompt_cache_breakpoint,
|
||||
prompt_cache_breakpoint_for_wire,
|
||||
|
|
@ -28,14 +32,19 @@ from litellm.types.llms.anthropic import (
|
|||
AnthropicMessagesRequest,
|
||||
AnthropicOutputConfig,
|
||||
AnthropicOutputSchema,
|
||||
ContextManagementResponse,
|
||||
)
|
||||
from litellm.types.llms.anthropic_messages.anthropic_response import (
|
||||
AnthropicMessagesResponse,
|
||||
)
|
||||
from litellm.types.llms.openai import ResponsesAPIResponse
|
||||
from litellm.types.utils import CountContextManagement
|
||||
from litellm.utils import ProviderConfigManager
|
||||
|
||||
from ..adapters.handler import request_safeguards_evaluator
|
||||
from ..adapters.handler import context_management_explicitly_dropped, request_safeguards_evaluator
|
||||
from ..context_management import AnthropicContextManagementError, PolyfillResult, apply_context_management
|
||||
from ..context_management.constants import COMPACT_EDIT_TYPE
|
||||
from ..context_management.dispatcher import normalize_spec
|
||||
from ..convertible_blocks import reject_malformed_tool_result_text
|
||||
from ..safeguards import with_safeguard_results
|
||||
from ..utils import litellm_logging_obj_from_kwargs, local_model_name
|
||||
|
|
@ -45,10 +54,111 @@ from .transformation import LiteLLMAnthropicToResponsesAPIAdapter
|
|||
AnthropicRequestMessages: TypeAlias = list[AllAnthropicMessageValues] | list[dict[str, object]]
|
||||
BlockRewrite: TypeAlias = Callable[[Mapping[str, object]], Mapping[str, object]]
|
||||
_BLOCK_LIST: Final = TypeAdapter(list[dict[str, object]])
|
||||
_SYSTEM_ITEMS: Final = TypeAdapter(list[object])
|
||||
_SPEC: Final = TypeAdapter(dict[str, object])
|
||||
_SplitEdits: TypeAlias = tuple[dict[str, object] | None, Sequence[Mapping[str, object]]]
|
||||
AnthropicContextManagementSpec: TypeAlias = dict[str, object] | list[dict[str, object]] | None
|
||||
|
||||
_ADAPTER: Final = LiteLLMAnthropicToResponsesAPIAdapter()
|
||||
|
||||
|
||||
def _split_context_management(
|
||||
spec: AnthropicContextManagementSpec,
|
||||
kwargs: Mapping[str, object],
|
||||
) -> _SplitEdits:
|
||||
"""Return (the edits the Responses API runs natively, the edits the gateway polyfill runs).
|
||||
|
||||
The Responses API compacts natively (``compact_20260112`` becomes its ``compaction``); any other
|
||||
edit, such as ``clear_tool_uses_20250919``, has no Responses API form and runs in the polyfill, as
|
||||
it does on the chat completions path. An OpenAI-style list spec is not Anthropic's shape and stays
|
||||
unsent, as it was before the polyfill ran here.
|
||||
"""
|
||||
if not isinstance(spec, dict):
|
||||
return None, ()
|
||||
if not spec or context_management_explicitly_dropped(kwargs.get("additional_drop_params")):
|
||||
return spec, ()
|
||||
edits: Final = normalize_spec(spec) or []
|
||||
native: Final = [edit for edit in edits if edit.get("type") == COMPACT_EDIT_TYPE]
|
||||
return ({"edits": native} if native else None), tuple(
|
||||
edit for edit in edits if edit.get("type") != COMPACT_EDIT_TYPE
|
||||
)
|
||||
|
||||
|
||||
def _system_blocks(system: object) -> str | Sequence[Mapping[str, object]] | None:
|
||||
if system is None or isinstance(system, str):
|
||||
return system
|
||||
try:
|
||||
items: Final = _SYSTEM_ITEMS.validate_python(system)
|
||||
except ValidationError:
|
||||
return None
|
||||
return tuple(block for block in map(as_str_mapping, items) if block is not None)
|
||||
|
||||
|
||||
async def _polyfilled(
|
||||
*,
|
||||
model: str,
|
||||
messages: AnthropicRequestMessages,
|
||||
tools: Sequence[AllAnthropicToolsValues | Mapping[str, object]] | None,
|
||||
system: object,
|
||||
edits: Sequence[Mapping[str, object]],
|
||||
kwargs: Mapping[str, object],
|
||||
) -> PolyfillResult | None:
|
||||
"""Run the edits the Responses API can't run natively; raise like the chat completions path does."""
|
||||
if not edits:
|
||||
return None
|
||||
reject_malformed_tool_result_text(messages, model, kwargs)
|
||||
drop_invalid_edits: Final = drops_unsupported_params(kwargs.get("drop_params"))
|
||||
blocks: Final = _system_blocks(system)
|
||||
try:
|
||||
return await apply_context_management(
|
||||
model=model,
|
||||
messages=_BLOCK_LIST.validate_python(messages),
|
||||
tools=_BLOCK_LIST.validate_python(tools) if tools else None,
|
||||
system=blocks if blocks is None or isinstance(blocks, str) else _BLOCK_LIST.validate_python(blocks),
|
||||
context_management_spec={"edits": [dict(edit) for edit in edits]},
|
||||
drop_invalid_edits=drop_invalid_edits,
|
||||
)
|
||||
except AnthropicContextManagementError:
|
||||
raise
|
||||
except Exception as e:
|
||||
verbose_logger.exception("context_management polyfill failed on the Responses API path: %s", e)
|
||||
raise AnthropicContextManagementError(
|
||||
status_code=500, message=f"context_management polyfill failed: {e}"
|
||||
) from e
|
||||
|
||||
|
||||
async def count_view_messages(
|
||||
*,
|
||||
model: str,
|
||||
messages: Sequence[Mapping[str, object]],
|
||||
tools: Sequence[Mapping[str, object]] | None,
|
||||
system: object,
|
||||
context_management: CountContextManagement | None,
|
||||
) -> Sequence[Mapping[str, object]]:
|
||||
"""The messages /v1/messages sends the Responses API once the polyfill has run the edits it can't run natively."""
|
||||
if context_management is None or not isinstance(context_management.spec, dict):
|
||||
return messages
|
||||
_, edits = _split_context_management(_SPEC.validate_python(context_management.spec), {})
|
||||
result: Final = await _polyfilled(
|
||||
model=model,
|
||||
messages=_BLOCK_LIST.validate_python(messages),
|
||||
tools=tools,
|
||||
system=system,
|
||||
edits=edits,
|
||||
kwargs={"drop_params": context_management.drop_invalid_edits},
|
||||
)
|
||||
return messages if result is None else result.messages
|
||||
|
||||
|
||||
def _with_applied_edits(
|
||||
response: AnthropicMessagesResponse, polyfill_result: PolyfillResult | None
|
||||
) -> AnthropicMessagesResponse:
|
||||
applied_edits: Final = polyfill_result.applied_edits_for_response() if polyfill_result else None
|
||||
if not applied_edits:
|
||||
return response
|
||||
return {**response, "context_management": ContextManagementResponse(applied_edits=list(applied_edits))}
|
||||
|
||||
|
||||
def _record_bridged_response(responses_kwargs: Mapping[str, object], result: ResponsesAPIResponse) -> None:
|
||||
logging_obj: Final = litellm_logging_obj_from_kwargs(responses_kwargs)
|
||||
if logging_obj is not None:
|
||||
|
|
@ -281,7 +391,7 @@ class LiteLLMMessagesToResponsesAPIHandler:
|
|||
max_tokens: int,
|
||||
messages: AnthropicRequestMessages,
|
||||
model: str,
|
||||
context_management: dict[str, object] | None = None,
|
||||
context_management: AnthropicContextManagementSpec = None,
|
||||
metadata: dict[str, object] | None = None,
|
||||
output_config: AnthropicOutputConfig | None = None,
|
||||
stop_sequences: list[str] | None = None,
|
||||
|
|
@ -296,11 +406,15 @@ class LiteLLMMessagesToResponsesAPIHandler:
|
|||
output_format: AnthropicOutputSchema | None = None,
|
||||
**kwargs: object,
|
||||
) -> AnthropicMessagesResponse | AsyncIterator[bytes]:
|
||||
native_context_management, polyfilled_edits = _split_context_management(context_management, kwargs)
|
||||
polyfill_result: Final = await _polyfilled(
|
||||
model=model, messages=messages, tools=tools, system=system, edits=polyfilled_edits, kwargs=kwargs
|
||||
)
|
||||
responses_kwargs: Final = _build_responses_kwargs(
|
||||
max_tokens=max_tokens,
|
||||
messages=messages,
|
||||
messages=polyfill_result.messages if polyfill_result is not None else messages,
|
||||
model=model,
|
||||
context_management=context_management,
|
||||
context_management=native_context_management,
|
||||
metadata=metadata,
|
||||
output_config=output_config,
|
||||
stop_sequences=stop_sequences,
|
||||
|
|
@ -325,13 +439,16 @@ class LiteLLMMessagesToResponsesAPIHandler:
|
|||
model=local_model_name(model, kwargs.get("custom_llm_provider")),
|
||||
litellm_logging_obj=litellm_logging_obj_from_kwargs(responses_kwargs),
|
||||
safeguards_evaluator=safeguards_evaluator,
|
||||
applied_edits=polyfill_result.applied_edits_for_response() if polyfill_result else None,
|
||||
)
|
||||
return wrapper.async_anthropic_sse_wrapper()
|
||||
|
||||
if not isinstance(result, ResponsesAPIResponse):
|
||||
raise ValueError(f"Expected ResponsesAPIResponse, got {type(result)}")
|
||||
|
||||
served: Final = await with_safeguard_results(_ADAPTER.translate_response(result), safeguards_evaluator)
|
||||
served: Final = await with_safeguard_results(
|
||||
_with_applied_edits(_ADAPTER.translate_response(result), polyfill_result), safeguards_evaluator
|
||||
)
|
||||
_record_bridged_response(responses_kwargs, result)
|
||||
return served
|
||||
|
||||
|
|
@ -340,7 +457,7 @@ class LiteLLMMessagesToResponsesAPIHandler:
|
|||
max_tokens: int,
|
||||
messages: AnthropicRequestMessages,
|
||||
model: str,
|
||||
context_management: dict[str, object] | None = None,
|
||||
context_management: AnthropicContextManagementSpec = None,
|
||||
metadata: dict[str, object] | None = None,
|
||||
output_config: AnthropicOutputConfig | None = None,
|
||||
stop_sequences: list[str] | None = None,
|
||||
|
|
@ -382,11 +499,26 @@ class LiteLLMMessagesToResponsesAPIHandler:
|
|||
)
|
||||
|
||||
# Sync path
|
||||
native_context_management, polyfilled_edits = _split_context_management(context_management, kwargs)
|
||||
ran: Final[object] = ( # pyright: ignore[reportUnknownVariableType] # run_async_function is untyped
|
||||
run_async_function(
|
||||
_polyfilled,
|
||||
model=model,
|
||||
messages=messages,
|
||||
tools=tools,
|
||||
system=system,
|
||||
edits=polyfilled_edits,
|
||||
kwargs=kwargs,
|
||||
)
|
||||
if polyfilled_edits
|
||||
else None
|
||||
)
|
||||
polyfill_result: Final = ran if isinstance(ran, PolyfillResult) else None
|
||||
responses_kwargs: Final = _build_responses_kwargs(
|
||||
max_tokens=max_tokens,
|
||||
messages=messages,
|
||||
messages=polyfill_result.messages if polyfill_result is not None else messages,
|
||||
model=model,
|
||||
context_management=context_management,
|
||||
context_management=native_context_management,
|
||||
metadata=metadata,
|
||||
output_config=output_config,
|
||||
stop_sequences=stop_sequences,
|
||||
|
|
@ -409,10 +541,11 @@ class LiteLLMMessagesToResponsesAPIHandler:
|
|||
responses_stream=result,
|
||||
model=local_model_name(model, kwargs.get("custom_llm_provider")),
|
||||
litellm_logging_obj=litellm_logging_obj_from_kwargs(responses_kwargs),
|
||||
applied_edits=polyfill_result.applied_edits_for_response() if polyfill_result else None,
|
||||
)
|
||||
return wrapper.async_anthropic_sse_wrapper()
|
||||
|
||||
if not isinstance(result, ResponsesAPIResponse):
|
||||
raise ValueError(f"Expected ResponsesAPIResponse, got {type(result)}")
|
||||
|
||||
return _ADAPTER.translate_response(result)
|
||||
return _with_applied_edits(_ADAPTER.translate_response(result), polyfill_result)
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@
|
|||
## Translates OpenAI call to Anthropic `/v1/messages` format
|
||||
import asyncio
|
||||
from collections import deque
|
||||
from collections.abc import AsyncGenerator, Awaitable, Callable, Iterator, Mapping
|
||||
from collections.abc import AsyncGenerator, Awaitable, Callable, Iterator, Mapping, Sequence
|
||||
from typing import TYPE_CHECKING, Any, Final
|
||||
|
||||
import anyio
|
||||
|
|
@ -27,6 +27,7 @@ from litellm.responses.streaming_iterator import (
|
|||
partial_responses_incomplete_event_on_disconnect,
|
||||
stream_error_status_and_message,
|
||||
)
|
||||
from litellm.types.llms.anthropic import AppliedEdit, ContextManagementResponse
|
||||
from litellm.types.llms.anthropic_messages.anthropic_response import AnthropicUsage
|
||||
from litellm.types.llms.base import LiteLLMBaseModel
|
||||
|
||||
|
|
@ -129,8 +130,10 @@ class AnthropicResponsesStreamWrapper:
|
|||
model: str,
|
||||
litellm_logging_obj: "LiteLLMLoggingObject | None" = None,
|
||||
safeguards_evaluator: "SafeguardsEvaluator | None" = None,
|
||||
applied_edits: Sequence[AppliedEdit] | None = None,
|
||||
) -> None:
|
||||
self.responses_stream = responses_stream
|
||||
self._applied_edits: Final = tuple(applied_edits or ())
|
||||
self.model = model
|
||||
self.litellm_logging_obj = litellm_logging_obj
|
||||
self.safeguards_evaluator: Final = safeguards_evaluator
|
||||
|
|
@ -445,6 +448,11 @@ class AnthropicResponsesStreamWrapper:
|
|||
"type": "message_delta",
|
||||
"delta": message_delta_payload,
|
||||
"usage": dict(anthropic_usage),
|
||||
**(
|
||||
{"context_management": ContextManagementResponse(applied_edits=list(self._applied_edits))}
|
||||
if self._applied_edits
|
||||
else {}
|
||||
),
|
||||
}
|
||||
)
|
||||
self._chunk_queue.append({"type": "message_stop"})
|
||||
|
|
|
|||
|
|
@ -15,7 +15,9 @@ from litellm._logging import verbose_logger
|
|||
from litellm.litellm_core_utils.asyncify import asyncify
|
||||
from litellm.litellm_core_utils.core_helpers import as_str_mapping
|
||||
from litellm.litellm_core_utils.tool_fields import DropPolicy
|
||||
from litellm.llms.anthropic.messages.context_management import AnthropicContextManagementError
|
||||
from litellm.llms.anthropic.messages.convertible_blocks import deployment_drops_params, unconvertible_blocks_error
|
||||
from litellm.llms.anthropic.messages.responses_adapters.handler import count_view_messages
|
||||
from litellm.llms.anthropic.messages.responses_adapters.transformation import (
|
||||
LiteLLMAnthropicToResponsesAPIAdapter,
|
||||
)
|
||||
|
|
@ -95,6 +97,32 @@ def _count_error(model_to_use: str, request_model: str, message: str, status_cod
|
|||
)
|
||||
|
||||
|
||||
async def _sent_messages(
|
||||
model: str,
|
||||
messages: Sequence[Mapping[str, object]],
|
||||
tools: Sequence[Mapping[str, object]] | None,
|
||||
system: object,
|
||||
messages_format: TokenCountMessagesFormat,
|
||||
context_management: CountContextManagement | None,
|
||||
) -> Sequence[Mapping[str, object]] | AnthropicContextManagementError:
|
||||
match messages_format:
|
||||
case "openai":
|
||||
return messages
|
||||
case "anthropic":
|
||||
try:
|
||||
return await count_view_messages(
|
||||
model=model,
|
||||
messages=messages,
|
||||
tools=tools,
|
||||
system=system,
|
||||
context_management=context_management,
|
||||
)
|
||||
except AnthropicContextManagementError as error:
|
||||
return error
|
||||
case _:
|
||||
assert_never(messages_format)
|
||||
|
||||
|
||||
class OpenAITokenCounter(BaseTokenCounter):
|
||||
"""Token counter implementation for OpenAI provider using the Responses API."""
|
||||
|
||||
|
|
@ -175,13 +203,28 @@ class OpenAITokenCounter(BaseTokenCounter):
|
|||
rejected_before_provider_call=True,
|
||||
)
|
||||
|
||||
sent_messages: Final = await _sent_messages(
|
||||
model_to_use, messages, tools, system, messages_format, context_management
|
||||
)
|
||||
if isinstance(sent_messages, AnthropicContextManagementError):
|
||||
return TokenCountResponse(
|
||||
total_tokens=0,
|
||||
request_model=request_model,
|
||||
model_used=model_to_use,
|
||||
tokenizer_type="openai_api",
|
||||
error=True,
|
||||
error_message=sent_messages.message,
|
||||
status_code=sent_messages.status_code,
|
||||
rejected_before_provider_call=True,
|
||||
)
|
||||
|
||||
resolved: Final = _count_credential(deployment, request_model or model_to_use)
|
||||
if isinstance(resolved, OpenAIError):
|
||||
return _count_error(model_to_use, request_model, resolved.message, resolved.status_code)
|
||||
|
||||
input_items, instructions, counted_tools, text = self._responses_request_parts(
|
||||
model_to_use=model_to_use,
|
||||
messages=messages,
|
||||
messages=sent_messages,
|
||||
tools=tools,
|
||||
system=system,
|
||||
messages_format=messages_format,
|
||||
|
|
|
|||
|
|
@ -523,6 +523,43 @@ def test_token_counter_skips_non_mapping_tools():
|
|||
assert mixed_tokens == valid_tokens
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("shorthand", "explicit"),
|
||||
[({"type": "array"}, {"type": "array", "items": {}}), (True, {})],
|
||||
ids=["array_without_items", "boolean_subschema"],
|
||||
)
|
||||
def test_token_counter_counts_json_schema_shorthands_like_their_explicit_form(shorthand: object, explicit: object) -> None:
|
||||
assert _count_with_property_schema(shorthand) == _count_with_property_schema(explicit)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("malformed", "without_keyword"),
|
||||
[
|
||||
(
|
||||
{"type": "object", "properties": {"id": {"type": "string"}}, "required": True},
|
||||
{"type": "object", "properties": {"id": {"type": "string"}}},
|
||||
),
|
||||
({"type": "object", "properties": [{"id": {"type": "string"}}]}, {"type": "object"}),
|
||||
({"type": "string", "enum": None}, {"type": "string"}),
|
||||
({"type": "integer", "enum": 3}, {"type": "integer"}),
|
||||
],
|
||||
ids=["required_true", "properties_list", "string_enum_null", "integer_enum_number"],
|
||||
)
|
||||
def test_token_counter_counts_a_malformed_schema_keyword_as_absent(malformed: object, without_keyword: object) -> None:
|
||||
assert _count_with_property_schema(malformed) == _count_with_property_schema(without_keyword)
|
||||
|
||||
|
||||
def _count_with_property_schema(property_schema: object) -> int:
|
||||
tool: Final = {
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "read_files",
|
||||
"parameters": {"type": "object", "properties": {"paths": property_schema}},
|
||||
},
|
||||
}
|
||||
return token_counter_new(model="gpt-4o", messages=[{"role": "user", "content": "Read the config files"}], tools=[tool])
|
||||
|
||||
|
||||
_WEATHER_FUNCTION: Final = {
|
||||
"name": "get_weather",
|
||||
"description": "Get the current weather for a city",
|
||||
|
|
@ -2132,6 +2169,20 @@ def test_token_counter_counts_anthropic_search_result_text() -> None:
|
|||
)
|
||||
|
||||
|
||||
def test_token_counter_counts_a_search_result_without_a_title() -> None:
|
||||
from litellm.litellm_core_utils.token_counter import _count_content_list
|
||||
|
||||
search_result: Final = {
|
||||
"type": "search_result",
|
||||
"source": "https://example.invalid/notes",
|
||||
"content": [{"type": "text", "text": "The code word is 7f3a."}],
|
||||
}
|
||||
|
||||
assert _count_content_list(len, [search_result], False, None) == len(
|
||||
"https://example.invalid/notesThe code word is 7f3a."
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("nested", (False, True))
|
||||
def test_unknown_content_type_remains_a_value_error_with_an_optional_fallback(nested: bool) -> None:
|
||||
from litellm.litellm_core_utils.token_counter import _count_content_list
|
||||
|
|
|
|||
|
|
@ -217,6 +217,106 @@ def test_cleared_input_tokens_is_nonnegative():
|
|||
assert applied["cleared_input_tokens"] >= 0
|
||||
|
||||
|
||||
SERVER_TOOL_TURN: Final = {
|
||||
"role": "assistant",
|
||||
"content": [
|
||||
{"type": "server_tool_use", "id": "srvtoolu_01", "name": "web_search", "input": {"query": "Mumbai rain"}},
|
||||
{
|
||||
"type": "web_search_tool_result",
|
||||
"tool_use_id": "srvtoolu_01",
|
||||
"content": [{"type": "web_search_result", "url": "https://example.com", "title": "rain " * 300}],
|
||||
},
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def _clear_above(
|
||||
messages: list[dict[str, object]], threshold: int
|
||||
) -> tuple[list[dict[str, object]], AppliedEdit | None]:
|
||||
return apply_clear_tool_uses_20250919(
|
||||
model=MODEL,
|
||||
messages=messages,
|
||||
tools=None,
|
||||
system=None,
|
||||
edit_spec={
|
||||
"type": "clear_tool_uses_20250919",
|
||||
"trigger": {"type": "input_tokens", "value": threshold},
|
||||
"keep": {"type": "tool_uses", "value": 1},
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def test_an_input_tokens_trigger_skips_blocks_no_adapter_sends() -> None:
|
||||
history: Final = _make_history(n_pairs=3)
|
||||
sent_tokens: Final = litellm.token_counter(model=MODEL, messages=history)
|
||||
turn: Final = SERVER_TOOL_TURN
|
||||
messages: Final = [history[0], turn, *history[1:]]
|
||||
|
||||
untouched, not_applied = _clear_above(messages, sent_tokens + 100)
|
||||
cleared, applied = _clear_above(messages, sent_tokens - 100)
|
||||
|
||||
assert not_applied is None
|
||||
assert untouched == messages
|
||||
assert applied is not None
|
||||
assert applied["cleared_tool_uses"] == 2
|
||||
assert _cleared_ids(cleared) == {"toolu_00", "toolu_01"}
|
||||
assert cleared[1] == turn
|
||||
|
||||
|
||||
def _first_result_with(blocks: list[dict[str, object]]) -> dict[str, object]:
|
||||
return {"role": "user", "content": [{"type": "tool_result", "tool_use_id": "toolu_00", "content": blocks}]}
|
||||
|
||||
|
||||
def test_an_input_tokens_trigger_skips_blocks_no_adapter_sends_inside_a_tool_result() -> None:
|
||||
history: Final = _make_history(n_pairs=3)
|
||||
result_text: Final = {"type": "text", "text": "Result 0: " + "x" * 200}
|
||||
search_result: Final = {"type": "web_search_result", "url": "https://example.com", "title": "rain " * 300}
|
||||
sent_tokens: Final = litellm.token_counter(
|
||||
model=MODEL, messages=[*history[:2], _first_result_with([result_text]), *history[3:]]
|
||||
)
|
||||
messages: Final = [*history[:2], _first_result_with([result_text, search_result]), *history[3:]]
|
||||
|
||||
untouched, not_applied = _clear_above(messages, sent_tokens + 100)
|
||||
cleared, applied = _clear_above(messages, sent_tokens - 100)
|
||||
|
||||
assert not_applied is None
|
||||
assert untouched == messages
|
||||
assert applied is not None
|
||||
assert _cleared_ids(cleared) == {"toolu_00", "toolu_01"}
|
||||
|
||||
|
||||
def _clear_at_least_one_token(
|
||||
messages: list[dict[str, object]],
|
||||
) -> tuple[list[dict[str, object]], AppliedEdit | None]:
|
||||
return apply_clear_tool_uses_20250919(
|
||||
model=MODEL,
|
||||
messages=messages,
|
||||
tools=None,
|
||||
system=None,
|
||||
edit_spec={
|
||||
"type": "clear_tool_uses_20250919",
|
||||
"trigger": {"type": "tool_uses", "value": 0},
|
||||
"keep": {"type": "tool_uses", "value": 1},
|
||||
"clear_at_least": {"type": "input_tokens", "value": 1},
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def test_clear_at_least_frees_only_blocks_an_adapter_sends_inside_a_tool_result() -> None:
|
||||
history: Final = _make_history(n_pairs=3)
|
||||
result_text: Final = {"type": "text", "text": "Result 0: " + "x" * 200}
|
||||
search_result: Final = {"type": "web_search_result", "url": "https://example.com", "title": "rain " * 300}
|
||||
|
||||
_, sent_only = _clear_at_least_one_token([*history[:2], _first_result_with([result_text]), *history[3:]])
|
||||
cleared, applied = _clear_at_least_one_token(
|
||||
[*history[:2], _first_result_with([result_text, search_result]), *history[3:]]
|
||||
)
|
||||
|
||||
assert applied is not None
|
||||
assert applied == sent_only
|
||||
assert _cleared_ids(cleared) == {"toolu_00", "toolu_01"}
|
||||
|
||||
|
||||
def _blocks_of(messages: list[dict[str, object]], block_type: str) -> tuple[dict[str, object], ...]:
|
||||
contents: Final = (message["content"] for message in messages)
|
||||
blocks: Final = chain.from_iterable(content for content in contents if isinstance(content, list))
|
||||
|
|
@ -559,6 +659,9 @@ async def _malformed_spec_error(edits: list[dict[str, object]]) -> AnthropicCont
|
|||
("exclude_tools", "get_weather"),
|
||||
("exclude_tools", [1]),
|
||||
("clear_tool_inputs", "get_weather"),
|
||||
("trigger", "soon"),
|
||||
("trigger", [1]),
|
||||
("keep", 5),
|
||||
],
|
||||
)
|
||||
async def test_malformed_fields_are_a_400_even_below_the_trigger(field: str, value: object) -> None:
|
||||
|
|
@ -576,12 +679,15 @@ async def test_malformed_fields_are_a_400_even_below_the_trigger(field: str, val
|
|||
("clear_at_least", 1200, "clear_at_least: Input does not match the expected shape."),
|
||||
("clear_at_least", {"type": "input_tokens", "value": "1200"}, "value: Input should be a valid integer"),
|
||||
("clear_tool_inputs", 5, "clear_tool_inputs.bool: Input should be a valid boolean, unable to interpret input"),
|
||||
("trigger", "soon", "trigger: Input should be an object"),
|
||||
("keep", 5, "keep: Input should be an object"),
|
||||
],
|
||||
)
|
||||
async def test_malformed_field_400_names_the_edit_and_field_as_the_api_does(
|
||||
field: str, value: object, message: str
|
||||
) -> None:
|
||||
# api.anthropic.com, measured 2026-10-19, answers these four with these messages for the edit at index 1.
|
||||
# api.anthropic.com answers these with these messages for the edit at index 1 (trigger and keep measured
|
||||
# 2026-10-28, the rest 2026-10-19).
|
||||
error = await _malformed_spec_error(
|
||||
[{"type": "clear_thinking_20251015"}, {"type": "clear_tool_uses_20250919", field: value}]
|
||||
)
|
||||
|
|
@ -625,6 +731,28 @@ async def test_drop_params_drops_only_the_malformed_field(field: str, value: obj
|
|||
assert _tool_inputs(result.messages) == {"toolu_00": {}, "toolu_01": {}, "toolu_02": {"location": "City2"}}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("field", "value"), [("trigger", "soon"), ("keep", 5)])
|
||||
async def test_drop_params_treats_a_malformed_trigger_or_keep_as_never_sent(field: str, value: object) -> None:
|
||||
async def run(edit: dict[str, object], drop_invalid_edits: bool) -> PolyfillResult | None:
|
||||
return await _run_polyfill_if_enabled(
|
||||
model=MODEL,
|
||||
messages=_make_history(n_pairs=5),
|
||||
tools=None,
|
||||
system=None,
|
||||
context_management_spec={"edits": [edit]},
|
||||
litellm_metadata={},
|
||||
additional_drop_params=None,
|
||||
llm_router=None,
|
||||
drop_invalid_edits=drop_invalid_edits,
|
||||
)
|
||||
|
||||
malformed: Final = {**_VALID_EDIT, field: value}
|
||||
with pytest.raises(AnthropicContextManagementError, match=rf"\.{field}: Input should be an object$"):
|
||||
await run(malformed, drop_invalid_edits=False)
|
||||
never_sent: Final = {key: spec for key, spec in _VALID_EDIT.items() if key != field}
|
||||
assert await run(malformed, drop_invalid_edits=True) == await run(never_sent, drop_invalid_edits=True)
|
||||
|
||||
|
||||
async def test_malformed_edit_is_rejected_before_a_compact_summary_is_paid_for(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
|
|
|
|||
|
|
@ -29,7 +29,7 @@ from litellm.llms.anthropic.messages.context_management.editors.compact import (
|
|||
_augment_system_with_summary,
|
||||
_extract_summary_text,
|
||||
_select_last_user_question,
|
||||
_slice_around_compaction_block,
|
||||
slice_around_compaction_block,
|
||||
_strip_compaction_blocks,
|
||||
apply_client_compaction_block_history,
|
||||
apply_compact_20260112,
|
||||
|
|
@ -153,7 +153,7 @@ def test_applied_edits_for_response_includes_compact_when_block_present():
|
|||
|
||||
def test_slice_around_compaction_block_found():
|
||||
messages = _messages_with_compaction("my summary")
|
||||
sliced, block = _slice_around_compaction_block(messages)
|
||||
sliced, block = slice_around_compaction_block(messages)
|
||||
assert block is not None
|
||||
assert block["type"] == "compaction"
|
||||
assert block["content"] == "my summary"
|
||||
|
|
@ -164,7 +164,7 @@ def test_slice_around_compaction_block_found():
|
|||
|
||||
def test_slice_around_compaction_block_not_found():
|
||||
messages = _simple_messages()
|
||||
sliced, block = _slice_around_compaction_block(messages)
|
||||
sliced, block = slice_around_compaction_block(messages)
|
||||
assert block is None
|
||||
assert sliced is messages # same object, no copy
|
||||
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ Unit tests for the context_management polyfill dispatcher.
|
|||
from litellm.llms.anthropic.messages.context_management import (
|
||||
apply_context_management,
|
||||
)
|
||||
from litellm.llms.anthropic.messages.convertible_blocks import drop_non_string_tool_result_texts
|
||||
|
||||
MODEL = "xai/grok-4"
|
||||
|
||||
|
|
@ -160,3 +161,57 @@ async def test_sync_editor_counts_tokens_off_the_event_loop():
|
|||
|
||||
assert result.messages == messages
|
||||
assert_loop_stayed_free(took, lags)
|
||||
|
||||
|
||||
def _history_with_null_text_beside_a_short_first_result():
|
||||
return [
|
||||
{"role": "user", "content": "Hi"},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": [{"type": "tool_use", "id": "t0", "name": "f", "input": {}}],
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{
|
||||
"type": "tool_result",
|
||||
"tool_use_id": "t0",
|
||||
"content": [{"type": "text", "text": "ok"}, {"type": "text", "text": None}],
|
||||
}
|
||||
],
|
||||
},
|
||||
*_history_with_two_tool_pairs()[1:],
|
||||
]
|
||||
|
||||
|
||||
async def test_drop_invalid_edits_clears_as_if_malformed_tool_result_text_was_never_sent():
|
||||
spec = {
|
||||
"edits": [
|
||||
{
|
||||
"type": "clear_tool_uses_20250919",
|
||||
"trigger": {"type": "tool_uses", "value": 2},
|
||||
"keep": {"type": "tool_uses", "value": 1},
|
||||
}
|
||||
]
|
||||
}
|
||||
history = _history_with_null_text_beside_a_short_first_result()
|
||||
|
||||
from_raw = await apply_context_management(
|
||||
model=MODEL,
|
||||
messages=history,
|
||||
tools=None,
|
||||
system=None,
|
||||
context_management_spec=spec,
|
||||
drop_invalid_edits=True,
|
||||
)
|
||||
from_sent = await apply_context_management(
|
||||
model=MODEL,
|
||||
messages=list(drop_non_string_tool_result_texts(history, True)),
|
||||
tools=None,
|
||||
system=None,
|
||||
context_management_spec=spec,
|
||||
drop_invalid_edits=True,
|
||||
)
|
||||
|
||||
assert from_raw.applied_edits == from_sent.applied_edits
|
||||
assert from_raw.messages == from_sent.messages
|
||||
|
|
|
|||
|
|
@ -2,7 +2,8 @@ import datetime
|
|||
import json
|
||||
import os
|
||||
import sys
|
||||
from collections.abc import Mapping
|
||||
from collections.abc import AsyncIterator, Mapping
|
||||
from itertools import chain
|
||||
from typing import Final, cast
|
||||
from unittest.mock import AsyncMock, patch
|
||||
|
||||
|
|
@ -761,3 +762,325 @@ async def test_recomputed_cost_of_a_served_bridge_response_prices_the_reasoning_
|
|||
assert logging_obj.cost_breakdown is not None
|
||||
assert logging_obj.cost_breakdown["output_cost"] == pytest.approx(expected_output)
|
||||
assert logging_obj.cost_breakdown["reasoning_cost"] == pytest.approx(expected_reasoning)
|
||||
|
||||
|
||||
TEXT_RESPONSE: Final = {
|
||||
"id": "resp_cleared",
|
||||
"object": "response",
|
||||
"created_at": 1,
|
||||
"status": "completed",
|
||||
"model": "gpt-5.6-luna",
|
||||
"output": [
|
||||
{
|
||||
"type": "message",
|
||||
"id": "msg_1",
|
||||
"role": "assistant",
|
||||
"status": "completed",
|
||||
"content": [{"type": "output_text", "text": "done", "annotations": []}],
|
||||
}
|
||||
],
|
||||
"parallel_tool_calls": True,
|
||||
"tool_choice": "auto",
|
||||
"tools": [],
|
||||
"usage": {"input_tokens": 3, "output_tokens": 4, "total_tokens": 7},
|
||||
}
|
||||
CLEAR_EDIT: Final = {
|
||||
"type": "clear_tool_uses_20250919",
|
||||
"trigger": {"type": "tool_uses", "value": 1},
|
||||
"keep": {"type": "tool_uses", "value": 1},
|
||||
}
|
||||
COMPACT_EDIT: Final = {"type": "compact_20260112", "trigger": {"type": "input_tokens", "value": 150000}}
|
||||
|
||||
|
||||
def _read(i: int) -> tuple[dict[str, object], dict[str, object]]:
|
||||
return (
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": [{"type": "tool_use", "id": f"toolu_{i}", "name": "Bash", "input": {"command": f"cat {i}"}}],
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": [{"type": "tool_result", "tool_use_id": f"toolu_{i}", "content": f"log {i} " * 40}],
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def _three_reads() -> list[dict[str, object]]:
|
||||
return [{"role": "user", "content": "Read the logs."}, *chain.from_iterable(_read(i) for i in range(3))]
|
||||
|
||||
|
||||
async def _cleared_bridge(
|
||||
respx_mock: respx.MockRouter,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
*,
|
||||
stream: bool,
|
||||
edits: list[dict[str, object]],
|
||||
drop_params: bool = False,
|
||||
system: object = None,
|
||||
) -> tuple[object, Mapping[str, object]]:
|
||||
monkeypatch.setenv("OPENAI_API_KEY", "sk-clear-test")
|
||||
monkeypatch.setenv("DISABLE_AIOHTTP_TRANSPORT", "True")
|
||||
litellm.in_memory_llm_clients_cache.flush_cache()
|
||||
upstream: Final = respx_mock.post("https://api.openai.com/v1/responses")
|
||||
if stream:
|
||||
upstream.respond(status_code=200, headers={"Content-Type": "text/event-stream"}, content=RESPONSES_SSE_BODY)
|
||||
else:
|
||||
upstream.respond(status_code=200, json=TEXT_RESPONSE)
|
||||
response: Final = await LiteLLMMessagesToResponsesAPIHandler.async_anthropic_messages_handler(
|
||||
max_tokens=1024,
|
||||
messages=_three_reads(),
|
||||
model="openai/gpt-5.6-luna",
|
||||
tools=[dict(BASH_TOOL)],
|
||||
stream=stream,
|
||||
custom_llm_provider="openai",
|
||||
context_management={"edits": edits},
|
||||
drop_params=drop_params,
|
||||
system=system,
|
||||
)
|
||||
body: Final = json.loads(upstream.calls.last.request.content)
|
||||
if not stream:
|
||||
return response, body
|
||||
return [json.loads(chunk.decode().split("data: ", 1)[1]) async for chunk in response], body
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_clear_tool_uses_clears_old_results_before_the_responses_api_call(
|
||||
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
from litellm.llms.anthropic.messages.context_management.constants import CLEARED_TOOL_RESULT_PLACEHOLDER
|
||||
|
||||
served, body = await _cleared_bridge(respx_mock, monkeypatch, stream=False, edits=[dict(CLEAR_EDIT)])
|
||||
|
||||
outputs: Final = {
|
||||
item["call_id"]: item["output"] for item in body["input"] if item.get("type") == "function_call_output"
|
||||
}
|
||||
assert [CLEARED_TOOL_RESULT_PLACEHOLDER in json.dumps(outputs[f"toolu_{i}"]) for i in range(3)] == [
|
||||
True,
|
||||
True,
|
||||
False,
|
||||
]
|
||||
assert "context_management" not in body
|
||||
assert isinstance(served, dict)
|
||||
assert served["context_management"]["applied_edits"][0]["cleared_tool_uses"] == 2
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_streamed_clear_tool_uses_reports_its_edits_on_the_final_message_delta(
|
||||
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
events, _ = await _cleared_bridge(respx_mock, monkeypatch, stream=True, edits=[dict(CLEAR_EDIT)])
|
||||
|
||||
assert isinstance(events, list)
|
||||
message_delta: Final = next(event for event in events if event["type"] == "message_delta")
|
||||
assert message_delta["context_management"]["applied_edits"][0]["cleared_tool_uses"] == 2
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_compact_stays_native_while_clear_tool_uses_runs_in_the_gateway(
|
||||
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
served, body = await _cleared_bridge(
|
||||
respx_mock, monkeypatch, stream=False, edits=[dict(CLEAR_EDIT), dict(COMPACT_EDIT)]
|
||||
)
|
||||
|
||||
assert body["context_management"] == [{"type": "compaction", "compact_threshold": 150000}]
|
||||
assert isinstance(served, dict)
|
||||
assert [edit["type"] for edit in served["context_management"]["applied_edits"]] == ["clear_tool_uses_20250919"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_malformed_clear_tool_uses_edit_is_a_400_before_the_responses_api_call(
|
||||
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
from litellm.llms.anthropic.messages.context_management import AnthropicContextManagementError
|
||||
|
||||
upstream: Final = respx_mock.post("https://api.openai.com/v1/responses").respond(
|
||||
status_code=200, json=TEXT_RESPONSE
|
||||
)
|
||||
monkeypatch.setenv("OPENAI_API_KEY", "sk-clear-test")
|
||||
with pytest.raises(AnthropicContextManagementError) as raised:
|
||||
await LiteLLMMessagesToResponsesAPIHandler.async_anthropic_messages_handler(
|
||||
max_tokens=1024,
|
||||
messages=_three_reads(),
|
||||
model="openai/gpt-5.6-luna",
|
||||
custom_llm_provider="openai",
|
||||
context_management={"edits": [{**CLEAR_EDIT, "exclude_tools": "Bash"}]},
|
||||
)
|
||||
assert raised.value.status_code == 400
|
||||
assert not upstream.called
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_malformed_text_in_a_result_clearing_would_drop_is_still_a_400(
|
||||
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
upstream: Final = respx_mock.post("https://api.openai.com/v1/responses").respond(
|
||||
status_code=200, json=TEXT_RESPONSE
|
||||
)
|
||||
monkeypatch.setenv("OPENAI_API_KEY", "sk-clear-test")
|
||||
reads: Final = _three_reads()
|
||||
malformed_oldest_result: Final = {
|
||||
"role": "user",
|
||||
"content": [{"type": "tool_result", "tool_use_id": "toolu_0", "content": [{"type": "text", "text": None}]}],
|
||||
}
|
||||
with pytest.raises(litellm.BadRequestError) as rejected:
|
||||
await LiteLLMMessagesToResponsesAPIHandler.async_anthropic_messages_handler(
|
||||
max_tokens=1024,
|
||||
messages=[*reads[:2], malformed_oldest_result, *reads[3:]],
|
||||
model="openai/gpt-5.6-luna",
|
||||
tools=[dict(BASH_TOOL)],
|
||||
custom_llm_provider="openai",
|
||||
context_management={"edits": [dict(CLEAR_EDIT)]},
|
||||
)
|
||||
assert rejected.value.status_code == 400
|
||||
assert "tool_result toolu_0 content.0.text: Input should be a valid string" in rejected.value.message
|
||||
assert not upstream.called
|
||||
|
||||
|
||||
def test_the_sync_path_clears_old_results_too(respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
from litellm.llms.anthropic.messages.context_management.constants import CLEARED_TOOL_RESULT_PLACEHOLDER
|
||||
|
||||
monkeypatch.setenv("OPENAI_API_KEY", "sk-clear-test")
|
||||
monkeypatch.setenv("DISABLE_AIOHTTP_TRANSPORT", "True")
|
||||
litellm.in_memory_llm_clients_cache.flush_cache()
|
||||
upstream: Final = respx_mock.post("https://api.openai.com/v1/responses").respond(
|
||||
status_code=200, json=TEXT_RESPONSE
|
||||
)
|
||||
|
||||
served: Final = LiteLLMMessagesToResponsesAPIHandler.anthropic_messages_handler(
|
||||
max_tokens=1024,
|
||||
messages=_three_reads(),
|
||||
model="openai/gpt-5.6-luna",
|
||||
tools=[dict(BASH_TOOL)],
|
||||
custom_llm_provider="openai",
|
||||
context_management={"edits": [dict(CLEAR_EDIT)]},
|
||||
)
|
||||
|
||||
body: Final = json.loads(upstream.calls.last.request.content)
|
||||
outputs: Final = {
|
||||
item["call_id"]: item["output"] for item in body["input"] if item.get("type") == "function_call_output"
|
||||
}
|
||||
assert [CLEARED_TOOL_RESULT_PLACEHOLDER in json.dumps(outputs[f"toolu_{i}"]) for i in range(3)] == [
|
||||
True,
|
||||
True,
|
||||
False,
|
||||
]
|
||||
assert isinstance(served, dict)
|
||||
assert served["context_management"]["applied_edits"][0]["cleared_tool_uses"] == 2
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_an_openai_style_context_management_list_is_still_not_sent_to_the_responses_api(
|
||||
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
upstream: Final = respx_mock.post("https://api.openai.com/v1/responses").respond(
|
||||
status_code=200, json=TEXT_RESPONSE
|
||||
)
|
||||
monkeypatch.setenv("OPENAI_API_KEY", "sk-clear-test")
|
||||
monkeypatch.setenv("DISABLE_AIOHTTP_TRANSPORT", "True")
|
||||
litellm.in_memory_llm_clients_cache.flush_cache()
|
||||
|
||||
served: Final = await LiteLLMMessagesToResponsesAPIHandler.async_anthropic_messages_handler(
|
||||
max_tokens=1024,
|
||||
messages=_three_reads(),
|
||||
model="openai/gpt-5.6-luna",
|
||||
custom_llm_provider="openai",
|
||||
context_management=[{"type": "compaction", "compact_threshold": 500}],
|
||||
)
|
||||
|
||||
assert "context_management" not in json.loads(upstream.calls.last.request.content)
|
||||
assert isinstance(served, dict)
|
||||
assert "context_management" not in served
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_under_drop_params_a_malformed_edit_still_clears_without_its_bad_field(
|
||||
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
from litellm.llms.anthropic.messages.context_management.constants import CLEARED_TOOL_RESULT_PLACEHOLDER
|
||||
|
||||
served, body = await _cleared_bridge(
|
||||
respx_mock, monkeypatch, stream=False, edits=[{**CLEAR_EDIT, "exclude_tools": "Bash"}], drop_params=True
|
||||
)
|
||||
|
||||
outputs: Final = {
|
||||
item["call_id"]: item["output"] for item in body["input"] if item.get("type") == "function_call_output"
|
||||
}
|
||||
assert [CLEARED_TOOL_RESULT_PLACEHOLDER in json.dumps(outputs[f"toolu_{i}"]) for i in range(3)] == [
|
||||
True,
|
||||
True,
|
||||
False,
|
||||
]
|
||||
assert isinstance(served, dict)
|
||||
assert served["context_management"]["applied_edits"][0]["cleared_tool_uses"] == 2
|
||||
|
||||
|
||||
def _cleared_outputs(body: Mapping[str, object]) -> list[bool]:
|
||||
from litellm.llms.anthropic.messages.context_management.constants import CLEARED_TOOL_RESULT_PLACEHOLDER
|
||||
|
||||
outputs: Final = {
|
||||
item["call_id"]: item["output"] for item in body["input"] if item.get("type") == "function_call_output"
|
||||
}
|
||||
return [CLEARED_TOOL_RESULT_PLACEHOLDER in json.dumps(outputs[f"toolu_{i}"]) for i in range(3)]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"system",
|
||||
[7, {"type": "text", "text": "Be brief."}, ["Be brief."], ["Be brief.", {"type": "text", "text": "Be kind."}]],
|
||||
ids=["number", "object", "strings", "mixed_list"],
|
||||
)
|
||||
@pytest.mark.parametrize("stream", [False, True], ids=["non_stream", "stream"])
|
||||
async def test_clear_tool_uses_still_clears_when_the_system_prompt_is_not_text_or_a_block_list(
|
||||
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, system: object, stream: bool
|
||||
) -> None:
|
||||
_, body = await _cleared_bridge(respx_mock, monkeypatch, stream=stream, edits=[dict(CLEAR_EDIT)], system=system)
|
||||
|
||||
assert _cleared_outputs(body) == [True, True, False]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("system", "expect_cleared"),
|
||||
[
|
||||
(["Stray.", {"type": "text", "text": "Follow the policy. " * 1500}], [True, True, False]),
|
||||
(["Follow the policy. " * 1500], [False, False, False]),
|
||||
],
|
||||
ids=["text_block_counts", "stray_string_does_not"],
|
||||
)
|
||||
async def test_input_tokens_trigger_counts_the_text_blocks_of_a_system_list_with_stray_items(
|
||||
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, system: object, expect_cleared: list[bool]
|
||||
) -> None:
|
||||
trigger_between_messages_and_system: Final = litellm.token_counter(model="gpt-4o", messages=_three_reads()) + 1000
|
||||
edit: Final = {**CLEAR_EDIT, "trigger": {"type": "input_tokens", "value": trigger_between_messages_and_system}}
|
||||
|
||||
_, body = await _cleared_bridge(respx_mock, monkeypatch, stream=False, edits=[edit], system=system)
|
||||
|
||||
assert _cleared_outputs(body) == expect_cleared
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_the_sync_path_reports_its_edits_on_the_streamed_message_delta(
|
||||
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
monkeypatch.setenv("OPENAI_API_KEY", "sk-clear-test")
|
||||
monkeypatch.setenv("DISABLE_AIOHTTP_TRANSPORT", "True")
|
||||
litellm.in_memory_llm_clients_cache.flush_cache()
|
||||
respx_mock.post("https://api.openai.com/v1/responses").respond(
|
||||
status_code=200, headers={"Content-Type": "text/event-stream"}, content=RESPONSES_SSE_BODY
|
||||
)
|
||||
|
||||
sse: Final = LiteLLMMessagesToResponsesAPIHandler.anthropic_messages_handler(
|
||||
max_tokens=1024,
|
||||
messages=_three_reads(),
|
||||
model="openai/gpt-5.6-luna",
|
||||
tools=[dict(BASH_TOOL)],
|
||||
stream=True,
|
||||
custom_llm_provider="openai",
|
||||
context_management={"edits": [dict(CLEAR_EDIT)]},
|
||||
)
|
||||
events: Final = [
|
||||
json.loads(chunk.decode().split("data: ", 1)[1]) async for chunk in cast(AsyncIterator[bytes], sse)
|
||||
]
|
||||
message_delta: Final = next(event for event in events if event["type"] == "message_delta")
|
||||
assert message_delta["context_management"]["applied_edits"][0]["cleared_tool_uses"] == 2
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@ import copy
|
|||
import json
|
||||
import operator
|
||||
from collections.abc import Mapping
|
||||
from itertools import chain
|
||||
from typing import Final
|
||||
|
||||
import httpx
|
||||
|
|
@ -11,9 +12,10 @@ from pydantic import JsonValue, ValidationError
|
|||
|
||||
import litellm
|
||||
from litellm.litellm_core_utils.tool_fields import DropPolicy
|
||||
from litellm.llms.anthropic.messages.context_management.constants import CLEARED_TOOL_RESULT_PLACEHOLDER
|
||||
from litellm.llms.openai.responses.count_tokens.token_counter import OpenAITokenCounter
|
||||
from litellm.responses.utils import ResponsesAPIRequestUtils
|
||||
from litellm.types.utils import TokenCountMessagesFormat, TokenCountResponse
|
||||
from litellm.types.utils import CountContextManagement, TokenCountMessagesFormat, TokenCountResponse
|
||||
|
||||
COUNT_URL: Final = "https://api.openai.com/v1/responses/input_tokens"
|
||||
|
||||
|
|
@ -484,6 +486,122 @@ async def test_anthropic_format_tool_result_text_block_without_text_still_counts
|
|||
assert {"type": "function_call_output", "call_id": "toolu_log", "output": "build ok\n\ndone"} in sent["input"]
|
||||
|
||||
|
||||
def _tool_pair(index: int) -> tuple[dict[str, object], dict[str, object]]:
|
||||
return (
|
||||
{"role": "assistant", "content": [{"type": "tool_use", "id": f"toolu_{index}", "name": "log", "input": {}}]},
|
||||
{
|
||||
"role": "user",
|
||||
"content": [{"type": "tool_result", "tool_use_id": f"toolu_{index}", "content": f"log {index} " * 40}],
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
TOOL_PAIRS: Final[list[dict[str, object]]] = [
|
||||
{"role": "user", "content": "Compare the logs."},
|
||||
*chain.from_iterable(_tool_pair(index) for index in range(3)),
|
||||
]
|
||||
|
||||
|
||||
async def _count_with_edit(
|
||||
monkeypatch: pytest.MonkeyPatch, edit: dict[str, JsonValue], drop_invalid_edits: bool, system: object = None
|
||||
) -> tuple[TokenCountResponse | None, respx.Route]:
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
with respx.mock:
|
||||
route: Final = respx.post(COUNT_URL).mock(return_value=httpx.Response(200, json={"input_tokens": 9}))
|
||||
result: Final = await OpenAITokenCounter().count_tokens(
|
||||
model_to_use="gpt-4o",
|
||||
messages=copy.deepcopy(TOOL_PAIRS),
|
||||
contents=None,
|
||||
deployment={"litellm_params": {"api_key": "sk-test"}},
|
||||
request_model="openai/gpt-4o",
|
||||
system=system,
|
||||
messages_format="anthropic",
|
||||
context_management=CountContextManagement(spec={"edits": [edit]}, drop_invalid_edits=drop_invalid_edits),
|
||||
)
|
||||
return result, route
|
||||
|
||||
|
||||
def _sent_outputs(route: respx.Route) -> dict[str, str]:
|
||||
sent: Final = json.loads(route.calls.last.request.content)
|
||||
return {item["call_id"]: item["output"] for item in sent["input"] if item["type"] == "function_call_output"}
|
||||
|
||||
|
||||
CLEAR_EDIT: Final[dict[str, JsonValue]] = {
|
||||
"type": "clear_tool_uses_20250919",
|
||||
"trigger": {"type": "tool_uses", "value": 1},
|
||||
"keep": {"type": "tool_uses", "value": 1},
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_anthropic_format_counts_the_history_the_clear_edit_leaves_for_inference(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
result, route = await _count_with_edit(monkeypatch, CLEAR_EDIT, drop_invalid_edits=False)
|
||||
|
||||
assert result is not None
|
||||
assert (result.error, result.total_tokens) == (False, 9)
|
||||
assert _sent_outputs(route) == {
|
||||
"toolu_0": CLEARED_TOOL_RESULT_PLACEHOLDER,
|
||||
"toolu_1": CLEARED_TOOL_RESULT_PLACEHOLDER,
|
||||
"toolu_2": "log 2 " * 40,
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"system",
|
||||
[
|
||||
"Follow the policy. " * 1500,
|
||||
[{"type": "text", "text": "Follow the policy. " * 1500}],
|
||||
["Stray.", {"type": "text", "text": "Follow the policy. " * 1500}],
|
||||
],
|
||||
ids=["string", "blocks", "blocks_beside_a_stray_string"],
|
||||
)
|
||||
async def test_anthropic_format_input_tokens_trigger_counts_the_system_prompt_like_inference(
|
||||
monkeypatch: pytest.MonkeyPatch, system: object
|
||||
) -> None:
|
||||
message_tokens: Final = litellm.token_counter(model="gpt-4o", messages=TOOL_PAIRS)
|
||||
trigger_between_messages_and_system: Final = message_tokens + 1000
|
||||
edit: Final[dict[str, JsonValue]] = {
|
||||
**CLEAR_EDIT,
|
||||
"trigger": {"type": "input_tokens", "value": trigger_between_messages_and_system},
|
||||
}
|
||||
|
||||
_, without_system = await _count_with_edit(monkeypatch, edit, drop_invalid_edits=False)
|
||||
_, with_system = await _count_with_edit(monkeypatch, edit, drop_invalid_edits=False, system=system)
|
||||
|
||||
assert _sent_outputs(without_system)["toolu_0"] == "log 0 " * 40
|
||||
assert _sent_outputs(with_system) == {
|
||||
"toolu_0": CLEARED_TOOL_RESULT_PLACEHOLDER,
|
||||
"toolu_1": CLEARED_TOOL_RESULT_PLACEHOLDER,
|
||||
"toolu_2": "log 2 " * 40,
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_anthropic_format_malformed_clear_edit_answers_400_without_calling_openai(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
result, route = await _count_with_edit(monkeypatch, {**CLEAR_EDIT, "exclude_tools": "log"}, drop_invalid_edits=False)
|
||||
|
||||
assert route.called is False
|
||||
assert result is not None
|
||||
assert (result.error, result.status_code) == (True, 400)
|
||||
assert "exclude_tools" in str(result.error_message)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_anthropic_format_drop_params_counts_the_clear_without_its_malformed_field(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
result, route = await _count_with_edit(monkeypatch, {**CLEAR_EDIT, "exclude_tools": "log"}, drop_invalid_edits=True)
|
||||
|
||||
assert result is not None
|
||||
assert result.error is False
|
||||
assert _sent_outputs(route)["toolu_0"] == CLEARED_TOOL_RESULT_PLACEHOLDER
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_count_uses_the_global_openai_key_chat_would_use(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
|
|
|
|||
|
|
@ -649,7 +649,7 @@ async def test_anthropic_view_counts_the_unedited_body_without_an_error_log_when
|
|||
assert not any(record.levelno >= logging.ERROR for record in caplog.records)
|
||||
|
||||
|
||||
async def test_anthropic_view_counts_the_turns_after_a_compaction_block_when_an_edit_before_compact_fails() -> None:
|
||||
async def test_anthropic_view_clears_only_the_turns_after_a_compaction_block_when_clear_runs_before_compact() -> None:
|
||||
clear_before_compact: Final[JsonValue] = {
|
||||
"edits": [
|
||||
{
|
||||
|
|
|
|||
|
|
@ -2064,22 +2064,6 @@ async def test_framing_keeps_the_400_for_blocks_the_provider_counter_cannot_conv
|
|||
},
|
||||
id="web_search_blocks",
|
||||
),
|
||||
pytest.param(
|
||||
{
|
||||
"messages": [{"role": "user", "content": "What is new?"}],
|
||||
"tools": [{"name": "search", "input_schema": {"type": "object", "properties": {"query": True}}}],
|
||||
},
|
||||
id="boolean_subschema_tool",
|
||||
),
|
||||
pytest.param(
|
||||
{
|
||||
"messages": [{"role": "user", "content": "What is new?"}],
|
||||
"tools": [
|
||||
{"name": "search", "input_schema": {"type": "object", "properties": {"tags": {"type": "array"}}}}
|
||||
],
|
||||
},
|
||||
id="array_schema_without_items",
|
||||
),
|
||||
pytest.param(
|
||||
{"messages": [], "contents": [{"role": "user", "parts": [{"text": "What is new?"}]}]},
|
||||
id="google_contents",
|
||||
|
|
@ -2138,6 +2122,44 @@ async def test_framed_estimate_that_cannot_read_the_body_answers_the_provider_er
|
|||
assert readable.count_source == "local_estimate"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("shorthand", "explicit"),
|
||||
[
|
||||
pytest.param({"type": "array"}, {"type": "array", "items": {}}, id="array_schema_without_items"),
|
||||
pytest.param(True, {}, id="boolean_subschema"),
|
||||
],
|
||||
)
|
||||
async def test_framed_estimate_counts_a_json_schema_shorthand_like_its_explicit_form(
|
||||
monkeypatch: pytest.MonkeyPatch, shorthand: object, explicit: dict[str, object]
|
||||
):
|
||||
counter: Final = _ScriptedCounter({"us.moonshotai.kimi-k3": _provider_reply("kimi", status_code=400)})
|
||||
monkeypatch.setattr(
|
||||
litellm.proxy.proxy_server,
|
||||
"llm_router",
|
||||
Router(model_list=[{"model_name": "kimi", "litellm_params": {"model": "bedrock/us.moonshotai.kimi-k3"}}]),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
litellm.proxy.proxy_server, "_get_provider_token_counter", lambda deployment, model: (counter, model, "bedrock")
|
||||
)
|
||||
|
||||
async def count(schema: object) -> TokenCountResponse:
|
||||
return await token_counter(
|
||||
request=TokenCountRequest(
|
||||
model="kimi",
|
||||
messages=[{"role": "user", "content": "What is new?"}],
|
||||
tools=[{"name": "search", "input_schema": {"type": "object", "properties": {"tags": schema}}}],
|
||||
),
|
||||
call_endpoint=True,
|
||||
)
|
||||
|
||||
from_shorthand: Final = await count(shorthand)
|
||||
from_explicit: Final = await count(explicit)
|
||||
|
||||
assert from_shorthand.count_source == "local_estimate"
|
||||
assert from_shorthand.total_tokens == from_explicit.total_tokens
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"contents",
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue