This commit is contained in:
devin-ai-integration[bot] 2026-10-30 00:20:46 +00:00 • committed by GitHub
commit 343ed79058
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
15 changed files with 520 additions and 82 deletions

View file

@ -15,6 +15,7 @@ import anyio
import anyio.lowlevel
import httpx
import tiktoken
from pydantic import TypeAdapter
from tiktoken.model import MODEL_TO_ENCODING
from typing_extensions import ParamSpec, TypeVar
@ -62,6 +63,7 @@ from litellm.types.llms.anthropic import (
AnthropicMessagesToolResultParam,
AnthropicMessagesToolUseParam,
AnthropicSearchResultParam,
CompactionBlock,
)
from litellm.types.llms.openai import (
AllMessageValues,
@ -843,6 +845,8 @@ def _count_messages(
use_default_image_token_count,
default_token_count,
)
elif key == "provider_specific_fields":
num_tokens += sum(map(params.count_function, _compaction_summaries(value)))
elif key == "search_results" and isinstance(value, list):
from litellm.litellm_core_utils.prompt_templates.common_utils import (
extract_search_results_text,
@ -857,6 +861,25 @@ def _count_messages(
return num_tokens
_OBJECT_LIST: Final = TypeAdapter(list[object])
def _compaction_summaries(provider_specific_fields: object) -> tuple[str, ...]:
fields: Final = as_str_mapping(provider_specific_fields)
blocks: Final = fields.get("compaction_blocks") if fields is not None else None
block_fields: Final = (
tuple(map(as_str_mapping, _OBJECT_LIST.validate_python(blocks))) if isinstance(blocks, list) else ()
)
return tuple(
content
for block in block_fields
if block is not None
and block.get("type") == "compaction"
and isinstance(content := block.get("content"), str)
and content
)
def _count_extra(
count_function: TokenCounterFunction,
tools: list[ChatCompletionToolParam] | None,
@ -1379,6 +1402,11 @@ def _count_anthropic_content(
return tokens
_TEXT_FIELD_BY_BLOCK_TYPE: Final = MappingProxyType(
{"text": "text", "thinking": "thinking", "redacted_thinking": "thinking", "compaction": "content"}
)
def _count_content_list(
count_function: TokenCounterFunction,
content_list: str
@ -1390,6 +1418,7 @@ def _count_content_list(
| AnthropicMessagesImageParam
| AnthropicMessagesDocumentParam
| AnthropicSearchResultParam
| CompactionBlock
],
use_default_image_token_count: bool,
default_token_count: int | None,
@ -1400,8 +1429,10 @@ def _count_content_list(
for c in content_list:
if isinstance(c, str):
num_tokens += count_function(c)
elif c["type"] == "text":
num_tokens += count_function(str(c.get("text", "")))
elif c["type"] in _TEXT_FIELD_BY_BLOCK_TYPE:
block_text = str(c.get(_TEXT_FIELD_BY_BLOCK_TYPE[c["type"]]) or "")
if block_text:
num_tokens += count_function(block_text)
elif c["type"] == "image_url":
image_url = c.get("image_url")
num_tokens += _count_image_tokens(image_url, use_default_image_token_count)
@ -1431,12 +1462,6 @@ def _count_content_list(
use_default_image_token_count,
default_token_count,
)
elif c["type"] in ("thinking", "redacted_thinking"):
# Claude extended thinking content block
# Count the thinking text and skip the opaque blobs (signature, redacted data)
thinking_text = str(c.get("thinking", ""))
if thinking_text:
num_tokens += count_function(thinking_text)
elif c["type"] == "tool_reference":
# Anthropic tool-search reference block: a lightweight pointer to
# a deferred tool, e.g. {"type": "tool_reference", "tool_name": ...}.
@ -1454,7 +1479,7 @@ def _count_content_list(
f"Invalid content item type: {content_type}. "
f"Expected str or dict with 'type' field "
f"(text, image_url, image, document, file, tool_use, tool_result, thinking, redacted_thinking, "
f"tool_reference, search_result)."
f"compaction, tool_reference, search_result)."
)
return num_tokens
except Exception as e:

View file

@ -8,12 +8,13 @@ import re
from collections.abc import Container, Mapping, MutableMapping, Sequence
from dataclasses import dataclass
from datetime import datetime, timezone
from itertools import chain
from types import MappingProxyType
from typing import Annotated, Any, ClassVar, Final, Literal, TypeVar
from urllib.parse import quote, urlparse
import httpx
from pydantic import BeforeValidator, ConfigDict, Field, StrictBool, TypeAdapter, ValidationError
from pydantic import BeforeValidator, ConfigDict, Field, JsonValue, StrictBool, TypeAdapter, ValidationError
from typing_extensions import ReadOnly, TypedDict
import litellm
@ -24,6 +25,7 @@ from litellm.constants import (
DEFAULT_REASONING_EFFORT_XHIGH_THINKING_BUDGET,
)
from litellm.exceptions import UnsupportedParamsError
from litellm.litellm_core_utils.core_helpers import as_str_mapping
from litellm.litellm_core_utils.prompt_templates.common_utils import (
get_file_ids_from_messages,
is_encrypted_reasoning_block,
@ -46,6 +48,7 @@ from litellm.llms.anthropic.wif import (
from litellm.llms.base_llm.base_utils import BaseLLMModelInfo, BaseTokenCounter
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.types.llms.anthropic import (
ANTHROPIC_BETA_HEADER_VALUES,
ANTHROPIC_HOSTED_TOOLS,
ANTHROPIC_MID_CONVERSATION_OUTPUT_CONFIG_BETA_HEADER,
ANTHROPIC_MID_CONVERSATION_TOOL_CHANGES_BETA_HEADER,
@ -118,6 +121,7 @@ _CLAUDE_CODE_BILLING_HEADER_PREFIX: Final = "x-anthropic-billing-header:"
OBJECT_MAPPING_ADAPTER: Final = TypeAdapter(dict[object, object])
OBJECT_LIST_ADAPTER: Final = TypeAdapter(list[object])
_DECODED_JSON: Final = TypeAdapter(object)
_JSON_VALUE: Final = TypeAdapter(JsonValue)
_CLAUDE_CODE_USER_AGENT_PREFIXES: Final = ("claude-cli/", "claude-code/")
@ -177,24 +181,68 @@ def is_anthropic_messages_url(url: str) -> bool:
return parsed_url.hostname == "api.anthropic.com" or parsed_url.path.removesuffix("/").endswith("/v1/messages")
def _message_blocks(message: object) -> Sequence[object]:
fields: Final = as_str_mapping(message)
blocks: Final = validated_object_list(fields.get("content")) if fields is not None else None
return blocks or ()
def _is_signed_compaction_block(block: object) -> bool:
fields: Final = as_str_mapping(block)
if fields is None or fields.get("type") != "compaction":
return False
signature: Final = fields.get("signature")
return isinstance(signature, str) and bool(signature)
def carries_signed_compaction_block(messages: Sequence[object]) -> bool:
return any(map(_is_signed_compaction_block, chain.from_iterable(map(_message_blocks, messages))))
def requires_native_compaction_beta(
custom_llm_provider: str,
optional_params: Mapping[str, object],
messages: Sequence[object],
) -> bool:
return custom_llm_provider == "anthropic" and (
optional_params.get("compaction") is not None
or any(
isinstance(block, Mapping)
and block.get("type") == "compaction"
and isinstance(block.get("signature"), str)
and bool(block.get("signature"))
for message in messages
if isinstance(message, Mapping)
for content in (message.get("content"),)
if isinstance(content, (list, tuple))
for block in content
optional_params.get("compaction") is not None or carries_signed_compaction_block(messages)
)
def anthropic_context_management(context_management: JsonValue) -> JsonValue:
if not isinstance(context_management, list):
return context_management
from litellm.llms.anthropic.chat.transformation import AnthropicConfig
edits: Final = AnthropicConfig.map_openai_context_management_to_anthropic(
[entry for entry in context_management if isinstance(entry, dict)]
)
return context_management if edits is None else _JSON_VALUE.validate_python(edits)
def _context_management_edits(spec: JsonValue) -> Sequence[object]:
mapping: Final = as_str_mapping(spec)
return (validated_object_list(mapping.get("edits")) if mapping is not None else None) or ()
def context_management_with_edits(context_management: JsonValue) -> JsonValue:
spec: Final = anthropic_context_management(context_management)
return spec if _context_management_edits(spec) else None
def context_management_betas(context_management: JsonValue) -> tuple[str, ...]:
edits: Final = _context_management_edits(anthropic_context_management(context_management))
edit_types: Final = tuple(edit.get("type") for edit in map(as_str_mapping, edits) if edit is not None)
return tuple(
beta
for beta, used in (
(ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_01_12.value, "compact_20260112" in edit_types),
(
ANTHROPIC_BETA_HEADER_VALUES.CONTEXT_MANAGEMENT_2025_06_27.value,
any(edit_type != "compact_20260112" for edit_type in edit_types),
),
)
if used
)
@ -1367,8 +1415,6 @@ class AnthropicModelInfo(BaseLLMModelInfo):
if is_vertex_request is True:
# Vertex AI requires web search beta header for web search to work
if web_search_tool_used:
from litellm.types.llms.anthropic import ANTHROPIC_BETA_HEADER_VALUES
headers["anthropic-beta"] = ANTHROPIC_BETA_HEADER_VALUES.WEB_SEARCH_2025_03_05.value
elif len(all_betas) > 0:
headers["anthropic-beta"] = ",".join(all_betas)

View file

@ -39,6 +39,7 @@ class AnthropicCountTokensHandler(AnthropicCountTokensConfig):
tools: list[dict[str, JsonValue]] | None = None,
system: JsonValue = None,
optional_params: Mapping[str, JsonValue] | None = None,
context_management: JsonValue = None,
drop_params: bool = False,
) -> dict[str, JsonValue]:
"""
@ -70,6 +71,7 @@ class AnthropicCountTokensHandler(AnthropicCountTokensConfig):
tools=tools,
system=system,
optional_params=optional_params,
context_management=context_management,
drop_params=drop_params,
)
@ -81,7 +83,7 @@ class AnthropicCountTokensHandler(AnthropicCountTokensConfig):
verbose_logger.debug("Making request to: %s", endpoint_url)
# Get required headers
headers: Final = self.get_count_tokens_headers(auth_header)
headers: Final = self.get_count_tokens_headers(auth_header, messages, context_management)
# Use LiteLLM's async httpx client
async_client: Final = get_async_httpx_client(llm_provider=litellm.LlmProviders.ANTHROPIC)

View file

@ -132,6 +132,7 @@ class AnthropicTokenCounter(BaseTokenCounter):
tools=tools,
system=system,
optional_params=count_options,
context_management=context_management.spec if context_management is not None else None,
drop_params=drops_unsupported_params(litellm_params.get("drop_params")),
)

View file

@ -27,11 +27,15 @@ from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import
)
from litellm.litellm_core_utils.url_utils import join_api_base
from litellm.llms.anthropic.common_utils import (
carries_signed_compaction_block,
context_management_betas,
context_management_with_edits,
is_claude_model_target,
merge_anthropic_beta_headers,
tools_without_custom_field,
)
from litellm.llms.anthropic.wif import resolve_anthropic_base
from litellm.types.llms.anthropic import ANTHROPIC_BETA_HEADER_VALUES
from litellm.types.llms.openai import AllMessageValues, ChatCompletionImageObject, ValidUserMessageContentTypes
from litellm.types.utils import TokenCountMessagesFormat, TokenCountResponse
@ -376,6 +380,7 @@ class AnthropicCountTokensConfig:
tools: list[dict[str, JsonValue]] | None = None,
system: JsonValue = None,
optional_params: Mapping[str, JsonValue] | None = None,
context_management: JsonValue = None,
drop_params: bool = False,
) -> dict[str, JsonValue]: # mutable-ok: provider transport requires JSON dictionaries
"""
@ -408,6 +413,7 @@ class AnthropicCountTokensConfig:
drop_custom=drop_params,
),
),
("context_management", context_management_with_edits(context_management)),
)
if value is not None
}
@ -419,18 +425,33 @@ class AnthropicCountTokensConfig:
)
)
def get_count_tokens_headers(self, auth_header: Mapping[str, str]) -> dict[str, str]:
def get_count_tokens_headers(
self,
auth_header: Mapping[str, str],
messages: Sequence[Mapping[str, JsonValue]],
context_management: JsonValue = None,
) -> dict[str, str]:
"""The count-tokens headers around a resolved Anthropic auth header
(``AnthropicModelInfo.get_auth_header``): x-api-key for a static key, an Authorization
bearer for ``ANTHROPIC_AUTH_TOKEN`` and for sk-ant-oat tokens, whose mandatory oauth beta
merges with the token-counting beta instead of replacing it."""
merges with the token-counting beta instead of replacing it. A history that carries a
signed compaction block also needs the compaction beta, or Anthropic rejects the block
as an unknown content type, and ``context_management`` edits need the betas inference sends."""
compaction_betas: Final = (
(ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_09_04.value,)
if carries_signed_compaction_block(messages)
else ()
)
request_betas: Final = (
ANTHROPIC_TOKEN_COUNTING_BETA_VERSION,
*compaction_betas,
*context_management_betas(context_management),
)
return {
"Content-Type": "application/json",
"anthropic-version": "2023-06-01",
**auth_header,
"anthropic-beta": merge_anthropic_beta_headers(
auth_header.get("anthropic-beta"), ANTHROPIC_TOKEN_COUNTING_BETA_VERSION
),
"anthropic-beta": merge_anthropic_beta_headers(auth_header.get("anthropic-beta"), request_betas),
}
def validate_request(

View file

@ -43,6 +43,7 @@ from ...common_utils import (
OBJECT_LIST_ADAPTER,
AnthropicError,
AnthropicModelInfo,
context_management_betas,
is_claude_model_target,
merge_anthropic_beta_headers,
optionally_handle_anthropic_oauth,
@ -745,6 +746,8 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
)
if transformed_context_management is not None:
anthropic_messages_optional_request_params["context_management"] = transformed_context_management
elif drops_unsupported_params(litellm_params.drop_params):
anthropic_messages_optional_request_params.pop("context_management")
####### get required params for all anthropic messages requests ######
# Lazy %s: the f-string previously stringified the entire messages
@ -873,28 +876,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
if requires_native_compaction_beta(custom_llm_provider, optional_params, messages):
beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_09_04.value)
# Check for context management
context_management_param: Final = optional_params.get("context_management")
if context_management_param is not None:
# Check edits array for compact_20260112 type
edits: Final = context_management_param.get("edits", ())
has_compact = False
has_other = False
for edit in edits:
edit_type = edit.get("type", "")
if edit_type == "compact_20260112":
has_compact = True
else:
has_other = True
# Add compact header if any compact edits exist
if has_compact:
beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_01_12.value)
# Add context management header if any other edits exist
if has_other:
beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.CONTEXT_MANAGEMENT_2025_06_27.value)
beta_values.update(context_management_betas(optional_params.get("context_management")))
# Check for structured outputs. Anthropic's newer request shape nests
# the schema under output_config.format; the older top-level

View file

@ -3,7 +3,7 @@ from typing import Final
from litellm.exceptions import UnsupportedParamsError
from litellm.litellm_core_utils.core_helpers import drops_unsupported_params
from litellm.llms.anthropic.common_utils import AnthropicModelInfo
from litellm.llms.anthropic.common_utils import AnthropicModelInfo, context_management_betas
from litellm.llms.anthropic.messages.native.transformation import (
AnthropicMessagesConfig,
messages_carry_output_config,
@ -171,28 +171,7 @@ class VertexAIPartnerModelsAnthropicMessagesConfig(AnthropicMessagesConfig, Vert
if existing_beta:
beta_values.update(b.strip() for b in existing_beta.split(","))
# Check for context management
context_management_param: Final = optional_params.get("context_management")
if context_management_param is not None:
# Check edits array for compact_20260112 type
edits: Final = context_management_param.get("edits", [])
has_compact = False
has_other = False
for edit in edits:
edit_type = edit.get("type", "")
if edit_type == "compact_20260112":
has_compact = True
else:
has_other = True
# Add compact header if any compact edits exist
if has_compact:
beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_01_12.value)
# Add context management header if any other edits exist
if has_other:
beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.CONTEXT_MANAGEMENT_2025_06_27.value)
beta_values.update(context_management_betas(optional_params.get("context_management")))
# Check for web search tool
for tool in tools:

View file

@ -8,7 +8,7 @@ from typing import Final
from fastapi import APIRouter, Depends, HTTPException, Request, Response, status
from fastapi.responses import JSONResponse
from pydantic import JsonValue, ValidationError
from pydantic import JsonValue, TypeAdapter, ValidationError
from typing_extensions import ReadOnly, TypedDict
import litellm
@ -54,6 +54,7 @@ from litellm.proxy.litellm_pre_call_utils import (
should_auto_drop_params_for_agentic_cli,
)
from litellm.router import Router
from litellm.types.utils import TokenCountResponse
router: Final = APIRouter()
@ -334,6 +335,15 @@ def _count_tokens_body_options(data: Mapping[str, JsonValue], auto_drop_params:
}
_PROVIDER_COUNT_REPLY: Final[TypeAdapter[Mapping[str, object] | None]] = TypeAdapter(Mapping[str, object] | None)
def _provider_context_management(response: TokenCountResponse) -> Mapping[str, object]:
original: Final = _PROVIDER_COUNT_REPLY.validate_python(response.original_response)
context_management: Final = original.get("context_management") if original is not None else None
return {} if context_management is None else {"context_management": context_management}
@router.post(
"/v1/messages/count_tokens",
tags=["[beta] Anthropic Messages Token Counting"],
@ -413,7 +423,11 @@ async def count_tokens(
log_llm_api_exception(e, litellm_call_id)
return _anthropic_error_json_response(proxy_exception_from_route_error(e), request)
else:
return {"input_tokens": token_response.total_tokens, **count_source_fields(token_response)}
return {
"input_tokens": token_response.total_tokens,
**_provider_context_management(token_response),
**count_source_fields(token_response),
}
@router.post(

View file

@ -1921,6 +1921,97 @@ def test_token_counter_with_redacted_thinking_content():
assert token_counter(model=model, messages=with_block) == token_counter(model=model, messages=without_block)
def test_token_counter_with_compaction_block():
"""
A replayed on-demand compaction block (Anthropic's signed summary of the earlier turns) counts its
summary text like a text block and nothing for its signature. It used to raise, which turned every
/v1/messages/count_tokens call on a compacted history into a 500 once the provider count fell back here.
"""
model = "anthropic/claude-sonnet-5-5"
summary = "The user is building a recipe app in Python and asked for one-sentence class descriptions."
follow_up = {"role": "user", "content": [{"type": "text", "text": "Now do the same for Ingredient."}]}
signed_block = {"type": "compaction", "content": summary, "signature": "EqQBCkYIBRgCKkBjZ2xhc3M" * 40}
as_compaction = [{"role": "assistant", "content": [signed_block]}, follow_up]
unsigned = [{"role": "assistant", "content": [{"type": "compaction", "content": summary}]}, follow_up]
as_text = [{"role": "assistant", "content": [{"type": "text", "text": summary}]}, follow_up]
assert token_counter(model=model, messages=as_compaction) == token_counter(model=model, messages=as_text)
assert token_counter(model=model, messages=unsigned) == token_counter(model=model, messages=as_text)
def test_token_counter_counts_chat_compaction_blocks_like_their_summary_text():
"""
Chat clients replay a compaction block in the assistant message's provider_specific_fields, and inference
sends it to Anthropic as a content block, so the local count includes its summary text and skips its signature.
"""
model = "anthropic/claude-sonnet-5-5"
summary = "The user is building a recipe app in Python and asked for one-sentence class descriptions."
follow_up = {"role": "user", "content": "Now do the same for Ingredient."}
signed_block = {"type": "compaction", "content": summary, "signature": "EqQBCkYIBRgCKkBjZ2xhc3M" * 40}
replayed = [
{"role": "assistant", "content": None, "provider_specific_fields": {"compaction_blocks": [signed_block]}},
follow_up,
]
as_text = [{"role": "assistant", "content": summary}, follow_up]
assert token_counter(model=model, messages=replayed) == token_counter(model=model, messages=as_text)
class _BosEncoder:
def encode_batch_fast(self, texts: list[str]) -> list[list[int]]:
return [[0] * (len(text.split()) + 1) for text in texts]
def test_token_counter_charges_nothing_for_a_chat_compaction_block_with_no_summary():
"""
A tokenizer that prepends a BOS token charges one token for the empty string, so a replayed
compaction block with an empty summary must not reach it, like an empty thinking block.
"""
model = "anthropic/claude-sonnet-5-5"
tokenizer = {"type": "huggingface_tokenizer", "tokenizer": _BosEncoder()}
follow_up = {"role": "user", "content": "Now do the same for Ingredient."}
replayed = [
{
"role": "assistant",
"content": None,
"provider_specific_fields": {"compaction_blocks": [{"type": "compaction", "content": ""}]},
},
follow_up,
]
bare = [{"role": "assistant", "content": None}, follow_up]
assert token_counter(model=model, messages=replayed, custom_tokenizer=tokenizer) == token_counter(
model=model, messages=bare, custom_tokenizer=tokenizer
)
def test_count_content_list_skips_the_tokenizer_for_empty_block_text():
"""
An empty thinking block, a redacted_thinking block (no thinking text at all) and a compaction
block with no summary carry no tokens, even under a tokenizer that charges for the empty string
(the llama-2 tokenizer prepends its BOS token to every encode call). Counting them through the
tokenizer added one token per block on such models.
"""
from litellm.litellm_core_utils.token_counter import _count_content_list
def charges_for_empty_text(text: str) -> int:
return len(text.split()) + 1
empty_blocks = [
{"type": "thinking", "thinking": "", "signature": "EqcLCkYICxgCKkCrqu6lP..."},
{"type": "thinking", "thinking": None, "signature": "EqcLCkYICxgCKkCrqu6lP..."},
{"type": "redacted_thinking", "data": "EmwKAhgBEgy3va3pzix/LafPsn4aDFIT2Xlxh0L5L8rLVyIw"},
{"type": "compaction", "content": ""},
{"type": "text", "text": ""},
]
assert _count_content_list(charges_for_empty_text, empty_blocks, False, None) == 0
assert _count_content_list(charges_for_empty_text, [{"type": "thinking", "thinking": "two words"}], False, None) == 3
def test_token_counter_with_tool_reference_block():
"""
Regression test: a message containing an Anthropic tool-search

View file

@ -24,7 +24,7 @@ from litellm.llms.tencent.messages.transformation import TencentAnthropicMessage
from litellm.llms.vertex_ai.vertex_ai_partner_models.anthropic.experimental_pass_through.transformation import (
VertexAIPartnerModelsAnthropicMessagesConfig,
)
from litellm.types.llms.anthropic import AnthropicFinishReason
from litellm.types.llms.anthropic import ANTHROPIC_BETA_HEADER_VALUES, AnthropicFinishReason
from litellm.types.router import GenericLiteLLMParams
from tests.unit.llms.anthropic.test_anthropic_wif import (
ScriptedPoster,
@ -321,6 +321,66 @@ def _derived_beta_header(config: AnthropicMessagesConfig) -> dict:
)
_COMPACT_EDIT_BETA: Final = ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_01_12.value
_CLEARING_EDIT_BETA: Final = ANTHROPIC_BETA_HEADER_VALUES.CONTEXT_MANAGEMENT_2025_06_27.value
@pytest.mark.parametrize(
("context_management", "betas"),
(
pytest.param({"edits": [{"type": "compact_20260112"}]}, {_COMPACT_EDIT_BETA}, id="compact"),
pytest.param({"edits": [{"type": "clear_tool_uses_20250919"}]}, {_CLEARING_EDIT_BETA}, id="clearing"),
pytest.param(
{"edits": [{"type": "compact_20260112"}, {"type": "clear_thinking_20251015"}]},
{_COMPACT_EDIT_BETA, _CLEARING_EDIT_BETA},
id="both",
),
pytest.param(
[{"type": "compaction", "compact_threshold": 150000}], {_COMPACT_EDIT_BETA}, id="openai_list_form"
),
),
)
def test_context_management_edits_add_the_beta_each_edit_needs(
context_management: JsonValue, betas: set[str], local_beta_headers_config: None
) -> None:
headers: Final = AnthropicMessagesConfig()._update_headers_with_anthropic_beta(
headers={}, optional_params={"context_management": context_management}, messages=[]
)
assert set(headers["anthropic-beta"].split(",")) == betas
_UNMAPPABLE_LIST_FORM: Final = [{"type": "clear_tool_uses_20250919"}]
@pytest.mark.parametrize("config_class", [AnthropicMessagesConfig, VertexAIPartnerModelsAnthropicMessagesConfig])
@pytest.mark.parametrize(
("context_management", "drop_params", "forwarded"),
(
pytest.param(_UNMAPPABLE_LIST_FORM, True, None, id="unmappable_dropped_under_drop_params"),
pytest.param(_UNMAPPABLE_LIST_FORM, False, _UNMAPPABLE_LIST_FORM, id="unmappable_kept_for_the_provider_400"),
pytest.param(
[{"type": "compaction"}],
True,
{"edits": [{"type": "compact_20260112"}]},
id="mappable_still_mapped_under_drop_params",
),
),
)
def test_context_management_the_list_form_cannot_map_is_dropped_only_under_drop_params(
config_class: type[AnthropicMessagesConfig], context_management: JsonValue, drop_params: bool, forwarded: JsonValue
) -> None:
request: Final = config_class().transform_anthropic_messages_request(
model="claude-haiku-5-5",
messages=[{"role": "user", "content": "Hi"}],
anthropic_messages_optional_request_params={"max_tokens": 50, "context_management": context_management},
litellm_params=GenericLiteLLMParams(drop_params=drop_params),
headers={},
)
assert request.get("context_management") == forwarded
@pytest.mark.parametrize("config", _classifier_configs(), ids=type)
def test_routes_whose_platform_runs_the_classifier_keep_safeguards_and_the_beta(
config: AnthropicMessagesConfig, local_beta_headers_config: None

View file

@ -26,7 +26,24 @@ from litellm.llms.azure_ai.anthropic.count_tokens.token_counter import AzureAIAn
from litellm.llms.azure_ai.anthropic.count_tokens.transformation import (
AzureAIAnthropicCountTokensConfig,
)
from litellm.types.utils import TokenCountMessagesFormat, TokenCountResponse
from litellm.types.utils import CountContextManagement, TokenCountMessagesFormat, TokenCountResponse
from litellm.constants import ANTHROPIC_TOKEN_COUNTING_BETA_VERSION
from litellm.types.llms.anthropic import ANTHROPIC_BETA_HEADER_VALUES
COMPACTION_BETA = ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_09_04.value
COMPACTED_HISTORY = [
{
"role": "assistant",
"content": [
{
"type": "compaction",
"content": "The user is building a recipe app and asked for one-sentence class descriptions.",
"signature": "EqQBCkYIBRgCKkBjZ2xhc3M" * 40,
}
],
},
{"role": "user", "content": "Now do the same for Ingredient."},
]
@pytest.mark.parametrize(
@ -890,3 +907,115 @@ async def test_counter_sends_the_structured_output_tool_choice_and_thinking_chat
inference["tool_choice"],
inference["tools"],
)
def test_headers_add_the_compaction_beta_only_for_a_compacted_history():
config = AnthropicCountTokensConfig()
auth_header = {"x-api-key": "sk-ant-api03-test-key"}
with_block = config.get_count_tokens_headers(auth_header, COMPACTED_HISTORY)["anthropic-beta"].split(",")
plain = config.get_count_tokens_headers(auth_header, [{"role": "user", "content": "hi"}])["anthropic-beta"]
assert COMPACTION_BETA in with_block
assert ANTHROPIC_TOKEN_COUNTING_BETA_VERSION in with_block
assert plain == ANTHROPIC_TOKEN_COUNTING_BETA_VERSION
@pytest.mark.asyncio
async def test_handler_sends_the_compaction_beta_with_a_compacted_history(httpx_transport_clients):
with respx.mock:
route = respx.post("https://gateway.example/v1/messages/count_tokens").mock(
return_value=httpx.Response(200, json={"input_tokens": 404})
)
result = await AnthropicCountTokensHandler().handle_count_tokens_request(
model="claude-sonnet-5-5",
messages=COMPACTED_HISTORY,
auth_header={"x-api-key": "sk-ant-api03-test-key"},
api_base="https://gateway.example",
)
assert result == {"input_tokens": 404}
betas = route.calls.last.request.headers["anthropic-beta"].split(",")
assert COMPACTION_BETA in betas
assert ANTHROPIC_TOKEN_COUNTING_BETA_VERSION in betas
@pytest.mark.asyncio
@pytest.mark.usefixtures("httpx_transport_clients")
@pytest.mark.parametrize(
("context_management", "forwarded", "sent_beta", "unsent_beta"),
(
pytest.param(
{"edits": [{"type": "compact_20260112"}]},
{"edits": [{"type": "compact_20260112"}]},
ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_01_12.value,
ANTHROPIC_BETA_HEADER_VALUES.CONTEXT_MANAGEMENT_2025_06_27.value,
id="compact",
),
pytest.param(
{"edits": [{"type": "clear_tool_uses_20250919"}]},
{"edits": [{"type": "clear_tool_uses_20250919"}]},
ANTHROPIC_BETA_HEADER_VALUES.CONTEXT_MANAGEMENT_2025_06_27.value,
ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_01_12.value,
id="clearing",
),
pytest.param(
[{"type": "compaction", "compact_threshold": 150000}],
{"edits": [{"type": "compact_20260112", "trigger": {"type": "input_tokens", "value": 150000}}]},
ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_01_12.value,
ANTHROPIC_BETA_HEADER_VALUES.CONTEXT_MANAGEMENT_2025_06_27.value,
id="openai_list_form",
),
),
)
async def test_counter_forwards_context_management_with_the_beta_inference_sends(
context_management: JsonValue, forwarded: JsonValue, sent_beta: str, unsent_beta: str
) -> None:
"""Anthropic counts an unsigned compaction block only under the compact_20260112 edit that inference sends,
and a clearing edit changes the count, so the count carries the request's context_management as inference
sends it, the OpenAI list form mapped to edits, and its beta."""
with respx.mock:
route: Final = respx.post("https://gateway.example/v1/messages/count_tokens").mock(
return_value=httpx.Response(200, json={"input_tokens": 259})
)
result: Final = await AnthropicTokenCounter().count_tokens(
model_to_use="claude-opus-5-5",
messages=[{"role": "assistant", "content": [{"type": "compaction", "content": "Recipe app summary."}]}],
contents=None,
deployment={"litellm_params": {"api_key": "sk-ant-api03-test-key", "api_base": "https://gateway.example"}},
context_management=CountContextManagement(spec=context_management, drop_invalid_edits=False),
)
sent: Final = route.calls.last.request
betas: Final = sent.headers["anthropic-beta"].split(",")
assert result is not None and result.total_tokens == 259
assert TypeAdapter(dict[str, JsonValue]).validate_json(sent.content)["context_management"] == forwarded
assert (sent_beta in betas, unsent_beta in betas) == (True, False)
@pytest.mark.asyncio
@pytest.mark.usefixtures("httpx_transport_clients")
@pytest.mark.parametrize("context_management", ({}, {"edits": []}, []), ids=("empty_object", "empty_edits", "empty_list"))
async def test_counter_leaves_out_context_management_without_edits(context_management: JsonValue) -> None:
"""Anthropic's count endpoint rejects a context_management with no edits, which inference accepts, so the count
leaves it out and adds no context-management beta."""
with respx.mock:
route: Final = respx.post("https://gateway.example/v1/messages/count_tokens").mock(
return_value=httpx.Response(200, json={"input_tokens": 11})
)
result: Final = await AnthropicTokenCounter().count_tokens(
model_to_use="claude-opus-5-5",
messages=[{"role": "user", "content": "Hello"}],
contents=None,
deployment={"litellm_params": {"api_key": "sk-ant-api03-test-key", "api_base": "https://gateway.example"}},
context_management=CountContextManagement(spec=context_management, drop_invalid_edits=False),
)
sent: Final = route.calls.last.request
betas: Final = frozenset(sent.headers.get("anthropic-beta", "").split(","))
assert result is not None and result.total_tokens == 11
assert "context_management" not in TypeAdapter(dict[str, JsonValue]).validate_json(sent.content)
assert not betas & {
ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_01_12.value,
ANTHROPIC_BETA_HEADER_VALUES.CONTEXT_MANAGEMENT_2025_06_27.value,
}

View file

@ -48,7 +48,7 @@ FEDERATED_DEPLOYMENT = {
def count_tokens_headers_for(api_key: str) -> dict[str, str]:
auth_header = AnthropicModelInfo.get_auth_header(api_key=api_key)
assert auth_header is not None
return AnthropicCountTokensConfig().get_count_tokens_headers(auth_header)
return AnthropicCountTokensConfig().get_count_tokens_headers(auth_header, [{"role": "user", "content": "hi"}])
@pytest.fixture

View file

@ -666,7 +666,8 @@ async def test_anthropic_view_counts_the_turns_after_a_compaction_block_when_an_
*_agent_turns(2, first_round=3),
)
view: Final = await inference_request_view(turns, None, "anthropic", _edits(clear_before_compact))
with patch.object(litellm, "token_counter", side_effect=RuntimeError("tokenizer unavailable")):
view: Final = await inference_request_view(turns, None, "anthropic", _edits(clear_before_compact))
assert isinstance(view, InferenceRequestView)
assert _tool_outputs_kept(view) == (True, True)

View file

@ -526,6 +526,29 @@ def test_both_compact_and_context_management_headers_added():
)
def test_openai_list_form_context_management_sends_the_compact_beta():
config: Final = VertexAIPartnerModelsAnthropicMessagesConfig()
with (
patch.object(config, "_ensure_access_token", return_value=("token", "test-project")),
patch.object(config, "get_complete_vertex_url", return_value="https://mock-url"),
):
updated_headers, _ = config.validate_anthropic_messages_environment(
headers={},
model="claude-vertex-ai-opus-4-6",
messages=[],
optional_params={"context_management": [{"type": "compaction", "compact_threshold": 150000}]},
litellm_params={
"vertex_ai_project": "test-project",
"vertex_ai_location": "us-central1",
"vertex_credentials": "{}",
},
api_base=None,
)
assert updated_headers["anthropic-beta"].split(",") == [ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_01_12.value]
def test_validate_environment_always_refreshes_token_ignoring_stale_bearer():
"""Regression: stale Authorization in shared deployment extra_headers must not
skip token refresh on /v1/messages — _ensure_access_token is always called."""

View file

@ -6,6 +6,7 @@ import json
import logging
import unittest
from collections.abc import AsyncGenerator, AsyncIterator, Callable, Mapping
from types import MappingProxyType
from typing import Final
from unittest.mock import AsyncMock, MagicMock, patch
@ -587,6 +588,48 @@ class TestErrorLogCarriesCallId:
assert call_id in record.getMessage()
class TestCountTokensCompactedHistory:
"""A history carrying Anthropic's signed compaction block used to fail /v1/messages/count_tokens once the count
ran on the local tokenizer, which rejected the block type."""
@pytest.mark.asyncio
async def test_count_tokens_counts_a_compaction_block_like_its_summary_text(self):
import litellm.proxy.anthropic_endpoints.endpoints as ep
import litellm.proxy.proxy_server as proxy_server
from litellm.proxy._types import UserAPIKeyAuth
summary = "The user is building a recipe app and asked for one-sentence class descriptions."
follow_up = {"role": "user", "content": "Now do the same for Ingredient."}
compacted = {
"model": "claude-sonnet-5-5",
"messages": [
{
"role": "assistant",
"content": [{"type": "compaction", "content": summary, "signature": "EqQBCkYIBRgCKkBjZ2xhc3M" * 40}],
},
follow_up,
],
}
as_text = {
"model": "claude-sonnet-5-5",
"messages": [{"role": "assistant", "content": [{"type": "text", "text": summary}]}, follow_up],
}
request = MagicMock()
request.headers = {}
async def count(body):
with (
patch.object(ep, "read_request_body", new=AsyncMock(return_value=body)), # test-quality-ok: endpoint reads the body via a module function; no injection seam
patch.object(proxy_server, "llm_router", None), # test-quality-ok: module global; with no router the count runs on the local tokenizer, the path that raised
):
return await ep.count_tokens(request=request, user_api_key_dict=UserAPIKeyAuth())
counted = await count(compacted)
assert counted["input_tokens"] > 0
assert counted == await count(as_text)
def _claude_router() -> Router:
return Router(
model_list=[
@ -759,7 +802,10 @@ async def test_count_tokens_drops_malformed_tools_the_counted_deployment_drops_e
def _count_on_claude(
monkeypatch: pytest.MonkeyPatch, peer: Callable[[httpx.Request], httpx.Response], headers: Mapping[str, str]
monkeypatch: pytest.MonkeyPatch,
peer: Callable[[httpx.Request], httpx.Response],
headers: Mapping[str, str],
extra_body: Mapping[str, object] = MappingProxyType({}),
) -> httpx.Response:
from fastapi import FastAPI
@ -799,7 +845,7 @@ def _count_on_claude(
app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth()
return TestClient(app).post(
"/v1/messages/count_tokens",
json={"model": "count-log-claude", "messages": [{"role": "user", "content": "Hello"}]},
json={"model": "count-log-claude", "messages": [{"role": "user", "content": "Hello"}], **extra_body},
headers=dict(headers),
)
@ -827,6 +873,24 @@ def test_count_tokens_logs_one_line_with_the_callers_call_id_model_and_count(
)
def test_count_tokens_returns_the_context_management_the_provider_applied(monkeypatch: pytest.MonkeyPatch) -> None:
applied: Final = {"original_input_tokens": 640}
response: Final = _count_on_claude(
monkeypatch,
lambda request: httpx.Response(
200, json={"input_tokens": 576, "context_management": applied}, request=request
),
{},
{"context_management": {"edits": [{"type": "clear_tool_uses_20250919"}]}},
)
assert response.json() == {
"input_tokens": 576,
"context_management": applied,
"litellm_count_source": "provider",
}, response.text
def test_count_tokens_provider_rejection_logs_its_status_and_answers_the_same_error(
monkeypatch: pytest.MonkeyPatch, token_count_log_lines: Callable[[], tuple[dict[str, object], ...]]
) -> None: