mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
Merge fcc88b195b into 68b5a737a1
This commit is contained in:
commit
343ed79058
15 changed files with 520 additions and 82 deletions
|
|
@ -15,6 +15,7 @@ import anyio
|
|||
import anyio.lowlevel
|
||||
import httpx
|
||||
import tiktoken
|
||||
from pydantic import TypeAdapter
|
||||
from tiktoken.model import MODEL_TO_ENCODING
|
||||
from typing_extensions import ParamSpec, TypeVar
|
||||
|
||||
|
|
@ -62,6 +63,7 @@ from litellm.types.llms.anthropic import (
|
|||
AnthropicMessagesToolResultParam,
|
||||
AnthropicMessagesToolUseParam,
|
||||
AnthropicSearchResultParam,
|
||||
CompactionBlock,
|
||||
)
|
||||
from litellm.types.llms.openai import (
|
||||
AllMessageValues,
|
||||
|
|
@ -843,6 +845,8 @@ def _count_messages(
|
|||
use_default_image_token_count,
|
||||
default_token_count,
|
||||
)
|
||||
elif key == "provider_specific_fields":
|
||||
num_tokens += sum(map(params.count_function, _compaction_summaries(value)))
|
||||
elif key == "search_results" and isinstance(value, list):
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
||||
extract_search_results_text,
|
||||
|
|
@ -857,6 +861,25 @@ def _count_messages(
|
|||
return num_tokens
|
||||
|
||||
|
||||
_OBJECT_LIST: Final = TypeAdapter(list[object])
|
||||
|
||||
|
||||
def _compaction_summaries(provider_specific_fields: object) -> tuple[str, ...]:
|
||||
fields: Final = as_str_mapping(provider_specific_fields)
|
||||
blocks: Final = fields.get("compaction_blocks") if fields is not None else None
|
||||
block_fields: Final = (
|
||||
tuple(map(as_str_mapping, _OBJECT_LIST.validate_python(blocks))) if isinstance(blocks, list) else ()
|
||||
)
|
||||
return tuple(
|
||||
content
|
||||
for block in block_fields
|
||||
if block is not None
|
||||
and block.get("type") == "compaction"
|
||||
and isinstance(content := block.get("content"), str)
|
||||
and content
|
||||
)
|
||||
|
||||
|
||||
def _count_extra(
|
||||
count_function: TokenCounterFunction,
|
||||
tools: list[ChatCompletionToolParam] | None,
|
||||
|
|
@ -1379,6 +1402,11 @@ def _count_anthropic_content(
|
|||
return tokens
|
||||
|
||||
|
||||
_TEXT_FIELD_BY_BLOCK_TYPE: Final = MappingProxyType(
|
||||
{"text": "text", "thinking": "thinking", "redacted_thinking": "thinking", "compaction": "content"}
|
||||
)
|
||||
|
||||
|
||||
def _count_content_list(
|
||||
count_function: TokenCounterFunction,
|
||||
content_list: str
|
||||
|
|
@ -1390,6 +1418,7 @@ def _count_content_list(
|
|||
| AnthropicMessagesImageParam
|
||||
| AnthropicMessagesDocumentParam
|
||||
| AnthropicSearchResultParam
|
||||
| CompactionBlock
|
||||
],
|
||||
use_default_image_token_count: bool,
|
||||
default_token_count: int | None,
|
||||
|
|
@ -1400,8 +1429,10 @@ def _count_content_list(
|
|||
for c in content_list:
|
||||
if isinstance(c, str):
|
||||
num_tokens += count_function(c)
|
||||
elif c["type"] == "text":
|
||||
num_tokens += count_function(str(c.get("text", "")))
|
||||
elif c["type"] in _TEXT_FIELD_BY_BLOCK_TYPE:
|
||||
block_text = str(c.get(_TEXT_FIELD_BY_BLOCK_TYPE[c["type"]]) or "")
|
||||
if block_text:
|
||||
num_tokens += count_function(block_text)
|
||||
elif c["type"] == "image_url":
|
||||
image_url = c.get("image_url")
|
||||
num_tokens += _count_image_tokens(image_url, use_default_image_token_count)
|
||||
|
|
@ -1431,12 +1462,6 @@ def _count_content_list(
|
|||
use_default_image_token_count,
|
||||
default_token_count,
|
||||
)
|
||||
elif c["type"] in ("thinking", "redacted_thinking"):
|
||||
# Claude extended thinking content block
|
||||
# Count the thinking text and skip the opaque blobs (signature, redacted data)
|
||||
thinking_text = str(c.get("thinking", ""))
|
||||
if thinking_text:
|
||||
num_tokens += count_function(thinking_text)
|
||||
elif c["type"] == "tool_reference":
|
||||
# Anthropic tool-search reference block: a lightweight pointer to
|
||||
# a deferred tool, e.g. {"type": "tool_reference", "tool_name": ...}.
|
||||
|
|
@ -1454,7 +1479,7 @@ def _count_content_list(
|
|||
f"Invalid content item type: {content_type}. "
|
||||
f"Expected str or dict with 'type' field "
|
||||
f"(text, image_url, image, document, file, tool_use, tool_result, thinking, redacted_thinking, "
|
||||
f"tool_reference, search_result)."
|
||||
f"compaction, tool_reference, search_result)."
|
||||
)
|
||||
return num_tokens
|
||||
except Exception as e:
|
||||
|
|
|
|||
|
|
@ -8,12 +8,13 @@ import re
|
|||
from collections.abc import Container, Mapping, MutableMapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timezone
|
||||
from itertools import chain
|
||||
from types import MappingProxyType
|
||||
from typing import Annotated, Any, ClassVar, Final, Literal, TypeVar
|
||||
from urllib.parse import quote, urlparse
|
||||
|
||||
import httpx
|
||||
from pydantic import BeforeValidator, ConfigDict, Field, StrictBool, TypeAdapter, ValidationError
|
||||
from pydantic import BeforeValidator, ConfigDict, Field, JsonValue, StrictBool, TypeAdapter, ValidationError
|
||||
from typing_extensions import ReadOnly, TypedDict
|
||||
|
||||
import litellm
|
||||
|
|
@ -24,6 +25,7 @@ from litellm.constants import (
|
|||
DEFAULT_REASONING_EFFORT_XHIGH_THINKING_BUDGET,
|
||||
)
|
||||
from litellm.exceptions import UnsupportedParamsError
|
||||
from litellm.litellm_core_utils.core_helpers import as_str_mapping
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
||||
get_file_ids_from_messages,
|
||||
is_encrypted_reasoning_block,
|
||||
|
|
@ -46,6 +48,7 @@ from litellm.llms.anthropic.wif import (
|
|||
from litellm.llms.base_llm.base_utils import BaseLLMModelInfo, BaseTokenCounter
|
||||
from litellm.llms.base_llm.chat.transformation import BaseLLMException
|
||||
from litellm.types.llms.anthropic import (
|
||||
ANTHROPIC_BETA_HEADER_VALUES,
|
||||
ANTHROPIC_HOSTED_TOOLS,
|
||||
ANTHROPIC_MID_CONVERSATION_OUTPUT_CONFIG_BETA_HEADER,
|
||||
ANTHROPIC_MID_CONVERSATION_TOOL_CHANGES_BETA_HEADER,
|
||||
|
|
@ -118,6 +121,7 @@ _CLAUDE_CODE_BILLING_HEADER_PREFIX: Final = "x-anthropic-billing-header:"
|
|||
OBJECT_MAPPING_ADAPTER: Final = TypeAdapter(dict[object, object])
|
||||
OBJECT_LIST_ADAPTER: Final = TypeAdapter(list[object])
|
||||
_DECODED_JSON: Final = TypeAdapter(object)
|
||||
_JSON_VALUE: Final = TypeAdapter(JsonValue)
|
||||
|
||||
|
||||
_CLAUDE_CODE_USER_AGENT_PREFIXES: Final = ("claude-cli/", "claude-code/")
|
||||
|
|
@ -177,24 +181,68 @@ def is_anthropic_messages_url(url: str) -> bool:
|
|||
return parsed_url.hostname == "api.anthropic.com" or parsed_url.path.removesuffix("/").endswith("/v1/messages")
|
||||
|
||||
|
||||
def _message_blocks(message: object) -> Sequence[object]:
|
||||
fields: Final = as_str_mapping(message)
|
||||
blocks: Final = validated_object_list(fields.get("content")) if fields is not None else None
|
||||
return blocks or ()
|
||||
|
||||
|
||||
def _is_signed_compaction_block(block: object) -> bool:
|
||||
fields: Final = as_str_mapping(block)
|
||||
if fields is None or fields.get("type") != "compaction":
|
||||
return False
|
||||
signature: Final = fields.get("signature")
|
||||
return isinstance(signature, str) and bool(signature)
|
||||
|
||||
|
||||
def carries_signed_compaction_block(messages: Sequence[object]) -> bool:
|
||||
return any(map(_is_signed_compaction_block, chain.from_iterable(map(_message_blocks, messages))))
|
||||
|
||||
|
||||
def requires_native_compaction_beta(
|
||||
custom_llm_provider: str,
|
||||
optional_params: Mapping[str, object],
|
||||
messages: Sequence[object],
|
||||
) -> bool:
|
||||
return custom_llm_provider == "anthropic" and (
|
||||
optional_params.get("compaction") is not None
|
||||
or any(
|
||||
isinstance(block, Mapping)
|
||||
and block.get("type") == "compaction"
|
||||
and isinstance(block.get("signature"), str)
|
||||
and bool(block.get("signature"))
|
||||
for message in messages
|
||||
if isinstance(message, Mapping)
|
||||
for content in (message.get("content"),)
|
||||
if isinstance(content, (list, tuple))
|
||||
for block in content
|
||||
optional_params.get("compaction") is not None or carries_signed_compaction_block(messages)
|
||||
)
|
||||
|
||||
|
||||
def anthropic_context_management(context_management: JsonValue) -> JsonValue:
|
||||
if not isinstance(context_management, list):
|
||||
return context_management
|
||||
from litellm.llms.anthropic.chat.transformation import AnthropicConfig
|
||||
|
||||
edits: Final = AnthropicConfig.map_openai_context_management_to_anthropic(
|
||||
[entry for entry in context_management if isinstance(entry, dict)]
|
||||
)
|
||||
return context_management if edits is None else _JSON_VALUE.validate_python(edits)
|
||||
|
||||
|
||||
def _context_management_edits(spec: JsonValue) -> Sequence[object]:
|
||||
mapping: Final = as_str_mapping(spec)
|
||||
return (validated_object_list(mapping.get("edits")) if mapping is not None else None) or ()
|
||||
|
||||
|
||||
def context_management_with_edits(context_management: JsonValue) -> JsonValue:
|
||||
spec: Final = anthropic_context_management(context_management)
|
||||
return spec if _context_management_edits(spec) else None
|
||||
|
||||
|
||||
def context_management_betas(context_management: JsonValue) -> tuple[str, ...]:
|
||||
edits: Final = _context_management_edits(anthropic_context_management(context_management))
|
||||
edit_types: Final = tuple(edit.get("type") for edit in map(as_str_mapping, edits) if edit is not None)
|
||||
return tuple(
|
||||
beta
|
||||
for beta, used in (
|
||||
(ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_01_12.value, "compact_20260112" in edit_types),
|
||||
(
|
||||
ANTHROPIC_BETA_HEADER_VALUES.CONTEXT_MANAGEMENT_2025_06_27.value,
|
||||
any(edit_type != "compact_20260112" for edit_type in edit_types),
|
||||
),
|
||||
)
|
||||
if used
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -1367,8 +1415,6 @@ class AnthropicModelInfo(BaseLLMModelInfo):
|
|||
if is_vertex_request is True:
|
||||
# Vertex AI requires web search beta header for web search to work
|
||||
if web_search_tool_used:
|
||||
from litellm.types.llms.anthropic import ANTHROPIC_BETA_HEADER_VALUES
|
||||
|
||||
headers["anthropic-beta"] = ANTHROPIC_BETA_HEADER_VALUES.WEB_SEARCH_2025_03_05.value
|
||||
elif len(all_betas) > 0:
|
||||
headers["anthropic-beta"] = ",".join(all_betas)
|
||||
|
|
|
|||
|
|
@ -39,6 +39,7 @@ class AnthropicCountTokensHandler(AnthropicCountTokensConfig):
|
|||
tools: list[dict[str, JsonValue]] | None = None,
|
||||
system: JsonValue = None,
|
||||
optional_params: Mapping[str, JsonValue] | None = None,
|
||||
context_management: JsonValue = None,
|
||||
drop_params: bool = False,
|
||||
) -> dict[str, JsonValue]:
|
||||
"""
|
||||
|
|
@ -70,6 +71,7 @@ class AnthropicCountTokensHandler(AnthropicCountTokensConfig):
|
|||
tools=tools,
|
||||
system=system,
|
||||
optional_params=optional_params,
|
||||
context_management=context_management,
|
||||
drop_params=drop_params,
|
||||
)
|
||||
|
||||
|
|
@ -81,7 +83,7 @@ class AnthropicCountTokensHandler(AnthropicCountTokensConfig):
|
|||
verbose_logger.debug("Making request to: %s", endpoint_url)
|
||||
|
||||
# Get required headers
|
||||
headers: Final = self.get_count_tokens_headers(auth_header)
|
||||
headers: Final = self.get_count_tokens_headers(auth_header, messages, context_management)
|
||||
|
||||
# Use LiteLLM's async httpx client
|
||||
async_client: Final = get_async_httpx_client(llm_provider=litellm.LlmProviders.ANTHROPIC)
|
||||
|
|
|
|||
|
|
@ -132,6 +132,7 @@ class AnthropicTokenCounter(BaseTokenCounter):
|
|||
tools=tools,
|
||||
system=system,
|
||||
optional_params=count_options,
|
||||
context_management=context_management.spec if context_management is not None else None,
|
||||
drop_params=drops_unsupported_params(litellm_params.get("drop_params")),
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -27,11 +27,15 @@ from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import
|
|||
)
|
||||
from litellm.litellm_core_utils.url_utils import join_api_base
|
||||
from litellm.llms.anthropic.common_utils import (
|
||||
carries_signed_compaction_block,
|
||||
context_management_betas,
|
||||
context_management_with_edits,
|
||||
is_claude_model_target,
|
||||
merge_anthropic_beta_headers,
|
||||
tools_without_custom_field,
|
||||
)
|
||||
from litellm.llms.anthropic.wif import resolve_anthropic_base
|
||||
from litellm.types.llms.anthropic import ANTHROPIC_BETA_HEADER_VALUES
|
||||
from litellm.types.llms.openai import AllMessageValues, ChatCompletionImageObject, ValidUserMessageContentTypes
|
||||
from litellm.types.utils import TokenCountMessagesFormat, TokenCountResponse
|
||||
|
||||
|
|
@ -376,6 +380,7 @@ class AnthropicCountTokensConfig:
|
|||
tools: list[dict[str, JsonValue]] | None = None,
|
||||
system: JsonValue = None,
|
||||
optional_params: Mapping[str, JsonValue] | None = None,
|
||||
context_management: JsonValue = None,
|
||||
drop_params: bool = False,
|
||||
) -> dict[str, JsonValue]: # mutable-ok: provider transport requires JSON dictionaries
|
||||
"""
|
||||
|
|
@ -408,6 +413,7 @@ class AnthropicCountTokensConfig:
|
|||
drop_custom=drop_params,
|
||||
),
|
||||
),
|
||||
("context_management", context_management_with_edits(context_management)),
|
||||
)
|
||||
if value is not None
|
||||
}
|
||||
|
|
@ -419,18 +425,33 @@ class AnthropicCountTokensConfig:
|
|||
)
|
||||
)
|
||||
|
||||
def get_count_tokens_headers(self, auth_header: Mapping[str, str]) -> dict[str, str]:
|
||||
def get_count_tokens_headers(
|
||||
self,
|
||||
auth_header: Mapping[str, str],
|
||||
messages: Sequence[Mapping[str, JsonValue]],
|
||||
context_management: JsonValue = None,
|
||||
) -> dict[str, str]:
|
||||
"""The count-tokens headers around a resolved Anthropic auth header
|
||||
(``AnthropicModelInfo.get_auth_header``): x-api-key for a static key, an Authorization
|
||||
bearer for ``ANTHROPIC_AUTH_TOKEN`` and for sk-ant-oat tokens, whose mandatory oauth beta
|
||||
merges with the token-counting beta instead of replacing it."""
|
||||
merges with the token-counting beta instead of replacing it. A history that carries a
|
||||
signed compaction block also needs the compaction beta, or Anthropic rejects the block
|
||||
as an unknown content type, and ``context_management`` edits need the betas inference sends."""
|
||||
compaction_betas: Final = (
|
||||
(ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_09_04.value,)
|
||||
if carries_signed_compaction_block(messages)
|
||||
else ()
|
||||
)
|
||||
request_betas: Final = (
|
||||
ANTHROPIC_TOKEN_COUNTING_BETA_VERSION,
|
||||
*compaction_betas,
|
||||
*context_management_betas(context_management),
|
||||
)
|
||||
return {
|
||||
"Content-Type": "application/json",
|
||||
"anthropic-version": "2023-06-01",
|
||||
**auth_header,
|
||||
"anthropic-beta": merge_anthropic_beta_headers(
|
||||
auth_header.get("anthropic-beta"), ANTHROPIC_TOKEN_COUNTING_BETA_VERSION
|
||||
),
|
||||
"anthropic-beta": merge_anthropic_beta_headers(auth_header.get("anthropic-beta"), request_betas),
|
||||
}
|
||||
|
||||
def validate_request(
|
||||
|
|
|
|||
|
|
@ -43,6 +43,7 @@ from ...common_utils import (
|
|||
OBJECT_LIST_ADAPTER,
|
||||
AnthropicError,
|
||||
AnthropicModelInfo,
|
||||
context_management_betas,
|
||||
is_claude_model_target,
|
||||
merge_anthropic_beta_headers,
|
||||
optionally_handle_anthropic_oauth,
|
||||
|
|
@ -745,6 +746,8 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
|
|||
)
|
||||
if transformed_context_management is not None:
|
||||
anthropic_messages_optional_request_params["context_management"] = transformed_context_management
|
||||
elif drops_unsupported_params(litellm_params.drop_params):
|
||||
anthropic_messages_optional_request_params.pop("context_management")
|
||||
|
||||
####### get required params for all anthropic messages requests ######
|
||||
# Lazy %s: the f-string previously stringified the entire messages
|
||||
|
|
@ -873,28 +876,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
|
|||
if requires_native_compaction_beta(custom_llm_provider, optional_params, messages):
|
||||
beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_09_04.value)
|
||||
|
||||
# Check for context management
|
||||
context_management_param: Final = optional_params.get("context_management")
|
||||
if context_management_param is not None:
|
||||
# Check edits array for compact_20260112 type
|
||||
edits: Final = context_management_param.get("edits", ())
|
||||
has_compact = False
|
||||
has_other = False
|
||||
|
||||
for edit in edits:
|
||||
edit_type = edit.get("type", "")
|
||||
if edit_type == "compact_20260112":
|
||||
has_compact = True
|
||||
else:
|
||||
has_other = True
|
||||
|
||||
# Add compact header if any compact edits exist
|
||||
if has_compact:
|
||||
beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_01_12.value)
|
||||
|
||||
# Add context management header if any other edits exist
|
||||
if has_other:
|
||||
beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.CONTEXT_MANAGEMENT_2025_06_27.value)
|
||||
beta_values.update(context_management_betas(optional_params.get("context_management")))
|
||||
|
||||
# Check for structured outputs. Anthropic's newer request shape nests
|
||||
# the schema under output_config.format; the older top-level
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@ from typing import Final
|
|||
|
||||
from litellm.exceptions import UnsupportedParamsError
|
||||
from litellm.litellm_core_utils.core_helpers import drops_unsupported_params
|
||||
from litellm.llms.anthropic.common_utils import AnthropicModelInfo
|
||||
from litellm.llms.anthropic.common_utils import AnthropicModelInfo, context_management_betas
|
||||
from litellm.llms.anthropic.messages.native.transformation import (
|
||||
AnthropicMessagesConfig,
|
||||
messages_carry_output_config,
|
||||
|
|
@ -171,28 +171,7 @@ class VertexAIPartnerModelsAnthropicMessagesConfig(AnthropicMessagesConfig, Vert
|
|||
if existing_beta:
|
||||
beta_values.update(b.strip() for b in existing_beta.split(","))
|
||||
|
||||
# Check for context management
|
||||
context_management_param: Final = optional_params.get("context_management")
|
||||
if context_management_param is not None:
|
||||
# Check edits array for compact_20260112 type
|
||||
edits: Final = context_management_param.get("edits", [])
|
||||
has_compact = False
|
||||
has_other = False
|
||||
|
||||
for edit in edits:
|
||||
edit_type = edit.get("type", "")
|
||||
if edit_type == "compact_20260112":
|
||||
has_compact = True
|
||||
else:
|
||||
has_other = True
|
||||
|
||||
# Add compact header if any compact edits exist
|
||||
if has_compact:
|
||||
beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_01_12.value)
|
||||
|
||||
# Add context management header if any other edits exist
|
||||
if has_other:
|
||||
beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.CONTEXT_MANAGEMENT_2025_06_27.value)
|
||||
beta_values.update(context_management_betas(optional_params.get("context_management")))
|
||||
|
||||
# Check for web search tool
|
||||
for tool in tools:
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@ from typing import Final
|
|||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Request, Response, status
|
||||
from fastapi.responses import JSONResponse
|
||||
from pydantic import JsonValue, ValidationError
|
||||
from pydantic import JsonValue, TypeAdapter, ValidationError
|
||||
from typing_extensions import ReadOnly, TypedDict
|
||||
|
||||
import litellm
|
||||
|
|
@ -54,6 +54,7 @@ from litellm.proxy.litellm_pre_call_utils import (
|
|||
should_auto_drop_params_for_agentic_cli,
|
||||
)
|
||||
from litellm.router import Router
|
||||
from litellm.types.utils import TokenCountResponse
|
||||
|
||||
router: Final = APIRouter()
|
||||
|
||||
|
|
@ -334,6 +335,15 @@ def _count_tokens_body_options(data: Mapping[str, JsonValue], auto_drop_params:
|
|||
}
|
||||
|
||||
|
||||
_PROVIDER_COUNT_REPLY: Final[TypeAdapter[Mapping[str, object] | None]] = TypeAdapter(Mapping[str, object] | None)
|
||||
|
||||
|
||||
def _provider_context_management(response: TokenCountResponse) -> Mapping[str, object]:
|
||||
original: Final = _PROVIDER_COUNT_REPLY.validate_python(response.original_response)
|
||||
context_management: Final = original.get("context_management") if original is not None else None
|
||||
return {} if context_management is None else {"context_management": context_management}
|
||||
|
||||
|
||||
@router.post(
|
||||
"/v1/messages/count_tokens",
|
||||
tags=["[beta] Anthropic Messages Token Counting"],
|
||||
|
|
@ -413,7 +423,11 @@ async def count_tokens(
|
|||
log_llm_api_exception(e, litellm_call_id)
|
||||
return _anthropic_error_json_response(proxy_exception_from_route_error(e), request)
|
||||
else:
|
||||
return {"input_tokens": token_response.total_tokens, **count_source_fields(token_response)}
|
||||
return {
|
||||
"input_tokens": token_response.total_tokens,
|
||||
**_provider_context_management(token_response),
|
||||
**count_source_fields(token_response),
|
||||
}
|
||||
|
||||
|
||||
@router.post(
|
||||
|
|
|
|||
|
|
@ -1921,6 +1921,97 @@ def test_token_counter_with_redacted_thinking_content():
|
|||
|
||||
assert token_counter(model=model, messages=with_block) == token_counter(model=model, messages=without_block)
|
||||
|
||||
|
||||
def test_token_counter_with_compaction_block():
|
||||
"""
|
||||
A replayed on-demand compaction block (Anthropic's signed summary of the earlier turns) counts its
|
||||
summary text like a text block and nothing for its signature. It used to raise, which turned every
|
||||
/v1/messages/count_tokens call on a compacted history into a 500 once the provider count fell back here.
|
||||
"""
|
||||
model = "anthropic/claude-sonnet-5-5"
|
||||
summary = "The user is building a recipe app in Python and asked for one-sentence class descriptions."
|
||||
follow_up = {"role": "user", "content": [{"type": "text", "text": "Now do the same for Ingredient."}]}
|
||||
signed_block = {"type": "compaction", "content": summary, "signature": "EqQBCkYIBRgCKkBjZ2xhc3M" * 40}
|
||||
|
||||
as_compaction = [{"role": "assistant", "content": [signed_block]}, follow_up]
|
||||
unsigned = [{"role": "assistant", "content": [{"type": "compaction", "content": summary}]}, follow_up]
|
||||
as_text = [{"role": "assistant", "content": [{"type": "text", "text": summary}]}, follow_up]
|
||||
|
||||
assert token_counter(model=model, messages=as_compaction) == token_counter(model=model, messages=as_text)
|
||||
assert token_counter(model=model, messages=unsigned) == token_counter(model=model, messages=as_text)
|
||||
|
||||
|
||||
def test_token_counter_counts_chat_compaction_blocks_like_their_summary_text():
|
||||
"""
|
||||
Chat clients replay a compaction block in the assistant message's provider_specific_fields, and inference
|
||||
sends it to Anthropic as a content block, so the local count includes its summary text and skips its signature.
|
||||
"""
|
||||
model = "anthropic/claude-sonnet-5-5"
|
||||
summary = "The user is building a recipe app in Python and asked for one-sentence class descriptions."
|
||||
follow_up = {"role": "user", "content": "Now do the same for Ingredient."}
|
||||
signed_block = {"type": "compaction", "content": summary, "signature": "EqQBCkYIBRgCKkBjZ2xhc3M" * 40}
|
||||
|
||||
replayed = [
|
||||
{"role": "assistant", "content": None, "provider_specific_fields": {"compaction_blocks": [signed_block]}},
|
||||
follow_up,
|
||||
]
|
||||
as_text = [{"role": "assistant", "content": summary}, follow_up]
|
||||
|
||||
assert token_counter(model=model, messages=replayed) == token_counter(model=model, messages=as_text)
|
||||
|
||||
|
||||
class _BosEncoder:
|
||||
def encode_batch_fast(self, texts: list[str]) -> list[list[int]]:
|
||||
return [[0] * (len(text.split()) + 1) for text in texts]
|
||||
|
||||
|
||||
def test_token_counter_charges_nothing_for_a_chat_compaction_block_with_no_summary():
|
||||
"""
|
||||
A tokenizer that prepends a BOS token charges one token for the empty string, so a replayed
|
||||
compaction block with an empty summary must not reach it, like an empty thinking block.
|
||||
"""
|
||||
model = "anthropic/claude-sonnet-5-5"
|
||||
tokenizer = {"type": "huggingface_tokenizer", "tokenizer": _BosEncoder()}
|
||||
follow_up = {"role": "user", "content": "Now do the same for Ingredient."}
|
||||
replayed = [
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": None,
|
||||
"provider_specific_fields": {"compaction_blocks": [{"type": "compaction", "content": ""}]},
|
||||
},
|
||||
follow_up,
|
||||
]
|
||||
bare = [{"role": "assistant", "content": None}, follow_up]
|
||||
|
||||
assert token_counter(model=model, messages=replayed, custom_tokenizer=tokenizer) == token_counter(
|
||||
model=model, messages=bare, custom_tokenizer=tokenizer
|
||||
)
|
||||
|
||||
|
||||
def test_count_content_list_skips_the_tokenizer_for_empty_block_text():
|
||||
"""
|
||||
An empty thinking block, a redacted_thinking block (no thinking text at all) and a compaction
|
||||
block with no summary carry no tokens, even under a tokenizer that charges for the empty string
|
||||
(the llama-2 tokenizer prepends its BOS token to every encode call). Counting them through the
|
||||
tokenizer added one token per block on such models.
|
||||
"""
|
||||
from litellm.litellm_core_utils.token_counter import _count_content_list
|
||||
|
||||
def charges_for_empty_text(text: str) -> int:
|
||||
return len(text.split()) + 1
|
||||
|
||||
empty_blocks = [
|
||||
{"type": "thinking", "thinking": "", "signature": "EqcLCkYICxgCKkCrqu6lP..."},
|
||||
{"type": "thinking", "thinking": None, "signature": "EqcLCkYICxgCKkCrqu6lP..."},
|
||||
{"type": "redacted_thinking", "data": "EmwKAhgBEgy3va3pzix/LafPsn4aDFIT2Xlxh0L5L8rLVyIw"},
|
||||
{"type": "compaction", "content": ""},
|
||||
{"type": "text", "text": ""},
|
||||
]
|
||||
|
||||
assert _count_content_list(charges_for_empty_text, empty_blocks, False, None) == 0
|
||||
assert _count_content_list(charges_for_empty_text, [{"type": "thinking", "thinking": "two words"}], False, None) == 3
|
||||
|
||||
|
||||
def test_token_counter_with_tool_reference_block():
|
||||
"""
|
||||
Regression test: a message containing an Anthropic tool-search
|
||||
|
|
|
|||
|
|
@ -24,7 +24,7 @@ from litellm.llms.tencent.messages.transformation import TencentAnthropicMessage
|
|||
from litellm.llms.vertex_ai.vertex_ai_partner_models.anthropic.experimental_pass_through.transformation import (
|
||||
VertexAIPartnerModelsAnthropicMessagesConfig,
|
||||
)
|
||||
from litellm.types.llms.anthropic import AnthropicFinishReason
|
||||
from litellm.types.llms.anthropic import ANTHROPIC_BETA_HEADER_VALUES, AnthropicFinishReason
|
||||
from litellm.types.router import GenericLiteLLMParams
|
||||
from tests.unit.llms.anthropic.test_anthropic_wif import (
|
||||
ScriptedPoster,
|
||||
|
|
@ -321,6 +321,66 @@ def _derived_beta_header(config: AnthropicMessagesConfig) -> dict:
|
|||
)
|
||||
|
||||
|
||||
_COMPACT_EDIT_BETA: Final = ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_01_12.value
|
||||
_CLEARING_EDIT_BETA: Final = ANTHROPIC_BETA_HEADER_VALUES.CONTEXT_MANAGEMENT_2025_06_27.value
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("context_management", "betas"),
|
||||
(
|
||||
pytest.param({"edits": [{"type": "compact_20260112"}]}, {_COMPACT_EDIT_BETA}, id="compact"),
|
||||
pytest.param({"edits": [{"type": "clear_tool_uses_20250919"}]}, {_CLEARING_EDIT_BETA}, id="clearing"),
|
||||
pytest.param(
|
||||
{"edits": [{"type": "compact_20260112"}, {"type": "clear_thinking_20251015"}]},
|
||||
{_COMPACT_EDIT_BETA, _CLEARING_EDIT_BETA},
|
||||
id="both",
|
||||
),
|
||||
pytest.param(
|
||||
[{"type": "compaction", "compact_threshold": 150000}], {_COMPACT_EDIT_BETA}, id="openai_list_form"
|
||||
),
|
||||
),
|
||||
)
|
||||
def test_context_management_edits_add_the_beta_each_edit_needs(
|
||||
context_management: JsonValue, betas: set[str], local_beta_headers_config: None
|
||||
) -> None:
|
||||
headers: Final = AnthropicMessagesConfig()._update_headers_with_anthropic_beta(
|
||||
headers={}, optional_params={"context_management": context_management}, messages=[]
|
||||
)
|
||||
|
||||
assert set(headers["anthropic-beta"].split(",")) == betas
|
||||
|
||||
|
||||
_UNMAPPABLE_LIST_FORM: Final = [{"type": "clear_tool_uses_20250919"}]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("config_class", [AnthropicMessagesConfig, VertexAIPartnerModelsAnthropicMessagesConfig])
|
||||
@pytest.mark.parametrize(
|
||||
("context_management", "drop_params", "forwarded"),
|
||||
(
|
||||
pytest.param(_UNMAPPABLE_LIST_FORM, True, None, id="unmappable_dropped_under_drop_params"),
|
||||
pytest.param(_UNMAPPABLE_LIST_FORM, False, _UNMAPPABLE_LIST_FORM, id="unmappable_kept_for_the_provider_400"),
|
||||
pytest.param(
|
||||
[{"type": "compaction"}],
|
||||
True,
|
||||
{"edits": [{"type": "compact_20260112"}]},
|
||||
id="mappable_still_mapped_under_drop_params",
|
||||
),
|
||||
),
|
||||
)
|
||||
def test_context_management_the_list_form_cannot_map_is_dropped_only_under_drop_params(
|
||||
config_class: type[AnthropicMessagesConfig], context_management: JsonValue, drop_params: bool, forwarded: JsonValue
|
||||
) -> None:
|
||||
request: Final = config_class().transform_anthropic_messages_request(
|
||||
model="claude-haiku-5-5",
|
||||
messages=[{"role": "user", "content": "Hi"}],
|
||||
anthropic_messages_optional_request_params={"max_tokens": 50, "context_management": context_management},
|
||||
litellm_params=GenericLiteLLMParams(drop_params=drop_params),
|
||||
headers={},
|
||||
)
|
||||
|
||||
assert request.get("context_management") == forwarded
|
||||
|
||||
|
||||
@pytest.mark.parametrize("config", _classifier_configs(), ids=type)
|
||||
def test_routes_whose_platform_runs_the_classifier_keep_safeguards_and_the_beta(
|
||||
config: AnthropicMessagesConfig, local_beta_headers_config: None
|
||||
|
|
|
|||
|
|
@ -26,7 +26,24 @@ from litellm.llms.azure_ai.anthropic.count_tokens.token_counter import AzureAIAn
|
|||
from litellm.llms.azure_ai.anthropic.count_tokens.transformation import (
|
||||
AzureAIAnthropicCountTokensConfig,
|
||||
)
|
||||
from litellm.types.utils import TokenCountMessagesFormat, TokenCountResponse
|
||||
from litellm.types.utils import CountContextManagement, TokenCountMessagesFormat, TokenCountResponse
|
||||
from litellm.constants import ANTHROPIC_TOKEN_COUNTING_BETA_VERSION
|
||||
from litellm.types.llms.anthropic import ANTHROPIC_BETA_HEADER_VALUES
|
||||
|
||||
COMPACTION_BETA = ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_09_04.value
|
||||
COMPACTED_HISTORY = [
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": [
|
||||
{
|
||||
"type": "compaction",
|
||||
"content": "The user is building a recipe app and asked for one-sentence class descriptions.",
|
||||
"signature": "EqQBCkYIBRgCKkBjZ2xhc3M" * 40,
|
||||
}
|
||||
],
|
||||
},
|
||||
{"role": "user", "content": "Now do the same for Ingredient."},
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
|
|
@ -890,3 +907,115 @@ async def test_counter_sends_the_structured_output_tool_choice_and_thinking_chat
|
|||
inference["tool_choice"],
|
||||
inference["tools"],
|
||||
)
|
||||
|
||||
|
||||
def test_headers_add_the_compaction_beta_only_for_a_compacted_history():
|
||||
config = AnthropicCountTokensConfig()
|
||||
auth_header = {"x-api-key": "sk-ant-api03-test-key"}
|
||||
|
||||
with_block = config.get_count_tokens_headers(auth_header, COMPACTED_HISTORY)["anthropic-beta"].split(",")
|
||||
plain = config.get_count_tokens_headers(auth_header, [{"role": "user", "content": "hi"}])["anthropic-beta"]
|
||||
|
||||
assert COMPACTION_BETA in with_block
|
||||
assert ANTHROPIC_TOKEN_COUNTING_BETA_VERSION in with_block
|
||||
assert plain == ANTHROPIC_TOKEN_COUNTING_BETA_VERSION
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_handler_sends_the_compaction_beta_with_a_compacted_history(httpx_transport_clients):
|
||||
with respx.mock:
|
||||
route = respx.post("https://gateway.example/v1/messages/count_tokens").mock(
|
||||
return_value=httpx.Response(200, json={"input_tokens": 404})
|
||||
)
|
||||
result = await AnthropicCountTokensHandler().handle_count_tokens_request(
|
||||
model="claude-sonnet-5-5",
|
||||
messages=COMPACTED_HISTORY,
|
||||
auth_header={"x-api-key": "sk-ant-api03-test-key"},
|
||||
api_base="https://gateway.example",
|
||||
)
|
||||
|
||||
assert result == {"input_tokens": 404}
|
||||
betas = route.calls.last.request.headers["anthropic-beta"].split(",")
|
||||
assert COMPACTION_BETA in betas
|
||||
assert ANTHROPIC_TOKEN_COUNTING_BETA_VERSION in betas
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.usefixtures("httpx_transport_clients")
|
||||
@pytest.mark.parametrize(
|
||||
("context_management", "forwarded", "sent_beta", "unsent_beta"),
|
||||
(
|
||||
pytest.param(
|
||||
{"edits": [{"type": "compact_20260112"}]},
|
||||
{"edits": [{"type": "compact_20260112"}]},
|
||||
ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_01_12.value,
|
||||
ANTHROPIC_BETA_HEADER_VALUES.CONTEXT_MANAGEMENT_2025_06_27.value,
|
||||
id="compact",
|
||||
),
|
||||
pytest.param(
|
||||
{"edits": [{"type": "clear_tool_uses_20250919"}]},
|
||||
{"edits": [{"type": "clear_tool_uses_20250919"}]},
|
||||
ANTHROPIC_BETA_HEADER_VALUES.CONTEXT_MANAGEMENT_2025_06_27.value,
|
||||
ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_01_12.value,
|
||||
id="clearing",
|
||||
),
|
||||
pytest.param(
|
||||
[{"type": "compaction", "compact_threshold": 150000}],
|
||||
{"edits": [{"type": "compact_20260112", "trigger": {"type": "input_tokens", "value": 150000}}]},
|
||||
ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_01_12.value,
|
||||
ANTHROPIC_BETA_HEADER_VALUES.CONTEXT_MANAGEMENT_2025_06_27.value,
|
||||
id="openai_list_form",
|
||||
),
|
||||
),
|
||||
)
|
||||
async def test_counter_forwards_context_management_with_the_beta_inference_sends(
|
||||
context_management: JsonValue, forwarded: JsonValue, sent_beta: str, unsent_beta: str
|
||||
) -> None:
|
||||
"""Anthropic counts an unsigned compaction block only under the compact_20260112 edit that inference sends,
|
||||
and a clearing edit changes the count, so the count carries the request's context_management as inference
|
||||
sends it, the OpenAI list form mapped to edits, and its beta."""
|
||||
with respx.mock:
|
||||
route: Final = respx.post("https://gateway.example/v1/messages/count_tokens").mock(
|
||||
return_value=httpx.Response(200, json={"input_tokens": 259})
|
||||
)
|
||||
result: Final = await AnthropicTokenCounter().count_tokens(
|
||||
model_to_use="claude-opus-5-5",
|
||||
messages=[{"role": "assistant", "content": [{"type": "compaction", "content": "Recipe app summary."}]}],
|
||||
contents=None,
|
||||
deployment={"litellm_params": {"api_key": "sk-ant-api03-test-key", "api_base": "https://gateway.example"}},
|
||||
context_management=CountContextManagement(spec=context_management, drop_invalid_edits=False),
|
||||
)
|
||||
|
||||
sent: Final = route.calls.last.request
|
||||
betas: Final = sent.headers["anthropic-beta"].split(",")
|
||||
assert result is not None and result.total_tokens == 259
|
||||
assert TypeAdapter(dict[str, JsonValue]).validate_json(sent.content)["context_management"] == forwarded
|
||||
assert (sent_beta in betas, unsent_beta in betas) == (True, False)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.usefixtures("httpx_transport_clients")
|
||||
@pytest.mark.parametrize("context_management", ({}, {"edits": []}, []), ids=("empty_object", "empty_edits", "empty_list"))
|
||||
async def test_counter_leaves_out_context_management_without_edits(context_management: JsonValue) -> None:
|
||||
"""Anthropic's count endpoint rejects a context_management with no edits, which inference accepts, so the count
|
||||
leaves it out and adds no context-management beta."""
|
||||
with respx.mock:
|
||||
route: Final = respx.post("https://gateway.example/v1/messages/count_tokens").mock(
|
||||
return_value=httpx.Response(200, json={"input_tokens": 11})
|
||||
)
|
||||
result: Final = await AnthropicTokenCounter().count_tokens(
|
||||
model_to_use="claude-opus-5-5",
|
||||
messages=[{"role": "user", "content": "Hello"}],
|
||||
contents=None,
|
||||
deployment={"litellm_params": {"api_key": "sk-ant-api03-test-key", "api_base": "https://gateway.example"}},
|
||||
context_management=CountContextManagement(spec=context_management, drop_invalid_edits=False),
|
||||
)
|
||||
|
||||
sent: Final = route.calls.last.request
|
||||
betas: Final = frozenset(sent.headers.get("anthropic-beta", "").split(","))
|
||||
assert result is not None and result.total_tokens == 11
|
||||
assert "context_management" not in TypeAdapter(dict[str, JsonValue]).validate_json(sent.content)
|
||||
assert not betas & {
|
||||
ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_01_12.value,
|
||||
ANTHROPIC_BETA_HEADER_VALUES.CONTEXT_MANAGEMENT_2025_06_27.value,
|
||||
}
|
||||
|
|
|
|||
|
|
@ -48,7 +48,7 @@ FEDERATED_DEPLOYMENT = {
|
|||
def count_tokens_headers_for(api_key: str) -> dict[str, str]:
|
||||
auth_header = AnthropicModelInfo.get_auth_header(api_key=api_key)
|
||||
assert auth_header is not None
|
||||
return AnthropicCountTokensConfig().get_count_tokens_headers(auth_header)
|
||||
return AnthropicCountTokensConfig().get_count_tokens_headers(auth_header, [{"role": "user", "content": "hi"}])
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
|
|
|
|||
|
|
@ -666,7 +666,8 @@ async def test_anthropic_view_counts_the_turns_after_a_compaction_block_when_an_
|
|||
*_agent_turns(2, first_round=3),
|
||||
)
|
||||
|
||||
view: Final = await inference_request_view(turns, None, "anthropic", _edits(clear_before_compact))
|
||||
with patch.object(litellm, "token_counter", side_effect=RuntimeError("tokenizer unavailable")):
|
||||
view: Final = await inference_request_view(turns, None, "anthropic", _edits(clear_before_compact))
|
||||
|
||||
assert isinstance(view, InferenceRequestView)
|
||||
assert _tool_outputs_kept(view) == (True, True)
|
||||
|
|
|
|||
|
|
@ -526,6 +526,29 @@ def test_both_compact_and_context_management_headers_added():
|
|||
)
|
||||
|
||||
|
||||
def test_openai_list_form_context_management_sends_the_compact_beta():
|
||||
config: Final = VertexAIPartnerModelsAnthropicMessagesConfig()
|
||||
|
||||
with (
|
||||
patch.object(config, "_ensure_access_token", return_value=("token", "test-project")),
|
||||
patch.object(config, "get_complete_vertex_url", return_value="https://mock-url"),
|
||||
):
|
||||
updated_headers, _ = config.validate_anthropic_messages_environment(
|
||||
headers={},
|
||||
model="claude-vertex-ai-opus-4-6",
|
||||
messages=[],
|
||||
optional_params={"context_management": [{"type": "compaction", "compact_threshold": 150000}]},
|
||||
litellm_params={
|
||||
"vertex_ai_project": "test-project",
|
||||
"vertex_ai_location": "us-central1",
|
||||
"vertex_credentials": "{}",
|
||||
},
|
||||
api_base=None,
|
||||
)
|
||||
|
||||
assert updated_headers["anthropic-beta"].split(",") == [ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_01_12.value]
|
||||
|
||||
|
||||
def test_validate_environment_always_refreshes_token_ignoring_stale_bearer():
|
||||
"""Regression: stale Authorization in shared deployment extra_headers must not
|
||||
skip token refresh on /v1/messages — _ensure_access_token is always called."""
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ import json
|
|||
import logging
|
||||
import unittest
|
||||
from collections.abc import AsyncGenerator, AsyncIterator, Callable, Mapping
|
||||
from types import MappingProxyType
|
||||
from typing import Final
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
|
|
@ -587,6 +588,48 @@ class TestErrorLogCarriesCallId:
|
|||
assert call_id in record.getMessage()
|
||||
|
||||
|
||||
|
||||
class TestCountTokensCompactedHistory:
|
||||
"""A history carrying Anthropic's signed compaction block used to fail /v1/messages/count_tokens once the count
|
||||
ran on the local tokenizer, which rejected the block type."""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_count_tokens_counts_a_compaction_block_like_its_summary_text(self):
|
||||
import litellm.proxy.anthropic_endpoints.endpoints as ep
|
||||
import litellm.proxy.proxy_server as proxy_server
|
||||
from litellm.proxy._types import UserAPIKeyAuth
|
||||
|
||||
summary = "The user is building a recipe app and asked for one-sentence class descriptions."
|
||||
follow_up = {"role": "user", "content": "Now do the same for Ingredient."}
|
||||
compacted = {
|
||||
"model": "claude-sonnet-5-5",
|
||||
"messages": [
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": [{"type": "compaction", "content": summary, "signature": "EqQBCkYIBRgCKkBjZ2xhc3M" * 40}],
|
||||
},
|
||||
follow_up,
|
||||
],
|
||||
}
|
||||
as_text = {
|
||||
"model": "claude-sonnet-5-5",
|
||||
"messages": [{"role": "assistant", "content": [{"type": "text", "text": summary}]}, follow_up],
|
||||
}
|
||||
request = MagicMock()
|
||||
request.headers = {}
|
||||
|
||||
async def count(body):
|
||||
with (
|
||||
patch.object(ep, "read_request_body", new=AsyncMock(return_value=body)), # test-quality-ok: endpoint reads the body via a module function; no injection seam
|
||||
patch.object(proxy_server, "llm_router", None), # test-quality-ok: module global; with no router the count runs on the local tokenizer, the path that raised
|
||||
):
|
||||
return await ep.count_tokens(request=request, user_api_key_dict=UserAPIKeyAuth())
|
||||
|
||||
counted = await count(compacted)
|
||||
|
||||
assert counted["input_tokens"] > 0
|
||||
assert counted == await count(as_text)
|
||||
|
||||
def _claude_router() -> Router:
|
||||
return Router(
|
||||
model_list=[
|
||||
|
|
@ -759,7 +802,10 @@ async def test_count_tokens_drops_malformed_tools_the_counted_deployment_drops_e
|
|||
|
||||
|
||||
def _count_on_claude(
|
||||
monkeypatch: pytest.MonkeyPatch, peer: Callable[[httpx.Request], httpx.Response], headers: Mapping[str, str]
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
peer: Callable[[httpx.Request], httpx.Response],
|
||||
headers: Mapping[str, str],
|
||||
extra_body: Mapping[str, object] = MappingProxyType({}),
|
||||
) -> httpx.Response:
|
||||
from fastapi import FastAPI
|
||||
|
||||
|
|
@ -799,7 +845,7 @@ def _count_on_claude(
|
|||
app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth()
|
||||
return TestClient(app).post(
|
||||
"/v1/messages/count_tokens",
|
||||
json={"model": "count-log-claude", "messages": [{"role": "user", "content": "Hello"}]},
|
||||
json={"model": "count-log-claude", "messages": [{"role": "user", "content": "Hello"}], **extra_body},
|
||||
headers=dict(headers),
|
||||
)
|
||||
|
||||
|
|
@ -827,6 +873,24 @@ def test_count_tokens_logs_one_line_with_the_callers_call_id_model_and_count(
|
|||
)
|
||||
|
||||
|
||||
def test_count_tokens_returns_the_context_management_the_provider_applied(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
applied: Final = {"original_input_tokens": 640}
|
||||
response: Final = _count_on_claude(
|
||||
monkeypatch,
|
||||
lambda request: httpx.Response(
|
||||
200, json={"input_tokens": 576, "context_management": applied}, request=request
|
||||
),
|
||||
{},
|
||||
{"context_management": {"edits": [{"type": "clear_tool_uses_20250919"}]}},
|
||||
)
|
||||
|
||||
assert response.json() == {
|
||||
"input_tokens": 576,
|
||||
"context_management": applied,
|
||||
"litellm_count_source": "provider",
|
||||
}, response.text
|
||||
|
||||
|
||||
def test_count_tokens_provider_rejection_logs_its_status_and_answers_the_same_error(
|
||||
monkeypatch: pytest.MonkeyPatch, token_count_log_lines: Callable[[], tuple[dict[str, object], ...]]
|
||||
) -> None:
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue