fix(count_tokens): count a malformed body or a failed Mantle fallback again without the options, and never retry a 401 or 403

The count sends inference's betas when the count forwards options, and it now filters them through update_headers_with_filtered_beta, the helper inference uses. A count without options still sends only the token-counting beta and the bare body

A body the count transform cannot read, like a tool whose type is not a string, now comes back as a translation error, so the proxy counts again without the options instead of answering 500. On Bedrock, when the runtime count rejects the forwarded options with a 400 or 404, the Mantle answer is used only if Mantle counted or rejected the request with a 4xx other than 401, 403 or 429. A Mantle 5xx, 429, auth error, timeout or connection error now returns the runtime rejection, and the proxy counts that again without the options

The proxy no longer counts again after a 401 or 403 because those are not about the request, and its warning names every option it drops. Tests cover a group where only one deployment drops context_management, Vertex's tool-search beta, and the exact betas each counter sends
This commit is contained in:
mateo-berri 2026-11-06 17:57:27 -08:00
parent f84c244f3e
commit 14b572d44b
8 changed files with 437 additions and 80 deletions

View file

@ -28,6 +28,7 @@ from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import
from litellm.litellm_core_utils.url_utils import join_api_base
from litellm.llms.anthropic.common_utils import merge_anthropic_beta_headers
from litellm.llms.anthropic.wif import resolve_anthropic_base
from litellm.llms.base_llm.anthropic_messages.transformation import BaseAnthropicMessagesConfig
from litellm.types.llms.openai import AllMessageValues, ChatCompletionImageObject, ValidUserMessageContentTypes
from litellm.types.utils import LlmProviders, TokenCountMessagesFormat, TokenCountResponse
@ -345,17 +346,61 @@ _COUNTED_MESSAGES: Final = TypeAdapter(tuple[Mapping[str, JsonValue], ...])
_MAX_TOKENS_ABOVE_ANY_BUDGET: Final = 2**31 - 1
def _with_provider_betas(headers: Mapping[str, str], provider: str) -> Mapping[str, str]:
from litellm.anthropic_beta_headers_manager import filter_and_transform_beta_headers
_STATUSES_NOT_CAUSED_BY_THE_REQUEST: Final = frozenset({401, 403, 429})
beta_header: Final = headers.get("anthropic-beta")
if beta_header is None:
return headers
supported: Final = filter_and_transform_beta_headers(
[beta.strip() for beta in beta_header.split(",") if beta.strip()], provider
def is_request_rejection(status_code: int) -> bool:
return 400 <= status_code < 500 and status_code not in _STATUSES_NOT_CAUSED_BY_THE_REQUEST
def _claude_count_options_for(
messages_config: BaseAnthropicMessagesConfig,
model: str,
provider: LlmProviders,
messages: Sequence[Mapping[str, object]],
tools: Sequence[Mapping[str, object]] | None,
sent: Mapping[str, JsonValue],
anthropic_beta: str | None,
deployment: Mapping[str, object] | None,
) -> ClaudeCountOptions:
from litellm.anthropic_beta_headers_manager import update_headers_with_filtered_beta
from litellm.types.router import GenericLiteLLMParams
resolved_deployment: Final = deployment or {}
deployment_params: Final = _INFERENCE_PARAMS.validate_python(resolved_deployment.get("litellm_params") or {})
leading_system, conversation = split_leading_system_run(messages)
sent_conversation: Final = copy.deepcopy([dict(message) for message in conversation])
sent_params: Final = {
**copy.deepcopy(sent),
**({"tools": copy.deepcopy([dict(tool) for tool in tools])} if tools else {}),
}
beta_headers: Final = messages_config.headers_with_anthropic_betas(
{"anthropic-beta": anthropic_beta} if anthropic_beta else {}, sent_params, sent_conversation
)
sent_headers: Final = (
update_headers_with_filtered_beta(headers=dict(beta_headers), provider=provider.value)
if messages_config.should_filter_anthropic_beta_headers()
else dict(beta_headers)
)
request: Final = messages_config.transform_anthropic_messages_request( # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # the base config signature takes and returns bare dicts
model=model,
messages=sent_conversation,
anthropic_messages_optional_request_params={**sent_params, "max_tokens": _MAX_TOKENS_ABOVE_ANY_BUDGET},
litellm_params=GenericLiteLLMParams.model_validate(
{
"drop_params": deployment_params.get("drop_params"),
"model_info": resolved_deployment.get("model_info"),
}
),
headers=sent_headers,
)
transformed: Final = _ANTHROPIC_CHAT_REQUEST.validate_python(request)
return ClaudeCountOptions(
options=_count_options(transformed),
betas=messages_config.sent_anthropic_betas(sent_headers, _INFERENCE_PARAMS.validate_python(request)),
messages=(*_COUNTED_MESSAGES.validate_python(leading_system), *transformed["messages"]),
tools=transformed.get("tools"),
)
other_headers: Final = {name: value for name, value in headers.items() if name != "anthropic-beta"}
return {**other_headers, "anthropic-beta": ",".join(supported)} if supported else other_headers
def claude_count_options(
@ -372,50 +417,19 @@ def claude_count_options(
return NO_CLAUDE_COUNT_OPTIONS
from litellm.exceptions import BadRequestError
from litellm.llms.anthropic.common_utils import AnthropicError
from litellm.types.router import GenericLiteLLMParams
from litellm.utils import ProviderConfigManager
messages_config: Final = ProviderConfigManager.get_provider_anthropic_messages_config(model, provider)
if messages_config is None:
return NO_CLAUDE_COUNT_OPTIONS
resolved_deployment: Final = deployment or {}
deployment_params: Final = _INFERENCE_PARAMS.validate_python(resolved_deployment.get("litellm_params") or {})
leading_system, conversation = split_leading_system_run(messages)
sent_conversation: Final = copy.deepcopy([dict(message) for message in conversation])
sent_params: Final = {
**copy.deepcopy(sent),
**({"tools": copy.deepcopy([dict(tool) for tool in tools])} if tools else {}),
}
beta_headers: Final = messages_config.headers_with_anthropic_betas(
{"anthropic-beta": anthropic_beta} if anthropic_beta else {}, sent_params, sent_conversation
)
sent_headers: Final = dict(
_with_provider_betas(beta_headers, provider.value)
if messages_config.should_filter_anthropic_beta_headers()
else beta_headers
)
try:
request: Final = messages_config.transform_anthropic_messages_request( # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # the base config signature takes and returns bare dicts
model=model,
messages=sent_conversation,
anthropic_messages_optional_request_params={**sent_params, "max_tokens": _MAX_TOKENS_ABOVE_ANY_BUDGET},
litellm_params=GenericLiteLLMParams.model_validate(
{
"drop_params": deployment_params.get("drop_params"),
"model_info": resolved_deployment.get("model_info"),
}
),
headers=sent_headers,
return _claude_count_options_for(
messages_config, model, provider, messages, tools, sent, anthropic_beta, deployment
)
except (AnthropicError, BadRequestError) as e:
return ChatCountTranslationError(message=str(e.message))
transformed: Final = _ANTHROPIC_CHAT_REQUEST.validate_python(request)
return ClaudeCountOptions(
options=_count_options(transformed),
betas=messages_config.sent_anthropic_betas(sent_headers, _INFERENCE_PARAMS.validate_python(request)),
messages=(*_COUNTED_MESSAGES.validate_python(leading_system), *transformed["messages"]),
tools=transformed.get("tools"),
)
except (AttributeError, IndexError, KeyError, TypeError, ValueError) as e:
return ChatCountTranslationError(message=f"Invalid request for token counting: {e}")
async def anthropic_count_fields(

View file

@ -19,6 +19,7 @@ from litellm.llms.anthropic.count_tokens.transformation import (
ClaudeCountOptions,
anthropic_count_fields,
claude_count_options,
is_request_rejection,
)
from litellm.llms.base_llm.base_utils import BaseTokenCounter
from litellm.llms.bedrock.common_utils import BedrockError, get_bedrock_base_model
@ -80,6 +81,10 @@ def _runtime_rejected_claude_model(outcome: CountTokensOutcome, resolved_model:
)
def _mantle_answered_for_runtime(fallback: CountTokensOutcome, forwarded_options: bool) -> bool:
return not forwarded_options or isinstance(fallback, CountedTokens) or is_request_rejection(fallback.status_code)
def _invoke_fields(claude_options: ClaudeCountOptions) -> Mapping[str, object]:
return {
**claude_options.options,
@ -225,11 +230,16 @@ class BedrockTokenCounter(BaseTokenCounter):
primary_tokenizer,
)
)
outcome: Final = (
fallback: Final = (
await self._count_with(
self._count_on_mantle(mantle_request, litellm_params, resolved_model), MANTLE_TOKENIZER_TYPE
)
if not use_mantle and _runtime_rejected_claude_model(primary, resolved_model)
else None
)
outcome: Final = (
fallback
if fallback is not None and _mantle_answered_for_runtime(fallback, forwarded_options=bool(invoke_fields))
else primary
)
match outcome:

View file

@ -8,6 +8,7 @@ from pydantic import BaseModel, ConfigDict, ValidationError
import litellm
from litellm.anthropic_interface.exceptions.exception_mapping_utils import AnthropicExceptionMapping
from litellm.litellm_core_utils.core_helpers import as_str_mapping
from litellm.llms.anthropic.count_tokens.transformation import is_request_rejection
from litellm.proxy._types import ProxyException
from litellm.types.token_count import TokenCountProvider, TokenCountSourceFields, TokenCountTokenizer
from litellm.types.utils import TokenCountResponse
@ -74,7 +75,7 @@ class ProviderCountFailure:
@property
def rejected_request(self) -> bool:
return 400 <= self.status_code < 500 and self.status_code != 429
return is_request_rejection(self.status_code)
def with_hint(self, hint: str) -> "ProviderCountFailure":
if self.rejected_before_provider_call:

View file

@ -14775,9 +14775,8 @@ _DEPLOYMENT_LITELLM_PARAMS: Final = TypeAdapter(Mapping[str, object])
def _deployment_with_request_drop_params(
deployment: dict[str, object] | None, # mutable-ok: BaseTokenCounter.count_tokens takes a dict deployment
drop_params: object,
) -> dict[str, object] | None: # mutable-ok: BaseTokenCounter.count_tokens takes a dict deployment
deployment: Mapping[str, object] | None, drop_params: bool | None
) -> Mapping[str, object] | None:
if drop_params is None:
return deployment
resolved_deployment: Final = deployment or {}
@ -14931,11 +14930,12 @@ def _retried_without_rejected_options(
) -> Callable[[str, str | None], Awaitable[Union["TokenCountResponse", ProviderCountFailure, None]]]:
if not request_options:
return count_with
return partial(_count_retrying_without_options, count_with)
return partial(_count_retrying_without_options, count_with, tuple(request_options))
async def _count_retrying_without_options(
count_with: "partial[Awaitable[TokenCountResponse | ProviderCountFailure | None]]",
forwarded_options: tuple[str, ...],
model: str,
region: str | None,
) -> Union["TokenCountResponse", ProviderCountFailure, None]:
@ -14943,10 +14943,10 @@ async def _count_retrying_without_options(
if not (isinstance(counted, ProviderCountFailure) and counted.rejected_request):
return counted
verbose_proxy_logger.warning(
"Token count for %s answered %s with the request's thinking, tool, and output options, "
"so it counts again without them",
"Token count for %s answered %s with the request's %s, so it counts again without them",
model,
counted.status_code,
", ".join(forwarded_options),
)
return await count_with(model, region, request_options=MappingProxyType({}))
@ -15475,7 +15475,9 @@ async def _count_request_tokens(
model_to_use = _model
sent_params: Final[Mapping[str, object]] = {**_router_default_litellm_params(llm_router), **request_litellm_params}
count_deployment: Final = _deployment_with_request_drop_params(deployment, sent_params.get("drop_params"))
count_deployment: Final = _deployment_with_request_drop_params(
deployment, normalize_drop_params(sent_params.get("drop_params"))
)
conversation_empty: Final = not messages and not contents
if prompt and not conversation_empty and not deployment_drops_params(count_deployment):
raise ProxyException(

View file

@ -916,21 +916,27 @@ async def _messages_inference_request(
return sent.get()
def _on_both_counters(
*betas: str,
) -> Mapping[type[AnthropicTokenCounter] | type[AzureAIAnthropicTokenCounter], frozenset[str]]:
return MappingProxyType({AnthropicTokenCounter: frozenset(betas), AzureAIAnthropicTokenCounter: frozenset(betas)})
def _betas(request: httpx.Request) -> frozenset[str]:
return frozenset(beta for beta in request.headers.get("anthropic-beta", "").split(",") if beta)
@pytest.mark.asyncio
@pytest.mark.usefixtures("httpx_transport_clients")
@pytest.mark.usefixtures("httpx_transport_clients", "local_beta_headers_config")
@_COUNTERS
@pytest.mark.parametrize(
("request_options", "expected_betas"),
(
({"thinking": _ENABLED_THINKING}, frozenset()),
({"tool_choice": {"type": "any"}}, frozenset()),
({"thinking": _ENABLED_THINKING}, _on_both_counters()),
({"tool_choice": {"type": "any"}}, _on_both_counters()),
(
{"output_config": {"effort": "low"}, "output_format": {"type": "json_schema", "schema": _ANSWER_SCHEMA}},
frozenset((ANTHROPIC_BETA_HEADER_VALUES.STRUCTURED_OUTPUT_2025_09_25.value,)),
_on_both_counters(ANTHROPIC_BETA_HEADER_VALUES.STRUCTURED_OUTPUT_2025_09_25.value),
),
(
{
@ -939,9 +945,17 @@ def _betas(request: httpx.Request) -> frozenset[str]:
"edits": [{"type": "clear_tool_uses_20250919"}, {"type": "clear_thinking_20251015"}]
},
},
frozenset((ANTHROPIC_BETA_HEADER_VALUES.CONTEXT_MANAGEMENT_2025_06_27.value,)),
_on_both_counters(ANTHROPIC_BETA_HEADER_VALUES.CONTEXT_MANAGEMENT_2025_06_27.value),
),
(
{"context_management": {"edits": [{"type": "compact_20260112"}]}},
MappingProxyType(
{
AnthropicTokenCounter: frozenset((ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_01_12.value,)),
AzureAIAnthropicTokenCounter: frozenset(),
}
),
),
({"context_management": {"edits": [{"type": "compact_20260112"}]}}, frozenset()),
),
ids=("thinking", "tool_choice", "output_config_and_output_format", "context_edits", "compact_edit"),
)
@ -949,7 +963,7 @@ async def test_counter_sends_the_options_and_context_edits_messages_inference_se
counter_type: type[AnthropicTokenCounter] | type[AzureAIAnthropicTokenCounter],
count_endpoint: str,
request_options: dict[str, JsonValue],
expected_betas: frozenset[str],
expected_betas: Mapping[type[AnthropicTokenCounter] | type[AzureAIAnthropicTokenCounter], frozenset[str]],
) -> None:
inference: Final = await _messages_inference_request(
counter_type, count_endpoint.removesuffix("/count_tokens"), request_options
@ -977,7 +991,7 @@ async def test_counter_sends_the_options_and_context_edits_messages_inference_se
assert {name: count_body.get(name) for name in ("thinking", "tool_choice", "context_management")} == {
name: inference_body.get(name) for name in ("thinking", "tool_choice", "context_management")
}
assert expected_betas <= _betas(count) - {ANTHROPIC_TOKEN_COUNTING_BETA_VERSION} == _betas(inference)
assert expected_betas[counter_type] == _betas(count) - {ANTHROPIC_TOKEN_COUNTING_BETA_VERSION} == _betas(inference)
@pytest.mark.asyncio
@ -1011,6 +1025,30 @@ async def test_counter_sends_the_thinking_display_and_caller_betas_messages_infe
assert {ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER, caller_beta} <= counted_betas
@pytest.mark.asyncio
@pytest.mark.usefixtures("httpx_transport_clients")
@_COUNTERS
async def test_counter_without_options_sends_only_the_token_counting_beta_and_the_bare_count_body(
counter_type: type[AnthropicTokenCounter] | type[AzureAIAnthropicTokenCounter], count_endpoint: str
) -> None:
_, (count,) = await _counted_requests(
counter_type,
count_endpoint,
[{"role": "user", "content": "Weather in SF?"}],
[_WEATHER_TOOL],
"anthropic",
anthropic_beta=ANTHROPIC_BETA_HEADER_VALUES.WEB_FETCH_2025_09_10.value,
model="claude-haiku-5-5",
)
assert _betas(count) == {ANTHROPIC_TOKEN_COUNTING_BETA_VERSION}
assert _JSON_BODY.validate_json(count.content) == {
"model": "claude-haiku-5-5",
"messages": [{"role": "user", "content": "Weather in SF?"}],
"tools": [_WEATHER_TOOL],
}
@pytest.mark.asyncio
@pytest.mark.usefixtures("httpx_transport_clients")
@_COUNTERS

View file

@ -1,6 +1,6 @@
import base64
import json
from collections.abc import Mapping
from collections.abc import Callable, Mapping
from typing import Final
import httpx
@ -489,6 +489,77 @@ async def test_mantle_count_sends_the_thinking_display_beta_mantle_inference_sen
assert ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER in _sent_betas(bedrock.requests[0])
def _mantle_answers(status_code: int, body: Mapping[str, JsonValue]) -> Callable[[httpx.Request], httpx.Response]:
return lambda request: httpx.Response(status_code, json=dict(body), request=request)
def _mantle_unreachable(error: type[httpx.TransportError]) -> Callable[[httpx.Request], httpx.Response]:
def unreachable(request: httpx.Request) -> httpx.Response:
raise error("bedrock-mantle is unreachable", request=request)
return unreachable
_MANTLE_REJECTION: Final[dict[str, JsonValue]] = {
"type": "error",
"error": {"type": "invalid_request_error", "message": "thinking: unsupported"},
}
_MANTLE_FAILURE: Final[dict[str, JsonValue]] = {"type": "error", "error": {"type": "api_error", "message": "down"}}
@pytest.mark.asyncio
@pytest.mark.parametrize(
("request_options", "mantle", "expected"),
(
(_DISPLAY_THINKING, _mantle_unreachable(httpx.ConnectError), (True, 400, "bedrock_api")),
(_DISPLAY_THINKING, _mantle_unreachable(httpx.ReadTimeout), (True, 400, "bedrock_api")),
(_DISPLAY_THINKING, _mantle_answers(503, _MANTLE_FAILURE), (True, 400, "bedrock_api")),
(_DISPLAY_THINKING, _mantle_answers(429, _MANTLE_FAILURE), (True, 400, "bedrock_api")),
(_DISPLAY_THINKING, _mantle_answers(403, _MANTLE_FAILURE), (True, 400, "bedrock_api")),
(_DISPLAY_THINKING, _mantle_answers(400, _MANTLE_REJECTION), (True, 400, "bedrock_mantle_api")),
(_DISPLAY_THINKING, _mantle_answers(422, _MANTLE_REJECTION), (True, 422, "bedrock_mantle_api")),
(_DISPLAY_THINKING, _mantle_answers(200, {"input_tokens": 2177}), (False, None, "bedrock_mantle_api")),
({}, _mantle_answers(503, _MANTLE_FAILURE), (True, 503, "bedrock_mantle_api")),
),
ids=(
"mantle-unreachable",
"mantle-timeout",
"mantle-5xx",
"mantle-rate-limit",
"mantle-denied",
"mantle-rejects-the-options",
"mantle-rejects-the-body",
"mantle-counts",
"no-options-mantle-5xx",
),
)
async def test_runtime_rejecting_the_options_is_answered_unless_mantle_counts_or_rejects_the_request(
request_options: dict[str, JsonValue],
mantle: Callable[[httpx.Request], httpx.Response],
expected: tuple[bool, int | None, str],
) -> None:
def bedrock(request: httpx.Request) -> httpx.Response:
if request.url.host == RUNTIME_HOST:
return httpx.Response(400, json=UNSUPPORTED, request=request)
return mantle(request)
result: Final = await BedrockTokenCounter(
client=AsyncHTTPHandler(transport=httpx.MockTransport(bedrock))
).count_tokens(
model_to_use="us.anthropic.claude-opus-5-5",
messages=MESSAGES,
contents=None,
deployment={
"litellm_params": {**DEPLOYMENT["litellm_params"], "model": "bedrock/us.anthropic.claude-opus-5-5"}
},
request_model="claude-opus-5-5",
request_options=request_options,
)
assert result is not None
assert (result.error, result.status_code, result.tokenizer_type) == expected
@pytest.mark.asyncio
async def test_runtime_count_sends_the_tools_invoke_inference_sends() -> None:
inference, bedrock = await _messages_inference_and_count(

View file

@ -33,6 +33,7 @@ from litellm.llms.vertex_ai.common_utils import (
)
from litellm.llms.vertex_ai.count_tokens import handler as vertex_count_tokens_handler
from litellm.llms.vertex_ai.vertex_ai_partner_models.anthropic.transformation import VertexAIAnthropicConfig
from litellm.types.llms.anthropic_tool_search import get_tool_search_beta_header
from litellm.types.utils import CountContextManagement, TokenCountMessagesFormat
@ -1970,20 +1971,13 @@ _PARTNER_CLAUDE_REPLY: Final[dict[str, JsonValue]] = {
}
@pytest.mark.asyncio
async def test_vertex_claude_count_sends_the_options_context_edits_and_beta_vertex_messages_inference_sends() -> None:
async def _vertex_claude_inference_and_count(
tools: list[dict[str, JsonValue]], request_options: Mapping[str, JsonValue]
) -> tuple[httpx.Request, httpx.Request]:
from litellm.llms.vertex_ai.vertex_ai_partner_models.count_tokens import handler
from litellm.llms.vertex_ai.vertex_llm_base import VertexBase
context_edits: Final[dict[str, JsonValue]] = {
"edits": [{"type": "clear_tool_uses_20250919"}, {"type": "clear_thinking_20251015"}]
}
options: Final[dict[str, JsonValue]] = {
"thinking": {"type": "enabled", "budget_tokens": 2000},
"tool_choice": {"type": "auto"},
}
messages: Final[list[dict[str, JsonValue]]] = [{"role": "user", "content": "Weather in SF?"}]
tools: Final[list[dict[str, JsonValue]]] = [{"name": "get_weather", "input_schema": {"type": "object"}}]
sent: Final[SimpleQueue[httpx.Request]] = SimpleQueue()
def vertex(request: httpx.Request) -> httpx.Response:
@ -2004,11 +1998,10 @@ async def test_vertex_claude_count_sends_the_options_context_edits_and_beta_vert
messages=messages,
tools=tools,
max_tokens=4096,
context_management=context_edits,
vertex_project="test-project",
vertex_location="us-east5",
client=client,
**options,
**request_options,
)
result: Final = await VertexAITokenCounter().count_tokens(
model_to_use="claude-haiku-5-5",
@ -2016,13 +2009,29 @@ async def test_vertex_claude_count_sends_the_options_context_edits_and_beta_vert
contents=None,
deployment={"litellm_params": {"vertex_project": "test-project", "vertex_location": "us-east5"}},
tools=tools,
request_options={**options, "context_management": context_edits},
request_options=request_options,
)
assert result is not None
assert result.total_tokens == 37
inference, count = tuple(sent.get() for _ in range(sent.qsize()))
assert count.url.path.endswith("count-tokens:rawPredict")
return inference, count
@pytest.mark.asyncio
async def test_vertex_claude_count_sends_the_options_context_edits_and_beta_vertex_messages_inference_sends() -> None:
inference, count = await _vertex_claude_inference_and_count(
[{"name": "get_weather", "input_schema": {"type": "object"}}],
{
"thinking": {"type": "enabled", "budget_tokens": 2000},
"tool_choice": {"type": "auto"},
"context_management": {
"edits": [{"type": "clear_tool_uses_20250919"}, {"type": "clear_thinking_20251015"}]
},
},
)
counted: Final = TypeAdapter(dict[str, JsonValue]).validate_json(count.content)
inferred: Final = TypeAdapter(dict[str, JsonValue]).validate_json(inference.content)
fields: Final = ("thinking", "tool_choice", "output_config", "context_management")
@ -2030,6 +2039,20 @@ async def test_vertex_claude_count_sends_the_options_context_edits_and_beta_vert
assert count.headers["anthropic-beta"] == inference.headers["anthropic-beta"]
@pytest.mark.asyncio
async def test_vertex_claude_count_sends_the_tool_search_beta_vertex_messages_inference_sends() -> None:
inference, count = await _vertex_claude_inference_and_count(
[
{"type": "tool_search_tool_regex_20251119", "name": "tool_search_tool_regex"},
{"name": "get_weather", "input_schema": {"type": "object"}, "defer_loading": True},
],
{"thinking": {"type": "enabled", "budget_tokens": 2000}},
)
assert count.headers["anthropic-beta"] == inference.headers["anthropic-beta"]
assert get_tool_search_beta_header("vertex_ai") in count.headers["anthropic-beta"].split(",")
@pytest.mark.asyncio
async def test_vertex_claude_count_refuses_options_vertex_messages_inference_refuses_before_calling_vertex() -> None:
result: Final = await VertexAITokenCounter().count_tokens(

View file

@ -83,6 +83,8 @@ from litellm.proxy.proxy_server import (
)
from litellm.responses.utils import ResponsesAPIRequestUtils
from litellm.rust_bridge.fork_guard import ForkedAfterNativeRuntimeStarted
from litellm.types.llms.anthropic import ANTHROPIC_BETA_HEADER_VALUES
from litellm.types.router import CustomRoutingStrategyBase
from litellm.types.secret_managers.main import KeyManagementSettings, KeyManagementSystem
from litellm.types.utils import (
ADDRESSED_RESPONSE_ID_FIELD,
@ -1739,7 +1741,7 @@ async def test_anthropic_count_tokens_on_a_claude_deployment_sends_the_options_i
)
_CALLER_BETA: Final = "web-fetch-2025-09-10"
_CALLER_BETA: Final = ANTHROPIC_BETA_HEADER_VALUES.WEB_FETCH_2025_09_10.value
async def _count_on_a_claude_deployment(
@ -1811,9 +1813,9 @@ async def test_anthropic_count_tokens_counts_again_without_the_options_when_the_
@pytest.mark.asyncio
@pytest.mark.parametrize("status_code", (401, 403, 404))
async def test_anthropic_count_tokens_answers_the_error_the_count_without_the_options_also_gets(
monkeypatch: pytest.MonkeyPatch, status_code: int
@pytest.mark.parametrize(("status_code", "counted_again"), ((401, False), (403, False), (404, True), (422, True)))
async def test_anthropic_count_tokens_counts_again_only_when_the_claude_count_rejects_the_request(
monkeypatch: pytest.MonkeyPatch, status_code: int, counted_again: bool
) -> None:
response, requests = await _count_on_a_claude_deployment(
monkeypatch,
@ -1824,7 +1826,30 @@ async def test_anthropic_count_tokens_answers_the_error_the_count_without_the_op
assert isinstance(response, JSONResponse)
assert response.status_code == status_code
assert [bool(_sent_option_paths(json.loads(sent.content))) for sent in requests] == [True, False]
assert [bool(_sent_option_paths(json.loads(sent.content))) for sent in requests] == [
True,
*((False,) if counted_again else ()),
]
@pytest.mark.asyncio
async def test_anthropic_count_tokens_warns_with_every_option_it_counts_again_without(
monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture
) -> None:
verbose_proxy_logger.addHandler(caplog.handler)
try:
with caplog.at_level(logging.WARNING, logger=verbose_proxy_logger.name):
await _count_on_a_claude_deployment(
monkeypatch, "anthropic/claude-haiku-5-5", _CLAUDE_COUNT_OPTIONS, _counts_only_without_options
)
finally:
verbose_proxy_logger.removeHandler(caplog.handler)
warnings: Final = {
record.getMessage() for record in caplog.records if "counts again without" in record.getMessage()
}
(warning,) = warnings
assert all(option in warning for option in _CLAUDE_COUNT_OPTIONS)
@pytest.mark.asyncio
@ -1843,6 +1868,179 @@ async def test_anthropic_count_tokens_does_not_count_again_after_a_rate_limit(
assert len(requests) == 1
_BEDROCK_RUNTIME_HOST: Final = "bedrock-runtime.us-east-1.amazonaws.com"
_BEDROCK_MANTLE_HOST: Final = "bedrock-mantle.us-east-1.api.aws"
_BEDROCK_CLAUDE_DEPLOYMENT: Final[dict[str, str]] = {
"model": "bedrock/us.anthropic.claude-count-probe-v1:0",
"aws_access_key_id": "AKIATESTACCESSKEY",
"aws_secret_access_key": "test-secret",
"aws_region_name": "us-east-1",
}
async def _count_on_a_bedrock_claude_deployment(
monkeypatch: pytest.MonkeyPatch,
body: Mapping[str, JsonValue],
runtime: Callable[[httpx.Request], httpx.Response],
mantle: Callable[[httpx.Request], httpx.Response],
) -> tuple[object, tuple[httpx.Request, ...]]:
import litellm.proxy.proxy_server as proxy_server
monkeypatch.delenv("AWS_BEARER_TOKEN_BEDROCK", raising=False)
monkeypatch.delenv("BEDROCK_MANTLE_API_BASE", raising=False)
router: Final = Router(model_list=[{"model_name": "claude-agent", "litellm_params": _BEDROCK_CLAUDE_DEPLOYMENT}])
request: Final = _json_request(
{"model": "claude-agent", "messages": [{"role": "user", "content": "Weather in SF?"}], **body},
"python-httpx/0.28.1",
)
monkeypatch.setattr(proxy_server, "llm_router", router)
monkeypatch.setattr(proxy_server, "proxy_config", proxy_server.ProxyConfig())
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
with respx.mock as bedrock:
bedrock.route(host=_BEDROCK_RUNTIME_HOST).mock(side_effect=runtime)
bedrock.route(host=_BEDROCK_MANTLE_HOST).mock(side_effect=mantle)
response: Final = await anthropic_count_tokens(request, UserAPIKeyAuth())
return response, tuple(call.request for call in bedrock.calls)
def _runtime_counts_only_without_options(request: httpx.Request) -> httpx.Response:
if "invokeModel" in json.loads(request.content)["input"]:
return httpx.Response(400, json={"message": "thinking: unsupported"})
return httpx.Response(200, json={"inputTokens": 9})
def _mantle_unreachable(request: httpx.Request) -> httpx.Response:
raise httpx.ConnectError("bedrock-mantle is unreachable", request=request)
@pytest.mark.asyncio
async def test_anthropic_count_tokens_on_bedrock_counts_a_malformed_tool_again_without_the_options(
monkeypatch: pytest.MonkeyPatch,
) -> None:
response, requests = await _count_on_a_bedrock_claude_deployment(
monkeypatch,
{
"tools": [{"type": 5, "name": "x", "input_schema": {"type": "object"}}],
"thinking": {"type": "enabled", "budget_tokens": 2000},
},
_runtime_counts_only_without_options,
_mantle_unreachable,
)
assert not isinstance(response, JSONResponse)
assert response["input_tokens"] == 9
assert [request.url.host for request in requests] == [_BEDROCK_RUNTIME_HOST]
assert "invokeModel" not in json.loads(requests[0].content)["input"]
@pytest.mark.asyncio
async def test_anthropic_count_tokens_on_bedrock_counts_again_without_the_options_when_mantle_is_unreachable(
monkeypatch: pytest.MonkeyPatch,
) -> None:
response, requests = await _count_on_a_bedrock_claude_deployment(
monkeypatch,
{"thinking": {"type": "enabled", "budget_tokens": 2000}},
_runtime_counts_only_without_options,
_mantle_unreachable,
)
assert not isinstance(response, JSONResponse)
assert response["input_tokens"] == 9
hosts: Final = [request.url.host for request in requests]
assert hosts[0] == hosts[-1] == _BEDROCK_RUNTIME_HOST
assert _BEDROCK_MANTLE_HOST in hosts
assert "invokeModel" not in json.loads(requests[-1].content)["input"]
_KEEPS_CONTEXT_MANAGEMENT: Final = "sk-ant-keeps-context-management"
_DROPS_CONTEXT_MANAGEMENT: Final = "sk-ant-drops-context-management"
_CONTEXT_EDITS: Final[dict[str, JsonValue]] = {"edits": [{"type": "clear_tool_uses_20250919"}]}
_CLAUDE_MESSAGE_REPLY: Final[dict[str, JsonValue]] = {
"id": "msg_1",
"type": "message",
"role": "assistant",
"model": "claude-haiku-5-5",
"content": [{"type": "text", "text": "ok"}],
"stop_reason": "end_turn",
"usage": {"input_tokens": 3, "output_tokens": 1},
}
class _RoutesTo(CustomRoutingStrategyBase):
def __init__(self, router: Router, api_key: str) -> None:
self._router: Final = router
self._api_key: Final = api_key
async def async_get_available_deployment(self, *args: object, **kwargs: object) -> Mapping[str, object]:
return next(
deployment
for deployment in self._router.model_list
if deployment["litellm_params"]["api_key"] == self._api_key
)
def _mixed_drop_group(routed_key: str) -> Router:
router: Final = Router(
model_list=[
{
"model_name": "claude-agent",
"litellm_params": {"model": "anthropic/claude-haiku-5-5", "api_key": _KEEPS_CONTEXT_MANAGEMENT},
},
{
"model_name": "claude-agent",
"litellm_params": {
"model": "anthropic/claude-haiku-5-5",
"api_key": _DROPS_CONTEXT_MANAGEMENT,
"additional_drop_params": ["context_management"],
},
},
]
)
router.set_custom_routing_strategy(_RoutesTo(router, routed_key))
return router
@pytest.mark.asyncio
@pytest.mark.parametrize("routed_key", (_KEEPS_CONTEXT_MANAGEMENT, _DROPS_CONTEXT_MANAGEMENT))
async def test_anthropic_count_tokens_in_a_mixed_drop_group_sends_context_management_when_the_routed_inference_does(
monkeypatch: pytest.MonkeyPatch, routed_key: str
) -> None:
import litellm.proxy.proxy_server as proxy_server
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
monkeypatch.setattr(proxy_server, "llm_router", _mixed_drop_group(routed_key))
monkeypatch.setattr(proxy_server, "proxy_config", proxy_server.ProxyConfig())
messages: Final = [{"role": "user", "content": "Weather in SF?"}]
with respx.mock:
inference_route: Final = respx.post("https://api.anthropic.com/v1/messages").mock(
return_value=httpx.Response(200, json=_CLAUDE_MESSAGE_REPLY)
)
count_route: Final = respx.post("https://api.anthropic.com/v1/messages/count_tokens").mock(
return_value=httpx.Response(200, json={"input_tokens": 7})
)
await _mixed_drop_group(routed_key).aanthropic_messages(
model="claude-agent", messages=messages, max_tokens=16, context_management=_CONTEXT_EDITS
)
response: Final = await anthropic_count_tokens(
_json_request(
{"model": "claude-agent", "messages": messages, "context_management": _CONTEXT_EDITS},
"python-httpx/0.28.1",
),
UserAPIKeyAuth(),
)
assert not isinstance(response, JSONResponse)
inference: Final = inference_route.calls.last.request
count: Final = count_route.calls.last.request
assert inference.headers["x-api-key"] == count.headers["x-api-key"] == routed_key
assert ("context_management" in json.loads(count.content)) == (
"context_management" in json.loads(inference.content)
)
assert ("context_management" in json.loads(count.content)) == (routed_key == _KEEPS_CONTEXT_MANAGEMENT)
async def _claude_inference_body(model: str, options: Mapping[str, JsonValue]) -> dict[str, JsonValue]:
with respx.mock:
route: Final = respx.post("https://api.anthropic.com/v1/messages").mock(