fix(count_tokens): filter count betas like inference, retry only forwarded Claude options, and keep Mantle's error on edit-only Bedrock counts

The count now builds its beta filter with the same helper inference uses, so it reads the deployment's api_base, filters an anthropic deployment on a Foundry host with the azure_ai row, drops unlisted betas only where the provider filters them, and keeps the betas the deployment's model_info supports, such as per-turn-control on Bedrock invoke

The proxy counts again without the request's options only when the counter forwarded Claude options or betas, so a Gemini or OpenAI count that answers 400 is no longer sent twice

On Bedrock, a count that forwards only context edits returns Mantle's 401, 403, 429, 5xx or connection error again when the runtime answers 400 or 404, like main does. A count that forwarded Claude options keeps the runtime's answer

The effort beta guard in the Bedrock runtime count is removed, since the invoke body already sends each beta once

Tests cover a request drop list replacing the deployment's, the edit merge into the count's options on a Bedrock invoke count, and an opus-5-5 drop_params case. The Bedrock invoke beta hook now calls the base no-op instead of repeating it. A leftover docstring line, an extra blank line, and long test lines are cleaned up
This commit is contained in:
mateo-berri 2026-11-06 23:56:45 -08:00
parent 54ec2c8330
commit 4bdba8bbd5
17 changed files with 356 additions and 65 deletions

View file

@ -28,6 +28,7 @@ import os
import threading
from collections.abc import Mapping
from concurrent.futures import ThreadPoolExecutor
from dataclasses import dataclass
from functools import cache
from importlib.resources import files
from typing import Final, Literal, TypeAlias
@ -474,6 +475,28 @@ def update_headers_with_filtered_beta(
return headers
@dataclass(frozen=True, slots=True)
class DeploymentBetaFilter:
provider: str
api_base: str | None
unlisted: UnlistedBetas
def filtered(self, headers: dict[str, str], keep: frozenset[str]) -> dict[str, str]:
return update_headers_with_filtered_beta(
headers=headers, provider=self.provider, api_base=self.api_base, unlisted=self.unlisted, keep=keep
)
def deployment_beta_filter(
provider: str, affinity_api_base: str | None, provider_filters_betas: bool
) -> DeploymentBetaFilter | None:
if provider_filters_betas:
return DeploymentBetaFilter(provider=provider, api_base=affinity_api_base, unlisted="drop")
if resolve_beta_provider(provider, affinity_api_base) != provider:
return DeploymentBetaFilter(provider=provider, api_base=affinity_api_base, unlisted="forward")
return None
def update_request_with_filtered_beta(
headers: dict,
request_data: dict,

View file

@ -1676,6 +1676,10 @@ class AnthropicModelInfo(BaseLLMModelInfo):
def get_api_base(api_base: str | None = None) -> str | None:
return AnthropicModelInfo.resolve_anthropic_api_base(api_base)
@staticmethod
def affinity_api_base(custom_llm_provider: str, api_base: str | None) -> str | None:
return AnthropicModelInfo.get_api_base(api_base) if custom_llm_provider == "anthropic" else api_base
@staticmethod
def get_api_key(api_key: str | None = None) -> str | None:
"""An empty or whitespace-only key counts as unset: it can never authenticate anything, and

View file

@ -168,6 +168,7 @@ class AnthropicTokenCounter(BaseTokenCounter):
error=True,
error_message=e.message,
status_code=e.status_code,
carried_request_options=claude_options.carries_request_options,
)
except Exception as e:
verbose_logger.warning("Error calling Anthropic CountTokens API: %s", e)

View file

@ -20,6 +20,7 @@ from typing import (
from pydantic import ConfigDict, JsonValue, TypeAdapter, with_config
from typing_extensions import NotRequired, ReadOnly, TypedDict
from litellm.anthropic_beta_headers_manager import DeploymentBetaFilter, deployment_beta_filter
from litellm.constants import ANTHROPIC_TOKEN_COUNTING_BETA_VERSION
from litellm.litellm_core_utils.asyncify import asyncify
from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import (
@ -114,6 +115,7 @@ class AnthropicCountFields:
@dataclass(frozen=True, slots=True)
class ChatCountTranslationError:
message: str
carried_request_options: bool = False
def response(self, request_model: str, model_used: str, tokenizer_type: str) -> TokenCountResponse:
return TokenCountResponse(
@ -125,6 +127,7 @@ class ChatCountTranslationError:
error_message=self.message,
status_code=400,
rejected_before_provider_call=True,
carried_request_options=self.carried_request_options,
)
@ -349,6 +352,10 @@ class ClaudeCountOptions:
messages: tuple[Mapping[str, JsonValue], ...] | None
tools: tuple[Mapping[str, JsonValue], ...] | None
@property
def carries_request_options(self) -> bool:
return bool(self.options) or bool(self.betas)
NO_CLAUDE_COUNT_OPTIONS: Final = ClaudeCountOptions(options=MappingProxyType({}), betas=(), messages=None, tools=None)
_COUNTED_MESSAGES: Final = TypeAdapter(tuple[Mapping[str, JsonValue], ...])
@ -379,6 +386,11 @@ def deployment_inference_model(deployment: Mapping[str, object] | None) -> str |
return inference_model
def _deployment_api_base(deployment: Mapping[str, object] | None) -> str | None:
api_base: Final = _INFERENCE_PARAMS.validate_python((deployment or {}).get("litellm_params") or {}).get("api_base")
return api_base if isinstance(api_base, str) else None
def _messages_transform(model: str, provider: LlmProviders) -> tuple[str, BaseAnthropicMessagesConfig] | None:
from litellm.utils import ProviderConfigManager
@ -403,14 +415,12 @@ def _claude_count_options_for(
messages_config: BaseAnthropicMessagesConfig,
model: str,
litellm_params: "GenericLiteLLMParams",
provider: LlmProviders,
beta_filter: DeploymentBetaFilter | None,
messages: Sequence[Mapping[str, object]],
tools: Sequence[Mapping[str, object]] | None,
sent: Mapping[str, JsonValue],
anthropic_beta: str | None,
) -> ClaudeCountOptions:
from litellm.anthropic_beta_headers_manager import update_headers_with_filtered_beta
leading_system, conversation = split_leading_system_run(messages)
sent_conversation: Final = copy.deepcopy([dict(message) for message in conversation])
sent_params: Final = {
@ -421,9 +431,11 @@ def _claude_count_options_for(
{"anthropic-beta": anthropic_beta} if anthropic_beta else {}, sent_params, sent_conversation
)
sent_headers: Final = (
update_headers_with_filtered_beta(headers=dict(beta_headers), provider=provider.value)
if messages_config.should_filter_anthropic_beta_headers()
else dict(beta_headers)
dict(beta_headers)
if beta_filter is None
else beta_filter.filtered(
dict(beta_headers), keep=messages_config.model_specific_anthropic_betas(model, litellm_params)
)
)
request: Final = messages_config.transform_anthropic_messages_request( # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # the base config signature takes and returns bare dicts
model=model,
@ -462,16 +474,23 @@ def claude_count_options(
transform_model, messages_config = transform
litellm_params: Final = _transform_litellm_params(deployment)
from litellm.exceptions import BadRequestError
from litellm.llms.anthropic.common_utils import AnthropicError
from litellm.llms.anthropic.common_utils import AnthropicError, AnthropicModelInfo
beta_filter: Final = deployment_beta_filter(
provider.value,
AnthropicModelInfo.affinity_api_base(provider.value, _deployment_api_base(deployment)),
messages_config.should_filter_anthropic_beta_headers(),
)
try:
return _claude_count_options_for(
messages_config, transform_model, litellm_params, provider, messages, tools, sent, anthropic_beta
messages_config, transform_model, litellm_params, beta_filter, messages, tools, sent, anthropic_beta
)
except (AnthropicError, BadRequestError) as e:
return ChatCountTranslationError(message=str(e.message))
return ChatCountTranslationError(message=str(e.message), carried_request_options=True)
except (AttributeError, KeyError, TypeError, ValueError) as e:
return ChatCountTranslationError(message=f"Invalid request for token counting: {e}")
return ChatCountTranslationError(
message=f"Invalid request for token counting: {e}", carried_request_options=True
)
async def anthropic_count_fields(

View file

@ -176,6 +176,7 @@ class AzureAIAnthropicTokenCounter(BaseTokenCounter):
error=True,
error_message=e.message,
status_code=e.status_code,
carried_request_options=claude_options.carries_request_options,
)
except Exception as e:
verbose_logger.warning("Error calling Azure AI Anthropic CountTokens API: %s", e)

View file

@ -89,12 +89,12 @@ def _runtime_rejected_claude_model(outcome: CountTokensOutcome, resolved_model:
def _mantle_retry_outcome(
runtime: CountTokensOutcome, mantle: CountTokensOutcome, forwarded: bool
runtime: CountTokensOutcome, mantle: CountTokensOutcome, forwarded: bool, carried_options: bool
) -> CountTokensOutcome:
match runtime, mantle:
case CountTokensFailure(status_code=400), CountTokensFailure(status_code=404) if forwarded:
return runtime
case _, CountTokensFailure(status_code=status_code) if forwarded and not is_request_rejection(status_code):
case _, CountTokensFailure(status_code=code) if carried_options and not is_request_rejection(code):
return runtime
case _:
return mantle
@ -117,7 +117,7 @@ def _sets_effort(options: Mapping[str, JsonValue]) -> bool:
def _runtime_count_betas(claude_options: ClaudeCountOptions) -> tuple[str, ...]:
if not _sets_effort(claude_options.options) or ANTHROPIC_EFFORT_BETA_HEADER in claude_options.betas:
if not _sets_effort(claude_options.options):
return claude_options.betas
return (*claude_options.betas, ANTHROPIC_EFFORT_BETA_HEADER)
@ -322,7 +322,14 @@ class BedrockTokenCounter(BaseTokenCounter):
else None
)
outcome: Final = (
primary if fallback is None else _mantle_retry_outcome(primary, fallback, forwarded=bool(runtime_fields))
primary
if fallback is None
else _mantle_retry_outcome(
primary,
fallback,
forwarded=bool(runtime_fields),
carried_options=claude_options.carries_request_options,
)
)
match outcome:
case CountedTokens():
@ -348,6 +355,7 @@ class BedrockTokenCounter(BaseTokenCounter):
error=True,
error_message=outcome.message,
status_code=outcome.status_code,
carried_request_options=claude_options.carries_request_options,
)
case _:
assert_never(outcome)

View file

@ -121,7 +121,7 @@ class AmazonAnthropicClaudeMessagesConfig(
optional_params: Mapping[str, object],
messages: Sequence[object],
) -> Mapping[str, str]:
return headers
return BaseAnthropicMessagesConfig.headers_with_anthropic_betas(self, headers, optional_params, messages)
def sent_anthropic_betas(self, headers: Mapping[str, str], request: Mapping[str, object]) -> tuple[str, ...]:
return _ANTHROPIC_BETAS.validate_python(request.get("anthropic_beta") or ())

View file

@ -39,11 +39,7 @@ import litellm.types
import litellm.types.utils
from litellm._internal_context import internal_sub_call
from litellm._logging import redact_string, verbose_logger
from litellm.anthropic_beta_headers_manager import (
aload_beta_headers_config,
resolve_beta_provider,
update_headers_with_filtered_beta,
)
from litellm.anthropic_beta_headers_manager import aload_beta_headers_config, deployment_beta_filter
from litellm.constants import MAX_FILE_LIST_LIMIT, REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES
from litellm.files.types import FileContentStreamingResult
from litellm.litellm_core_utils.agentic_followup_kwargs import build_agentic_followup_kwargs
@ -2623,15 +2619,13 @@ class BaseLLMHTTPHandler:
litellm_params.get("additional_drop_params") or (),
anthropic_messages_optional_request_params,
)
affinity_api_base: Final = (
AnthropicModelInfo.get_api_base(api_base) if custom_llm_provider == "anthropic" else api_base
affinity_api_base: Final = AnthropicModelInfo.affinity_api_base(custom_llm_provider, api_base)
beta_filter: Final = deployment_beta_filter(
custom_llm_provider,
affinity_api_base,
anthropic_messages_provider_config.should_filter_anthropic_beta_headers(),
)
takes_another_beta_row: Final = (
resolve_beta_provider(custom_llm_provider, affinity_api_base) != custom_llm_provider
)
provider_filters_betas: Final = anthropic_messages_provider_config.should_filter_anthropic_beta_headers()
filters_beta_headers: Final = provider_filters_betas or takes_another_beta_row
if request_params.get("safeguards") is not None and filters_beta_headers:
if request_params.get("safeguards") is not None and beta_filter is not None:
await aload_beta_headers_config()
default_affinity_header: Final = get_provider_affinity_header(affinity_api_base)
(
@ -2651,14 +2645,10 @@ class BaseLLMHTTPHandler:
api_base=api_base,
)
if filters_beta_headers and headers.get("anthropic-beta") is not None:
if beta_filter is not None and headers.get("anthropic-beta") is not None:
await aload_beta_headers_config()
headers = update_headers_with_filtered_beta(
headers=headers,
provider=custom_llm_provider,
api_base=affinity_api_base,
unlisted="drop" if provider_filters_betas else "forward",
keep=anthropic_messages_provider_config.model_specific_anthropic_betas(model, litellm_params),
headers = beta_filter.filtered(
headers, keep=anthropic_messages_provider_config.model_specific_anthropic_betas(model, litellm_params)
)
from litellm.llms.vertex_ai.vertex_llm_base import VertexBase

View file

@ -1581,6 +1581,7 @@ class VertexAITokenCounter(BaseTokenCounter):
error=True,
error_message=error_message,
status_code=status_code,
carried_request_options=claude_options.carries_request_options,
)
if result is not None:

View file

@ -153,12 +153,6 @@ class VertexAIPartnerModelsTokenCounter(VertexBase):
"""
Handle token counting request for a Vertex AI partner model.
Args:
model: The model name
request_data: Request payload (Anthropic Messages API format)
litellm_params: LiteLLM parameters containing credentials, project, location
betas: anthropic-beta values the counted request needs, sent as the anthropic-beta header
Returns:
Dict containing token count information

View file

@ -80,6 +80,7 @@ class ProviderCountFailure:
status_code: int
message: str
rejected_before_provider_call: bool = False
carried_request_options: bool = False
@property
def cannot_count_model(self) -> bool:
@ -90,8 +91,8 @@ class ProviderCountFailure:
return not self.rejected_before_provider_call and self.status_code in REGION_DENIED_STATUSES
@property
def rejected_request(self) -> bool:
return is_request_rejection(self.status_code)
def rejected_request_options(self) -> bool:
return self.carried_request_options and is_request_rejection(self.status_code)
def with_hint(self, hint: str) -> "ProviderCountFailure":
if self.rejected_before_provider_call:

View file

@ -15269,6 +15269,7 @@ def _provider_count_outcome(
status_code=result.status_code or 500,
message=result.error_message or "Token counting failed",
rejected_before_provider_call=result.rejected_before_provider_call,
carried_request_options=result.carried_request_options,
)
return result
@ -15469,7 +15470,7 @@ async def _count_retrying_without_options(
region: str | None,
) -> Union["TokenCountResponse", ProviderCountFailure, None]:
counted: Final = await count_with(model, region)
if not (isinstance(counted, ProviderCountFailure) and counted.rejected_request):
if not (isinstance(counted, ProviderCountFailure) and counted.rejected_request_options):
return counted
verbose_proxy_logger.warning(
"Token count for %s %s with the request's %s, so it counts again without them and the caller's betas",

View file

@ -4912,6 +4912,7 @@ class TokenCountResponse(LiteLLMPydanticObjectBase):
"""
status_code: int | None = None
rejected_before_provider_call: bool = Field(default=False, exclude=True)
carried_request_options: bool = Field(default=False, exclude=True)
class CustomHuggingfaceTokenizer(TypedDict):

View file

@ -567,6 +567,7 @@ async def _counted_requests(
request_options: Mapping[str, JsonValue] = MappingProxyType({}),
anthropic_beta: str | None = None,
model: str = "claude-opus-5-5",
api_base: str = "https://gateway.example",
) -> tuple[TokenCountResponse | None, tuple[httpx.Request, ...]]:
sent: Final[SimpleQueue[httpx.Request]] = SimpleQueue()
@ -580,7 +581,7 @@ async def _counted_requests(
model_to_use=model,
messages=messages,
contents=None,
deployment={"litellm_params": {"api_key": "sk-ant-api03-test-key", "api_base": "https://gateway.example"}},
deployment={"litellm_params": {"api_key": "sk-ant-api03-test-key", "api_base": api_base}},
request_model=model,
tools=tools,
messages_format=messages_format,
@ -972,6 +973,7 @@ async def _messages_inference_request(
request_options: Mapping[str, JsonValue],
model: str = "claude-haiku-5-5",
extra_headers: Mapping[str, str] = MappingProxyType({}),
api_base: str = "https://gateway.example",
) -> httpx.Request:
sent: Final[SimpleQueue[httpx.Request]] = SimpleQueue()
@ -987,7 +989,7 @@ async def _messages_inference_request(
tools=[_WEATHER_TOOL],
max_tokens=4096,
api_key="sk-ant-api03-test-key",
api_base="https://gateway.example",
api_base=api_base,
extra_headers=dict(extra_headers),
**request_options,
)
@ -1059,7 +1061,10 @@ async def test_counter_sends_the_options_and_context_edits_messages_inference_se
assert count_body.get("output_config") == (
inference_output_config
if inference_output_format is None
else {**(inference_output_config if isinstance(inference_output_config, dict) else {}), "format": inference_output_format}
else {
**(inference_output_config if isinstance(inference_output_config, dict) else {}),
"format": inference_output_format,
}
)
assert {name: count_body.get(name) for name in ("thinking", "tool_choice", "context_management")} == {
name: inference_body.get(name) for name in ("thinking", "tool_choice", "context_management")
@ -1098,6 +1103,44 @@ async def test_counter_sends_the_thinking_display_and_caller_betas_messages_infe
assert {ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER, caller_beta} <= counted_betas
_FOUNDRY_BASE: Final = "https://my-resource.services.ai.azure.com/anthropic"
_UNLISTED_BETA: Final = "unlisted-beta-2099-01-01"
@pytest.mark.asyncio
@pytest.mark.usefixtures("httpx_transport_clients", "local_beta_headers_config")
@pytest.mark.parametrize(
("counter_type", "expected_betas"),
((AnthropicTokenCounter, frozenset({_UNLISTED_BETA})), (AzureAIAnthropicTokenCounter, frozenset())),
ids=("anthropic-deployment-forwards-unlisted", "azure-ai-deployment-drops-unlisted"),
)
async def test_counter_on_a_foundry_host_drops_the_caller_betas_messages_inference_drops_there(
counter_type: type[AnthropicTokenCounter] | type[AzureAIAnthropicTokenCounter], expected_betas: frozenset[str]
) -> None:
caller_betas: Final = f"mcp-servers-2025-12-04,{_UNLISTED_BETA}"
inference: Final = await _messages_inference_request(
counter_type,
f"{_FOUNDRY_BASE}/v1/messages",
{"thinking": _ENABLED_THINKING},
extra_headers={"anthropic-beta": caller_betas},
api_base=_FOUNDRY_BASE,
)
_, (count,) = await _counted_requests(
counter_type,
f"{_FOUNDRY_BASE}/v1/messages/count_tokens",
[{"role": "user", "content": "Weather in SF?"}],
[_WEATHER_TOOL],
"anthropic",
request_options={"thinking": _ENABLED_THINKING},
anthropic_beta=caller_betas,
model="claude-haiku-5-5",
api_base=_FOUNDRY_BASE,
)
counted_betas: Final = _betas(count) - {ANTHROPIC_TOKEN_COUNTING_BETA_VERSION}
assert counted_betas == _betas(inference) == expected_betas
@pytest.mark.asyncio
@pytest.mark.usefixtures("httpx_transport_clients")
@_COUNTERS
@ -1151,7 +1194,12 @@ _ADVISOR_TOOL: Final[dict[str, JsonValue]] = {
"name": "advisor",
"model": "claude-opus-5-5",
}
_ADVISOR_CALL: Final[dict[str, JsonValue]] = {"type": "server_tool_use", "id": "srvtoolu_1", "name": "advisor", "input": {}}
_ADVISOR_CALL: Final[dict[str, JsonValue]] = {
"type": "server_tool_use",
"id": "srvtoolu_1",
"name": "advisor",
"input": {},
}
_ADVISOR_RESULT: Final[dict[str, JsonValue]] = {
"type": "advisor_tool_result",
"tool_use_id": "srvtoolu_1",

View file

@ -11,6 +11,7 @@ import litellm
from litellm.llms.bedrock.count_tokens.bedrock_token_counter import BedrockTokenCounter
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
from litellm.types.llms.anthropic import (
ANTHROPIC_BETA_HEADER_VALUES,
ANTHROPIC_EFFORT_BETA_HEADER,
ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER,
)
@ -457,10 +458,15 @@ _CITY_SCHEMA: Final[dict[str, JsonValue]] = {
"properties": {"city": {"type": "string"}},
"required": ["city"],
}
_CITY_FORMAT: Final[dict[str, JsonValue]] = {"output_config": {"format": {"type": "json_schema", "schema": _CITY_SCHEMA}}}
_CITY_FORMAT: Final[dict[str, JsonValue]] = {
"output_config": {"format": {"type": "json_schema", "schema": _CITY_SCHEMA}}
}
_CONTEXT_EDITING_OPTIONS: Final[dict[str, JsonValue]] = {**_THINKING_AND_TOOL_CHOICE, "context_management": _CONTEXT_EDITS}
_CONTEXT_EDITING_OPTIONS: Final[dict[str, JsonValue]] = {
**_THINKING_AND_TOOL_CHOICE,
"context_management": _CONTEXT_EDITS,
}
async def _messages_inference_and_count(
@ -565,7 +571,11 @@ def _sent_betas(request: httpx.Request) -> tuple[str, ...]:
{"output_config": {"effort": "low"}},
ANTHROPIC_EFFORT_BETA_HEADER,
),
("bedrock/invoke/us.anthropic.claude-opus-5-5", _DISPLAY_THINKING, ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER),
(
"bedrock/invoke/us.anthropic.claude-opus-5-5",
_DISPLAY_THINKING,
ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER,
),
),
ids=("effort", "thinking_display"),
)
@ -654,6 +664,53 @@ async def test_runtime_count_of_an_output_format_sends_only_the_betas_invoke_inf
assert counted.get("anthropic_beta") == sent.get("anthropic_beta")
_PER_TURN_CONTROL: Final = ANTHROPIC_BETA_HEADER_VALUES.PER_TURN_CONTROL_2026_07_01.value
@pytest.mark.asyncio
@pytest.mark.usefixtures("local_beta_headers_config")
@pytest.mark.parametrize(
("caller_beta", "model_info", "kept"),
(
(_PER_TURN_CONTROL, {"supports_per_turn_control": True}, True),
(_PER_TURN_CONTROL, {"supports_per_turn_control": False}, False),
("unlisted-beta-2099-01-01", {}, False),
),
ids=("deployment-supports-per-turn-control", "deployment-lacks-per-turn-control", "unlisted-beta"),
)
async def test_runtime_count_keeps_and_drops_the_caller_betas_invoke_inference_keeps_and_drops(
caller_beta: str, model_info: dict[str, JsonValue], kept: bool
) -> None:
deployment_model: Final = "bedrock/invoke/us.anthropic.claude-opus-5-5"
credentials: Final = DEPLOYMENT["litellm_params"]
inference: Final = _Bedrock(runtime=(200, _CLAUDE_REPLY), mantle=(200, _CLAUDE_REPLY))
await litellm.anthropic_messages(
model=deployment_model,
messages=MESSAGES,
max_tokens=4096,
extra_headers={"anthropic-beta": caller_beta},
model_info=model_info,
client=AsyncHTTPHandler(transport=httpx.MockTransport(inference)),
**_THINKING,
**credentials,
)
bedrock: Final = _Bedrock(runtime=(200, {"inputTokens": 412}), mantle=(200, {"input_tokens": 2177}))
result: Final = await _counter(bedrock).count_tokens(
model_to_use="us.anthropic.claude-opus-5-5",
messages=MESSAGES,
contents=None,
deployment={"litellm_params": {**credentials, "model": deployment_model}, "model_info": model_info},
request_model="claude",
request_options=_THINKING,
anthropic_beta=caller_beta,
)
assert result is not None and result.error is False
counted: Final = _betas_besides_effort(_invoke_body(bedrock.requests[0]))
assert counted == _betas_besides_effort(_JSON_BODY.validate_json(inference.requests[0].content))
assert (caller_beta in counted) is kept
_CONVERSE_REPLY: Final = {
"output": {"message": {"role": "assistant", "content": [{"text": "ok"}]}},
"stopReason": "end_turn",
@ -787,6 +844,51 @@ async def test_runtime_rejecting_the_options_is_answered_unless_mantle_counts_or
assert (result.error, result.status_code, result.tokenizer_type) == expected
_CLEAR_TOOL_USES: Final[dict[str, JsonValue]] = {"edits": [{"type": "clear_tool_uses_20250919"}]}
@pytest.mark.asyncio
@pytest.mark.parametrize(
("request_options", "mantle", "expected"),
(
({}, _mantle_answers(403, _MANTLE_FAILURE), (403, "bedrock_mantle_api")),
({}, _mantle_unreachable(httpx.ConnectError), (500, "bedrock_mantle_api")),
(
{**_DISPLAY_THINKING, "context_management": _CLEAR_TOOL_USES},
_mantle_answers(403, _MANTLE_FAILURE),
(400, "bedrock_api"),
),
),
ids=("edit-only-mantle-denied", "edit-only-mantle-unreachable", "edit-and-thinking-mantle-denied"),
)
async def test_runtime_400_on_an_edit_reports_the_mantle_failure_unless_options_were_forwarded(
request_options: dict[str, JsonValue],
mantle: Callable[[httpx.Request], httpx.Response],
expected: tuple[int, str],
) -> None:
def bedrock(request: httpx.Request) -> httpx.Response:
if request.url.host == RUNTIME_HOST:
return httpx.Response(400, json=UNSUPPORTED, request=request)
return mantle(request)
result: Final = await BedrockTokenCounter(
client=AsyncHTTPHandler(transport=httpx.MockTransport(bedrock))
).count_tokens(
model_to_use="us.anthropic.claude-opus-5-5",
messages=MESSAGES,
contents=None,
deployment={
"litellm_params": {**DEPLOYMENT["litellm_params"], "model": "bedrock/us.anthropic.claude-opus-5-5"}
},
request_model="claude-opus-5-5",
context_management=CountContextManagement(spec=_CLEAR_TOOL_USES, drop_invalid_edits=False),
request_options=request_options,
)
assert result is not None
assert (result.error, result.status_code, result.tokenizer_type) == (True, *expected)
@pytest.mark.asyncio
async def test_runtime_count_sends_the_tools_invoke_inference_sends() -> None:
inference, bedrock = await _messages_inference_and_count(
@ -797,7 +899,8 @@ async def test_runtime_count_sends_the_tools_invoke_inference_sends() -> None:
)
assert bedrock.posted_hosts() == (RUNTIME_HOST,)
assert _invoke_body(bedrock.requests[0])["tools"] == _JSON_BODY.validate_json(inference.requests[0].content)["tools"]
sent_tools: Final = _JSON_BODY.validate_json(inference.requests[0].content)["tools"]
assert _invoke_body(bedrock.requests[0])["tools"] == sent_tools
def _texts(message: JsonValue) -> tuple[str, ...]:
@ -866,7 +969,13 @@ _THINKING: Final[dict[str, JsonValue]] = {"thinking": {"type": "enabled", "budge
([{"role": "user", "content": 5}], None, _THINKING),
([{"role": "user", "content": [{"text": "hi"}]}], None, _THINKING),
),
ids=("option_refused", "tool_type_not_a_string", "tool_not_a_mapping", "content_not_iterable", "block_without_type"),
ids=(
"option_refused",
"tool_type_not_a_string",
"tool_not_a_mapping",
"content_not_iterable",
"block_without_type",
),
)
async def test_claude_count_refuses_options_bedrock_messages_inference_refuses_without_calling_bedrock(
messages: list[dict[str, JsonValue]], tools: list[JsonValue] | None, request_options: dict[str, JsonValue]

View file

@ -2082,9 +2082,8 @@ async def _vertex_claude_inference_and_count(
def vertex(request: httpx.Request) -> httpx.Response:
sent.put(request)
return httpx.Response(
200, json={"input_tokens": 37} if request.url.path.endswith("count-tokens:rawPredict") else _PARTNER_CLAUDE_REPLY
)
counted: Final = request.url.path.endswith("count-tokens:rawPredict")
return httpx.Response(200, json={"input_tokens": 37} if counted else _PARTNER_CLAUDE_REPLY)
client: Final = AsyncHTTPHandler(transport=httpx.MockTransport(vertex))
with (

View file

@ -1698,7 +1698,6 @@ async def test_anthropic_count_tokens_on_a_claude_deployment_drops_a_malformed_t
assert sent["messages"][2]["content"][0]["content"] == [{"type": "text", "text": "build ok"}]
_CLAUDE_COUNT_OPTIONS: Final[dict[str, JsonValue]] = {
"thinking": {"type": "adaptive"},
"tool_choice": {"type": "auto"},
@ -1756,12 +1755,18 @@ def _sent_option_paths(body: Mapping[str, JsonValue]) -> frozenset[str]:
{"additional_drop_params": ["thinking", "context_management"]},
{"tool_choice", "output_config", "output_config.effort", "output_config.format"},
),
(
["tool_choice"],
{"additional_drop_params": ["thinking"]},
{"tool_choice", "output_config", "output_config.effort", "output_config.format", "context_management"},
),
),
ids=(
"nothing-dropped",
"deployment-drops-tool-choice-and-the-format",
"deployment-drops-context-management",
"request-drops-thinking-and-context-management",
"request-drop-list-replaces-the-deployments",
),
)
async def test_anthropic_count_tokens_on_a_claude_deployment_sends_the_options_inference_sends(
@ -1770,7 +1775,7 @@ async def test_anthropic_count_tokens_on_a_claude_deployment_sends_the_options_i
request_params: dict[str, JsonValue],
expected_paths: set[str],
) -> None:
import litellm.proxy.proxy_server as proxy_server
from litellm.proxy import proxy_server
router: Final = Router(
model_list=[
@ -1834,7 +1839,7 @@ async def _count_on_a_claude_deployment(
user_agent: str = "python-httpx/0.28.1",
deployment_params: Mapping[str, JsonValue] = MappingProxyType({}),
) -> tuple[object, tuple[httpx.Request, ...]]:
import litellm.proxy.proxy_server as proxy_server
from litellm.proxy import proxy_server
router: Final = _claude_router(deployment_model, deployment_params)
request: Final = _json_request(
@ -1954,6 +1959,45 @@ async def test_anthropic_count_tokens_does_not_count_again_after_a_rate_limit(
assert len(requests) == 1
@pytest.mark.asyncio
@pytest.mark.parametrize(
("deployment_model", "count_host", "provider_calls"),
(
("gemini/gemini-3.8-flash", "generativelanguage.googleapis.com", 1),
("anthropic/claude-haiku-5-5", "api.anthropic.com", 2),
),
ids=("counter-without-claude-options", "claude-counter-forwarding-the-options"),
)
async def test_anthropic_count_tokens_counts_again_without_the_options_only_when_the_counter_forwarded_them(
monkeypatch: pytest.MonkeyPatch, deployment_model: str, count_host: str, provider_calls: int
) -> None:
from litellm.proxy import proxy_server
request: Final = _json_request(
{
"model": "claude-agent",
"messages": [{"role": "user", "content": "Weather in SF?"}],
"thinking": {"type": "enabled", "budget_tokens": 2000},
},
"python-httpx/0.28.1",
)
monkeypatch.setattr(proxy_server, "llm_router", _claude_router(deployment_model, {}))
monkeypatch.setattr(proxy_server, "proxy_config", proxy_server.ProxyConfig())
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
with respx.mock:
route: Final = respx.route(host=count_host).mock(
return_value=httpx.Response(
400, json={"type": "error", "error": {"type": "invalid_request_error", "message": "rejected"}}
)
)
response: Final = await anthropic_count_tokens(request, UserAPIKeyAuth())
assert isinstance(response, JSONResponse)
assert response.status_code == 400
assert route.call_count == provider_calls
_BEDROCK_RUNTIME_HOST: Final = "bedrock-runtime.us-east-1.amazonaws.com"
_BEDROCK_MANTLE_HOST: Final = "bedrock-mantle.us-east-1.api.aws"
_BEDROCK_CLAUDE_DEPLOYMENT: Final[dict[str, str]] = {
@ -1969,12 +2013,17 @@ async def _count_on_a_bedrock_claude_deployment(
body: Mapping[str, JsonValue],
runtime: Callable[[httpx.Request], httpx.Response],
mantle: Callable[[httpx.Request], httpx.Response],
deployment_model: str = _BEDROCK_CLAUDE_DEPLOYMENT["model"],
) -> tuple[object, tuple[httpx.Request, ...]]:
import litellm.proxy.proxy_server as proxy_server
from litellm.proxy import proxy_server
monkeypatch.delenv("AWS_BEARER_TOKEN_BEDROCK", raising=False)
monkeypatch.delenv("BEDROCK_MANTLE_API_BASE", raising=False)
router: Final = Router(model_list=[{"model_name": "claude-agent", "litellm_params": _BEDROCK_CLAUDE_DEPLOYMENT}])
router: Final = Router(
model_list=[
{"model_name": "claude-agent", "litellm_params": {**_BEDROCK_CLAUDE_DEPLOYMENT, "model": deployment_model}}
]
)
request: Final = _json_request(
{"model": "claude-agent", "messages": [{"role": "user", "content": "Weather in SF?"}], **body},
"python-httpx/0.28.1",
@ -2045,6 +2094,48 @@ async def test_anthropic_count_tokens_on_bedrock_counts_again_without_the_option
assert "invokeModel" not in json.loads(requests[-1].content)["input"]
@pytest.mark.asyncio
async def test_anthropic_count_tokens_on_bedrock_sends_the_thinking_a_clear_thinking_edit_makes_inference_send(
monkeypatch: pytest.MonkeyPatch,
) -> None:
deployment_model: Final = "bedrock/invoke/us.anthropic.claude-sonnet-4-6"
options_and_edit: Final[dict[str, JsonValue]] = {
"tools": [{"name": "get_weather", "input_schema": {"type": "object"}}],
"tool_choice": {"type": "auto"},
"context_management": {"edits": [{"type": "clear_thinking_20251015"}]},
}
credentials: Final = {name: value for name, value in _BEDROCK_CLAUDE_DEPLOYMENT.items() if name != "model"}
monkeypatch.delenv("AWS_BEARER_TOKEN_BEDROCK", raising=False)
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
with respx.mock:
inference_route: Final = respx.route(host=_BEDROCK_RUNTIME_HOST).mock(
return_value=httpx.Response(200, json=_CLAUDE_MESSAGE_REPLY)
)
await litellm.anthropic_messages(
model=deployment_model,
messages=[{"role": "user", "content": "Weather in SF?"}],
max_tokens=4096,
**options_and_edit,
**credentials,
)
sent: Final = json.loads(inference_route.calls.last.request.content)
response, (count,) = await _count_on_a_bedrock_claude_deployment(
monkeypatch,
options_and_edit,
lambda _: httpx.Response(200, json={"inputTokens": 9}),
_mantle_unreachable,
deployment_model,
)
assert not isinstance(response, JSONResponse)
counted: Final = json.loads(_bedrock_counted_request(count.content))
assert sent.get("thinking") is not None
assert {name: counted.get(name) for name in ("thinking", "output_config", "context_management")} == {
name: sent.get(name) for name in ("thinking", "output_config", "context_management")
}
_KEEPS_CONTEXT_MANAGEMENT: Final = "sk-ant-keeps-context-management"
_DROPS_CONTEXT_MANAGEMENT: Final = "sk-ant-drops-context-management"
_CONTEXT_EDITS: Final[dict[str, JsonValue]] = {"edits": [{"type": "clear_tool_uses_20250919"}]}
@ -2098,7 +2189,7 @@ def _mixed_drop_group(routed_key: str) -> Router:
async def test_anthropic_count_tokens_in_a_mixed_drop_group_counts_without_context_management_on_any_deployment(
monkeypatch: pytest.MonkeyPatch, routed_key: str
) -> None:
import litellm.proxy.proxy_server as proxy_server
from litellm.proxy import proxy_server
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
monkeypatch.setattr(proxy_server, "llm_router", _mixed_drop_group(routed_key))
@ -2170,13 +2261,13 @@ async def test_anthropic_count_tokens_on_a_claude_deployment_sends_the_thinking_
) -> None:
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
thinking: Final[dict[str, JsonValue]] = {"type": "enabled", "budget_tokens": 2000, "summary": "auto"}
dropped: Final = await _claude_inference_body("claude-opus-4-7", {"thinking": thinking, "drop_params": True})
kept: Final = await _claude_inference_body("claude-opus-4-7", {"thinking": thinking})
dropped: Final = await _claude_inference_body("claude-opus-5-5", {"thinking": thinking, "drop_params": True})
kept: Final = await _claude_inference_body("claude-opus-5-5", {"thinking": thinking})
assert dropped["thinking"] != kept["thinking"]
response, (count,) = await _count_on_a_claude_deployment(
monkeypatch,
"anthropic/claude-opus-4-7",
"anthropic/claude-opus-5-5",
{"thinking": thinking, **request_drop_params},
lambda _: httpx.Response(200, json={"input_tokens": 7}),
user_agent,