From 4bdba8bbd5d40a5b078dd1ac8e089a4746e1c6e8 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 6 Nov 2026 23:56:45 -0800 Subject: [PATCH] fix(count_tokens): filter count betas like inference, retry only forwarded Claude options, and keep Mantle's error on edit-only Bedrock counts The count now builds its beta filter with the same helper inference uses, so it reads the deployment's api_base, filters an anthropic deployment on a Foundry host with the azure_ai row, drops unlisted betas only where the provider filters them, and keeps the betas the deployment's model_info supports, such as per-turn-control on Bedrock invoke The proxy counts again without the request's options only when the counter forwarded Claude options or betas, so a Gemini or OpenAI count that answers 400 is no longer sent twice On Bedrock, a count that forwards only context edits returns Mantle's 401, 403, 429, 5xx or connection error again when the runtime answers 400 or 404, like main does. A count that forwarded Claude options keeps the runtime's answer The effort beta guard in the Bedrock runtime count is removed, since the invoke body already sends each beta once Tests cover a request drop list replacing the deployment's, the edit merge into the count's options on a Bedrock invoke count, and an opus-5-5 drop_params case. The Bedrock invoke beta hook now calls the base no-op instead of repeating it. A leftover docstring line, an extra blank line, and long test lines are cleaned up --- litellm/anthropic_beta_headers_manager.py | 23 ++++ litellm/llms/anthropic/common_utils.py | 4 + .../anthropic/count_tokens/token_counter.py | 1 + .../anthropic/count_tokens/transformation.py | 39 ++++-- .../anthropic/count_tokens/token_counter.py | 1 + .../count_tokens/bedrock_token_counter.py | 16 ++- .../anthropic_claude_transformation.py | 2 +- litellm/llms/custom_httpx/llm_http_handler.py | 30 ++--- litellm/llms/vertex_ai/common_utils.py | 1 + .../count_tokens/handler.py | 6 - .../common_utils/token_count_settings.py | 5 +- litellm/proxy/proxy_server.py | 3 +- litellm/types/utils.py | 1 + ...t_anthropic_count_tokens_transformation.py | 56 ++++++++- .../test_bedrock_token_counter.py | 119 +++++++++++++++++- .../vertex_ai/test_vertex_ai_common_utils.py | 5 +- tests/unit/proxy/test_proxy_token_counter.py | 109 ++++++++++++++-- 17 files changed, 356 insertions(+), 65 deletions(-) diff --git a/litellm/anthropic_beta_headers_manager.py b/litellm/anthropic_beta_headers_manager.py index fd72bf7d2e7..f7a8e92df29 100644 --- a/litellm/anthropic_beta_headers_manager.py +++ b/litellm/anthropic_beta_headers_manager.py @@ -28,6 +28,7 @@ import os import threading from collections.abc import Mapping from concurrent.futures import ThreadPoolExecutor +from dataclasses import dataclass from functools import cache from importlib.resources import files from typing import Final, Literal, TypeAlias @@ -474,6 +475,28 @@ def update_headers_with_filtered_beta( return headers +@dataclass(frozen=True, slots=True) +class DeploymentBetaFilter: + provider: str + api_base: str | None + unlisted: UnlistedBetas + + def filtered(self, headers: dict[str, str], keep: frozenset[str]) -> dict[str, str]: + return update_headers_with_filtered_beta( + headers=headers, provider=self.provider, api_base=self.api_base, unlisted=self.unlisted, keep=keep + ) + + +def deployment_beta_filter( + provider: str, affinity_api_base: str | None, provider_filters_betas: bool +) -> DeploymentBetaFilter | None: + if provider_filters_betas: + return DeploymentBetaFilter(provider=provider, api_base=affinity_api_base, unlisted="drop") + if resolve_beta_provider(provider, affinity_api_base) != provider: + return DeploymentBetaFilter(provider=provider, api_base=affinity_api_base, unlisted="forward") + return None + + def update_request_with_filtered_beta( headers: dict, request_data: dict, diff --git a/litellm/llms/anthropic/common_utils.py b/litellm/llms/anthropic/common_utils.py index 1c42dd15853..af53778c35c 100644 --- a/litellm/llms/anthropic/common_utils.py +++ b/litellm/llms/anthropic/common_utils.py @@ -1676,6 +1676,10 @@ class AnthropicModelInfo(BaseLLMModelInfo): def get_api_base(api_base: str | None = None) -> str | None: return AnthropicModelInfo.resolve_anthropic_api_base(api_base) + @staticmethod + def affinity_api_base(custom_llm_provider: str, api_base: str | None) -> str | None: + return AnthropicModelInfo.get_api_base(api_base) if custom_llm_provider == "anthropic" else api_base + @staticmethod def get_api_key(api_key: str | None = None) -> str | None: """An empty or whitespace-only key counts as unset: it can never authenticate anything, and diff --git a/litellm/llms/anthropic/count_tokens/token_counter.py b/litellm/llms/anthropic/count_tokens/token_counter.py index 5c7f7a57a52..97ba430d14a 100644 --- a/litellm/llms/anthropic/count_tokens/token_counter.py +++ b/litellm/llms/anthropic/count_tokens/token_counter.py @@ -168,6 +168,7 @@ class AnthropicTokenCounter(BaseTokenCounter): error=True, error_message=e.message, status_code=e.status_code, + carried_request_options=claude_options.carries_request_options, ) except Exception as e: verbose_logger.warning("Error calling Anthropic CountTokens API: %s", e) diff --git a/litellm/llms/anthropic/count_tokens/transformation.py b/litellm/llms/anthropic/count_tokens/transformation.py index 82ccf3b6cc5..26defadfda1 100644 --- a/litellm/llms/anthropic/count_tokens/transformation.py +++ b/litellm/llms/anthropic/count_tokens/transformation.py @@ -20,6 +20,7 @@ from typing import ( from pydantic import ConfigDict, JsonValue, TypeAdapter, with_config from typing_extensions import NotRequired, ReadOnly, TypedDict +from litellm.anthropic_beta_headers_manager import DeploymentBetaFilter, deployment_beta_filter from litellm.constants import ANTHROPIC_TOKEN_COUNTING_BETA_VERSION from litellm.litellm_core_utils.asyncify import asyncify from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import ( @@ -114,6 +115,7 @@ class AnthropicCountFields: @dataclass(frozen=True, slots=True) class ChatCountTranslationError: message: str + carried_request_options: bool = False def response(self, request_model: str, model_used: str, tokenizer_type: str) -> TokenCountResponse: return TokenCountResponse( @@ -125,6 +127,7 @@ class ChatCountTranslationError: error_message=self.message, status_code=400, rejected_before_provider_call=True, + carried_request_options=self.carried_request_options, ) @@ -349,6 +352,10 @@ class ClaudeCountOptions: messages: tuple[Mapping[str, JsonValue], ...] | None tools: tuple[Mapping[str, JsonValue], ...] | None + @property + def carries_request_options(self) -> bool: + return bool(self.options) or bool(self.betas) + NO_CLAUDE_COUNT_OPTIONS: Final = ClaudeCountOptions(options=MappingProxyType({}), betas=(), messages=None, tools=None) _COUNTED_MESSAGES: Final = TypeAdapter(tuple[Mapping[str, JsonValue], ...]) @@ -379,6 +386,11 @@ def deployment_inference_model(deployment: Mapping[str, object] | None) -> str | return inference_model +def _deployment_api_base(deployment: Mapping[str, object] | None) -> str | None: + api_base: Final = _INFERENCE_PARAMS.validate_python((deployment or {}).get("litellm_params") or {}).get("api_base") + return api_base if isinstance(api_base, str) else None + + def _messages_transform(model: str, provider: LlmProviders) -> tuple[str, BaseAnthropicMessagesConfig] | None: from litellm.utils import ProviderConfigManager @@ -403,14 +415,12 @@ def _claude_count_options_for( messages_config: BaseAnthropicMessagesConfig, model: str, litellm_params: "GenericLiteLLMParams", - provider: LlmProviders, + beta_filter: DeploymentBetaFilter | None, messages: Sequence[Mapping[str, object]], tools: Sequence[Mapping[str, object]] | None, sent: Mapping[str, JsonValue], anthropic_beta: str | None, ) -> ClaudeCountOptions: - from litellm.anthropic_beta_headers_manager import update_headers_with_filtered_beta - leading_system, conversation = split_leading_system_run(messages) sent_conversation: Final = copy.deepcopy([dict(message) for message in conversation]) sent_params: Final = { @@ -421,9 +431,11 @@ def _claude_count_options_for( {"anthropic-beta": anthropic_beta} if anthropic_beta else {}, sent_params, sent_conversation ) sent_headers: Final = ( - update_headers_with_filtered_beta(headers=dict(beta_headers), provider=provider.value) - if messages_config.should_filter_anthropic_beta_headers() - else dict(beta_headers) + dict(beta_headers) + if beta_filter is None + else beta_filter.filtered( + dict(beta_headers), keep=messages_config.model_specific_anthropic_betas(model, litellm_params) + ) ) request: Final = messages_config.transform_anthropic_messages_request( # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # the base config signature takes and returns bare dicts model=model, @@ -462,16 +474,23 @@ def claude_count_options( transform_model, messages_config = transform litellm_params: Final = _transform_litellm_params(deployment) from litellm.exceptions import BadRequestError - from litellm.llms.anthropic.common_utils import AnthropicError + from litellm.llms.anthropic.common_utils import AnthropicError, AnthropicModelInfo + beta_filter: Final = deployment_beta_filter( + provider.value, + AnthropicModelInfo.affinity_api_base(provider.value, _deployment_api_base(deployment)), + messages_config.should_filter_anthropic_beta_headers(), + ) try: return _claude_count_options_for( - messages_config, transform_model, litellm_params, provider, messages, tools, sent, anthropic_beta + messages_config, transform_model, litellm_params, beta_filter, messages, tools, sent, anthropic_beta ) except (AnthropicError, BadRequestError) as e: - return ChatCountTranslationError(message=str(e.message)) + return ChatCountTranslationError(message=str(e.message), carried_request_options=True) except (AttributeError, KeyError, TypeError, ValueError) as e: - return ChatCountTranslationError(message=f"Invalid request for token counting: {e}") + return ChatCountTranslationError( + message=f"Invalid request for token counting: {e}", carried_request_options=True + ) async def anthropic_count_fields( diff --git a/litellm/llms/azure_ai/anthropic/count_tokens/token_counter.py b/litellm/llms/azure_ai/anthropic/count_tokens/token_counter.py index 31082e1e241..ebe7bdb664e 100644 --- a/litellm/llms/azure_ai/anthropic/count_tokens/token_counter.py +++ b/litellm/llms/azure_ai/anthropic/count_tokens/token_counter.py @@ -176,6 +176,7 @@ class AzureAIAnthropicTokenCounter(BaseTokenCounter): error=True, error_message=e.message, status_code=e.status_code, + carried_request_options=claude_options.carries_request_options, ) except Exception as e: verbose_logger.warning("Error calling Azure AI Anthropic CountTokens API: %s", e) diff --git a/litellm/llms/bedrock/count_tokens/bedrock_token_counter.py b/litellm/llms/bedrock/count_tokens/bedrock_token_counter.py index ca0409e824c..fa1cbe9ead9 100644 --- a/litellm/llms/bedrock/count_tokens/bedrock_token_counter.py +++ b/litellm/llms/bedrock/count_tokens/bedrock_token_counter.py @@ -89,12 +89,12 @@ def _runtime_rejected_claude_model(outcome: CountTokensOutcome, resolved_model: def _mantle_retry_outcome( - runtime: CountTokensOutcome, mantle: CountTokensOutcome, forwarded: bool + runtime: CountTokensOutcome, mantle: CountTokensOutcome, forwarded: bool, carried_options: bool ) -> CountTokensOutcome: match runtime, mantle: case CountTokensFailure(status_code=400), CountTokensFailure(status_code=404) if forwarded: return runtime - case _, CountTokensFailure(status_code=status_code) if forwarded and not is_request_rejection(status_code): + case _, CountTokensFailure(status_code=code) if carried_options and not is_request_rejection(code): return runtime case _: return mantle @@ -117,7 +117,7 @@ def _sets_effort(options: Mapping[str, JsonValue]) -> bool: def _runtime_count_betas(claude_options: ClaudeCountOptions) -> tuple[str, ...]: - if not _sets_effort(claude_options.options) or ANTHROPIC_EFFORT_BETA_HEADER in claude_options.betas: + if not _sets_effort(claude_options.options): return claude_options.betas return (*claude_options.betas, ANTHROPIC_EFFORT_BETA_HEADER) @@ -322,7 +322,14 @@ class BedrockTokenCounter(BaseTokenCounter): else None ) outcome: Final = ( - primary if fallback is None else _mantle_retry_outcome(primary, fallback, forwarded=bool(runtime_fields)) + primary + if fallback is None + else _mantle_retry_outcome( + primary, + fallback, + forwarded=bool(runtime_fields), + carried_options=claude_options.carries_request_options, + ) ) match outcome: case CountedTokens(): @@ -348,6 +355,7 @@ class BedrockTokenCounter(BaseTokenCounter): error=True, error_message=outcome.message, status_code=outcome.status_code, + carried_request_options=claude_options.carries_request_options, ) case _: assert_never(outcome) diff --git a/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude_transformation.py b/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude_transformation.py index 824a075309e..ebb91cb230a 100644 --- a/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude_transformation.py +++ b/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude_transformation.py @@ -121,7 +121,7 @@ class AmazonAnthropicClaudeMessagesConfig( optional_params: Mapping[str, object], messages: Sequence[object], ) -> Mapping[str, str]: - return headers + return BaseAnthropicMessagesConfig.headers_with_anthropic_betas(self, headers, optional_params, messages) def sent_anthropic_betas(self, headers: Mapping[str, str], request: Mapping[str, object]) -> tuple[str, ...]: return _ANTHROPIC_BETAS.validate_python(request.get("anthropic_beta") or ()) diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index 8c9de272c0d..62644959a01 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -39,11 +39,7 @@ import litellm.types import litellm.types.utils from litellm._internal_context import internal_sub_call from litellm._logging import redact_string, verbose_logger -from litellm.anthropic_beta_headers_manager import ( - aload_beta_headers_config, - resolve_beta_provider, - update_headers_with_filtered_beta, -) +from litellm.anthropic_beta_headers_manager import aload_beta_headers_config, deployment_beta_filter from litellm.constants import MAX_FILE_LIST_LIMIT, REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES from litellm.files.types import FileContentStreamingResult from litellm.litellm_core_utils.agentic_followup_kwargs import build_agentic_followup_kwargs @@ -2623,15 +2619,13 @@ class BaseLLMHTTPHandler: litellm_params.get("additional_drop_params") or (), anthropic_messages_optional_request_params, ) - affinity_api_base: Final = ( - AnthropicModelInfo.get_api_base(api_base) if custom_llm_provider == "anthropic" else api_base + affinity_api_base: Final = AnthropicModelInfo.affinity_api_base(custom_llm_provider, api_base) + beta_filter: Final = deployment_beta_filter( + custom_llm_provider, + affinity_api_base, + anthropic_messages_provider_config.should_filter_anthropic_beta_headers(), ) - takes_another_beta_row: Final = ( - resolve_beta_provider(custom_llm_provider, affinity_api_base) != custom_llm_provider - ) - provider_filters_betas: Final = anthropic_messages_provider_config.should_filter_anthropic_beta_headers() - filters_beta_headers: Final = provider_filters_betas or takes_another_beta_row - if request_params.get("safeguards") is not None and filters_beta_headers: + if request_params.get("safeguards") is not None and beta_filter is not None: await aload_beta_headers_config() default_affinity_header: Final = get_provider_affinity_header(affinity_api_base) ( @@ -2651,14 +2645,10 @@ class BaseLLMHTTPHandler: api_base=api_base, ) - if filters_beta_headers and headers.get("anthropic-beta") is not None: + if beta_filter is not None and headers.get("anthropic-beta") is not None: await aload_beta_headers_config() - headers = update_headers_with_filtered_beta( - headers=headers, - provider=custom_llm_provider, - api_base=affinity_api_base, - unlisted="drop" if provider_filters_betas else "forward", - keep=anthropic_messages_provider_config.model_specific_anthropic_betas(model, litellm_params), + headers = beta_filter.filtered( + headers, keep=anthropic_messages_provider_config.model_specific_anthropic_betas(model, litellm_params) ) from litellm.llms.vertex_ai.vertex_llm_base import VertexBase diff --git a/litellm/llms/vertex_ai/common_utils.py b/litellm/llms/vertex_ai/common_utils.py index 1ab84da6c1b..ae2af8b3448 100644 --- a/litellm/llms/vertex_ai/common_utils.py +++ b/litellm/llms/vertex_ai/common_utils.py @@ -1581,6 +1581,7 @@ class VertexAITokenCounter(BaseTokenCounter): error=True, error_message=error_message, status_code=status_code, + carried_request_options=claude_options.carries_request_options, ) if result is not None: diff --git a/litellm/llms/vertex_ai/vertex_ai_partner_models/count_tokens/handler.py b/litellm/llms/vertex_ai/vertex_ai_partner_models/count_tokens/handler.py index d4350553999..522da59c71c 100644 --- a/litellm/llms/vertex_ai/vertex_ai_partner_models/count_tokens/handler.py +++ b/litellm/llms/vertex_ai/vertex_ai_partner_models/count_tokens/handler.py @@ -153,12 +153,6 @@ class VertexAIPartnerModelsTokenCounter(VertexBase): """ Handle token counting request for a Vertex AI partner model. - Args: - model: The model name - request_data: Request payload (Anthropic Messages API format) - litellm_params: LiteLLM parameters containing credentials, project, location - betas: anthropic-beta values the counted request needs, sent as the anthropic-beta header - Returns: Dict containing token count information diff --git a/litellm/proxy/common_utils/token_count_settings.py b/litellm/proxy/common_utils/token_count_settings.py index 544b5b0f5e0..171b5ea93a5 100644 --- a/litellm/proxy/common_utils/token_count_settings.py +++ b/litellm/proxy/common_utils/token_count_settings.py @@ -80,6 +80,7 @@ class ProviderCountFailure: status_code: int message: str rejected_before_provider_call: bool = False + carried_request_options: bool = False @property def cannot_count_model(self) -> bool: @@ -90,8 +91,8 @@ class ProviderCountFailure: return not self.rejected_before_provider_call and self.status_code in REGION_DENIED_STATUSES @property - def rejected_request(self) -> bool: - return is_request_rejection(self.status_code) + def rejected_request_options(self) -> bool: + return self.carried_request_options and is_request_rejection(self.status_code) def with_hint(self, hint: str) -> "ProviderCountFailure": if self.rejected_before_provider_call: diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 649236e78af..6a8f2cb28de 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -15269,6 +15269,7 @@ def _provider_count_outcome( status_code=result.status_code or 500, message=result.error_message or "Token counting failed", rejected_before_provider_call=result.rejected_before_provider_call, + carried_request_options=result.carried_request_options, ) return result @@ -15469,7 +15470,7 @@ async def _count_retrying_without_options( region: str | None, ) -> Union["TokenCountResponse", ProviderCountFailure, None]: counted: Final = await count_with(model, region) - if not (isinstance(counted, ProviderCountFailure) and counted.rejected_request): + if not (isinstance(counted, ProviderCountFailure) and counted.rejected_request_options): return counted verbose_proxy_logger.warning( "Token count for %s %s with the request's %s, so it counts again without them and the caller's betas", diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 3ddeb118c33..4dc9abedcb1 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -4912,6 +4912,7 @@ class TokenCountResponse(LiteLLMPydanticObjectBase): """ status_code: int | None = None rejected_before_provider_call: bool = Field(default=False, exclude=True) + carried_request_options: bool = Field(default=False, exclude=True) class CustomHuggingfaceTokenizer(TypedDict): diff --git a/tests/unit/llms/anthropic/test_anthropic_count_tokens_transformation.py b/tests/unit/llms/anthropic/test_anthropic_count_tokens_transformation.py index 91e2b03dd1d..1f731faa724 100644 --- a/tests/unit/llms/anthropic/test_anthropic_count_tokens_transformation.py +++ b/tests/unit/llms/anthropic/test_anthropic_count_tokens_transformation.py @@ -567,6 +567,7 @@ async def _counted_requests( request_options: Mapping[str, JsonValue] = MappingProxyType({}), anthropic_beta: str | None = None, model: str = "claude-opus-5-5", + api_base: str = "https://gateway.example", ) -> tuple[TokenCountResponse | None, tuple[httpx.Request, ...]]: sent: Final[SimpleQueue[httpx.Request]] = SimpleQueue() @@ -580,7 +581,7 @@ async def _counted_requests( model_to_use=model, messages=messages, contents=None, - deployment={"litellm_params": {"api_key": "sk-ant-api03-test-key", "api_base": "https://gateway.example"}}, + deployment={"litellm_params": {"api_key": "sk-ant-api03-test-key", "api_base": api_base}}, request_model=model, tools=tools, messages_format=messages_format, @@ -972,6 +973,7 @@ async def _messages_inference_request( request_options: Mapping[str, JsonValue], model: str = "claude-haiku-5-5", extra_headers: Mapping[str, str] = MappingProxyType({}), + api_base: str = "https://gateway.example", ) -> httpx.Request: sent: Final[SimpleQueue[httpx.Request]] = SimpleQueue() @@ -987,7 +989,7 @@ async def _messages_inference_request( tools=[_WEATHER_TOOL], max_tokens=4096, api_key="sk-ant-api03-test-key", - api_base="https://gateway.example", + api_base=api_base, extra_headers=dict(extra_headers), **request_options, ) @@ -1059,7 +1061,10 @@ async def test_counter_sends_the_options_and_context_edits_messages_inference_se assert count_body.get("output_config") == ( inference_output_config if inference_output_format is None - else {**(inference_output_config if isinstance(inference_output_config, dict) else {}), "format": inference_output_format} + else { + **(inference_output_config if isinstance(inference_output_config, dict) else {}), + "format": inference_output_format, + } ) assert {name: count_body.get(name) for name in ("thinking", "tool_choice", "context_management")} == { name: inference_body.get(name) for name in ("thinking", "tool_choice", "context_management") @@ -1098,6 +1103,44 @@ async def test_counter_sends_the_thinking_display_and_caller_betas_messages_infe assert {ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER, caller_beta} <= counted_betas +_FOUNDRY_BASE: Final = "https://my-resource.services.ai.azure.com/anthropic" +_UNLISTED_BETA: Final = "unlisted-beta-2099-01-01" + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("httpx_transport_clients", "local_beta_headers_config") +@pytest.mark.parametrize( + ("counter_type", "expected_betas"), + ((AnthropicTokenCounter, frozenset({_UNLISTED_BETA})), (AzureAIAnthropicTokenCounter, frozenset())), + ids=("anthropic-deployment-forwards-unlisted", "azure-ai-deployment-drops-unlisted"), +) +async def test_counter_on_a_foundry_host_drops_the_caller_betas_messages_inference_drops_there( + counter_type: type[AnthropicTokenCounter] | type[AzureAIAnthropicTokenCounter], expected_betas: frozenset[str] +) -> None: + caller_betas: Final = f"mcp-servers-2025-12-04,{_UNLISTED_BETA}" + inference: Final = await _messages_inference_request( + counter_type, + f"{_FOUNDRY_BASE}/v1/messages", + {"thinking": _ENABLED_THINKING}, + extra_headers={"anthropic-beta": caller_betas}, + api_base=_FOUNDRY_BASE, + ) + _, (count,) = await _counted_requests( + counter_type, + f"{_FOUNDRY_BASE}/v1/messages/count_tokens", + [{"role": "user", "content": "Weather in SF?"}], + [_WEATHER_TOOL], + "anthropic", + request_options={"thinking": _ENABLED_THINKING}, + anthropic_beta=caller_betas, + model="claude-haiku-5-5", + api_base=_FOUNDRY_BASE, + ) + + counted_betas: Final = _betas(count) - {ANTHROPIC_TOKEN_COUNTING_BETA_VERSION} + assert counted_betas == _betas(inference) == expected_betas + + @pytest.mark.asyncio @pytest.mark.usefixtures("httpx_transport_clients") @_COUNTERS @@ -1151,7 +1194,12 @@ _ADVISOR_TOOL: Final[dict[str, JsonValue]] = { "name": "advisor", "model": "claude-opus-5-5", } -_ADVISOR_CALL: Final[dict[str, JsonValue]] = {"type": "server_tool_use", "id": "srvtoolu_1", "name": "advisor", "input": {}} +_ADVISOR_CALL: Final[dict[str, JsonValue]] = { + "type": "server_tool_use", + "id": "srvtoolu_1", + "name": "advisor", + "input": {}, +} _ADVISOR_RESULT: Final[dict[str, JsonValue]] = { "type": "advisor_tool_result", "tool_use_id": "srvtoolu_1", diff --git a/tests/unit/llms/bedrock/count_tokens/test_bedrock_token_counter.py b/tests/unit/llms/bedrock/count_tokens/test_bedrock_token_counter.py index 0b7ea57a5c3..94188c260fd 100644 --- a/tests/unit/llms/bedrock/count_tokens/test_bedrock_token_counter.py +++ b/tests/unit/llms/bedrock/count_tokens/test_bedrock_token_counter.py @@ -11,6 +11,7 @@ import litellm from litellm.llms.bedrock.count_tokens.bedrock_token_counter import BedrockTokenCounter from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler from litellm.types.llms.anthropic import ( + ANTHROPIC_BETA_HEADER_VALUES, ANTHROPIC_EFFORT_BETA_HEADER, ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER, ) @@ -457,10 +458,15 @@ _CITY_SCHEMA: Final[dict[str, JsonValue]] = { "properties": {"city": {"type": "string"}}, "required": ["city"], } -_CITY_FORMAT: Final[dict[str, JsonValue]] = {"output_config": {"format": {"type": "json_schema", "schema": _CITY_SCHEMA}}} +_CITY_FORMAT: Final[dict[str, JsonValue]] = { + "output_config": {"format": {"type": "json_schema", "schema": _CITY_SCHEMA}} +} -_CONTEXT_EDITING_OPTIONS: Final[dict[str, JsonValue]] = {**_THINKING_AND_TOOL_CHOICE, "context_management": _CONTEXT_EDITS} +_CONTEXT_EDITING_OPTIONS: Final[dict[str, JsonValue]] = { + **_THINKING_AND_TOOL_CHOICE, + "context_management": _CONTEXT_EDITS, +} async def _messages_inference_and_count( @@ -565,7 +571,11 @@ def _sent_betas(request: httpx.Request) -> tuple[str, ...]: {"output_config": {"effort": "low"}}, ANTHROPIC_EFFORT_BETA_HEADER, ), - ("bedrock/invoke/us.anthropic.claude-opus-5-5", _DISPLAY_THINKING, ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER), + ( + "bedrock/invoke/us.anthropic.claude-opus-5-5", + _DISPLAY_THINKING, + ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER, + ), ), ids=("effort", "thinking_display"), ) @@ -654,6 +664,53 @@ async def test_runtime_count_of_an_output_format_sends_only_the_betas_invoke_inf assert counted.get("anthropic_beta") == sent.get("anthropic_beta") +_PER_TURN_CONTROL: Final = ANTHROPIC_BETA_HEADER_VALUES.PER_TURN_CONTROL_2026_07_01.value + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("local_beta_headers_config") +@pytest.mark.parametrize( + ("caller_beta", "model_info", "kept"), + ( + (_PER_TURN_CONTROL, {"supports_per_turn_control": True}, True), + (_PER_TURN_CONTROL, {"supports_per_turn_control": False}, False), + ("unlisted-beta-2099-01-01", {}, False), + ), + ids=("deployment-supports-per-turn-control", "deployment-lacks-per-turn-control", "unlisted-beta"), +) +async def test_runtime_count_keeps_and_drops_the_caller_betas_invoke_inference_keeps_and_drops( + caller_beta: str, model_info: dict[str, JsonValue], kept: bool +) -> None: + deployment_model: Final = "bedrock/invoke/us.anthropic.claude-opus-5-5" + credentials: Final = DEPLOYMENT["litellm_params"] + inference: Final = _Bedrock(runtime=(200, _CLAUDE_REPLY), mantle=(200, _CLAUDE_REPLY)) + await litellm.anthropic_messages( + model=deployment_model, + messages=MESSAGES, + max_tokens=4096, + extra_headers={"anthropic-beta": caller_beta}, + model_info=model_info, + client=AsyncHTTPHandler(transport=httpx.MockTransport(inference)), + **_THINKING, + **credentials, + ) + bedrock: Final = _Bedrock(runtime=(200, {"inputTokens": 412}), mantle=(200, {"input_tokens": 2177})) + result: Final = await _counter(bedrock).count_tokens( + model_to_use="us.anthropic.claude-opus-5-5", + messages=MESSAGES, + contents=None, + deployment={"litellm_params": {**credentials, "model": deployment_model}, "model_info": model_info}, + request_model="claude", + request_options=_THINKING, + anthropic_beta=caller_beta, + ) + + assert result is not None and result.error is False + counted: Final = _betas_besides_effort(_invoke_body(bedrock.requests[0])) + assert counted == _betas_besides_effort(_JSON_BODY.validate_json(inference.requests[0].content)) + assert (caller_beta in counted) is kept + + _CONVERSE_REPLY: Final = { "output": {"message": {"role": "assistant", "content": [{"text": "ok"}]}}, "stopReason": "end_turn", @@ -787,6 +844,51 @@ async def test_runtime_rejecting_the_options_is_answered_unless_mantle_counts_or assert (result.error, result.status_code, result.tokenizer_type) == expected +_CLEAR_TOOL_USES: Final[dict[str, JsonValue]] = {"edits": [{"type": "clear_tool_uses_20250919"}]} + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("request_options", "mantle", "expected"), + ( + ({}, _mantle_answers(403, _MANTLE_FAILURE), (403, "bedrock_mantle_api")), + ({}, _mantle_unreachable(httpx.ConnectError), (500, "bedrock_mantle_api")), + ( + {**_DISPLAY_THINKING, "context_management": _CLEAR_TOOL_USES}, + _mantle_answers(403, _MANTLE_FAILURE), + (400, "bedrock_api"), + ), + ), + ids=("edit-only-mantle-denied", "edit-only-mantle-unreachable", "edit-and-thinking-mantle-denied"), +) +async def test_runtime_400_on_an_edit_reports_the_mantle_failure_unless_options_were_forwarded( + request_options: dict[str, JsonValue], + mantle: Callable[[httpx.Request], httpx.Response], + expected: tuple[int, str], +) -> None: + def bedrock(request: httpx.Request) -> httpx.Response: + if request.url.host == RUNTIME_HOST: + return httpx.Response(400, json=UNSUPPORTED, request=request) + return mantle(request) + + result: Final = await BedrockTokenCounter( + client=AsyncHTTPHandler(transport=httpx.MockTransport(bedrock)) + ).count_tokens( + model_to_use="us.anthropic.claude-opus-5-5", + messages=MESSAGES, + contents=None, + deployment={ + "litellm_params": {**DEPLOYMENT["litellm_params"], "model": "bedrock/us.anthropic.claude-opus-5-5"} + }, + request_model="claude-opus-5-5", + context_management=CountContextManagement(spec=_CLEAR_TOOL_USES, drop_invalid_edits=False), + request_options=request_options, + ) + + assert result is not None + assert (result.error, result.status_code, result.tokenizer_type) == (True, *expected) + + @pytest.mark.asyncio async def test_runtime_count_sends_the_tools_invoke_inference_sends() -> None: inference, bedrock = await _messages_inference_and_count( @@ -797,7 +899,8 @@ async def test_runtime_count_sends_the_tools_invoke_inference_sends() -> None: ) assert bedrock.posted_hosts() == (RUNTIME_HOST,) - assert _invoke_body(bedrock.requests[0])["tools"] == _JSON_BODY.validate_json(inference.requests[0].content)["tools"] + sent_tools: Final = _JSON_BODY.validate_json(inference.requests[0].content)["tools"] + assert _invoke_body(bedrock.requests[0])["tools"] == sent_tools def _texts(message: JsonValue) -> tuple[str, ...]: @@ -866,7 +969,13 @@ _THINKING: Final[dict[str, JsonValue]] = {"thinking": {"type": "enabled", "budge ([{"role": "user", "content": 5}], None, _THINKING), ([{"role": "user", "content": [{"text": "hi"}]}], None, _THINKING), ), - ids=("option_refused", "tool_type_not_a_string", "tool_not_a_mapping", "content_not_iterable", "block_without_type"), + ids=( + "option_refused", + "tool_type_not_a_string", + "tool_not_a_mapping", + "content_not_iterable", + "block_without_type", + ), ) async def test_claude_count_refuses_options_bedrock_messages_inference_refuses_without_calling_bedrock( messages: list[dict[str, JsonValue]], tools: list[JsonValue] | None, request_options: dict[str, JsonValue] diff --git a/tests/unit/llms/vertex_ai/test_vertex_ai_common_utils.py b/tests/unit/llms/vertex_ai/test_vertex_ai_common_utils.py index 2c5f2511dbd..4ac90259267 100644 --- a/tests/unit/llms/vertex_ai/test_vertex_ai_common_utils.py +++ b/tests/unit/llms/vertex_ai/test_vertex_ai_common_utils.py @@ -2082,9 +2082,8 @@ async def _vertex_claude_inference_and_count( def vertex(request: httpx.Request) -> httpx.Response: sent.put(request) - return httpx.Response( - 200, json={"input_tokens": 37} if request.url.path.endswith("count-tokens:rawPredict") else _PARTNER_CLAUDE_REPLY - ) + counted: Final = request.url.path.endswith("count-tokens:rawPredict") + return httpx.Response(200, json={"input_tokens": 37} if counted else _PARTNER_CLAUDE_REPLY) client: Final = AsyncHTTPHandler(transport=httpx.MockTransport(vertex)) with ( diff --git a/tests/unit/proxy/test_proxy_token_counter.py b/tests/unit/proxy/test_proxy_token_counter.py index 1bfb0d4cdaf..45199f48cdd 100644 --- a/tests/unit/proxy/test_proxy_token_counter.py +++ b/tests/unit/proxy/test_proxy_token_counter.py @@ -1698,7 +1698,6 @@ async def test_anthropic_count_tokens_on_a_claude_deployment_drops_a_malformed_t assert sent["messages"][2]["content"][0]["content"] == [{"type": "text", "text": "build ok"}] - _CLAUDE_COUNT_OPTIONS: Final[dict[str, JsonValue]] = { "thinking": {"type": "adaptive"}, "tool_choice": {"type": "auto"}, @@ -1756,12 +1755,18 @@ def _sent_option_paths(body: Mapping[str, JsonValue]) -> frozenset[str]: {"additional_drop_params": ["thinking", "context_management"]}, {"tool_choice", "output_config", "output_config.effort", "output_config.format"}, ), + ( + ["tool_choice"], + {"additional_drop_params": ["thinking"]}, + {"tool_choice", "output_config", "output_config.effort", "output_config.format", "context_management"}, + ), ), ids=( "nothing-dropped", "deployment-drops-tool-choice-and-the-format", "deployment-drops-context-management", "request-drops-thinking-and-context-management", + "request-drop-list-replaces-the-deployments", ), ) async def test_anthropic_count_tokens_on_a_claude_deployment_sends_the_options_inference_sends( @@ -1770,7 +1775,7 @@ async def test_anthropic_count_tokens_on_a_claude_deployment_sends_the_options_i request_params: dict[str, JsonValue], expected_paths: set[str], ) -> None: - import litellm.proxy.proxy_server as proxy_server + from litellm.proxy import proxy_server router: Final = Router( model_list=[ @@ -1834,7 +1839,7 @@ async def _count_on_a_claude_deployment( user_agent: str = "python-httpx/0.28.1", deployment_params: Mapping[str, JsonValue] = MappingProxyType({}), ) -> tuple[object, tuple[httpx.Request, ...]]: - import litellm.proxy.proxy_server as proxy_server + from litellm.proxy import proxy_server router: Final = _claude_router(deployment_model, deployment_params) request: Final = _json_request( @@ -1954,6 +1959,45 @@ async def test_anthropic_count_tokens_does_not_count_again_after_a_rate_limit( assert len(requests) == 1 +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("deployment_model", "count_host", "provider_calls"), + ( + ("gemini/gemini-3.8-flash", "generativelanguage.googleapis.com", 1), + ("anthropic/claude-haiku-5-5", "api.anthropic.com", 2), + ), + ids=("counter-without-claude-options", "claude-counter-forwarding-the-options"), +) +async def test_anthropic_count_tokens_counts_again_without_the_options_only_when_the_counter_forwarded_them( + monkeypatch: pytest.MonkeyPatch, deployment_model: str, count_host: str, provider_calls: int +) -> None: + from litellm.proxy import proxy_server + + request: Final = _json_request( + { + "model": "claude-agent", + "messages": [{"role": "user", "content": "Weather in SF?"}], + "thinking": {"type": "enabled", "budget_tokens": 2000}, + }, + "python-httpx/0.28.1", + ) + monkeypatch.setattr(proxy_server, "llm_router", _claude_router(deployment_model, {})) + monkeypatch.setattr(proxy_server, "proxy_config", proxy_server.ProxyConfig()) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + + with respx.mock: + route: Final = respx.route(host=count_host).mock( + return_value=httpx.Response( + 400, json={"type": "error", "error": {"type": "invalid_request_error", "message": "rejected"}} + ) + ) + response: Final = await anthropic_count_tokens(request, UserAPIKeyAuth()) + + assert isinstance(response, JSONResponse) + assert response.status_code == 400 + assert route.call_count == provider_calls + + _BEDROCK_RUNTIME_HOST: Final = "bedrock-runtime.us-east-1.amazonaws.com" _BEDROCK_MANTLE_HOST: Final = "bedrock-mantle.us-east-1.api.aws" _BEDROCK_CLAUDE_DEPLOYMENT: Final[dict[str, str]] = { @@ -1969,12 +2013,17 @@ async def _count_on_a_bedrock_claude_deployment( body: Mapping[str, JsonValue], runtime: Callable[[httpx.Request], httpx.Response], mantle: Callable[[httpx.Request], httpx.Response], + deployment_model: str = _BEDROCK_CLAUDE_DEPLOYMENT["model"], ) -> tuple[object, tuple[httpx.Request, ...]]: - import litellm.proxy.proxy_server as proxy_server + from litellm.proxy import proxy_server monkeypatch.delenv("AWS_BEARER_TOKEN_BEDROCK", raising=False) monkeypatch.delenv("BEDROCK_MANTLE_API_BASE", raising=False) - router: Final = Router(model_list=[{"model_name": "claude-agent", "litellm_params": _BEDROCK_CLAUDE_DEPLOYMENT}]) + router: Final = Router( + model_list=[ + {"model_name": "claude-agent", "litellm_params": {**_BEDROCK_CLAUDE_DEPLOYMENT, "model": deployment_model}} + ] + ) request: Final = _json_request( {"model": "claude-agent", "messages": [{"role": "user", "content": "Weather in SF?"}], **body}, "python-httpx/0.28.1", @@ -2045,6 +2094,48 @@ async def test_anthropic_count_tokens_on_bedrock_counts_again_without_the_option assert "invokeModel" not in json.loads(requests[-1].content)["input"] +@pytest.mark.asyncio +async def test_anthropic_count_tokens_on_bedrock_sends_the_thinking_a_clear_thinking_edit_makes_inference_send( + monkeypatch: pytest.MonkeyPatch, +) -> None: + deployment_model: Final = "bedrock/invoke/us.anthropic.claude-sonnet-4-6" + options_and_edit: Final[dict[str, JsonValue]] = { + "tools": [{"name": "get_weather", "input_schema": {"type": "object"}}], + "tool_choice": {"type": "auto"}, + "context_management": {"edits": [{"type": "clear_thinking_20251015"}]}, + } + credentials: Final = {name: value for name, value in _BEDROCK_CLAUDE_DEPLOYMENT.items() if name != "model"} + monkeypatch.delenv("AWS_BEARER_TOKEN_BEDROCK", raising=False) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + with respx.mock: + inference_route: Final = respx.route(host=_BEDROCK_RUNTIME_HOST).mock( + return_value=httpx.Response(200, json=_CLAUDE_MESSAGE_REPLY) + ) + await litellm.anthropic_messages( + model=deployment_model, + messages=[{"role": "user", "content": "Weather in SF?"}], + max_tokens=4096, + **options_and_edit, + **credentials, + ) + sent: Final = json.loads(inference_route.calls.last.request.content) + + response, (count,) = await _count_on_a_bedrock_claude_deployment( + monkeypatch, + options_and_edit, + lambda _: httpx.Response(200, json={"inputTokens": 9}), + _mantle_unreachable, + deployment_model, + ) + + assert not isinstance(response, JSONResponse) + counted: Final = json.loads(_bedrock_counted_request(count.content)) + assert sent.get("thinking") is not None + assert {name: counted.get(name) for name in ("thinking", "output_config", "context_management")} == { + name: sent.get(name) for name in ("thinking", "output_config", "context_management") + } + + _KEEPS_CONTEXT_MANAGEMENT: Final = "sk-ant-keeps-context-management" _DROPS_CONTEXT_MANAGEMENT: Final = "sk-ant-drops-context-management" _CONTEXT_EDITS: Final[dict[str, JsonValue]] = {"edits": [{"type": "clear_tool_uses_20250919"}]} @@ -2098,7 +2189,7 @@ def _mixed_drop_group(routed_key: str) -> Router: async def test_anthropic_count_tokens_in_a_mixed_drop_group_counts_without_context_management_on_any_deployment( monkeypatch: pytest.MonkeyPatch, routed_key: str ) -> None: - import litellm.proxy.proxy_server as proxy_server + from litellm.proxy import proxy_server monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) monkeypatch.setattr(proxy_server, "llm_router", _mixed_drop_group(routed_key)) @@ -2170,13 +2261,13 @@ async def test_anthropic_count_tokens_on_a_claude_deployment_sends_the_thinking_ ) -> None: monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) thinking: Final[dict[str, JsonValue]] = {"type": "enabled", "budget_tokens": 2000, "summary": "auto"} - dropped: Final = await _claude_inference_body("claude-opus-4-7", {"thinking": thinking, "drop_params": True}) - kept: Final = await _claude_inference_body("claude-opus-4-7", {"thinking": thinking}) + dropped: Final = await _claude_inference_body("claude-opus-5-5", {"thinking": thinking, "drop_params": True}) + kept: Final = await _claude_inference_body("claude-opus-5-5", {"thinking": thinking}) assert dropped["thinking"] != kept["thinking"] response, (count,) = await _count_on_a_claude_deployment( monkeypatch, - "anthropic/claude-opus-4-7", + "anthropic/claude-opus-5-5", {"thinking": thinking, **request_drop_params}, lambda _: httpx.Response(200, json={"input_tokens": 7}), user_agent,