diff --git a/litellm/anthropic_beta_headers_manager.py b/litellm/anthropic_beta_headers_manager.py index fd72bf7d2e7..f7a8e92df29 100644 --- a/litellm/anthropic_beta_headers_manager.py +++ b/litellm/anthropic_beta_headers_manager.py @@ -28,6 +28,7 @@ import os import threading from collections.abc import Mapping from concurrent.futures import ThreadPoolExecutor +from dataclasses import dataclass from functools import cache from importlib.resources import files from typing import Final, Literal, TypeAlias @@ -474,6 +475,28 @@ def update_headers_with_filtered_beta( return headers +@dataclass(frozen=True, slots=True) +class DeploymentBetaFilter: + provider: str + api_base: str | None + unlisted: UnlistedBetas + + def filtered(self, headers: dict[str, str], keep: frozenset[str]) -> dict[str, str]: + return update_headers_with_filtered_beta( + headers=headers, provider=self.provider, api_base=self.api_base, unlisted=self.unlisted, keep=keep + ) + + +def deployment_beta_filter( + provider: str, affinity_api_base: str | None, provider_filters_betas: bool +) -> DeploymentBetaFilter | None: + if provider_filters_betas: + return DeploymentBetaFilter(provider=provider, api_base=affinity_api_base, unlisted="drop") + if resolve_beta_provider(provider, affinity_api_base) != provider: + return DeploymentBetaFilter(provider=provider, api_base=affinity_api_base, unlisted="forward") + return None + + def update_request_with_filtered_beta( headers: dict, request_data: dict, diff --git a/litellm/llms/anthropic/common_utils.py b/litellm/llms/anthropic/common_utils.py index 1c42dd15853..af53778c35c 100644 --- a/litellm/llms/anthropic/common_utils.py +++ b/litellm/llms/anthropic/common_utils.py @@ -1676,6 +1676,10 @@ class AnthropicModelInfo(BaseLLMModelInfo): def get_api_base(api_base: str | None = None) -> str | None: return AnthropicModelInfo.resolve_anthropic_api_base(api_base) + @staticmethod + def affinity_api_base(custom_llm_provider: str, api_base: str | None) -> str | None: + return AnthropicModelInfo.get_api_base(api_base) if custom_llm_provider == "anthropic" else api_base + @staticmethod def get_api_key(api_key: str | None = None) -> str | None: """An empty or whitespace-only key counts as unset: it can never authenticate anything, and diff --git a/litellm/llms/anthropic/count_tokens/token_counter.py b/litellm/llms/anthropic/count_tokens/token_counter.py index 5c7f7a57a52..97ba430d14a 100644 --- a/litellm/llms/anthropic/count_tokens/token_counter.py +++ b/litellm/llms/anthropic/count_tokens/token_counter.py @@ -168,6 +168,7 @@ class AnthropicTokenCounter(BaseTokenCounter): error=True, error_message=e.message, status_code=e.status_code, + carried_request_options=claude_options.carries_request_options, ) except Exception as e: verbose_logger.warning("Error calling Anthropic CountTokens API: %s", e) diff --git a/litellm/llms/anthropic/count_tokens/transformation.py b/litellm/llms/anthropic/count_tokens/transformation.py index 82ccf3b6cc5..26defadfda1 100644 --- a/litellm/llms/anthropic/count_tokens/transformation.py +++ b/litellm/llms/anthropic/count_tokens/transformation.py @@ -20,6 +20,7 @@ from typing import ( from pydantic import ConfigDict, JsonValue, TypeAdapter, with_config from typing_extensions import NotRequired, ReadOnly, TypedDict +from litellm.anthropic_beta_headers_manager import DeploymentBetaFilter, deployment_beta_filter from litellm.constants import ANTHROPIC_TOKEN_COUNTING_BETA_VERSION from litellm.litellm_core_utils.asyncify import asyncify from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import ( @@ -114,6 +115,7 @@ class AnthropicCountFields: @dataclass(frozen=True, slots=True) class ChatCountTranslationError: message: str + carried_request_options: bool = False def response(self, request_model: str, model_used: str, tokenizer_type: str) -> TokenCountResponse: return TokenCountResponse( @@ -125,6 +127,7 @@ class ChatCountTranslationError: error_message=self.message, status_code=400, rejected_before_provider_call=True, + carried_request_options=self.carried_request_options, ) @@ -349,6 +352,10 @@ class ClaudeCountOptions: messages: tuple[Mapping[str, JsonValue], ...] | None tools: tuple[Mapping[str, JsonValue], ...] | None + @property + def carries_request_options(self) -> bool: + return bool(self.options) or bool(self.betas) + NO_CLAUDE_COUNT_OPTIONS: Final = ClaudeCountOptions(options=MappingProxyType({}), betas=(), messages=None, tools=None) _COUNTED_MESSAGES: Final = TypeAdapter(tuple[Mapping[str, JsonValue], ...]) @@ -379,6 +386,11 @@ def deployment_inference_model(deployment: Mapping[str, object] | None) -> str | return inference_model +def _deployment_api_base(deployment: Mapping[str, object] | None) -> str | None: + api_base: Final = _INFERENCE_PARAMS.validate_python((deployment or {}).get("litellm_params") or {}).get("api_base") + return api_base if isinstance(api_base, str) else None + + def _messages_transform(model: str, provider: LlmProviders) -> tuple[str, BaseAnthropicMessagesConfig] | None: from litellm.utils import ProviderConfigManager @@ -403,14 +415,12 @@ def _claude_count_options_for( messages_config: BaseAnthropicMessagesConfig, model: str, litellm_params: "GenericLiteLLMParams", - provider: LlmProviders, + beta_filter: DeploymentBetaFilter | None, messages: Sequence[Mapping[str, object]], tools: Sequence[Mapping[str, object]] | None, sent: Mapping[str, JsonValue], anthropic_beta: str | None, ) -> ClaudeCountOptions: - from litellm.anthropic_beta_headers_manager import update_headers_with_filtered_beta - leading_system, conversation = split_leading_system_run(messages) sent_conversation: Final = copy.deepcopy([dict(message) for message in conversation]) sent_params: Final = { @@ -421,9 +431,11 @@ def _claude_count_options_for( {"anthropic-beta": anthropic_beta} if anthropic_beta else {}, sent_params, sent_conversation ) sent_headers: Final = ( - update_headers_with_filtered_beta(headers=dict(beta_headers), provider=provider.value) - if messages_config.should_filter_anthropic_beta_headers() - else dict(beta_headers) + dict(beta_headers) + if beta_filter is None + else beta_filter.filtered( + dict(beta_headers), keep=messages_config.model_specific_anthropic_betas(model, litellm_params) + ) ) request: Final = messages_config.transform_anthropic_messages_request( # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # the base config signature takes and returns bare dicts model=model, @@ -462,16 +474,23 @@ def claude_count_options( transform_model, messages_config = transform litellm_params: Final = _transform_litellm_params(deployment) from litellm.exceptions import BadRequestError - from litellm.llms.anthropic.common_utils import AnthropicError + from litellm.llms.anthropic.common_utils import AnthropicError, AnthropicModelInfo + beta_filter: Final = deployment_beta_filter( + provider.value, + AnthropicModelInfo.affinity_api_base(provider.value, _deployment_api_base(deployment)), + messages_config.should_filter_anthropic_beta_headers(), + ) try: return _claude_count_options_for( - messages_config, transform_model, litellm_params, provider, messages, tools, sent, anthropic_beta + messages_config, transform_model, litellm_params, beta_filter, messages, tools, sent, anthropic_beta ) except (AnthropicError, BadRequestError) as e: - return ChatCountTranslationError(message=str(e.message)) + return ChatCountTranslationError(message=str(e.message), carried_request_options=True) except (AttributeError, KeyError, TypeError, ValueError) as e: - return ChatCountTranslationError(message=f"Invalid request for token counting: {e}") + return ChatCountTranslationError( + message=f"Invalid request for token counting: {e}", carried_request_options=True + ) async def anthropic_count_fields( diff --git a/litellm/llms/azure_ai/anthropic/count_tokens/token_counter.py b/litellm/llms/azure_ai/anthropic/count_tokens/token_counter.py index 31082e1e241..ebe7bdb664e 100644 --- a/litellm/llms/azure_ai/anthropic/count_tokens/token_counter.py +++ b/litellm/llms/azure_ai/anthropic/count_tokens/token_counter.py @@ -176,6 +176,7 @@ class AzureAIAnthropicTokenCounter(BaseTokenCounter): error=True, error_message=e.message, status_code=e.status_code, + carried_request_options=claude_options.carries_request_options, ) except Exception as e: verbose_logger.warning("Error calling Azure AI Anthropic CountTokens API: %s", e) diff --git a/litellm/llms/bedrock/count_tokens/bedrock_token_counter.py b/litellm/llms/bedrock/count_tokens/bedrock_token_counter.py index ca0409e824c..fa1cbe9ead9 100644 --- a/litellm/llms/bedrock/count_tokens/bedrock_token_counter.py +++ b/litellm/llms/bedrock/count_tokens/bedrock_token_counter.py @@ -89,12 +89,12 @@ def _runtime_rejected_claude_model(outcome: CountTokensOutcome, resolved_model: def _mantle_retry_outcome( - runtime: CountTokensOutcome, mantle: CountTokensOutcome, forwarded: bool + runtime: CountTokensOutcome, mantle: CountTokensOutcome, forwarded: bool, carried_options: bool ) -> CountTokensOutcome: match runtime, mantle: case CountTokensFailure(status_code=400), CountTokensFailure(status_code=404) if forwarded: return runtime - case _, CountTokensFailure(status_code=status_code) if forwarded and not is_request_rejection(status_code): + case _, CountTokensFailure(status_code=code) if carried_options and not is_request_rejection(code): return runtime case _: return mantle @@ -117,7 +117,7 @@ def _sets_effort(options: Mapping[str, JsonValue]) -> bool: def _runtime_count_betas(claude_options: ClaudeCountOptions) -> tuple[str, ...]: - if not _sets_effort(claude_options.options) or ANTHROPIC_EFFORT_BETA_HEADER in claude_options.betas: + if not _sets_effort(claude_options.options): return claude_options.betas return (*claude_options.betas, ANTHROPIC_EFFORT_BETA_HEADER) @@ -322,7 +322,14 @@ class BedrockTokenCounter(BaseTokenCounter): else None ) outcome: Final = ( - primary if fallback is None else _mantle_retry_outcome(primary, fallback, forwarded=bool(runtime_fields)) + primary + if fallback is None + else _mantle_retry_outcome( + primary, + fallback, + forwarded=bool(runtime_fields), + carried_options=claude_options.carries_request_options, + ) ) match outcome: case CountedTokens(): @@ -348,6 +355,7 @@ class BedrockTokenCounter(BaseTokenCounter): error=True, error_message=outcome.message, status_code=outcome.status_code, + carried_request_options=claude_options.carries_request_options, ) case _: assert_never(outcome) diff --git a/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude_transformation.py b/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude_transformation.py index 824a075309e..ebb91cb230a 100644 --- a/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude_transformation.py +++ b/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude_transformation.py @@ -121,7 +121,7 @@ class AmazonAnthropicClaudeMessagesConfig( optional_params: Mapping[str, object], messages: Sequence[object], ) -> Mapping[str, str]: - return headers + return BaseAnthropicMessagesConfig.headers_with_anthropic_betas(self, headers, optional_params, messages) def sent_anthropic_betas(self, headers: Mapping[str, str], request: Mapping[str, object]) -> tuple[str, ...]: return _ANTHROPIC_BETAS.validate_python(request.get("anthropic_beta") or ()) diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index 8c9de272c0d..62644959a01 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -39,11 +39,7 @@ import litellm.types import litellm.types.utils from litellm._internal_context import internal_sub_call from litellm._logging import redact_string, verbose_logger -from litellm.anthropic_beta_headers_manager import ( - aload_beta_headers_config, - resolve_beta_provider, - update_headers_with_filtered_beta, -) +from litellm.anthropic_beta_headers_manager import aload_beta_headers_config, deployment_beta_filter from litellm.constants import MAX_FILE_LIST_LIMIT, REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES from litellm.files.types import FileContentStreamingResult from litellm.litellm_core_utils.agentic_followup_kwargs import build_agentic_followup_kwargs @@ -2623,15 +2619,13 @@ class BaseLLMHTTPHandler: litellm_params.get("additional_drop_params") or (), anthropic_messages_optional_request_params, ) - affinity_api_base: Final = ( - AnthropicModelInfo.get_api_base(api_base) if custom_llm_provider == "anthropic" else api_base + affinity_api_base: Final = AnthropicModelInfo.affinity_api_base(custom_llm_provider, api_base) + beta_filter: Final = deployment_beta_filter( + custom_llm_provider, + affinity_api_base, + anthropic_messages_provider_config.should_filter_anthropic_beta_headers(), ) - takes_another_beta_row: Final = ( - resolve_beta_provider(custom_llm_provider, affinity_api_base) != custom_llm_provider - ) - provider_filters_betas: Final = anthropic_messages_provider_config.should_filter_anthropic_beta_headers() - filters_beta_headers: Final = provider_filters_betas or takes_another_beta_row - if request_params.get("safeguards") is not None and filters_beta_headers: + if request_params.get("safeguards") is not None and beta_filter is not None: await aload_beta_headers_config() default_affinity_header: Final = get_provider_affinity_header(affinity_api_base) ( @@ -2651,14 +2645,10 @@ class BaseLLMHTTPHandler: api_base=api_base, ) - if filters_beta_headers and headers.get("anthropic-beta") is not None: + if beta_filter is not None and headers.get("anthropic-beta") is not None: await aload_beta_headers_config() - headers = update_headers_with_filtered_beta( - headers=headers, - provider=custom_llm_provider, - api_base=affinity_api_base, - unlisted="drop" if provider_filters_betas else "forward", - keep=anthropic_messages_provider_config.model_specific_anthropic_betas(model, litellm_params), + headers = beta_filter.filtered( + headers, keep=anthropic_messages_provider_config.model_specific_anthropic_betas(model, litellm_params) ) from litellm.llms.vertex_ai.vertex_llm_base import VertexBase diff --git a/litellm/llms/vertex_ai/common_utils.py b/litellm/llms/vertex_ai/common_utils.py index 1ab84da6c1b..ae2af8b3448 100644 --- a/litellm/llms/vertex_ai/common_utils.py +++ b/litellm/llms/vertex_ai/common_utils.py @@ -1581,6 +1581,7 @@ class VertexAITokenCounter(BaseTokenCounter): error=True, error_message=error_message, status_code=status_code, + carried_request_options=claude_options.carries_request_options, ) if result is not None: diff --git a/litellm/llms/vertex_ai/vertex_ai_partner_models/count_tokens/handler.py b/litellm/llms/vertex_ai/vertex_ai_partner_models/count_tokens/handler.py index d4350553999..522da59c71c 100644 --- a/litellm/llms/vertex_ai/vertex_ai_partner_models/count_tokens/handler.py +++ b/litellm/llms/vertex_ai/vertex_ai_partner_models/count_tokens/handler.py @@ -153,12 +153,6 @@ class VertexAIPartnerModelsTokenCounter(VertexBase): """ Handle token counting request for a Vertex AI partner model. - Args: - model: The model name - request_data: Request payload (Anthropic Messages API format) - litellm_params: LiteLLM parameters containing credentials, project, location - betas: anthropic-beta values the counted request needs, sent as the anthropic-beta header - Returns: Dict containing token count information diff --git a/litellm/proxy/common_utils/token_count_settings.py b/litellm/proxy/common_utils/token_count_settings.py index 544b5b0f5e0..171b5ea93a5 100644 --- a/litellm/proxy/common_utils/token_count_settings.py +++ b/litellm/proxy/common_utils/token_count_settings.py @@ -80,6 +80,7 @@ class ProviderCountFailure: status_code: int message: str rejected_before_provider_call: bool = False + carried_request_options: bool = False @property def cannot_count_model(self) -> bool: @@ -90,8 +91,8 @@ class ProviderCountFailure: return not self.rejected_before_provider_call and self.status_code in REGION_DENIED_STATUSES @property - def rejected_request(self) -> bool: - return is_request_rejection(self.status_code) + def rejected_request_options(self) -> bool: + return self.carried_request_options and is_request_rejection(self.status_code) def with_hint(self, hint: str) -> "ProviderCountFailure": if self.rejected_before_provider_call: diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 649236e78af..6a8f2cb28de 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -15269,6 +15269,7 @@ def _provider_count_outcome( status_code=result.status_code or 500, message=result.error_message or "Token counting failed", rejected_before_provider_call=result.rejected_before_provider_call, + carried_request_options=result.carried_request_options, ) return result @@ -15469,7 +15470,7 @@ async def _count_retrying_without_options( region: str | None, ) -> Union["TokenCountResponse", ProviderCountFailure, None]: counted: Final = await count_with(model, region) - if not (isinstance(counted, ProviderCountFailure) and counted.rejected_request): + if not (isinstance(counted, ProviderCountFailure) and counted.rejected_request_options): return counted verbose_proxy_logger.warning( "Token count for %s %s with the request's %s, so it counts again without them and the caller's betas", diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 3ddeb118c33..4dc9abedcb1 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -4912,6 +4912,7 @@ class TokenCountResponse(LiteLLMPydanticObjectBase): """ status_code: int | None = None rejected_before_provider_call: bool = Field(default=False, exclude=True) + carried_request_options: bool = Field(default=False, exclude=True) class CustomHuggingfaceTokenizer(TypedDict): diff --git a/tests/unit/llms/anthropic/test_anthropic_count_tokens_transformation.py b/tests/unit/llms/anthropic/test_anthropic_count_tokens_transformation.py index 91e2b03dd1d..1f731faa724 100644 --- a/tests/unit/llms/anthropic/test_anthropic_count_tokens_transformation.py +++ b/tests/unit/llms/anthropic/test_anthropic_count_tokens_transformation.py @@ -567,6 +567,7 @@ async def _counted_requests( request_options: Mapping[str, JsonValue] = MappingProxyType({}), anthropic_beta: str | None = None, model: str = "claude-opus-5-5", + api_base: str = "https://gateway.example", ) -> tuple[TokenCountResponse | None, tuple[httpx.Request, ...]]: sent: Final[SimpleQueue[httpx.Request]] = SimpleQueue() @@ -580,7 +581,7 @@ async def _counted_requests( model_to_use=model, messages=messages, contents=None, - deployment={"litellm_params": {"api_key": "sk-ant-api03-test-key", "api_base": "https://gateway.example"}}, + deployment={"litellm_params": {"api_key": "sk-ant-api03-test-key", "api_base": api_base}}, request_model=model, tools=tools, messages_format=messages_format, @@ -972,6 +973,7 @@ async def _messages_inference_request( request_options: Mapping[str, JsonValue], model: str = "claude-haiku-5-5", extra_headers: Mapping[str, str] = MappingProxyType({}), + api_base: str = "https://gateway.example", ) -> httpx.Request: sent: Final[SimpleQueue[httpx.Request]] = SimpleQueue() @@ -987,7 +989,7 @@ async def _messages_inference_request( tools=[_WEATHER_TOOL], max_tokens=4096, api_key="sk-ant-api03-test-key", - api_base="https://gateway.example", + api_base=api_base, extra_headers=dict(extra_headers), **request_options, ) @@ -1059,7 +1061,10 @@ async def test_counter_sends_the_options_and_context_edits_messages_inference_se assert count_body.get("output_config") == ( inference_output_config if inference_output_format is None - else {**(inference_output_config if isinstance(inference_output_config, dict) else {}), "format": inference_output_format} + else { + **(inference_output_config if isinstance(inference_output_config, dict) else {}), + "format": inference_output_format, + } ) assert {name: count_body.get(name) for name in ("thinking", "tool_choice", "context_management")} == { name: inference_body.get(name) for name in ("thinking", "tool_choice", "context_management") @@ -1098,6 +1103,44 @@ async def test_counter_sends_the_thinking_display_and_caller_betas_messages_infe assert {ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER, caller_beta} <= counted_betas +_FOUNDRY_BASE: Final = "https://my-resource.services.ai.azure.com/anthropic" +_UNLISTED_BETA: Final = "unlisted-beta-2099-01-01" + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("httpx_transport_clients", "local_beta_headers_config") +@pytest.mark.parametrize( + ("counter_type", "expected_betas"), + ((AnthropicTokenCounter, frozenset({_UNLISTED_BETA})), (AzureAIAnthropicTokenCounter, frozenset())), + ids=("anthropic-deployment-forwards-unlisted", "azure-ai-deployment-drops-unlisted"), +) +async def test_counter_on_a_foundry_host_drops_the_caller_betas_messages_inference_drops_there( + counter_type: type[AnthropicTokenCounter] | type[AzureAIAnthropicTokenCounter], expected_betas: frozenset[str] +) -> None: + caller_betas: Final = f"mcp-servers-2025-12-04,{_UNLISTED_BETA}" + inference: Final = await _messages_inference_request( + counter_type, + f"{_FOUNDRY_BASE}/v1/messages", + {"thinking": _ENABLED_THINKING}, + extra_headers={"anthropic-beta": caller_betas}, + api_base=_FOUNDRY_BASE, + ) + _, (count,) = await _counted_requests( + counter_type, + f"{_FOUNDRY_BASE}/v1/messages/count_tokens", + [{"role": "user", "content": "Weather in SF?"}], + [_WEATHER_TOOL], + "anthropic", + request_options={"thinking": _ENABLED_THINKING}, + anthropic_beta=caller_betas, + model="claude-haiku-5-5", + api_base=_FOUNDRY_BASE, + ) + + counted_betas: Final = _betas(count) - {ANTHROPIC_TOKEN_COUNTING_BETA_VERSION} + assert counted_betas == _betas(inference) == expected_betas + + @pytest.mark.asyncio @pytest.mark.usefixtures("httpx_transport_clients") @_COUNTERS @@ -1151,7 +1194,12 @@ _ADVISOR_TOOL: Final[dict[str, JsonValue]] = { "name": "advisor", "model": "claude-opus-5-5", } -_ADVISOR_CALL: Final[dict[str, JsonValue]] = {"type": "server_tool_use", "id": "srvtoolu_1", "name": "advisor", "input": {}} +_ADVISOR_CALL: Final[dict[str, JsonValue]] = { + "type": "server_tool_use", + "id": "srvtoolu_1", + "name": "advisor", + "input": {}, +} _ADVISOR_RESULT: Final[dict[str, JsonValue]] = { "type": "advisor_tool_result", "tool_use_id": "srvtoolu_1", diff --git a/tests/unit/llms/bedrock/count_tokens/test_bedrock_token_counter.py b/tests/unit/llms/bedrock/count_tokens/test_bedrock_token_counter.py index 0b7ea57a5c3..94188c260fd 100644 --- a/tests/unit/llms/bedrock/count_tokens/test_bedrock_token_counter.py +++ b/tests/unit/llms/bedrock/count_tokens/test_bedrock_token_counter.py @@ -11,6 +11,7 @@ import litellm from litellm.llms.bedrock.count_tokens.bedrock_token_counter import BedrockTokenCounter from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler from litellm.types.llms.anthropic import ( + ANTHROPIC_BETA_HEADER_VALUES, ANTHROPIC_EFFORT_BETA_HEADER, ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER, ) @@ -457,10 +458,15 @@ _CITY_SCHEMA: Final[dict[str, JsonValue]] = { "properties": {"city": {"type": "string"}}, "required": ["city"], } -_CITY_FORMAT: Final[dict[str, JsonValue]] = {"output_config": {"format": {"type": "json_schema", "schema": _CITY_SCHEMA}}} +_CITY_FORMAT: Final[dict[str, JsonValue]] = { + "output_config": {"format": {"type": "json_schema", "schema": _CITY_SCHEMA}} +} -_CONTEXT_EDITING_OPTIONS: Final[dict[str, JsonValue]] = {**_THINKING_AND_TOOL_CHOICE, "context_management": _CONTEXT_EDITS} +_CONTEXT_EDITING_OPTIONS: Final[dict[str, JsonValue]] = { + **_THINKING_AND_TOOL_CHOICE, + "context_management": _CONTEXT_EDITS, +} async def _messages_inference_and_count( @@ -565,7 +571,11 @@ def _sent_betas(request: httpx.Request) -> tuple[str, ...]: {"output_config": {"effort": "low"}}, ANTHROPIC_EFFORT_BETA_HEADER, ), - ("bedrock/invoke/us.anthropic.claude-opus-5-5", _DISPLAY_THINKING, ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER), + ( + "bedrock/invoke/us.anthropic.claude-opus-5-5", + _DISPLAY_THINKING, + ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER, + ), ), ids=("effort", "thinking_display"), ) @@ -654,6 +664,53 @@ async def test_runtime_count_of_an_output_format_sends_only_the_betas_invoke_inf assert counted.get("anthropic_beta") == sent.get("anthropic_beta") +_PER_TURN_CONTROL: Final = ANTHROPIC_BETA_HEADER_VALUES.PER_TURN_CONTROL_2026_07_01.value + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("local_beta_headers_config") +@pytest.mark.parametrize( + ("caller_beta", "model_info", "kept"), + ( + (_PER_TURN_CONTROL, {"supports_per_turn_control": True}, True), + (_PER_TURN_CONTROL, {"supports_per_turn_control": False}, False), + ("unlisted-beta-2099-01-01", {}, False), + ), + ids=("deployment-supports-per-turn-control", "deployment-lacks-per-turn-control", "unlisted-beta"), +) +async def test_runtime_count_keeps_and_drops_the_caller_betas_invoke_inference_keeps_and_drops( + caller_beta: str, model_info: dict[str, JsonValue], kept: bool +) -> None: + deployment_model: Final = "bedrock/invoke/us.anthropic.claude-opus-5-5" + credentials: Final = DEPLOYMENT["litellm_params"] + inference: Final = _Bedrock(runtime=(200, _CLAUDE_REPLY), mantle=(200, _CLAUDE_REPLY)) + await litellm.anthropic_messages( + model=deployment_model, + messages=MESSAGES, + max_tokens=4096, + extra_headers={"anthropic-beta": caller_beta}, + model_info=model_info, + client=AsyncHTTPHandler(transport=httpx.MockTransport(inference)), + **_THINKING, + **credentials, + ) + bedrock: Final = _Bedrock(runtime=(200, {"inputTokens": 412}), mantle=(200, {"input_tokens": 2177})) + result: Final = await _counter(bedrock).count_tokens( + model_to_use="us.anthropic.claude-opus-5-5", + messages=MESSAGES, + contents=None, + deployment={"litellm_params": {**credentials, "model": deployment_model}, "model_info": model_info}, + request_model="claude", + request_options=_THINKING, + anthropic_beta=caller_beta, + ) + + assert result is not None and result.error is False + counted: Final = _betas_besides_effort(_invoke_body(bedrock.requests[0])) + assert counted == _betas_besides_effort(_JSON_BODY.validate_json(inference.requests[0].content)) + assert (caller_beta in counted) is kept + + _CONVERSE_REPLY: Final = { "output": {"message": {"role": "assistant", "content": [{"text": "ok"}]}}, "stopReason": "end_turn", @@ -787,6 +844,51 @@ async def test_runtime_rejecting_the_options_is_answered_unless_mantle_counts_or assert (result.error, result.status_code, result.tokenizer_type) == expected +_CLEAR_TOOL_USES: Final[dict[str, JsonValue]] = {"edits": [{"type": "clear_tool_uses_20250919"}]} + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("request_options", "mantle", "expected"), + ( + ({}, _mantle_answers(403, _MANTLE_FAILURE), (403, "bedrock_mantle_api")), + ({}, _mantle_unreachable(httpx.ConnectError), (500, "bedrock_mantle_api")), + ( + {**_DISPLAY_THINKING, "context_management": _CLEAR_TOOL_USES}, + _mantle_answers(403, _MANTLE_FAILURE), + (400, "bedrock_api"), + ), + ), + ids=("edit-only-mantle-denied", "edit-only-mantle-unreachable", "edit-and-thinking-mantle-denied"), +) +async def test_runtime_400_on_an_edit_reports_the_mantle_failure_unless_options_were_forwarded( + request_options: dict[str, JsonValue], + mantle: Callable[[httpx.Request], httpx.Response], + expected: tuple[int, str], +) -> None: + def bedrock(request: httpx.Request) -> httpx.Response: + if request.url.host == RUNTIME_HOST: + return httpx.Response(400, json=UNSUPPORTED, request=request) + return mantle(request) + + result: Final = await BedrockTokenCounter( + client=AsyncHTTPHandler(transport=httpx.MockTransport(bedrock)) + ).count_tokens( + model_to_use="us.anthropic.claude-opus-5-5", + messages=MESSAGES, + contents=None, + deployment={ + "litellm_params": {**DEPLOYMENT["litellm_params"], "model": "bedrock/us.anthropic.claude-opus-5-5"} + }, + request_model="claude-opus-5-5", + context_management=CountContextManagement(spec=_CLEAR_TOOL_USES, drop_invalid_edits=False), + request_options=request_options, + ) + + assert result is not None + assert (result.error, result.status_code, result.tokenizer_type) == (True, *expected) + + @pytest.mark.asyncio async def test_runtime_count_sends_the_tools_invoke_inference_sends() -> None: inference, bedrock = await _messages_inference_and_count( @@ -797,7 +899,8 @@ async def test_runtime_count_sends_the_tools_invoke_inference_sends() -> None: ) assert bedrock.posted_hosts() == (RUNTIME_HOST,) - assert _invoke_body(bedrock.requests[0])["tools"] == _JSON_BODY.validate_json(inference.requests[0].content)["tools"] + sent_tools: Final = _JSON_BODY.validate_json(inference.requests[0].content)["tools"] + assert _invoke_body(bedrock.requests[0])["tools"] == sent_tools def _texts(message: JsonValue) -> tuple[str, ...]: @@ -866,7 +969,13 @@ _THINKING: Final[dict[str, JsonValue]] = {"thinking": {"type": "enabled", "budge ([{"role": "user", "content": 5}], None, _THINKING), ([{"role": "user", "content": [{"text": "hi"}]}], None, _THINKING), ), - ids=("option_refused", "tool_type_not_a_string", "tool_not_a_mapping", "content_not_iterable", "block_without_type"), + ids=( + "option_refused", + "tool_type_not_a_string", + "tool_not_a_mapping", + "content_not_iterable", + "block_without_type", + ), ) async def test_claude_count_refuses_options_bedrock_messages_inference_refuses_without_calling_bedrock( messages: list[dict[str, JsonValue]], tools: list[JsonValue] | None, request_options: dict[str, JsonValue] diff --git a/tests/unit/llms/vertex_ai/test_vertex_ai_common_utils.py b/tests/unit/llms/vertex_ai/test_vertex_ai_common_utils.py index 2c5f2511dbd..4ac90259267 100644 --- a/tests/unit/llms/vertex_ai/test_vertex_ai_common_utils.py +++ b/tests/unit/llms/vertex_ai/test_vertex_ai_common_utils.py @@ -2082,9 +2082,8 @@ async def _vertex_claude_inference_and_count( def vertex(request: httpx.Request) -> httpx.Response: sent.put(request) - return httpx.Response( - 200, json={"input_tokens": 37} if request.url.path.endswith("count-tokens:rawPredict") else _PARTNER_CLAUDE_REPLY - ) + counted: Final = request.url.path.endswith("count-tokens:rawPredict") + return httpx.Response(200, json={"input_tokens": 37} if counted else _PARTNER_CLAUDE_REPLY) client: Final = AsyncHTTPHandler(transport=httpx.MockTransport(vertex)) with ( diff --git a/tests/unit/proxy/test_proxy_token_counter.py b/tests/unit/proxy/test_proxy_token_counter.py index 1bfb0d4cdaf..45199f48cdd 100644 --- a/tests/unit/proxy/test_proxy_token_counter.py +++ b/tests/unit/proxy/test_proxy_token_counter.py @@ -1698,7 +1698,6 @@ async def test_anthropic_count_tokens_on_a_claude_deployment_drops_a_malformed_t assert sent["messages"][2]["content"][0]["content"] == [{"type": "text", "text": "build ok"}] - _CLAUDE_COUNT_OPTIONS: Final[dict[str, JsonValue]] = { "thinking": {"type": "adaptive"}, "tool_choice": {"type": "auto"}, @@ -1756,12 +1755,18 @@ def _sent_option_paths(body: Mapping[str, JsonValue]) -> frozenset[str]: {"additional_drop_params": ["thinking", "context_management"]}, {"tool_choice", "output_config", "output_config.effort", "output_config.format"}, ), + ( + ["tool_choice"], + {"additional_drop_params": ["thinking"]}, + {"tool_choice", "output_config", "output_config.effort", "output_config.format", "context_management"}, + ), ), ids=( "nothing-dropped", "deployment-drops-tool-choice-and-the-format", "deployment-drops-context-management", "request-drops-thinking-and-context-management", + "request-drop-list-replaces-the-deployments", ), ) async def test_anthropic_count_tokens_on_a_claude_deployment_sends_the_options_inference_sends( @@ -1770,7 +1775,7 @@ async def test_anthropic_count_tokens_on_a_claude_deployment_sends_the_options_i request_params: dict[str, JsonValue], expected_paths: set[str], ) -> None: - import litellm.proxy.proxy_server as proxy_server + from litellm.proxy import proxy_server router: Final = Router( model_list=[ @@ -1834,7 +1839,7 @@ async def _count_on_a_claude_deployment( user_agent: str = "python-httpx/0.28.1", deployment_params: Mapping[str, JsonValue] = MappingProxyType({}), ) -> tuple[object, tuple[httpx.Request, ...]]: - import litellm.proxy.proxy_server as proxy_server + from litellm.proxy import proxy_server router: Final = _claude_router(deployment_model, deployment_params) request: Final = _json_request( @@ -1954,6 +1959,45 @@ async def test_anthropic_count_tokens_does_not_count_again_after_a_rate_limit( assert len(requests) == 1 +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("deployment_model", "count_host", "provider_calls"), + ( + ("gemini/gemini-3.8-flash", "generativelanguage.googleapis.com", 1), + ("anthropic/claude-haiku-5-5", "api.anthropic.com", 2), + ), + ids=("counter-without-claude-options", "claude-counter-forwarding-the-options"), +) +async def test_anthropic_count_tokens_counts_again_without_the_options_only_when_the_counter_forwarded_them( + monkeypatch: pytest.MonkeyPatch, deployment_model: str, count_host: str, provider_calls: int +) -> None: + from litellm.proxy import proxy_server + + request: Final = _json_request( + { + "model": "claude-agent", + "messages": [{"role": "user", "content": "Weather in SF?"}], + "thinking": {"type": "enabled", "budget_tokens": 2000}, + }, + "python-httpx/0.28.1", + ) + monkeypatch.setattr(proxy_server, "llm_router", _claude_router(deployment_model, {})) + monkeypatch.setattr(proxy_server, "proxy_config", proxy_server.ProxyConfig()) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + + with respx.mock: + route: Final = respx.route(host=count_host).mock( + return_value=httpx.Response( + 400, json={"type": "error", "error": {"type": "invalid_request_error", "message": "rejected"}} + ) + ) + response: Final = await anthropic_count_tokens(request, UserAPIKeyAuth()) + + assert isinstance(response, JSONResponse) + assert response.status_code == 400 + assert route.call_count == provider_calls + + _BEDROCK_RUNTIME_HOST: Final = "bedrock-runtime.us-east-1.amazonaws.com" _BEDROCK_MANTLE_HOST: Final = "bedrock-mantle.us-east-1.api.aws" _BEDROCK_CLAUDE_DEPLOYMENT: Final[dict[str, str]] = { @@ -1969,12 +2013,17 @@ async def _count_on_a_bedrock_claude_deployment( body: Mapping[str, JsonValue], runtime: Callable[[httpx.Request], httpx.Response], mantle: Callable[[httpx.Request], httpx.Response], + deployment_model: str = _BEDROCK_CLAUDE_DEPLOYMENT["model"], ) -> tuple[object, tuple[httpx.Request, ...]]: - import litellm.proxy.proxy_server as proxy_server + from litellm.proxy import proxy_server monkeypatch.delenv("AWS_BEARER_TOKEN_BEDROCK", raising=False) monkeypatch.delenv("BEDROCK_MANTLE_API_BASE", raising=False) - router: Final = Router(model_list=[{"model_name": "claude-agent", "litellm_params": _BEDROCK_CLAUDE_DEPLOYMENT}]) + router: Final = Router( + model_list=[ + {"model_name": "claude-agent", "litellm_params": {**_BEDROCK_CLAUDE_DEPLOYMENT, "model": deployment_model}} + ] + ) request: Final = _json_request( {"model": "claude-agent", "messages": [{"role": "user", "content": "Weather in SF?"}], **body}, "python-httpx/0.28.1", @@ -2045,6 +2094,48 @@ async def test_anthropic_count_tokens_on_bedrock_counts_again_without_the_option assert "invokeModel" not in json.loads(requests[-1].content)["input"] +@pytest.mark.asyncio +async def test_anthropic_count_tokens_on_bedrock_sends_the_thinking_a_clear_thinking_edit_makes_inference_send( + monkeypatch: pytest.MonkeyPatch, +) -> None: + deployment_model: Final = "bedrock/invoke/us.anthropic.claude-sonnet-4-6" + options_and_edit: Final[dict[str, JsonValue]] = { + "tools": [{"name": "get_weather", "input_schema": {"type": "object"}}], + "tool_choice": {"type": "auto"}, + "context_management": {"edits": [{"type": "clear_thinking_20251015"}]}, + } + credentials: Final = {name: value for name, value in _BEDROCK_CLAUDE_DEPLOYMENT.items() if name != "model"} + monkeypatch.delenv("AWS_BEARER_TOKEN_BEDROCK", raising=False) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + with respx.mock: + inference_route: Final = respx.route(host=_BEDROCK_RUNTIME_HOST).mock( + return_value=httpx.Response(200, json=_CLAUDE_MESSAGE_REPLY) + ) + await litellm.anthropic_messages( + model=deployment_model, + messages=[{"role": "user", "content": "Weather in SF?"}], + max_tokens=4096, + **options_and_edit, + **credentials, + ) + sent: Final = json.loads(inference_route.calls.last.request.content) + + response, (count,) = await _count_on_a_bedrock_claude_deployment( + monkeypatch, + options_and_edit, + lambda _: httpx.Response(200, json={"inputTokens": 9}), + _mantle_unreachable, + deployment_model, + ) + + assert not isinstance(response, JSONResponse) + counted: Final = json.loads(_bedrock_counted_request(count.content)) + assert sent.get("thinking") is not None + assert {name: counted.get(name) for name in ("thinking", "output_config", "context_management")} == { + name: sent.get(name) for name in ("thinking", "output_config", "context_management") + } + + _KEEPS_CONTEXT_MANAGEMENT: Final = "sk-ant-keeps-context-management" _DROPS_CONTEXT_MANAGEMENT: Final = "sk-ant-drops-context-management" _CONTEXT_EDITS: Final[dict[str, JsonValue]] = {"edits": [{"type": "clear_tool_uses_20250919"}]} @@ -2098,7 +2189,7 @@ def _mixed_drop_group(routed_key: str) -> Router: async def test_anthropic_count_tokens_in_a_mixed_drop_group_counts_without_context_management_on_any_deployment( monkeypatch: pytest.MonkeyPatch, routed_key: str ) -> None: - import litellm.proxy.proxy_server as proxy_server + from litellm.proxy import proxy_server monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) monkeypatch.setattr(proxy_server, "llm_router", _mixed_drop_group(routed_key)) @@ -2170,13 +2261,13 @@ async def test_anthropic_count_tokens_on_a_claude_deployment_sends_the_thinking_ ) -> None: monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) thinking: Final[dict[str, JsonValue]] = {"type": "enabled", "budget_tokens": 2000, "summary": "auto"} - dropped: Final = await _claude_inference_body("claude-opus-4-7", {"thinking": thinking, "drop_params": True}) - kept: Final = await _claude_inference_body("claude-opus-4-7", {"thinking": thinking}) + dropped: Final = await _claude_inference_body("claude-opus-5-5", {"thinking": thinking, "drop_params": True}) + kept: Final = await _claude_inference_body("claude-opus-5-5", {"thinking": thinking}) assert dropped["thinking"] != kept["thinking"] response, (count,) = await _count_on_a_claude_deployment( monkeypatch, - "anthropic/claude-opus-4-7", + "anthropic/claude-opus-5-5", {"thinking": thinking, **request_drop_params}, lambda _: httpx.Response(200, json={"input_tokens": 7}), user_agent,