diff --git a/litellm/llms/anthropic/count_tokens/transformation.py b/litellm/llms/anthropic/count_tokens/transformation.py index 3f6d68f1e4c..b9427216637 100644 --- a/litellm/llms/anthropic/count_tokens/transformation.py +++ b/litellm/llms/anthropic/count_tokens/transformation.py @@ -28,6 +28,7 @@ from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.anthropic.common_utils import merge_anthropic_beta_headers from litellm.llms.anthropic.wif import resolve_anthropic_base +from litellm.llms.base_llm.anthropic_messages.transformation import BaseAnthropicMessagesConfig from litellm.types.llms.openai import AllMessageValues, ChatCompletionImageObject, ValidUserMessageContentTypes from litellm.types.utils import LlmProviders, TokenCountMessagesFormat, TokenCountResponse @@ -345,17 +346,61 @@ _COUNTED_MESSAGES: Final = TypeAdapter(tuple[Mapping[str, JsonValue], ...]) _MAX_TOKENS_ABOVE_ANY_BUDGET: Final = 2**31 - 1 -def _with_provider_betas(headers: Mapping[str, str], provider: str) -> Mapping[str, str]: - from litellm.anthropic_beta_headers_manager import filter_and_transform_beta_headers +_STATUSES_NOT_CAUSED_BY_THE_REQUEST: Final = frozenset({401, 403, 429}) - beta_header: Final = headers.get("anthropic-beta") - if beta_header is None: - return headers - supported: Final = filter_and_transform_beta_headers( - [beta.strip() for beta in beta_header.split(",") if beta.strip()], provider + +def is_request_rejection(status_code: int) -> bool: + return 400 <= status_code < 500 and status_code not in _STATUSES_NOT_CAUSED_BY_THE_REQUEST + + +def _claude_count_options_for( + messages_config: BaseAnthropicMessagesConfig, + model: str, + provider: LlmProviders, + messages: Sequence[Mapping[str, object]], + tools: Sequence[Mapping[str, object]] | None, + sent: Mapping[str, JsonValue], + anthropic_beta: str | None, + deployment: Mapping[str, object] | None, +) -> ClaudeCountOptions: + from litellm.anthropic_beta_headers_manager import update_headers_with_filtered_beta + from litellm.types.router import GenericLiteLLMParams + + resolved_deployment: Final = deployment or {} + deployment_params: Final = _INFERENCE_PARAMS.validate_python(resolved_deployment.get("litellm_params") or {}) + leading_system, conversation = split_leading_system_run(messages) + sent_conversation: Final = copy.deepcopy([dict(message) for message in conversation]) + sent_params: Final = { + **copy.deepcopy(sent), + **({"tools": copy.deepcopy([dict(tool) for tool in tools])} if tools else {}), + } + beta_headers: Final = messages_config.headers_with_anthropic_betas( + {"anthropic-beta": anthropic_beta} if anthropic_beta else {}, sent_params, sent_conversation + ) + sent_headers: Final = ( + update_headers_with_filtered_beta(headers=dict(beta_headers), provider=provider.value) + if messages_config.should_filter_anthropic_beta_headers() + else dict(beta_headers) + ) + request: Final = messages_config.transform_anthropic_messages_request( # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # the base config signature takes and returns bare dicts + model=model, + messages=sent_conversation, + anthropic_messages_optional_request_params={**sent_params, "max_tokens": _MAX_TOKENS_ABOVE_ANY_BUDGET}, + litellm_params=GenericLiteLLMParams.model_validate( + { + "drop_params": deployment_params.get("drop_params"), + "model_info": resolved_deployment.get("model_info"), + } + ), + headers=sent_headers, + ) + transformed: Final = _ANTHROPIC_CHAT_REQUEST.validate_python(request) + return ClaudeCountOptions( + options=_count_options(transformed), + betas=messages_config.sent_anthropic_betas(sent_headers, _INFERENCE_PARAMS.validate_python(request)), + messages=(*_COUNTED_MESSAGES.validate_python(leading_system), *transformed["messages"]), + tools=transformed.get("tools"), ) - other_headers: Final = {name: value for name, value in headers.items() if name != "anthropic-beta"} - return {**other_headers, "anthropic-beta": ",".join(supported)} if supported else other_headers def claude_count_options( @@ -372,50 +417,19 @@ def claude_count_options( return NO_CLAUDE_COUNT_OPTIONS from litellm.exceptions import BadRequestError from litellm.llms.anthropic.common_utils import AnthropicError - from litellm.types.router import GenericLiteLLMParams from litellm.utils import ProviderConfigManager messages_config: Final = ProviderConfigManager.get_provider_anthropic_messages_config(model, provider) if messages_config is None: return NO_CLAUDE_COUNT_OPTIONS - resolved_deployment: Final = deployment or {} - deployment_params: Final = _INFERENCE_PARAMS.validate_python(resolved_deployment.get("litellm_params") or {}) - leading_system, conversation = split_leading_system_run(messages) - sent_conversation: Final = copy.deepcopy([dict(message) for message in conversation]) - sent_params: Final = { - **copy.deepcopy(sent), - **({"tools": copy.deepcopy([dict(tool) for tool in tools])} if tools else {}), - } - beta_headers: Final = messages_config.headers_with_anthropic_betas( - {"anthropic-beta": anthropic_beta} if anthropic_beta else {}, sent_params, sent_conversation - ) - sent_headers: Final = dict( - _with_provider_betas(beta_headers, provider.value) - if messages_config.should_filter_anthropic_beta_headers() - else beta_headers - ) try: - request: Final = messages_config.transform_anthropic_messages_request( # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # the base config signature takes and returns bare dicts - model=model, - messages=sent_conversation, - anthropic_messages_optional_request_params={**sent_params, "max_tokens": _MAX_TOKENS_ABOVE_ANY_BUDGET}, - litellm_params=GenericLiteLLMParams.model_validate( - { - "drop_params": deployment_params.get("drop_params"), - "model_info": resolved_deployment.get("model_info"), - } - ), - headers=sent_headers, + return _claude_count_options_for( + messages_config, model, provider, messages, tools, sent, anthropic_beta, deployment ) except (AnthropicError, BadRequestError) as e: return ChatCountTranslationError(message=str(e.message)) - transformed: Final = _ANTHROPIC_CHAT_REQUEST.validate_python(request) - return ClaudeCountOptions( - options=_count_options(transformed), - betas=messages_config.sent_anthropic_betas(sent_headers, _INFERENCE_PARAMS.validate_python(request)), - messages=(*_COUNTED_MESSAGES.validate_python(leading_system), *transformed["messages"]), - tools=transformed.get("tools"), - ) + except (AttributeError, IndexError, KeyError, TypeError, ValueError) as e: + return ChatCountTranslationError(message=f"Invalid request for token counting: {e}") async def anthropic_count_fields( diff --git a/litellm/llms/bedrock/count_tokens/bedrock_token_counter.py b/litellm/llms/bedrock/count_tokens/bedrock_token_counter.py index b1b95e9079b..2846ad858fa 100644 --- a/litellm/llms/bedrock/count_tokens/bedrock_token_counter.py +++ b/litellm/llms/bedrock/count_tokens/bedrock_token_counter.py @@ -19,6 +19,7 @@ from litellm.llms.anthropic.count_tokens.transformation import ( ClaudeCountOptions, anthropic_count_fields, claude_count_options, + is_request_rejection, ) from litellm.llms.base_llm.base_utils import BaseTokenCounter from litellm.llms.bedrock.common_utils import BedrockError, get_bedrock_base_model @@ -80,6 +81,10 @@ def _runtime_rejected_claude_model(outcome: CountTokensOutcome, resolved_model: ) +def _mantle_answered_for_runtime(fallback: CountTokensOutcome, forwarded_options: bool) -> bool: + return not forwarded_options or isinstance(fallback, CountedTokens) or is_request_rejection(fallback.status_code) + + def _invoke_fields(claude_options: ClaudeCountOptions) -> Mapping[str, object]: return { **claude_options.options, @@ -225,11 +230,16 @@ class BedrockTokenCounter(BaseTokenCounter): primary_tokenizer, ) ) - outcome: Final = ( + fallback: Final = ( await self._count_with( self._count_on_mantle(mantle_request, litellm_params, resolved_model), MANTLE_TOKENIZER_TYPE ) if not use_mantle and _runtime_rejected_claude_model(primary, resolved_model) + else None + ) + outcome: Final = ( + fallback + if fallback is not None and _mantle_answered_for_runtime(fallback, forwarded_options=bool(invoke_fields)) else primary ) match outcome: diff --git a/litellm/proxy/common_utils/token_count_settings.py b/litellm/proxy/common_utils/token_count_settings.py index 5763f0b2067..48c8e3211d3 100644 --- a/litellm/proxy/common_utils/token_count_settings.py +++ b/litellm/proxy/common_utils/token_count_settings.py @@ -8,6 +8,7 @@ from pydantic import BaseModel, ConfigDict, ValidationError import litellm from litellm.anthropic_interface.exceptions.exception_mapping_utils import AnthropicExceptionMapping from litellm.litellm_core_utils.core_helpers import as_str_mapping +from litellm.llms.anthropic.count_tokens.transformation import is_request_rejection from litellm.proxy._types import ProxyException from litellm.types.token_count import TokenCountProvider, TokenCountSourceFields, TokenCountTokenizer from litellm.types.utils import TokenCountResponse @@ -74,7 +75,7 @@ class ProviderCountFailure: @property def rejected_request(self) -> bool: - return 400 <= self.status_code < 500 and self.status_code != 429 + return is_request_rejection(self.status_code) def with_hint(self, hint: str) -> "ProviderCountFailure": if self.rejected_before_provider_call: diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 7db5b6e5a29..3f5b02ac63c 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -14775,9 +14775,8 @@ _DEPLOYMENT_LITELLM_PARAMS: Final = TypeAdapter(Mapping[str, object]) def _deployment_with_request_drop_params( - deployment: dict[str, object] | None, # mutable-ok: BaseTokenCounter.count_tokens takes a dict deployment - drop_params: object, -) -> dict[str, object] | None: # mutable-ok: BaseTokenCounter.count_tokens takes a dict deployment + deployment: Mapping[str, object] | None, drop_params: bool | None +) -> Mapping[str, object] | None: if drop_params is None: return deployment resolved_deployment: Final = deployment or {} @@ -14931,11 +14930,12 @@ def _retried_without_rejected_options( ) -> Callable[[str, str | None], Awaitable[Union["TokenCountResponse", ProviderCountFailure, None]]]: if not request_options: return count_with - return partial(_count_retrying_without_options, count_with) + return partial(_count_retrying_without_options, count_with, tuple(request_options)) async def _count_retrying_without_options( count_with: "partial[Awaitable[TokenCountResponse | ProviderCountFailure | None]]", + forwarded_options: tuple[str, ...], model: str, region: str | None, ) -> Union["TokenCountResponse", ProviderCountFailure, None]: @@ -14943,10 +14943,10 @@ async def _count_retrying_without_options( if not (isinstance(counted, ProviderCountFailure) and counted.rejected_request): return counted verbose_proxy_logger.warning( - "Token count for %s answered %s with the request's thinking, tool, and output options, " - "so it counts again without them", + "Token count for %s answered %s with the request's %s, so it counts again without them", model, counted.status_code, + ", ".join(forwarded_options), ) return await count_with(model, region, request_options=MappingProxyType({})) @@ -15475,7 +15475,9 @@ async def _count_request_tokens( model_to_use = _model sent_params: Final[Mapping[str, object]] = {**_router_default_litellm_params(llm_router), **request_litellm_params} - count_deployment: Final = _deployment_with_request_drop_params(deployment, sent_params.get("drop_params")) + count_deployment: Final = _deployment_with_request_drop_params( + deployment, normalize_drop_params(sent_params.get("drop_params")) + ) conversation_empty: Final = not messages and not contents if prompt and not conversation_empty and not deployment_drops_params(count_deployment): raise ProxyException( diff --git a/tests/unit/llms/anthropic/test_anthropic_count_tokens_transformation.py b/tests/unit/llms/anthropic/test_anthropic_count_tokens_transformation.py index 4391617295e..35c4b6e96c2 100644 --- a/tests/unit/llms/anthropic/test_anthropic_count_tokens_transformation.py +++ b/tests/unit/llms/anthropic/test_anthropic_count_tokens_transformation.py @@ -916,21 +916,27 @@ async def _messages_inference_request( return sent.get() +def _on_both_counters( + *betas: str, +) -> Mapping[type[AnthropicTokenCounter] | type[AzureAIAnthropicTokenCounter], frozenset[str]]: + return MappingProxyType({AnthropicTokenCounter: frozenset(betas), AzureAIAnthropicTokenCounter: frozenset(betas)}) + + def _betas(request: httpx.Request) -> frozenset[str]: return frozenset(beta for beta in request.headers.get("anthropic-beta", "").split(",") if beta) @pytest.mark.asyncio -@pytest.mark.usefixtures("httpx_transport_clients") +@pytest.mark.usefixtures("httpx_transport_clients", "local_beta_headers_config") @_COUNTERS @pytest.mark.parametrize( ("request_options", "expected_betas"), ( - ({"thinking": _ENABLED_THINKING}, frozenset()), - ({"tool_choice": {"type": "any"}}, frozenset()), + ({"thinking": _ENABLED_THINKING}, _on_both_counters()), + ({"tool_choice": {"type": "any"}}, _on_both_counters()), ( {"output_config": {"effort": "low"}, "output_format": {"type": "json_schema", "schema": _ANSWER_SCHEMA}}, - frozenset((ANTHROPIC_BETA_HEADER_VALUES.STRUCTURED_OUTPUT_2025_09_25.value,)), + _on_both_counters(ANTHROPIC_BETA_HEADER_VALUES.STRUCTURED_OUTPUT_2025_09_25.value), ), ( { @@ -939,9 +945,17 @@ def _betas(request: httpx.Request) -> frozenset[str]: "edits": [{"type": "clear_tool_uses_20250919"}, {"type": "clear_thinking_20251015"}] }, }, - frozenset((ANTHROPIC_BETA_HEADER_VALUES.CONTEXT_MANAGEMENT_2025_06_27.value,)), + _on_both_counters(ANTHROPIC_BETA_HEADER_VALUES.CONTEXT_MANAGEMENT_2025_06_27.value), + ), + ( + {"context_management": {"edits": [{"type": "compact_20260112"}]}}, + MappingProxyType( + { + AnthropicTokenCounter: frozenset((ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_01_12.value,)), + AzureAIAnthropicTokenCounter: frozenset(), + } + ), ), - ({"context_management": {"edits": [{"type": "compact_20260112"}]}}, frozenset()), ), ids=("thinking", "tool_choice", "output_config_and_output_format", "context_edits", "compact_edit"), ) @@ -949,7 +963,7 @@ async def test_counter_sends_the_options_and_context_edits_messages_inference_se counter_type: type[AnthropicTokenCounter] | type[AzureAIAnthropicTokenCounter], count_endpoint: str, request_options: dict[str, JsonValue], - expected_betas: frozenset[str], + expected_betas: Mapping[type[AnthropicTokenCounter] | type[AzureAIAnthropicTokenCounter], frozenset[str]], ) -> None: inference: Final = await _messages_inference_request( counter_type, count_endpoint.removesuffix("/count_tokens"), request_options @@ -977,7 +991,7 @@ async def test_counter_sends_the_options_and_context_edits_messages_inference_se assert {name: count_body.get(name) for name in ("thinking", "tool_choice", "context_management")} == { name: inference_body.get(name) for name in ("thinking", "tool_choice", "context_management") } - assert expected_betas <= _betas(count) - {ANTHROPIC_TOKEN_COUNTING_BETA_VERSION} == _betas(inference) + assert expected_betas[counter_type] == _betas(count) - {ANTHROPIC_TOKEN_COUNTING_BETA_VERSION} == _betas(inference) @pytest.mark.asyncio @@ -1011,6 +1025,30 @@ async def test_counter_sends_the_thinking_display_and_caller_betas_messages_infe assert {ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER, caller_beta} <= counted_betas +@pytest.mark.asyncio +@pytest.mark.usefixtures("httpx_transport_clients") +@_COUNTERS +async def test_counter_without_options_sends_only_the_token_counting_beta_and_the_bare_count_body( + counter_type: type[AnthropicTokenCounter] | type[AzureAIAnthropicTokenCounter], count_endpoint: str +) -> None: + _, (count,) = await _counted_requests( + counter_type, + count_endpoint, + [{"role": "user", "content": "Weather in SF?"}], + [_WEATHER_TOOL], + "anthropic", + anthropic_beta=ANTHROPIC_BETA_HEADER_VALUES.WEB_FETCH_2025_09_10.value, + model="claude-haiku-5-5", + ) + + assert _betas(count) == {ANTHROPIC_TOKEN_COUNTING_BETA_VERSION} + assert _JSON_BODY.validate_json(count.content) == { + "model": "claude-haiku-5-5", + "messages": [{"role": "user", "content": "Weather in SF?"}], + "tools": [_WEATHER_TOOL], + } + + @pytest.mark.asyncio @pytest.mark.usefixtures("httpx_transport_clients") @_COUNTERS diff --git a/tests/unit/llms/bedrock/count_tokens/test_bedrock_token_counter.py b/tests/unit/llms/bedrock/count_tokens/test_bedrock_token_counter.py index 425953c6f51..61ee1729821 100644 --- a/tests/unit/llms/bedrock/count_tokens/test_bedrock_token_counter.py +++ b/tests/unit/llms/bedrock/count_tokens/test_bedrock_token_counter.py @@ -1,6 +1,6 @@ import base64 import json -from collections.abc import Mapping +from collections.abc import Callable, Mapping from typing import Final import httpx @@ -489,6 +489,77 @@ async def test_mantle_count_sends_the_thinking_display_beta_mantle_inference_sen assert ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER in _sent_betas(bedrock.requests[0]) +def _mantle_answers(status_code: int, body: Mapping[str, JsonValue]) -> Callable[[httpx.Request], httpx.Response]: + return lambda request: httpx.Response(status_code, json=dict(body), request=request) + + +def _mantle_unreachable(error: type[httpx.TransportError]) -> Callable[[httpx.Request], httpx.Response]: + def unreachable(request: httpx.Request) -> httpx.Response: + raise error("bedrock-mantle is unreachable", request=request) + + return unreachable + + +_MANTLE_REJECTION: Final[dict[str, JsonValue]] = { + "type": "error", + "error": {"type": "invalid_request_error", "message": "thinking: unsupported"}, +} +_MANTLE_FAILURE: Final[dict[str, JsonValue]] = {"type": "error", "error": {"type": "api_error", "message": "down"}} + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("request_options", "mantle", "expected"), + ( + (_DISPLAY_THINKING, _mantle_unreachable(httpx.ConnectError), (True, 400, "bedrock_api")), + (_DISPLAY_THINKING, _mantle_unreachable(httpx.ReadTimeout), (True, 400, "bedrock_api")), + (_DISPLAY_THINKING, _mantle_answers(503, _MANTLE_FAILURE), (True, 400, "bedrock_api")), + (_DISPLAY_THINKING, _mantle_answers(429, _MANTLE_FAILURE), (True, 400, "bedrock_api")), + (_DISPLAY_THINKING, _mantle_answers(403, _MANTLE_FAILURE), (True, 400, "bedrock_api")), + (_DISPLAY_THINKING, _mantle_answers(400, _MANTLE_REJECTION), (True, 400, "bedrock_mantle_api")), + (_DISPLAY_THINKING, _mantle_answers(422, _MANTLE_REJECTION), (True, 422, "bedrock_mantle_api")), + (_DISPLAY_THINKING, _mantle_answers(200, {"input_tokens": 2177}), (False, None, "bedrock_mantle_api")), + ({}, _mantle_answers(503, _MANTLE_FAILURE), (True, 503, "bedrock_mantle_api")), + ), + ids=( + "mantle-unreachable", + "mantle-timeout", + "mantle-5xx", + "mantle-rate-limit", + "mantle-denied", + "mantle-rejects-the-options", + "mantle-rejects-the-body", + "mantle-counts", + "no-options-mantle-5xx", + ), +) +async def test_runtime_rejecting_the_options_is_answered_unless_mantle_counts_or_rejects_the_request( + request_options: dict[str, JsonValue], + mantle: Callable[[httpx.Request], httpx.Response], + expected: tuple[bool, int | None, str], +) -> None: + def bedrock(request: httpx.Request) -> httpx.Response: + if request.url.host == RUNTIME_HOST: + return httpx.Response(400, json=UNSUPPORTED, request=request) + return mantle(request) + + result: Final = await BedrockTokenCounter( + client=AsyncHTTPHandler(transport=httpx.MockTransport(bedrock)) + ).count_tokens( + model_to_use="us.anthropic.claude-opus-5-5", + messages=MESSAGES, + contents=None, + deployment={ + "litellm_params": {**DEPLOYMENT["litellm_params"], "model": "bedrock/us.anthropic.claude-opus-5-5"} + }, + request_model="claude-opus-5-5", + request_options=request_options, + ) + + assert result is not None + assert (result.error, result.status_code, result.tokenizer_type) == expected + + @pytest.mark.asyncio async def test_runtime_count_sends_the_tools_invoke_inference_sends() -> None: inference, bedrock = await _messages_inference_and_count( diff --git a/tests/unit/llms/vertex_ai/test_vertex_ai_common_utils.py b/tests/unit/llms/vertex_ai/test_vertex_ai_common_utils.py index 782074a344c..7c36eb47d9c 100644 --- a/tests/unit/llms/vertex_ai/test_vertex_ai_common_utils.py +++ b/tests/unit/llms/vertex_ai/test_vertex_ai_common_utils.py @@ -33,6 +33,7 @@ from litellm.llms.vertex_ai.common_utils import ( ) from litellm.llms.vertex_ai.count_tokens import handler as vertex_count_tokens_handler from litellm.llms.vertex_ai.vertex_ai_partner_models.anthropic.transformation import VertexAIAnthropicConfig +from litellm.types.llms.anthropic_tool_search import get_tool_search_beta_header from litellm.types.utils import CountContextManagement, TokenCountMessagesFormat @@ -1970,20 +1971,13 @@ _PARTNER_CLAUDE_REPLY: Final[dict[str, JsonValue]] = { } -@pytest.mark.asyncio -async def test_vertex_claude_count_sends_the_options_context_edits_and_beta_vertex_messages_inference_sends() -> None: +async def _vertex_claude_inference_and_count( + tools: list[dict[str, JsonValue]], request_options: Mapping[str, JsonValue] +) -> tuple[httpx.Request, httpx.Request]: from litellm.llms.vertex_ai.vertex_ai_partner_models.count_tokens import handler from litellm.llms.vertex_ai.vertex_llm_base import VertexBase - context_edits: Final[dict[str, JsonValue]] = { - "edits": [{"type": "clear_tool_uses_20250919"}, {"type": "clear_thinking_20251015"}] - } - options: Final[dict[str, JsonValue]] = { - "thinking": {"type": "enabled", "budget_tokens": 2000}, - "tool_choice": {"type": "auto"}, - } messages: Final[list[dict[str, JsonValue]]] = [{"role": "user", "content": "Weather in SF?"}] - tools: Final[list[dict[str, JsonValue]]] = [{"name": "get_weather", "input_schema": {"type": "object"}}] sent: Final[SimpleQueue[httpx.Request]] = SimpleQueue() def vertex(request: httpx.Request) -> httpx.Response: @@ -2004,11 +1998,10 @@ async def test_vertex_claude_count_sends_the_options_context_edits_and_beta_vert messages=messages, tools=tools, max_tokens=4096, - context_management=context_edits, vertex_project="test-project", vertex_location="us-east5", client=client, - **options, + **request_options, ) result: Final = await VertexAITokenCounter().count_tokens( model_to_use="claude-haiku-5-5", @@ -2016,13 +2009,29 @@ async def test_vertex_claude_count_sends_the_options_context_edits_and_beta_vert contents=None, deployment={"litellm_params": {"vertex_project": "test-project", "vertex_location": "us-east5"}}, tools=tools, - request_options={**options, "context_management": context_edits}, + request_options=request_options, ) assert result is not None assert result.total_tokens == 37 inference, count = tuple(sent.get() for _ in range(sent.qsize())) assert count.url.path.endswith("count-tokens:rawPredict") + return inference, count + + +@pytest.mark.asyncio +async def test_vertex_claude_count_sends_the_options_context_edits_and_beta_vertex_messages_inference_sends() -> None: + inference, count = await _vertex_claude_inference_and_count( + [{"name": "get_weather", "input_schema": {"type": "object"}}], + { + "thinking": {"type": "enabled", "budget_tokens": 2000}, + "tool_choice": {"type": "auto"}, + "context_management": { + "edits": [{"type": "clear_tool_uses_20250919"}, {"type": "clear_thinking_20251015"}] + }, + }, + ) + counted: Final = TypeAdapter(dict[str, JsonValue]).validate_json(count.content) inferred: Final = TypeAdapter(dict[str, JsonValue]).validate_json(inference.content) fields: Final = ("thinking", "tool_choice", "output_config", "context_management") @@ -2030,6 +2039,20 @@ async def test_vertex_claude_count_sends_the_options_context_edits_and_beta_vert assert count.headers["anthropic-beta"] == inference.headers["anthropic-beta"] +@pytest.mark.asyncio +async def test_vertex_claude_count_sends_the_tool_search_beta_vertex_messages_inference_sends() -> None: + inference, count = await _vertex_claude_inference_and_count( + [ + {"type": "tool_search_tool_regex_20251119", "name": "tool_search_tool_regex"}, + {"name": "get_weather", "input_schema": {"type": "object"}, "defer_loading": True}, + ], + {"thinking": {"type": "enabled", "budget_tokens": 2000}}, + ) + + assert count.headers["anthropic-beta"] == inference.headers["anthropic-beta"] + assert get_tool_search_beta_header("vertex_ai") in count.headers["anthropic-beta"].split(",") + + @pytest.mark.asyncio async def test_vertex_claude_count_refuses_options_vertex_messages_inference_refuses_before_calling_vertex() -> None: result: Final = await VertexAITokenCounter().count_tokens( diff --git a/tests/unit/proxy/test_proxy_token_counter.py b/tests/unit/proxy/test_proxy_token_counter.py index cbae1aac1fb..ae272cef1a9 100644 --- a/tests/unit/proxy/test_proxy_token_counter.py +++ b/tests/unit/proxy/test_proxy_token_counter.py @@ -83,6 +83,8 @@ from litellm.proxy.proxy_server import ( ) from litellm.responses.utils import ResponsesAPIRequestUtils from litellm.rust_bridge.fork_guard import ForkedAfterNativeRuntimeStarted +from litellm.types.llms.anthropic import ANTHROPIC_BETA_HEADER_VALUES +from litellm.types.router import CustomRoutingStrategyBase from litellm.types.secret_managers.main import KeyManagementSettings, KeyManagementSystem from litellm.types.utils import ( ADDRESSED_RESPONSE_ID_FIELD, @@ -1739,7 +1741,7 @@ async def test_anthropic_count_tokens_on_a_claude_deployment_sends_the_options_i ) -_CALLER_BETA: Final = "web-fetch-2025-09-10" +_CALLER_BETA: Final = ANTHROPIC_BETA_HEADER_VALUES.WEB_FETCH_2025_09_10.value async def _count_on_a_claude_deployment( @@ -1811,9 +1813,9 @@ async def test_anthropic_count_tokens_counts_again_without_the_options_when_the_ @pytest.mark.asyncio -@pytest.mark.parametrize("status_code", (401, 403, 404)) -async def test_anthropic_count_tokens_answers_the_error_the_count_without_the_options_also_gets( - monkeypatch: pytest.MonkeyPatch, status_code: int +@pytest.mark.parametrize(("status_code", "counted_again"), ((401, False), (403, False), (404, True), (422, True))) +async def test_anthropic_count_tokens_counts_again_only_when_the_claude_count_rejects_the_request( + monkeypatch: pytest.MonkeyPatch, status_code: int, counted_again: bool ) -> None: response, requests = await _count_on_a_claude_deployment( monkeypatch, @@ -1824,7 +1826,30 @@ async def test_anthropic_count_tokens_answers_the_error_the_count_without_the_op assert isinstance(response, JSONResponse) assert response.status_code == status_code - assert [bool(_sent_option_paths(json.loads(sent.content))) for sent in requests] == [True, False] + assert [bool(_sent_option_paths(json.loads(sent.content))) for sent in requests] == [ + True, + *((False,) if counted_again else ()), + ] + + +@pytest.mark.asyncio +async def test_anthropic_count_tokens_warns_with_every_option_it_counts_again_without( + monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture +) -> None: + verbose_proxy_logger.addHandler(caplog.handler) + try: + with caplog.at_level(logging.WARNING, logger=verbose_proxy_logger.name): + await _count_on_a_claude_deployment( + monkeypatch, "anthropic/claude-haiku-5-5", _CLAUDE_COUNT_OPTIONS, _counts_only_without_options + ) + finally: + verbose_proxy_logger.removeHandler(caplog.handler) + + warnings: Final = { + record.getMessage() for record in caplog.records if "counts again without" in record.getMessage() + } + (warning,) = warnings + assert all(option in warning for option in _CLAUDE_COUNT_OPTIONS) @pytest.mark.asyncio @@ -1843,6 +1868,179 @@ async def test_anthropic_count_tokens_does_not_count_again_after_a_rate_limit( assert len(requests) == 1 +_BEDROCK_RUNTIME_HOST: Final = "bedrock-runtime.us-east-1.amazonaws.com" +_BEDROCK_MANTLE_HOST: Final = "bedrock-mantle.us-east-1.api.aws" +_BEDROCK_CLAUDE_DEPLOYMENT: Final[dict[str, str]] = { + "model": "bedrock/us.anthropic.claude-count-probe-v1:0", + "aws_access_key_id": "AKIATESTACCESSKEY", + "aws_secret_access_key": "test-secret", + "aws_region_name": "us-east-1", +} + + +async def _count_on_a_bedrock_claude_deployment( + monkeypatch: pytest.MonkeyPatch, + body: Mapping[str, JsonValue], + runtime: Callable[[httpx.Request], httpx.Response], + mantle: Callable[[httpx.Request], httpx.Response], +) -> tuple[object, tuple[httpx.Request, ...]]: + import litellm.proxy.proxy_server as proxy_server + + monkeypatch.delenv("AWS_BEARER_TOKEN_BEDROCK", raising=False) + monkeypatch.delenv("BEDROCK_MANTLE_API_BASE", raising=False) + router: Final = Router(model_list=[{"model_name": "claude-agent", "litellm_params": _BEDROCK_CLAUDE_DEPLOYMENT}]) + request: Final = _json_request( + {"model": "claude-agent", "messages": [{"role": "user", "content": "Weather in SF?"}], **body}, + "python-httpx/0.28.1", + ) + monkeypatch.setattr(proxy_server, "llm_router", router) + monkeypatch.setattr(proxy_server, "proxy_config", proxy_server.ProxyConfig()) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + + with respx.mock as bedrock: + bedrock.route(host=_BEDROCK_RUNTIME_HOST).mock(side_effect=runtime) + bedrock.route(host=_BEDROCK_MANTLE_HOST).mock(side_effect=mantle) + response: Final = await anthropic_count_tokens(request, UserAPIKeyAuth()) + return response, tuple(call.request for call in bedrock.calls) + + +def _runtime_counts_only_without_options(request: httpx.Request) -> httpx.Response: + if "invokeModel" in json.loads(request.content)["input"]: + return httpx.Response(400, json={"message": "thinking: unsupported"}) + return httpx.Response(200, json={"inputTokens": 9}) + + +def _mantle_unreachable(request: httpx.Request) -> httpx.Response: + raise httpx.ConnectError("bedrock-mantle is unreachable", request=request) + + +@pytest.mark.asyncio +async def test_anthropic_count_tokens_on_bedrock_counts_a_malformed_tool_again_without_the_options( + monkeypatch: pytest.MonkeyPatch, +) -> None: + response, requests = await _count_on_a_bedrock_claude_deployment( + monkeypatch, + { + "tools": [{"type": 5, "name": "x", "input_schema": {"type": "object"}}], + "thinking": {"type": "enabled", "budget_tokens": 2000}, + }, + _runtime_counts_only_without_options, + _mantle_unreachable, + ) + + assert not isinstance(response, JSONResponse) + assert response["input_tokens"] == 9 + assert [request.url.host for request in requests] == [_BEDROCK_RUNTIME_HOST] + assert "invokeModel" not in json.loads(requests[0].content)["input"] + + +@pytest.mark.asyncio +async def test_anthropic_count_tokens_on_bedrock_counts_again_without_the_options_when_mantle_is_unreachable( + monkeypatch: pytest.MonkeyPatch, +) -> None: + response, requests = await _count_on_a_bedrock_claude_deployment( + monkeypatch, + {"thinking": {"type": "enabled", "budget_tokens": 2000}}, + _runtime_counts_only_without_options, + _mantle_unreachable, + ) + + assert not isinstance(response, JSONResponse) + assert response["input_tokens"] == 9 + hosts: Final = [request.url.host for request in requests] + assert hosts[0] == hosts[-1] == _BEDROCK_RUNTIME_HOST + assert _BEDROCK_MANTLE_HOST in hosts + assert "invokeModel" not in json.loads(requests[-1].content)["input"] + + +_KEEPS_CONTEXT_MANAGEMENT: Final = "sk-ant-keeps-context-management" +_DROPS_CONTEXT_MANAGEMENT: Final = "sk-ant-drops-context-management" +_CONTEXT_EDITS: Final[dict[str, JsonValue]] = {"edits": [{"type": "clear_tool_uses_20250919"}]} +_CLAUDE_MESSAGE_REPLY: Final[dict[str, JsonValue]] = { + "id": "msg_1", + "type": "message", + "role": "assistant", + "model": "claude-haiku-5-5", + "content": [{"type": "text", "text": "ok"}], + "stop_reason": "end_turn", + "usage": {"input_tokens": 3, "output_tokens": 1}, +} + + +class _RoutesTo(CustomRoutingStrategyBase): + def __init__(self, router: Router, api_key: str) -> None: + self._router: Final = router + self._api_key: Final = api_key + + async def async_get_available_deployment(self, *args: object, **kwargs: object) -> Mapping[str, object]: + return next( + deployment + for deployment in self._router.model_list + if deployment["litellm_params"]["api_key"] == self._api_key + ) + + +def _mixed_drop_group(routed_key: str) -> Router: + router: Final = Router( + model_list=[ + { + "model_name": "claude-agent", + "litellm_params": {"model": "anthropic/claude-haiku-5-5", "api_key": _KEEPS_CONTEXT_MANAGEMENT}, + }, + { + "model_name": "claude-agent", + "litellm_params": { + "model": "anthropic/claude-haiku-5-5", + "api_key": _DROPS_CONTEXT_MANAGEMENT, + "additional_drop_params": ["context_management"], + }, + }, + ] + ) + router.set_custom_routing_strategy(_RoutesTo(router, routed_key)) + return router + + +@pytest.mark.asyncio +@pytest.mark.parametrize("routed_key", (_KEEPS_CONTEXT_MANAGEMENT, _DROPS_CONTEXT_MANAGEMENT)) +async def test_anthropic_count_tokens_in_a_mixed_drop_group_sends_context_management_when_the_routed_inference_does( + monkeypatch: pytest.MonkeyPatch, routed_key: str +) -> None: + import litellm.proxy.proxy_server as proxy_server + + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + monkeypatch.setattr(proxy_server, "llm_router", _mixed_drop_group(routed_key)) + monkeypatch.setattr(proxy_server, "proxy_config", proxy_server.ProxyConfig()) + messages: Final = [{"role": "user", "content": "Weather in SF?"}] + + with respx.mock: + inference_route: Final = respx.post("https://api.anthropic.com/v1/messages").mock( + return_value=httpx.Response(200, json=_CLAUDE_MESSAGE_REPLY) + ) + count_route: Final = respx.post("https://api.anthropic.com/v1/messages/count_tokens").mock( + return_value=httpx.Response(200, json={"input_tokens": 7}) + ) + await _mixed_drop_group(routed_key).aanthropic_messages( + model="claude-agent", messages=messages, max_tokens=16, context_management=_CONTEXT_EDITS + ) + response: Final = await anthropic_count_tokens( + _json_request( + {"model": "claude-agent", "messages": messages, "context_management": _CONTEXT_EDITS}, + "python-httpx/0.28.1", + ), + UserAPIKeyAuth(), + ) + + assert not isinstance(response, JSONResponse) + inference: Final = inference_route.calls.last.request + count: Final = count_route.calls.last.request + assert inference.headers["x-api-key"] == count.headers["x-api-key"] == routed_key + assert ("context_management" in json.loads(count.content)) == ( + "context_management" in json.loads(inference.content) + ) + assert ("context_management" in json.loads(count.content)) == (routed_key == _KEEPS_CONTEXT_MANAGEMENT) + + async def _claude_inference_body(model: str, options: Mapping[str, JsonValue]) -> dict[str, JsonValue]: with respx.mock: route: Final = respx.post("https://api.anthropic.com/v1/messages").mock(