diff --git a/litellm/llms/anthropic/count_tokens/token_counter.py b/litellm/llms/anthropic/count_tokens/token_counter.py index a5c19728664..08a12e69ba8 100644 --- a/litellm/llms/anthropic/count_tokens/token_counter.py +++ b/litellm/llms/anthropic/count_tokens/token_counter.py @@ -126,6 +126,7 @@ class AnthropicTokenCounter(BaseTokenCounter): error=True, error_message=e.message, status_code=e.status_code, + carried_request_options=claude_options.carries_request_options, ) except Exception as e: verbose_logger.warning("Error calling Anthropic CountTokens API: %s", e) diff --git a/litellm/llms/anthropic/count_tokens/transformation.py b/litellm/llms/anthropic/count_tokens/transformation.py index 7516a7aa903..372ea980f76 100644 --- a/litellm/llms/anthropic/count_tokens/transformation.py +++ b/litellm/llms/anthropic/count_tokens/transformation.py @@ -85,6 +85,7 @@ def _lift_leading_system( @dataclass(frozen=True, slots=True) class ChatCountTranslationError: message: str + carried_request_options: bool = False def response(self, request_model: str, model_used: str, tokenizer_type: str) -> TokenCountResponse: return TokenCountResponse( @@ -96,6 +97,7 @@ class ChatCountTranslationError: error_message=self.message, status_code=400, rejected_before_provider_call=True, + carried_request_options=self.carried_request_options, ) @@ -147,6 +149,10 @@ class ClaudeCountOptions: messages: tuple[Mapping[str, JsonValue], ...] | None tools: tuple[Mapping[str, JsonValue], ...] | None + @property + def carries_request_options(self) -> bool: + return bool(self.options) or bool(self.betas) + NO_CLAUDE_COUNT_OPTIONS: Final = ClaudeCountOptions(options=MappingProxyType({}), betas=(), messages=None, tools=None) _COUNTED_MESSAGES: Final = TypeAdapter(tuple[Mapping[str, JsonValue], ...]) @@ -273,9 +279,11 @@ def claude_count_options( messages_config, transform_model, litellm_params, provider, messages, tools, sent, anthropic_beta ) except (AnthropicError, BadRequestError) as e: - return ChatCountTranslationError(message=str(e.message)) + return ChatCountTranslationError(message=str(e.message), carried_request_options=True) except (AttributeError, KeyError, TypeError, ValueError) as e: - return ChatCountTranslationError(message=f"Invalid request for token counting: {e}") + return ChatCountTranslationError( + message=f"Invalid request for token counting: {e}", carried_request_options=True + ) class AnthropicCountTokensConfig: diff --git a/litellm/llms/azure_ai/anthropic/count_tokens/token_counter.py b/litellm/llms/azure_ai/anthropic/count_tokens/token_counter.py index 0f10030a76e..577075cc14d 100644 --- a/litellm/llms/azure_ai/anthropic/count_tokens/token_counter.py +++ b/litellm/llms/azure_ai/anthropic/count_tokens/token_counter.py @@ -142,6 +142,7 @@ class AzureAIAnthropicTokenCounter(BaseTokenCounter): error=True, error_message=e.message, status_code=e.status_code, + carried_request_options=claude_options.carries_request_options, ) except Exception as e: verbose_logger.warning("Error calling Azure AI Anthropic CountTokens API: %s", e) diff --git a/litellm/llms/bedrock/count_tokens/bedrock_token_counter.py b/litellm/llms/bedrock/count_tokens/bedrock_token_counter.py index 624961eb628..1e8c28abd46 100644 --- a/litellm/llms/bedrock/count_tokens/bedrock_token_counter.py +++ b/litellm/llms/bedrock/count_tokens/bedrock_token_counter.py @@ -259,6 +259,7 @@ class BedrockTokenCounter(BaseTokenCounter): error=True, error_message=outcome.message, status_code=outcome.status_code, + carried_request_options=claude_options.carries_request_options, ) case _: assert_never(outcome) diff --git a/litellm/llms/vertex_ai/common_utils.py b/litellm/llms/vertex_ai/common_utils.py index 5441fd4deb7..9eba632f9a8 100644 --- a/litellm/llms/vertex_ai/common_utils.py +++ b/litellm/llms/vertex_ai/common_utils.py @@ -1382,6 +1382,7 @@ class VertexAITokenCounter(BaseTokenCounter): error=True, error_message=error_message, status_code=status_code, + carried_request_options=claude_options.carries_request_options, ) if result is not None: diff --git a/litellm/proxy/common_utils/token_count_settings.py b/litellm/proxy/common_utils/token_count_settings.py index 8e556678c98..55ad901b560 100644 --- a/litellm/proxy/common_utils/token_count_settings.py +++ b/litellm/proxy/common_utils/token_count_settings.py @@ -62,14 +62,15 @@ class ProviderCountFailure: status_code: int message: str rejected_before_provider_call: bool = False + carried_request_options: bool = False @property def cannot_count_model(self) -> bool: return not self.rejected_before_provider_call and self.status_code in CANNOT_COUNT_MODEL_STATUSES @property - def rejected_request(self) -> bool: - return is_request_rejection(self.status_code) + def rejected_request_options(self) -> bool: + return self.carried_request_options and is_request_rejection(self.status_code) def with_hint(self, hint: str) -> "ProviderCountFailure": return replace(self, message=_with_hint(self.message, hint)) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 0e64b2c5433..a2ea9062b1d 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -14326,6 +14326,7 @@ async def _try_provider_token_count( status_code=result.status_code or 500, message=result.error_message or "Token counting failed", rejected_before_provider_call=result.rejected_before_provider_call, + carried_request_options=result.carried_request_options, ) return result @@ -14384,7 +14385,7 @@ async def _count_retrying_without_options( region: str | None, ) -> Union["TokenCountResponse", ProviderCountFailure, None]: counted: Final = await count_with(model, region) - if not (isinstance(counted, ProviderCountFailure) and counted.rejected_request): + if not (isinstance(counted, ProviderCountFailure) and counted.rejected_request_options): return counted verbose_proxy_logger.warning( "Token count for %s %s with the request's %s, so it counts again without them and the caller's betas", diff --git a/litellm/types/utils.py b/litellm/types/utils.py index bbe581d5e5b..e1173ef0fe5 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -4557,6 +4557,7 @@ class TokenCountResponse(LiteLLMPydanticObjectBase): """ status_code: int | None = None rejected_before_provider_call: bool = Field(default=False, exclude=True) + carried_request_options: bool = Field(default=False, exclude=True) class CustomHuggingfaceTokenizer(TypedDict): diff --git a/tests/unit/proxy/test_proxy_token_counter.py b/tests/unit/proxy/test_proxy_token_counter.py index 0ede4f059ab..1d19524c665 100644 --- a/tests/unit/proxy/test_proxy_token_counter.py +++ b/tests/unit/proxy/test_proxy_token_counter.py @@ -1569,6 +1569,59 @@ async def test_anthropic_count_tokens_does_not_count_again_after_a_rate_limit( assert len(requests) == 1 +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("deployment_model", "deployment_params", "count_host", "provider_calls"), + ( + ("gemini/gemini-3.8-flash", {}, "generativelanguage.googleapis.com", 1), + ("anthropic/claude-haiku-5-5", {}, "api.anthropic.com", 2), + ( + "azure_ai/claude-haiku-5-5", + {"api_base": "https://example-resource.services.ai.azure.com/anthropic"}, + "example-resource.services.ai.azure.com", + 2, + ), + ), + ids=( + "counter-without-claude-options", + "claude-counter-forwarding-the-options", + "azure-ai-claude-counter-forwarding-the-options", + ), +) +async def test_anthropic_count_tokens_counts_again_without_the_options_only_when_the_counter_forwarded_them( + monkeypatch: pytest.MonkeyPatch, + deployment_model: str, + deployment_params: dict[str, JsonValue], + count_host: str, + provider_calls: int, +) -> None: + from litellm.proxy import proxy_server + + request: Final = _json_request( + { + "model": "claude-agent", + "messages": [{"role": "user", "content": "Weather in SF?"}], + "thinking": {"type": "enabled", "budget_tokens": 2000}, + }, + "python-httpx/0.28.1", + ) + monkeypatch.setattr(proxy_server, "llm_router", _claude_router(deployment_model, deployment_params)) + monkeypatch.setattr(proxy_server, "proxy_config", proxy_server.ProxyConfig()) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + + with respx.mock: + route: Final = respx.route(host=count_host).mock( + return_value=httpx.Response( + 400, json={"type": "error", "error": {"type": "invalid_request_error", "message": "rejected"}} + ) + ) + response: Final = await anthropic_count_tokens(request, UserAPIKeyAuth()) + + assert isinstance(response, JSONResponse) + assert response.status_code == 400 + assert route.call_count == provider_calls + + _BEDROCK_RUNTIME_HOST: Final = "bedrock-runtime.us-east-1.amazonaws.com" _BEDROCK_MANTLE_HOST: Final = "bedrock-mantle.us-east-1.api.aws" _BEDROCK_CLAUDE_DEPLOYMENT: Final[dict[str, str]] = {