fix(count_tokens): retry a count without options only when the counter sent Claude options

The proxy counted again without the request options on any 4xx other than 401, 403 and 429, so a Gemini, OpenAI Responses, non-Claude Bedrock or Vertex Gemini count that answered 400 was sent a second identical time, with a warning saying options were forwarded

The Claude counters (Anthropic, Azure AI, Vertex AI partner and Bedrock) now flag an error response that carried options or betas, a count transform error carries the flag too, and the proxy retries only a request rejection that carries it
This commit is contained in:
mateo-berri 2026-11-07 00:27:22 -08:00
parent 30bfb30dd0
commit af7e65b86e
9 changed files with 73 additions and 5 deletions

View file

@ -126,6 +126,7 @@ class AnthropicTokenCounter(BaseTokenCounter):
error=True,
error_message=e.message,
status_code=e.status_code,
carried_request_options=claude_options.carries_request_options,
)
except Exception as e:
verbose_logger.warning("Error calling Anthropic CountTokens API: %s", e)

View file

@ -85,6 +85,7 @@ def _lift_leading_system(
@dataclass(frozen=True, slots=True)
class ChatCountTranslationError:
message: str
carried_request_options: bool = False
def response(self, request_model: str, model_used: str, tokenizer_type: str) -> TokenCountResponse:
return TokenCountResponse(
@ -96,6 +97,7 @@ class ChatCountTranslationError:
error_message=self.message,
status_code=400,
rejected_before_provider_call=True,
carried_request_options=self.carried_request_options,
)
@ -147,6 +149,10 @@ class ClaudeCountOptions:
messages: tuple[Mapping[str, JsonValue], ...] | None
tools: tuple[Mapping[str, JsonValue], ...] | None
@property
def carries_request_options(self) -> bool:
return bool(self.options) or bool(self.betas)
NO_CLAUDE_COUNT_OPTIONS: Final = ClaudeCountOptions(options=MappingProxyType({}), betas=(), messages=None, tools=None)
_COUNTED_MESSAGES: Final = TypeAdapter(tuple[Mapping[str, JsonValue], ...])
@ -273,9 +279,11 @@ def claude_count_options(
messages_config, transform_model, litellm_params, provider, messages, tools, sent, anthropic_beta
)
except (AnthropicError, BadRequestError) as e:
return ChatCountTranslationError(message=str(e.message))
return ChatCountTranslationError(message=str(e.message), carried_request_options=True)
except (AttributeError, KeyError, TypeError, ValueError) as e:
return ChatCountTranslationError(message=f"Invalid request for token counting: {e}")
return ChatCountTranslationError(
message=f"Invalid request for token counting: {e}", carried_request_options=True
)
class AnthropicCountTokensConfig:

View file

@ -142,6 +142,7 @@ class AzureAIAnthropicTokenCounter(BaseTokenCounter):
error=True,
error_message=e.message,
status_code=e.status_code,
carried_request_options=claude_options.carries_request_options,
)
except Exception as e:
verbose_logger.warning("Error calling Azure AI Anthropic CountTokens API: %s", e)

View file

@ -259,6 +259,7 @@ class BedrockTokenCounter(BaseTokenCounter):
error=True,
error_message=outcome.message,
status_code=outcome.status_code,
carried_request_options=claude_options.carries_request_options,
)
case _:
assert_never(outcome)

View file

@ -1382,6 +1382,7 @@ class VertexAITokenCounter(BaseTokenCounter):
error=True,
error_message=error_message,
status_code=status_code,
carried_request_options=claude_options.carries_request_options,
)
if result is not None:

View file

@ -62,14 +62,15 @@ class ProviderCountFailure:
status_code: int
message: str
rejected_before_provider_call: bool = False
carried_request_options: bool = False
@property
def cannot_count_model(self) -> bool:
return not self.rejected_before_provider_call and self.status_code in CANNOT_COUNT_MODEL_STATUSES
@property
def rejected_request(self) -> bool:
return is_request_rejection(self.status_code)
def rejected_request_options(self) -> bool:
return self.carried_request_options and is_request_rejection(self.status_code)
def with_hint(self, hint: str) -> "ProviderCountFailure":
return replace(self, message=_with_hint(self.message, hint))

View file

@ -14326,6 +14326,7 @@ async def _try_provider_token_count(
status_code=result.status_code or 500,
message=result.error_message or "Token counting failed",
rejected_before_provider_call=result.rejected_before_provider_call,
carried_request_options=result.carried_request_options,
)
return result
@ -14384,7 +14385,7 @@ async def _count_retrying_without_options(
region: str | None,
) -> Union["TokenCountResponse", ProviderCountFailure, None]:
counted: Final = await count_with(model, region)
if not (isinstance(counted, ProviderCountFailure) and counted.rejected_request):
if not (isinstance(counted, ProviderCountFailure) and counted.rejected_request_options):
return counted
verbose_proxy_logger.warning(
"Token count for %s %s with the request's %s, so it counts again without them and the caller's betas",

View file

@ -4557,6 +4557,7 @@ class TokenCountResponse(LiteLLMPydanticObjectBase):
"""
status_code: int | None = None
rejected_before_provider_call: bool = Field(default=False, exclude=True)
carried_request_options: bool = Field(default=False, exclude=True)
class CustomHuggingfaceTokenizer(TypedDict):

View file

@ -1569,6 +1569,59 @@ async def test_anthropic_count_tokens_does_not_count_again_after_a_rate_limit(
assert len(requests) == 1
@pytest.mark.asyncio
@pytest.mark.parametrize(
("deployment_model", "deployment_params", "count_host", "provider_calls"),
(
("gemini/gemini-3.8-flash", {}, "generativelanguage.googleapis.com", 1),
("anthropic/claude-haiku-5-5", {}, "api.anthropic.com", 2),
(
"azure_ai/claude-haiku-5-5",
{"api_base": "https://example-resource.services.ai.azure.com/anthropic"},
"example-resource.services.ai.azure.com",
2,
),
),
ids=(
"counter-without-claude-options",
"claude-counter-forwarding-the-options",
"azure-ai-claude-counter-forwarding-the-options",
),
)
async def test_anthropic_count_tokens_counts_again_without_the_options_only_when_the_counter_forwarded_them(
monkeypatch: pytest.MonkeyPatch,
deployment_model: str,
deployment_params: dict[str, JsonValue],
count_host: str,
provider_calls: int,
) -> None:
from litellm.proxy import proxy_server
request: Final = _json_request(
{
"model": "claude-agent",
"messages": [{"role": "user", "content": "Weather in SF?"}],
"thinking": {"type": "enabled", "budget_tokens": 2000},
},
"python-httpx/0.28.1",
)
monkeypatch.setattr(proxy_server, "llm_router", _claude_router(deployment_model, deployment_params))
monkeypatch.setattr(proxy_server, "proxy_config", proxy_server.ProxyConfig())
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
with respx.mock:
route: Final = respx.route(host=count_host).mock(
return_value=httpx.Response(
400, json={"type": "error", "error": {"type": "invalid_request_error", "message": "rejected"}}
)
)
response: Final = await anthropic_count_tokens(request, UserAPIKeyAuth())
assert isinstance(response, JSONResponse)
assert response.status_code == 400
assert route.call_count == provider_calls
_BEDROCK_RUNTIME_HOST: Final = "bedrock-runtime.us-east-1.amazonaws.com"
_BEDROCK_MANTLE_HOST: Final = "bedrock-mantle.us-east-1.api.aws"
_BEDROCK_CLAUDE_DEPLOYMENT: Final[dict[str, str]] = {