mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
fix(count_tokens): retry a count without options only when the counter sent Claude options
The proxy counted again without the request options on any 4xx other than 401, 403 and 429, so a Gemini, OpenAI Responses, non-Claude Bedrock or Vertex Gemini count that answered 400 was sent a second identical time, with a warning saying options were forwarded The Claude counters (Anthropic, Azure AI, Vertex AI partner and Bedrock) now flag an error response that carried options or betas, a count transform error carries the flag too, and the proxy retries only a request rejection that carries it
This commit is contained in:
parent
30bfb30dd0
commit
af7e65b86e
9 changed files with 73 additions and 5 deletions
|
|
@ -126,6 +126,7 @@ class AnthropicTokenCounter(BaseTokenCounter):
|
|||
error=True,
|
||||
error_message=e.message,
|
||||
status_code=e.status_code,
|
||||
carried_request_options=claude_options.carries_request_options,
|
||||
)
|
||||
except Exception as e:
|
||||
verbose_logger.warning("Error calling Anthropic CountTokens API: %s", e)
|
||||
|
|
|
|||
|
|
@ -85,6 +85,7 @@ def _lift_leading_system(
|
|||
@dataclass(frozen=True, slots=True)
|
||||
class ChatCountTranslationError:
|
||||
message: str
|
||||
carried_request_options: bool = False
|
||||
|
||||
def response(self, request_model: str, model_used: str, tokenizer_type: str) -> TokenCountResponse:
|
||||
return TokenCountResponse(
|
||||
|
|
@ -96,6 +97,7 @@ class ChatCountTranslationError:
|
|||
error_message=self.message,
|
||||
status_code=400,
|
||||
rejected_before_provider_call=True,
|
||||
carried_request_options=self.carried_request_options,
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -147,6 +149,10 @@ class ClaudeCountOptions:
|
|||
messages: tuple[Mapping[str, JsonValue], ...] | None
|
||||
tools: tuple[Mapping[str, JsonValue], ...] | None
|
||||
|
||||
@property
|
||||
def carries_request_options(self) -> bool:
|
||||
return bool(self.options) or bool(self.betas)
|
||||
|
||||
|
||||
NO_CLAUDE_COUNT_OPTIONS: Final = ClaudeCountOptions(options=MappingProxyType({}), betas=(), messages=None, tools=None)
|
||||
_COUNTED_MESSAGES: Final = TypeAdapter(tuple[Mapping[str, JsonValue], ...])
|
||||
|
|
@ -273,9 +279,11 @@ def claude_count_options(
|
|||
messages_config, transform_model, litellm_params, provider, messages, tools, sent, anthropic_beta
|
||||
)
|
||||
except (AnthropicError, BadRequestError) as e:
|
||||
return ChatCountTranslationError(message=str(e.message))
|
||||
return ChatCountTranslationError(message=str(e.message), carried_request_options=True)
|
||||
except (AttributeError, KeyError, TypeError, ValueError) as e:
|
||||
return ChatCountTranslationError(message=f"Invalid request for token counting: {e}")
|
||||
return ChatCountTranslationError(
|
||||
message=f"Invalid request for token counting: {e}", carried_request_options=True
|
||||
)
|
||||
|
||||
|
||||
class AnthropicCountTokensConfig:
|
||||
|
|
|
|||
|
|
@ -142,6 +142,7 @@ class AzureAIAnthropicTokenCounter(BaseTokenCounter):
|
|||
error=True,
|
||||
error_message=e.message,
|
||||
status_code=e.status_code,
|
||||
carried_request_options=claude_options.carries_request_options,
|
||||
)
|
||||
except Exception as e:
|
||||
verbose_logger.warning("Error calling Azure AI Anthropic CountTokens API: %s", e)
|
||||
|
|
|
|||
|
|
@ -259,6 +259,7 @@ class BedrockTokenCounter(BaseTokenCounter):
|
|||
error=True,
|
||||
error_message=outcome.message,
|
||||
status_code=outcome.status_code,
|
||||
carried_request_options=claude_options.carries_request_options,
|
||||
)
|
||||
case _:
|
||||
assert_never(outcome)
|
||||
|
|
|
|||
|
|
@ -1382,6 +1382,7 @@ class VertexAITokenCounter(BaseTokenCounter):
|
|||
error=True,
|
||||
error_message=error_message,
|
||||
status_code=status_code,
|
||||
carried_request_options=claude_options.carries_request_options,
|
||||
)
|
||||
|
||||
if result is not None:
|
||||
|
|
|
|||
|
|
@ -62,14 +62,15 @@ class ProviderCountFailure:
|
|||
status_code: int
|
||||
message: str
|
||||
rejected_before_provider_call: bool = False
|
||||
carried_request_options: bool = False
|
||||
|
||||
@property
|
||||
def cannot_count_model(self) -> bool:
|
||||
return not self.rejected_before_provider_call and self.status_code in CANNOT_COUNT_MODEL_STATUSES
|
||||
|
||||
@property
|
||||
def rejected_request(self) -> bool:
|
||||
return is_request_rejection(self.status_code)
|
||||
def rejected_request_options(self) -> bool:
|
||||
return self.carried_request_options and is_request_rejection(self.status_code)
|
||||
|
||||
def with_hint(self, hint: str) -> "ProviderCountFailure":
|
||||
return replace(self, message=_with_hint(self.message, hint))
|
||||
|
|
|
|||
|
|
@ -14326,6 +14326,7 @@ async def _try_provider_token_count(
|
|||
status_code=result.status_code or 500,
|
||||
message=result.error_message or "Token counting failed",
|
||||
rejected_before_provider_call=result.rejected_before_provider_call,
|
||||
carried_request_options=result.carried_request_options,
|
||||
)
|
||||
return result
|
||||
|
||||
|
|
@ -14384,7 +14385,7 @@ async def _count_retrying_without_options(
|
|||
region: str | None,
|
||||
) -> Union["TokenCountResponse", ProviderCountFailure, None]:
|
||||
counted: Final = await count_with(model, region)
|
||||
if not (isinstance(counted, ProviderCountFailure) and counted.rejected_request):
|
||||
if not (isinstance(counted, ProviderCountFailure) and counted.rejected_request_options):
|
||||
return counted
|
||||
verbose_proxy_logger.warning(
|
||||
"Token count for %s %s with the request's %s, so it counts again without them and the caller's betas",
|
||||
|
|
|
|||
|
|
@ -4557,6 +4557,7 @@ class TokenCountResponse(LiteLLMPydanticObjectBase):
|
|||
"""
|
||||
status_code: int | None = None
|
||||
rejected_before_provider_call: bool = Field(default=False, exclude=True)
|
||||
carried_request_options: bool = Field(default=False, exclude=True)
|
||||
|
||||
|
||||
class CustomHuggingfaceTokenizer(TypedDict):
|
||||
|
|
|
|||
|
|
@ -1569,6 +1569,59 @@ async def test_anthropic_count_tokens_does_not_count_again_after_a_rate_limit(
|
|||
assert len(requests) == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("deployment_model", "deployment_params", "count_host", "provider_calls"),
|
||||
(
|
||||
("gemini/gemini-3.8-flash", {}, "generativelanguage.googleapis.com", 1),
|
||||
("anthropic/claude-haiku-5-5", {}, "api.anthropic.com", 2),
|
||||
(
|
||||
"azure_ai/claude-haiku-5-5",
|
||||
{"api_base": "https://example-resource.services.ai.azure.com/anthropic"},
|
||||
"example-resource.services.ai.azure.com",
|
||||
2,
|
||||
),
|
||||
),
|
||||
ids=(
|
||||
"counter-without-claude-options",
|
||||
"claude-counter-forwarding-the-options",
|
||||
"azure-ai-claude-counter-forwarding-the-options",
|
||||
),
|
||||
)
|
||||
async def test_anthropic_count_tokens_counts_again_without_the_options_only_when_the_counter_forwarded_them(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
deployment_model: str,
|
||||
deployment_params: dict[str, JsonValue],
|
||||
count_host: str,
|
||||
provider_calls: int,
|
||||
) -> None:
|
||||
from litellm.proxy import proxy_server
|
||||
|
||||
request: Final = _json_request(
|
||||
{
|
||||
"model": "claude-agent",
|
||||
"messages": [{"role": "user", "content": "Weather in SF?"}],
|
||||
"thinking": {"type": "enabled", "budget_tokens": 2000},
|
||||
},
|
||||
"python-httpx/0.28.1",
|
||||
)
|
||||
monkeypatch.setattr(proxy_server, "llm_router", _claude_router(deployment_model, deployment_params))
|
||||
monkeypatch.setattr(proxy_server, "proxy_config", proxy_server.ProxyConfig())
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
|
||||
with respx.mock:
|
||||
route: Final = respx.route(host=count_host).mock(
|
||||
return_value=httpx.Response(
|
||||
400, json={"type": "error", "error": {"type": "invalid_request_error", "message": "rejected"}}
|
||||
)
|
||||
)
|
||||
response: Final = await anthropic_count_tokens(request, UserAPIKeyAuth())
|
||||
|
||||
assert isinstance(response, JSONResponse)
|
||||
assert response.status_code == 400
|
||||
assert route.call_count == provider_calls
|
||||
|
||||
|
||||
_BEDROCK_RUNTIME_HOST: Final = "bedrock-runtime.us-east-1.amazonaws.com"
|
||||
_BEDROCK_MANTLE_HOST: Final = "bedrock-mantle.us-east-1.api.aws"
|
||||
_BEDROCK_CLAUDE_DEPLOYMENT: Final[dict[str, str]] = {
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue