mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
fix(count_tokens): filter count betas like inference, retry only forwarded Claude options, and keep Mantle's error on edit-only Bedrock counts
The count now builds its beta filter with the same helper inference uses, so it reads the deployment's api_base, filters an anthropic deployment on a Foundry host with the azure_ai row, drops unlisted betas only where the provider filters them, and keeps the betas the deployment's model_info supports, such as per-turn-control on Bedrock invoke The proxy counts again without the request's options only when the counter forwarded Claude options or betas, so a Gemini or OpenAI count that answers 400 is no longer sent twice On Bedrock, a count that forwards only context edits returns Mantle's 401, 403, 429, 5xx or connection error again when the runtime answers 400 or 404, like main does. A count that forwarded Claude options keeps the runtime's answer The effort beta guard in the Bedrock runtime count is removed, since the invoke body already sends each beta once Tests cover a request drop list replacing the deployment's, the edit merge into the count's options on a Bedrock invoke count, and an opus-5-5 drop_params case. The Bedrock invoke beta hook now calls the base no-op instead of repeating it. A leftover docstring line, an extra blank line, and long test lines are cleaned up
This commit is contained in:
parent
54ec2c8330
commit
4bdba8bbd5
17 changed files with 356 additions and 65 deletions
|
|
@ -28,6 +28,7 @@ import os
|
|||
import threading
|
||||
from collections.abc import Mapping
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from dataclasses import dataclass
|
||||
from functools import cache
|
||||
from importlib.resources import files
|
||||
from typing import Final, Literal, TypeAlias
|
||||
|
|
@ -474,6 +475,28 @@ def update_headers_with_filtered_beta(
|
|||
return headers
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class DeploymentBetaFilter:
|
||||
provider: str
|
||||
api_base: str | None
|
||||
unlisted: UnlistedBetas
|
||||
|
||||
def filtered(self, headers: dict[str, str], keep: frozenset[str]) -> dict[str, str]:
|
||||
return update_headers_with_filtered_beta(
|
||||
headers=headers, provider=self.provider, api_base=self.api_base, unlisted=self.unlisted, keep=keep
|
||||
)
|
||||
|
||||
|
||||
def deployment_beta_filter(
|
||||
provider: str, affinity_api_base: str | None, provider_filters_betas: bool
|
||||
) -> DeploymentBetaFilter | None:
|
||||
if provider_filters_betas:
|
||||
return DeploymentBetaFilter(provider=provider, api_base=affinity_api_base, unlisted="drop")
|
||||
if resolve_beta_provider(provider, affinity_api_base) != provider:
|
||||
return DeploymentBetaFilter(provider=provider, api_base=affinity_api_base, unlisted="forward")
|
||||
return None
|
||||
|
||||
|
||||
def update_request_with_filtered_beta(
|
||||
headers: dict,
|
||||
request_data: dict,
|
||||
|
|
|
|||
|
|
@ -1676,6 +1676,10 @@ class AnthropicModelInfo(BaseLLMModelInfo):
|
|||
def get_api_base(api_base: str | None = None) -> str | None:
|
||||
return AnthropicModelInfo.resolve_anthropic_api_base(api_base)
|
||||
|
||||
@staticmethod
|
||||
def affinity_api_base(custom_llm_provider: str, api_base: str | None) -> str | None:
|
||||
return AnthropicModelInfo.get_api_base(api_base) if custom_llm_provider == "anthropic" else api_base
|
||||
|
||||
@staticmethod
|
||||
def get_api_key(api_key: str | None = None) -> str | None:
|
||||
"""An empty or whitespace-only key counts as unset: it can never authenticate anything, and
|
||||
|
|
|
|||
|
|
@ -168,6 +168,7 @@ class AnthropicTokenCounter(BaseTokenCounter):
|
|||
error=True,
|
||||
error_message=e.message,
|
||||
status_code=e.status_code,
|
||||
carried_request_options=claude_options.carries_request_options,
|
||||
)
|
||||
except Exception as e:
|
||||
verbose_logger.warning("Error calling Anthropic CountTokens API: %s", e)
|
||||
|
|
|
|||
|
|
@ -20,6 +20,7 @@ from typing import (
|
|||
from pydantic import ConfigDict, JsonValue, TypeAdapter, with_config
|
||||
from typing_extensions import NotRequired, ReadOnly, TypedDict
|
||||
|
||||
from litellm.anthropic_beta_headers_manager import DeploymentBetaFilter, deployment_beta_filter
|
||||
from litellm.constants import ANTHROPIC_TOKEN_COUNTING_BETA_VERSION
|
||||
from litellm.litellm_core_utils.asyncify import asyncify
|
||||
from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import (
|
||||
|
|
@ -114,6 +115,7 @@ class AnthropicCountFields:
|
|||
@dataclass(frozen=True, slots=True)
|
||||
class ChatCountTranslationError:
|
||||
message: str
|
||||
carried_request_options: bool = False
|
||||
|
||||
def response(self, request_model: str, model_used: str, tokenizer_type: str) -> TokenCountResponse:
|
||||
return TokenCountResponse(
|
||||
|
|
@ -125,6 +127,7 @@ class ChatCountTranslationError:
|
|||
error_message=self.message,
|
||||
status_code=400,
|
||||
rejected_before_provider_call=True,
|
||||
carried_request_options=self.carried_request_options,
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -349,6 +352,10 @@ class ClaudeCountOptions:
|
|||
messages: tuple[Mapping[str, JsonValue], ...] | None
|
||||
tools: tuple[Mapping[str, JsonValue], ...] | None
|
||||
|
||||
@property
|
||||
def carries_request_options(self) -> bool:
|
||||
return bool(self.options) or bool(self.betas)
|
||||
|
||||
|
||||
NO_CLAUDE_COUNT_OPTIONS: Final = ClaudeCountOptions(options=MappingProxyType({}), betas=(), messages=None, tools=None)
|
||||
_COUNTED_MESSAGES: Final = TypeAdapter(tuple[Mapping[str, JsonValue], ...])
|
||||
|
|
@ -379,6 +386,11 @@ def deployment_inference_model(deployment: Mapping[str, object] | None) -> str |
|
|||
return inference_model
|
||||
|
||||
|
||||
def _deployment_api_base(deployment: Mapping[str, object] | None) -> str | None:
|
||||
api_base: Final = _INFERENCE_PARAMS.validate_python((deployment or {}).get("litellm_params") or {}).get("api_base")
|
||||
return api_base if isinstance(api_base, str) else None
|
||||
|
||||
|
||||
def _messages_transform(model: str, provider: LlmProviders) -> tuple[str, BaseAnthropicMessagesConfig] | None:
|
||||
from litellm.utils import ProviderConfigManager
|
||||
|
||||
|
|
@ -403,14 +415,12 @@ def _claude_count_options_for(
|
|||
messages_config: BaseAnthropicMessagesConfig,
|
||||
model: str,
|
||||
litellm_params: "GenericLiteLLMParams",
|
||||
provider: LlmProviders,
|
||||
beta_filter: DeploymentBetaFilter | None,
|
||||
messages: Sequence[Mapping[str, object]],
|
||||
tools: Sequence[Mapping[str, object]] | None,
|
||||
sent: Mapping[str, JsonValue],
|
||||
anthropic_beta: str | None,
|
||||
) -> ClaudeCountOptions:
|
||||
from litellm.anthropic_beta_headers_manager import update_headers_with_filtered_beta
|
||||
|
||||
leading_system, conversation = split_leading_system_run(messages)
|
||||
sent_conversation: Final = copy.deepcopy([dict(message) for message in conversation])
|
||||
sent_params: Final = {
|
||||
|
|
@ -421,9 +431,11 @@ def _claude_count_options_for(
|
|||
{"anthropic-beta": anthropic_beta} if anthropic_beta else {}, sent_params, sent_conversation
|
||||
)
|
||||
sent_headers: Final = (
|
||||
update_headers_with_filtered_beta(headers=dict(beta_headers), provider=provider.value)
|
||||
if messages_config.should_filter_anthropic_beta_headers()
|
||||
else dict(beta_headers)
|
||||
dict(beta_headers)
|
||||
if beta_filter is None
|
||||
else beta_filter.filtered(
|
||||
dict(beta_headers), keep=messages_config.model_specific_anthropic_betas(model, litellm_params)
|
||||
)
|
||||
)
|
||||
request: Final = messages_config.transform_anthropic_messages_request( # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # the base config signature takes and returns bare dicts
|
||||
model=model,
|
||||
|
|
@ -462,16 +474,23 @@ def claude_count_options(
|
|||
transform_model, messages_config = transform
|
||||
litellm_params: Final = _transform_litellm_params(deployment)
|
||||
from litellm.exceptions import BadRequestError
|
||||
from litellm.llms.anthropic.common_utils import AnthropicError
|
||||
from litellm.llms.anthropic.common_utils import AnthropicError, AnthropicModelInfo
|
||||
|
||||
beta_filter: Final = deployment_beta_filter(
|
||||
provider.value,
|
||||
AnthropicModelInfo.affinity_api_base(provider.value, _deployment_api_base(deployment)),
|
||||
messages_config.should_filter_anthropic_beta_headers(),
|
||||
)
|
||||
try:
|
||||
return _claude_count_options_for(
|
||||
messages_config, transform_model, litellm_params, provider, messages, tools, sent, anthropic_beta
|
||||
messages_config, transform_model, litellm_params, beta_filter, messages, tools, sent, anthropic_beta
|
||||
)
|
||||
except (AnthropicError, BadRequestError) as e:
|
||||
return ChatCountTranslationError(message=str(e.message))
|
||||
return ChatCountTranslationError(message=str(e.message), carried_request_options=True)
|
||||
except (AttributeError, KeyError, TypeError, ValueError) as e:
|
||||
return ChatCountTranslationError(message=f"Invalid request for token counting: {e}")
|
||||
return ChatCountTranslationError(
|
||||
message=f"Invalid request for token counting: {e}", carried_request_options=True
|
||||
)
|
||||
|
||||
|
||||
async def anthropic_count_fields(
|
||||
|
|
|
|||
|
|
@ -176,6 +176,7 @@ class AzureAIAnthropicTokenCounter(BaseTokenCounter):
|
|||
error=True,
|
||||
error_message=e.message,
|
||||
status_code=e.status_code,
|
||||
carried_request_options=claude_options.carries_request_options,
|
||||
)
|
||||
except Exception as e:
|
||||
verbose_logger.warning("Error calling Azure AI Anthropic CountTokens API: %s", e)
|
||||
|
|
|
|||
|
|
@ -89,12 +89,12 @@ def _runtime_rejected_claude_model(outcome: CountTokensOutcome, resolved_model:
|
|||
|
||||
|
||||
def _mantle_retry_outcome(
|
||||
runtime: CountTokensOutcome, mantle: CountTokensOutcome, forwarded: bool
|
||||
runtime: CountTokensOutcome, mantle: CountTokensOutcome, forwarded: bool, carried_options: bool
|
||||
) -> CountTokensOutcome:
|
||||
match runtime, mantle:
|
||||
case CountTokensFailure(status_code=400), CountTokensFailure(status_code=404) if forwarded:
|
||||
return runtime
|
||||
case _, CountTokensFailure(status_code=status_code) if forwarded and not is_request_rejection(status_code):
|
||||
case _, CountTokensFailure(status_code=code) if carried_options and not is_request_rejection(code):
|
||||
return runtime
|
||||
case _:
|
||||
return mantle
|
||||
|
|
@ -117,7 +117,7 @@ def _sets_effort(options: Mapping[str, JsonValue]) -> bool:
|
|||
|
||||
|
||||
def _runtime_count_betas(claude_options: ClaudeCountOptions) -> tuple[str, ...]:
|
||||
if not _sets_effort(claude_options.options) or ANTHROPIC_EFFORT_BETA_HEADER in claude_options.betas:
|
||||
if not _sets_effort(claude_options.options):
|
||||
return claude_options.betas
|
||||
return (*claude_options.betas, ANTHROPIC_EFFORT_BETA_HEADER)
|
||||
|
||||
|
|
@ -322,7 +322,14 @@ class BedrockTokenCounter(BaseTokenCounter):
|
|||
else None
|
||||
)
|
||||
outcome: Final = (
|
||||
primary if fallback is None else _mantle_retry_outcome(primary, fallback, forwarded=bool(runtime_fields))
|
||||
primary
|
||||
if fallback is None
|
||||
else _mantle_retry_outcome(
|
||||
primary,
|
||||
fallback,
|
||||
forwarded=bool(runtime_fields),
|
||||
carried_options=claude_options.carries_request_options,
|
||||
)
|
||||
)
|
||||
match outcome:
|
||||
case CountedTokens():
|
||||
|
|
@ -348,6 +355,7 @@ class BedrockTokenCounter(BaseTokenCounter):
|
|||
error=True,
|
||||
error_message=outcome.message,
|
||||
status_code=outcome.status_code,
|
||||
carried_request_options=claude_options.carries_request_options,
|
||||
)
|
||||
case _:
|
||||
assert_never(outcome)
|
||||
|
|
|
|||
|
|
@ -121,7 +121,7 @@ class AmazonAnthropicClaudeMessagesConfig(
|
|||
optional_params: Mapping[str, object],
|
||||
messages: Sequence[object],
|
||||
) -> Mapping[str, str]:
|
||||
return headers
|
||||
return BaseAnthropicMessagesConfig.headers_with_anthropic_betas(self, headers, optional_params, messages)
|
||||
|
||||
def sent_anthropic_betas(self, headers: Mapping[str, str], request: Mapping[str, object]) -> tuple[str, ...]:
|
||||
return _ANTHROPIC_BETAS.validate_python(request.get("anthropic_beta") or ())
|
||||
|
|
|
|||
|
|
@ -39,11 +39,7 @@ import litellm.types
|
|||
import litellm.types.utils
|
||||
from litellm._internal_context import internal_sub_call
|
||||
from litellm._logging import redact_string, verbose_logger
|
||||
from litellm.anthropic_beta_headers_manager import (
|
||||
aload_beta_headers_config,
|
||||
resolve_beta_provider,
|
||||
update_headers_with_filtered_beta,
|
||||
)
|
||||
from litellm.anthropic_beta_headers_manager import aload_beta_headers_config, deployment_beta_filter
|
||||
from litellm.constants import MAX_FILE_LIST_LIMIT, REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES
|
||||
from litellm.files.types import FileContentStreamingResult
|
||||
from litellm.litellm_core_utils.agentic_followup_kwargs import build_agentic_followup_kwargs
|
||||
|
|
@ -2623,15 +2619,13 @@ class BaseLLMHTTPHandler:
|
|||
litellm_params.get("additional_drop_params") or (),
|
||||
anthropic_messages_optional_request_params,
|
||||
)
|
||||
affinity_api_base: Final = (
|
||||
AnthropicModelInfo.get_api_base(api_base) if custom_llm_provider == "anthropic" else api_base
|
||||
affinity_api_base: Final = AnthropicModelInfo.affinity_api_base(custom_llm_provider, api_base)
|
||||
beta_filter: Final = deployment_beta_filter(
|
||||
custom_llm_provider,
|
||||
affinity_api_base,
|
||||
anthropic_messages_provider_config.should_filter_anthropic_beta_headers(),
|
||||
)
|
||||
takes_another_beta_row: Final = (
|
||||
resolve_beta_provider(custom_llm_provider, affinity_api_base) != custom_llm_provider
|
||||
)
|
||||
provider_filters_betas: Final = anthropic_messages_provider_config.should_filter_anthropic_beta_headers()
|
||||
filters_beta_headers: Final = provider_filters_betas or takes_another_beta_row
|
||||
if request_params.get("safeguards") is not None and filters_beta_headers:
|
||||
if request_params.get("safeguards") is not None and beta_filter is not None:
|
||||
await aload_beta_headers_config()
|
||||
default_affinity_header: Final = get_provider_affinity_header(affinity_api_base)
|
||||
(
|
||||
|
|
@ -2651,14 +2645,10 @@ class BaseLLMHTTPHandler:
|
|||
api_base=api_base,
|
||||
)
|
||||
|
||||
if filters_beta_headers and headers.get("anthropic-beta") is not None:
|
||||
if beta_filter is not None and headers.get("anthropic-beta") is not None:
|
||||
await aload_beta_headers_config()
|
||||
headers = update_headers_with_filtered_beta(
|
||||
headers=headers,
|
||||
provider=custom_llm_provider,
|
||||
api_base=affinity_api_base,
|
||||
unlisted="drop" if provider_filters_betas else "forward",
|
||||
keep=anthropic_messages_provider_config.model_specific_anthropic_betas(model, litellm_params),
|
||||
headers = beta_filter.filtered(
|
||||
headers, keep=anthropic_messages_provider_config.model_specific_anthropic_betas(model, litellm_params)
|
||||
)
|
||||
|
||||
from litellm.llms.vertex_ai.vertex_llm_base import VertexBase
|
||||
|
|
|
|||
|
|
@ -1581,6 +1581,7 @@ class VertexAITokenCounter(BaseTokenCounter):
|
|||
error=True,
|
||||
error_message=error_message,
|
||||
status_code=status_code,
|
||||
carried_request_options=claude_options.carries_request_options,
|
||||
)
|
||||
|
||||
if result is not None:
|
||||
|
|
|
|||
|
|
@ -153,12 +153,6 @@ class VertexAIPartnerModelsTokenCounter(VertexBase):
|
|||
"""
|
||||
Handle token counting request for a Vertex AI partner model.
|
||||
|
||||
Args:
|
||||
model: The model name
|
||||
request_data: Request payload (Anthropic Messages API format)
|
||||
litellm_params: LiteLLM parameters containing credentials, project, location
|
||||
betas: anthropic-beta values the counted request needs, sent as the anthropic-beta header
|
||||
|
||||
Returns:
|
||||
Dict containing token count information
|
||||
|
||||
|
|
|
|||
|
|
@ -80,6 +80,7 @@ class ProviderCountFailure:
|
|||
status_code: int
|
||||
message: str
|
||||
rejected_before_provider_call: bool = False
|
||||
carried_request_options: bool = False
|
||||
|
||||
@property
|
||||
def cannot_count_model(self) -> bool:
|
||||
|
|
@ -90,8 +91,8 @@ class ProviderCountFailure:
|
|||
return not self.rejected_before_provider_call and self.status_code in REGION_DENIED_STATUSES
|
||||
|
||||
@property
|
||||
def rejected_request(self) -> bool:
|
||||
return is_request_rejection(self.status_code)
|
||||
def rejected_request_options(self) -> bool:
|
||||
return self.carried_request_options and is_request_rejection(self.status_code)
|
||||
|
||||
def with_hint(self, hint: str) -> "ProviderCountFailure":
|
||||
if self.rejected_before_provider_call:
|
||||
|
|
|
|||
|
|
@ -15269,6 +15269,7 @@ def _provider_count_outcome(
|
|||
status_code=result.status_code or 500,
|
||||
message=result.error_message or "Token counting failed",
|
||||
rejected_before_provider_call=result.rejected_before_provider_call,
|
||||
carried_request_options=result.carried_request_options,
|
||||
)
|
||||
return result
|
||||
|
||||
|
|
@ -15469,7 +15470,7 @@ async def _count_retrying_without_options(
|
|||
region: str | None,
|
||||
) -> Union["TokenCountResponse", ProviderCountFailure, None]:
|
||||
counted: Final = await count_with(model, region)
|
||||
if not (isinstance(counted, ProviderCountFailure) and counted.rejected_request):
|
||||
if not (isinstance(counted, ProviderCountFailure) and counted.rejected_request_options):
|
||||
return counted
|
||||
verbose_proxy_logger.warning(
|
||||
"Token count for %s %s with the request's %s, so it counts again without them and the caller's betas",
|
||||
|
|
|
|||
|
|
@ -4912,6 +4912,7 @@ class TokenCountResponse(LiteLLMPydanticObjectBase):
|
|||
"""
|
||||
status_code: int | None = None
|
||||
rejected_before_provider_call: bool = Field(default=False, exclude=True)
|
||||
carried_request_options: bool = Field(default=False, exclude=True)
|
||||
|
||||
|
||||
class CustomHuggingfaceTokenizer(TypedDict):
|
||||
|
|
|
|||
|
|
@ -567,6 +567,7 @@ async def _counted_requests(
|
|||
request_options: Mapping[str, JsonValue] = MappingProxyType({}),
|
||||
anthropic_beta: str | None = None,
|
||||
model: str = "claude-opus-5-5",
|
||||
api_base: str = "https://gateway.example",
|
||||
) -> tuple[TokenCountResponse | None, tuple[httpx.Request, ...]]:
|
||||
sent: Final[SimpleQueue[httpx.Request]] = SimpleQueue()
|
||||
|
||||
|
|
@ -580,7 +581,7 @@ async def _counted_requests(
|
|||
model_to_use=model,
|
||||
messages=messages,
|
||||
contents=None,
|
||||
deployment={"litellm_params": {"api_key": "sk-ant-api03-test-key", "api_base": "https://gateway.example"}},
|
||||
deployment={"litellm_params": {"api_key": "sk-ant-api03-test-key", "api_base": api_base}},
|
||||
request_model=model,
|
||||
tools=tools,
|
||||
messages_format=messages_format,
|
||||
|
|
@ -972,6 +973,7 @@ async def _messages_inference_request(
|
|||
request_options: Mapping[str, JsonValue],
|
||||
model: str = "claude-haiku-5-5",
|
||||
extra_headers: Mapping[str, str] = MappingProxyType({}),
|
||||
api_base: str = "https://gateway.example",
|
||||
) -> httpx.Request:
|
||||
sent: Final[SimpleQueue[httpx.Request]] = SimpleQueue()
|
||||
|
||||
|
|
@ -987,7 +989,7 @@ async def _messages_inference_request(
|
|||
tools=[_WEATHER_TOOL],
|
||||
max_tokens=4096,
|
||||
api_key="sk-ant-api03-test-key",
|
||||
api_base="https://gateway.example",
|
||||
api_base=api_base,
|
||||
extra_headers=dict(extra_headers),
|
||||
**request_options,
|
||||
)
|
||||
|
|
@ -1059,7 +1061,10 @@ async def test_counter_sends_the_options_and_context_edits_messages_inference_se
|
|||
assert count_body.get("output_config") == (
|
||||
inference_output_config
|
||||
if inference_output_format is None
|
||||
else {**(inference_output_config if isinstance(inference_output_config, dict) else {}), "format": inference_output_format}
|
||||
else {
|
||||
**(inference_output_config if isinstance(inference_output_config, dict) else {}),
|
||||
"format": inference_output_format,
|
||||
}
|
||||
)
|
||||
assert {name: count_body.get(name) for name in ("thinking", "tool_choice", "context_management")} == {
|
||||
name: inference_body.get(name) for name in ("thinking", "tool_choice", "context_management")
|
||||
|
|
@ -1098,6 +1103,44 @@ async def test_counter_sends_the_thinking_display_and_caller_betas_messages_infe
|
|||
assert {ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER, caller_beta} <= counted_betas
|
||||
|
||||
|
||||
_FOUNDRY_BASE: Final = "https://my-resource.services.ai.azure.com/anthropic"
|
||||
_UNLISTED_BETA: Final = "unlisted-beta-2099-01-01"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.usefixtures("httpx_transport_clients", "local_beta_headers_config")
|
||||
@pytest.mark.parametrize(
|
||||
("counter_type", "expected_betas"),
|
||||
((AnthropicTokenCounter, frozenset({_UNLISTED_BETA})), (AzureAIAnthropicTokenCounter, frozenset())),
|
||||
ids=("anthropic-deployment-forwards-unlisted", "azure-ai-deployment-drops-unlisted"),
|
||||
)
|
||||
async def test_counter_on_a_foundry_host_drops_the_caller_betas_messages_inference_drops_there(
|
||||
counter_type: type[AnthropicTokenCounter] | type[AzureAIAnthropicTokenCounter], expected_betas: frozenset[str]
|
||||
) -> None:
|
||||
caller_betas: Final = f"mcp-servers-2025-12-04,{_UNLISTED_BETA}"
|
||||
inference: Final = await _messages_inference_request(
|
||||
counter_type,
|
||||
f"{_FOUNDRY_BASE}/v1/messages",
|
||||
{"thinking": _ENABLED_THINKING},
|
||||
extra_headers={"anthropic-beta": caller_betas},
|
||||
api_base=_FOUNDRY_BASE,
|
||||
)
|
||||
_, (count,) = await _counted_requests(
|
||||
counter_type,
|
||||
f"{_FOUNDRY_BASE}/v1/messages/count_tokens",
|
||||
[{"role": "user", "content": "Weather in SF?"}],
|
||||
[_WEATHER_TOOL],
|
||||
"anthropic",
|
||||
request_options={"thinking": _ENABLED_THINKING},
|
||||
anthropic_beta=caller_betas,
|
||||
model="claude-haiku-5-5",
|
||||
api_base=_FOUNDRY_BASE,
|
||||
)
|
||||
|
||||
counted_betas: Final = _betas(count) - {ANTHROPIC_TOKEN_COUNTING_BETA_VERSION}
|
||||
assert counted_betas == _betas(inference) == expected_betas
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.usefixtures("httpx_transport_clients")
|
||||
@_COUNTERS
|
||||
|
|
@ -1151,7 +1194,12 @@ _ADVISOR_TOOL: Final[dict[str, JsonValue]] = {
|
|||
"name": "advisor",
|
||||
"model": "claude-opus-5-5",
|
||||
}
|
||||
_ADVISOR_CALL: Final[dict[str, JsonValue]] = {"type": "server_tool_use", "id": "srvtoolu_1", "name": "advisor", "input": {}}
|
||||
_ADVISOR_CALL: Final[dict[str, JsonValue]] = {
|
||||
"type": "server_tool_use",
|
||||
"id": "srvtoolu_1",
|
||||
"name": "advisor",
|
||||
"input": {},
|
||||
}
|
||||
_ADVISOR_RESULT: Final[dict[str, JsonValue]] = {
|
||||
"type": "advisor_tool_result",
|
||||
"tool_use_id": "srvtoolu_1",
|
||||
|
|
|
|||
|
|
@ -11,6 +11,7 @@ import litellm
|
|||
from litellm.llms.bedrock.count_tokens.bedrock_token_counter import BedrockTokenCounter
|
||||
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
|
||||
from litellm.types.llms.anthropic import (
|
||||
ANTHROPIC_BETA_HEADER_VALUES,
|
||||
ANTHROPIC_EFFORT_BETA_HEADER,
|
||||
ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER,
|
||||
)
|
||||
|
|
@ -457,10 +458,15 @@ _CITY_SCHEMA: Final[dict[str, JsonValue]] = {
|
|||
"properties": {"city": {"type": "string"}},
|
||||
"required": ["city"],
|
||||
}
|
||||
_CITY_FORMAT: Final[dict[str, JsonValue]] = {"output_config": {"format": {"type": "json_schema", "schema": _CITY_SCHEMA}}}
|
||||
_CITY_FORMAT: Final[dict[str, JsonValue]] = {
|
||||
"output_config": {"format": {"type": "json_schema", "schema": _CITY_SCHEMA}}
|
||||
}
|
||||
|
||||
|
||||
_CONTEXT_EDITING_OPTIONS: Final[dict[str, JsonValue]] = {**_THINKING_AND_TOOL_CHOICE, "context_management": _CONTEXT_EDITS}
|
||||
_CONTEXT_EDITING_OPTIONS: Final[dict[str, JsonValue]] = {
|
||||
**_THINKING_AND_TOOL_CHOICE,
|
||||
"context_management": _CONTEXT_EDITS,
|
||||
}
|
||||
|
||||
|
||||
async def _messages_inference_and_count(
|
||||
|
|
@ -565,7 +571,11 @@ def _sent_betas(request: httpx.Request) -> tuple[str, ...]:
|
|||
{"output_config": {"effort": "low"}},
|
||||
ANTHROPIC_EFFORT_BETA_HEADER,
|
||||
),
|
||||
("bedrock/invoke/us.anthropic.claude-opus-5-5", _DISPLAY_THINKING, ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER),
|
||||
(
|
||||
"bedrock/invoke/us.anthropic.claude-opus-5-5",
|
||||
_DISPLAY_THINKING,
|
||||
ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER,
|
||||
),
|
||||
),
|
||||
ids=("effort", "thinking_display"),
|
||||
)
|
||||
|
|
@ -654,6 +664,53 @@ async def test_runtime_count_of_an_output_format_sends_only_the_betas_invoke_inf
|
|||
assert counted.get("anthropic_beta") == sent.get("anthropic_beta")
|
||||
|
||||
|
||||
_PER_TURN_CONTROL: Final = ANTHROPIC_BETA_HEADER_VALUES.PER_TURN_CONTROL_2026_07_01.value
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.usefixtures("local_beta_headers_config")
|
||||
@pytest.mark.parametrize(
|
||||
("caller_beta", "model_info", "kept"),
|
||||
(
|
||||
(_PER_TURN_CONTROL, {"supports_per_turn_control": True}, True),
|
||||
(_PER_TURN_CONTROL, {"supports_per_turn_control": False}, False),
|
||||
("unlisted-beta-2099-01-01", {}, False),
|
||||
),
|
||||
ids=("deployment-supports-per-turn-control", "deployment-lacks-per-turn-control", "unlisted-beta"),
|
||||
)
|
||||
async def test_runtime_count_keeps_and_drops_the_caller_betas_invoke_inference_keeps_and_drops(
|
||||
caller_beta: str, model_info: dict[str, JsonValue], kept: bool
|
||||
) -> None:
|
||||
deployment_model: Final = "bedrock/invoke/us.anthropic.claude-opus-5-5"
|
||||
credentials: Final = DEPLOYMENT["litellm_params"]
|
||||
inference: Final = _Bedrock(runtime=(200, _CLAUDE_REPLY), mantle=(200, _CLAUDE_REPLY))
|
||||
await litellm.anthropic_messages(
|
||||
model=deployment_model,
|
||||
messages=MESSAGES,
|
||||
max_tokens=4096,
|
||||
extra_headers={"anthropic-beta": caller_beta},
|
||||
model_info=model_info,
|
||||
client=AsyncHTTPHandler(transport=httpx.MockTransport(inference)),
|
||||
**_THINKING,
|
||||
**credentials,
|
||||
)
|
||||
bedrock: Final = _Bedrock(runtime=(200, {"inputTokens": 412}), mantle=(200, {"input_tokens": 2177}))
|
||||
result: Final = await _counter(bedrock).count_tokens(
|
||||
model_to_use="us.anthropic.claude-opus-5-5",
|
||||
messages=MESSAGES,
|
||||
contents=None,
|
||||
deployment={"litellm_params": {**credentials, "model": deployment_model}, "model_info": model_info},
|
||||
request_model="claude",
|
||||
request_options=_THINKING,
|
||||
anthropic_beta=caller_beta,
|
||||
)
|
||||
|
||||
assert result is not None and result.error is False
|
||||
counted: Final = _betas_besides_effort(_invoke_body(bedrock.requests[0]))
|
||||
assert counted == _betas_besides_effort(_JSON_BODY.validate_json(inference.requests[0].content))
|
||||
assert (caller_beta in counted) is kept
|
||||
|
||||
|
||||
_CONVERSE_REPLY: Final = {
|
||||
"output": {"message": {"role": "assistant", "content": [{"text": "ok"}]}},
|
||||
"stopReason": "end_turn",
|
||||
|
|
@ -787,6 +844,51 @@ async def test_runtime_rejecting_the_options_is_answered_unless_mantle_counts_or
|
|||
assert (result.error, result.status_code, result.tokenizer_type) == expected
|
||||
|
||||
|
||||
_CLEAR_TOOL_USES: Final[dict[str, JsonValue]] = {"edits": [{"type": "clear_tool_uses_20250919"}]}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("request_options", "mantle", "expected"),
|
||||
(
|
||||
({}, _mantle_answers(403, _MANTLE_FAILURE), (403, "bedrock_mantle_api")),
|
||||
({}, _mantle_unreachable(httpx.ConnectError), (500, "bedrock_mantle_api")),
|
||||
(
|
||||
{**_DISPLAY_THINKING, "context_management": _CLEAR_TOOL_USES},
|
||||
_mantle_answers(403, _MANTLE_FAILURE),
|
||||
(400, "bedrock_api"),
|
||||
),
|
||||
),
|
||||
ids=("edit-only-mantle-denied", "edit-only-mantle-unreachable", "edit-and-thinking-mantle-denied"),
|
||||
)
|
||||
async def test_runtime_400_on_an_edit_reports_the_mantle_failure_unless_options_were_forwarded(
|
||||
request_options: dict[str, JsonValue],
|
||||
mantle: Callable[[httpx.Request], httpx.Response],
|
||||
expected: tuple[int, str],
|
||||
) -> None:
|
||||
def bedrock(request: httpx.Request) -> httpx.Response:
|
||||
if request.url.host == RUNTIME_HOST:
|
||||
return httpx.Response(400, json=UNSUPPORTED, request=request)
|
||||
return mantle(request)
|
||||
|
||||
result: Final = await BedrockTokenCounter(
|
||||
client=AsyncHTTPHandler(transport=httpx.MockTransport(bedrock))
|
||||
).count_tokens(
|
||||
model_to_use="us.anthropic.claude-opus-5-5",
|
||||
messages=MESSAGES,
|
||||
contents=None,
|
||||
deployment={
|
||||
"litellm_params": {**DEPLOYMENT["litellm_params"], "model": "bedrock/us.anthropic.claude-opus-5-5"}
|
||||
},
|
||||
request_model="claude-opus-5-5",
|
||||
context_management=CountContextManagement(spec=_CLEAR_TOOL_USES, drop_invalid_edits=False),
|
||||
request_options=request_options,
|
||||
)
|
||||
|
||||
assert result is not None
|
||||
assert (result.error, result.status_code, result.tokenizer_type) == (True, *expected)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_runtime_count_sends_the_tools_invoke_inference_sends() -> None:
|
||||
inference, bedrock = await _messages_inference_and_count(
|
||||
|
|
@ -797,7 +899,8 @@ async def test_runtime_count_sends_the_tools_invoke_inference_sends() -> None:
|
|||
)
|
||||
|
||||
assert bedrock.posted_hosts() == (RUNTIME_HOST,)
|
||||
assert _invoke_body(bedrock.requests[0])["tools"] == _JSON_BODY.validate_json(inference.requests[0].content)["tools"]
|
||||
sent_tools: Final = _JSON_BODY.validate_json(inference.requests[0].content)["tools"]
|
||||
assert _invoke_body(bedrock.requests[0])["tools"] == sent_tools
|
||||
|
||||
|
||||
def _texts(message: JsonValue) -> tuple[str, ...]:
|
||||
|
|
@ -866,7 +969,13 @@ _THINKING: Final[dict[str, JsonValue]] = {"thinking": {"type": "enabled", "budge
|
|||
([{"role": "user", "content": 5}], None, _THINKING),
|
||||
([{"role": "user", "content": [{"text": "hi"}]}], None, _THINKING),
|
||||
),
|
||||
ids=("option_refused", "tool_type_not_a_string", "tool_not_a_mapping", "content_not_iterable", "block_without_type"),
|
||||
ids=(
|
||||
"option_refused",
|
||||
"tool_type_not_a_string",
|
||||
"tool_not_a_mapping",
|
||||
"content_not_iterable",
|
||||
"block_without_type",
|
||||
),
|
||||
)
|
||||
async def test_claude_count_refuses_options_bedrock_messages_inference_refuses_without_calling_bedrock(
|
||||
messages: list[dict[str, JsonValue]], tools: list[JsonValue] | None, request_options: dict[str, JsonValue]
|
||||
|
|
|
|||
|
|
@ -2082,9 +2082,8 @@ async def _vertex_claude_inference_and_count(
|
|||
|
||||
def vertex(request: httpx.Request) -> httpx.Response:
|
||||
sent.put(request)
|
||||
return httpx.Response(
|
||||
200, json={"input_tokens": 37} if request.url.path.endswith("count-tokens:rawPredict") else _PARTNER_CLAUDE_REPLY
|
||||
)
|
||||
counted: Final = request.url.path.endswith("count-tokens:rawPredict")
|
||||
return httpx.Response(200, json={"input_tokens": 37} if counted else _PARTNER_CLAUDE_REPLY)
|
||||
|
||||
client: Final = AsyncHTTPHandler(transport=httpx.MockTransport(vertex))
|
||||
with (
|
||||
|
|
|
|||
|
|
@ -1698,7 +1698,6 @@ async def test_anthropic_count_tokens_on_a_claude_deployment_drops_a_malformed_t
|
|||
assert sent["messages"][2]["content"][0]["content"] == [{"type": "text", "text": "build ok"}]
|
||||
|
||||
|
||||
|
||||
_CLAUDE_COUNT_OPTIONS: Final[dict[str, JsonValue]] = {
|
||||
"thinking": {"type": "adaptive"},
|
||||
"tool_choice": {"type": "auto"},
|
||||
|
|
@ -1756,12 +1755,18 @@ def _sent_option_paths(body: Mapping[str, JsonValue]) -> frozenset[str]:
|
|||
{"additional_drop_params": ["thinking", "context_management"]},
|
||||
{"tool_choice", "output_config", "output_config.effort", "output_config.format"},
|
||||
),
|
||||
(
|
||||
["tool_choice"],
|
||||
{"additional_drop_params": ["thinking"]},
|
||||
{"tool_choice", "output_config", "output_config.effort", "output_config.format", "context_management"},
|
||||
),
|
||||
),
|
||||
ids=(
|
||||
"nothing-dropped",
|
||||
"deployment-drops-tool-choice-and-the-format",
|
||||
"deployment-drops-context-management",
|
||||
"request-drops-thinking-and-context-management",
|
||||
"request-drop-list-replaces-the-deployments",
|
||||
),
|
||||
)
|
||||
async def test_anthropic_count_tokens_on_a_claude_deployment_sends_the_options_inference_sends(
|
||||
|
|
@ -1770,7 +1775,7 @@ async def test_anthropic_count_tokens_on_a_claude_deployment_sends_the_options_i
|
|||
request_params: dict[str, JsonValue],
|
||||
expected_paths: set[str],
|
||||
) -> None:
|
||||
import litellm.proxy.proxy_server as proxy_server
|
||||
from litellm.proxy import proxy_server
|
||||
|
||||
router: Final = Router(
|
||||
model_list=[
|
||||
|
|
@ -1834,7 +1839,7 @@ async def _count_on_a_claude_deployment(
|
|||
user_agent: str = "python-httpx/0.28.1",
|
||||
deployment_params: Mapping[str, JsonValue] = MappingProxyType({}),
|
||||
) -> tuple[object, tuple[httpx.Request, ...]]:
|
||||
import litellm.proxy.proxy_server as proxy_server
|
||||
from litellm.proxy import proxy_server
|
||||
|
||||
router: Final = _claude_router(deployment_model, deployment_params)
|
||||
request: Final = _json_request(
|
||||
|
|
@ -1954,6 +1959,45 @@ async def test_anthropic_count_tokens_does_not_count_again_after_a_rate_limit(
|
|||
assert len(requests) == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("deployment_model", "count_host", "provider_calls"),
|
||||
(
|
||||
("gemini/gemini-3.8-flash", "generativelanguage.googleapis.com", 1),
|
||||
("anthropic/claude-haiku-5-5", "api.anthropic.com", 2),
|
||||
),
|
||||
ids=("counter-without-claude-options", "claude-counter-forwarding-the-options"),
|
||||
)
|
||||
async def test_anthropic_count_tokens_counts_again_without_the_options_only_when_the_counter_forwarded_them(
|
||||
monkeypatch: pytest.MonkeyPatch, deployment_model: str, count_host: str, provider_calls: int
|
||||
) -> None:
|
||||
from litellm.proxy import proxy_server
|
||||
|
||||
request: Final = _json_request(
|
||||
{
|
||||
"model": "claude-agent",
|
||||
"messages": [{"role": "user", "content": "Weather in SF?"}],
|
||||
"thinking": {"type": "enabled", "budget_tokens": 2000},
|
||||
},
|
||||
"python-httpx/0.28.1",
|
||||
)
|
||||
monkeypatch.setattr(proxy_server, "llm_router", _claude_router(deployment_model, {}))
|
||||
monkeypatch.setattr(proxy_server, "proxy_config", proxy_server.ProxyConfig())
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
|
||||
with respx.mock:
|
||||
route: Final = respx.route(host=count_host).mock(
|
||||
return_value=httpx.Response(
|
||||
400, json={"type": "error", "error": {"type": "invalid_request_error", "message": "rejected"}}
|
||||
)
|
||||
)
|
||||
response: Final = await anthropic_count_tokens(request, UserAPIKeyAuth())
|
||||
|
||||
assert isinstance(response, JSONResponse)
|
||||
assert response.status_code == 400
|
||||
assert route.call_count == provider_calls
|
||||
|
||||
|
||||
_BEDROCK_RUNTIME_HOST: Final = "bedrock-runtime.us-east-1.amazonaws.com"
|
||||
_BEDROCK_MANTLE_HOST: Final = "bedrock-mantle.us-east-1.api.aws"
|
||||
_BEDROCK_CLAUDE_DEPLOYMENT: Final[dict[str, str]] = {
|
||||
|
|
@ -1969,12 +2013,17 @@ async def _count_on_a_bedrock_claude_deployment(
|
|||
body: Mapping[str, JsonValue],
|
||||
runtime: Callable[[httpx.Request], httpx.Response],
|
||||
mantle: Callable[[httpx.Request], httpx.Response],
|
||||
deployment_model: str = _BEDROCK_CLAUDE_DEPLOYMENT["model"],
|
||||
) -> tuple[object, tuple[httpx.Request, ...]]:
|
||||
import litellm.proxy.proxy_server as proxy_server
|
||||
from litellm.proxy import proxy_server
|
||||
|
||||
monkeypatch.delenv("AWS_BEARER_TOKEN_BEDROCK", raising=False)
|
||||
monkeypatch.delenv("BEDROCK_MANTLE_API_BASE", raising=False)
|
||||
router: Final = Router(model_list=[{"model_name": "claude-agent", "litellm_params": _BEDROCK_CLAUDE_DEPLOYMENT}])
|
||||
router: Final = Router(
|
||||
model_list=[
|
||||
{"model_name": "claude-agent", "litellm_params": {**_BEDROCK_CLAUDE_DEPLOYMENT, "model": deployment_model}}
|
||||
]
|
||||
)
|
||||
request: Final = _json_request(
|
||||
{"model": "claude-agent", "messages": [{"role": "user", "content": "Weather in SF?"}], **body},
|
||||
"python-httpx/0.28.1",
|
||||
|
|
@ -2045,6 +2094,48 @@ async def test_anthropic_count_tokens_on_bedrock_counts_again_without_the_option
|
|||
assert "invokeModel" not in json.loads(requests[-1].content)["input"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_anthropic_count_tokens_on_bedrock_sends_the_thinking_a_clear_thinking_edit_makes_inference_send(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
deployment_model: Final = "bedrock/invoke/us.anthropic.claude-sonnet-4-6"
|
||||
options_and_edit: Final[dict[str, JsonValue]] = {
|
||||
"tools": [{"name": "get_weather", "input_schema": {"type": "object"}}],
|
||||
"tool_choice": {"type": "auto"},
|
||||
"context_management": {"edits": [{"type": "clear_thinking_20251015"}]},
|
||||
}
|
||||
credentials: Final = {name: value for name, value in _BEDROCK_CLAUDE_DEPLOYMENT.items() if name != "model"}
|
||||
monkeypatch.delenv("AWS_BEARER_TOKEN_BEDROCK", raising=False)
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
with respx.mock:
|
||||
inference_route: Final = respx.route(host=_BEDROCK_RUNTIME_HOST).mock(
|
||||
return_value=httpx.Response(200, json=_CLAUDE_MESSAGE_REPLY)
|
||||
)
|
||||
await litellm.anthropic_messages(
|
||||
model=deployment_model,
|
||||
messages=[{"role": "user", "content": "Weather in SF?"}],
|
||||
max_tokens=4096,
|
||||
**options_and_edit,
|
||||
**credentials,
|
||||
)
|
||||
sent: Final = json.loads(inference_route.calls.last.request.content)
|
||||
|
||||
response, (count,) = await _count_on_a_bedrock_claude_deployment(
|
||||
monkeypatch,
|
||||
options_and_edit,
|
||||
lambda _: httpx.Response(200, json={"inputTokens": 9}),
|
||||
_mantle_unreachable,
|
||||
deployment_model,
|
||||
)
|
||||
|
||||
assert not isinstance(response, JSONResponse)
|
||||
counted: Final = json.loads(_bedrock_counted_request(count.content))
|
||||
assert sent.get("thinking") is not None
|
||||
assert {name: counted.get(name) for name in ("thinking", "output_config", "context_management")} == {
|
||||
name: sent.get(name) for name in ("thinking", "output_config", "context_management")
|
||||
}
|
||||
|
||||
|
||||
_KEEPS_CONTEXT_MANAGEMENT: Final = "sk-ant-keeps-context-management"
|
||||
_DROPS_CONTEXT_MANAGEMENT: Final = "sk-ant-drops-context-management"
|
||||
_CONTEXT_EDITS: Final[dict[str, JsonValue]] = {"edits": [{"type": "clear_tool_uses_20250919"}]}
|
||||
|
|
@ -2098,7 +2189,7 @@ def _mixed_drop_group(routed_key: str) -> Router:
|
|||
async def test_anthropic_count_tokens_in_a_mixed_drop_group_counts_without_context_management_on_any_deployment(
|
||||
monkeypatch: pytest.MonkeyPatch, routed_key: str
|
||||
) -> None:
|
||||
import litellm.proxy.proxy_server as proxy_server
|
||||
from litellm.proxy import proxy_server
|
||||
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
monkeypatch.setattr(proxy_server, "llm_router", _mixed_drop_group(routed_key))
|
||||
|
|
@ -2170,13 +2261,13 @@ async def test_anthropic_count_tokens_on_a_claude_deployment_sends_the_thinking_
|
|||
) -> None:
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
thinking: Final[dict[str, JsonValue]] = {"type": "enabled", "budget_tokens": 2000, "summary": "auto"}
|
||||
dropped: Final = await _claude_inference_body("claude-opus-4-7", {"thinking": thinking, "drop_params": True})
|
||||
kept: Final = await _claude_inference_body("claude-opus-4-7", {"thinking": thinking})
|
||||
dropped: Final = await _claude_inference_body("claude-opus-5-5", {"thinking": thinking, "drop_params": True})
|
||||
kept: Final = await _claude_inference_body("claude-opus-5-5", {"thinking": thinking})
|
||||
assert dropped["thinking"] != kept["thinking"]
|
||||
|
||||
response, (count,) = await _count_on_a_claude_deployment(
|
||||
monkeypatch,
|
||||
"anthropic/claude-opus-4-7",
|
||||
"anthropic/claude-opus-5-5",
|
||||
{"thinking": thinking, **request_drop_params},
|
||||
lambda _: httpx.Response(200, json={"input_tokens": 7}),
|
||||
user_agent,
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue