mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
fix(count_tokens): count a malformed body or a failed Mantle fallback again without the options, and never retry a 401 or 403
The count sends inference's betas when the count forwards options, and it now filters them through update_headers_with_filtered_beta, the helper inference uses. A count without options still sends only the token-counting beta and the bare body A body the count transform cannot read, like a tool whose type is not a string, now comes back as a translation error, so the proxy counts again without the options instead of answering 500. On Bedrock, when the runtime count rejects the forwarded options with a 400 or 404, the Mantle answer is used only if Mantle counted or rejected the request with a 4xx other than 401, 403 or 429. A Mantle 5xx, 429, auth error, timeout or connection error now returns the runtime rejection, and the proxy counts that again without the options The proxy no longer counts again after a 401 or 403 because those are not about the request, and its warning names every option it drops. Tests cover a group where only one deployment drops context_management, Vertex's tool-search beta, and the exact betas each counter sends
This commit is contained in:
parent
f84c244f3e
commit
14b572d44b
8 changed files with 437 additions and 80 deletions
|
|
@ -28,6 +28,7 @@ from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import
|
|||
from litellm.litellm_core_utils.url_utils import join_api_base
|
||||
from litellm.llms.anthropic.common_utils import merge_anthropic_beta_headers
|
||||
from litellm.llms.anthropic.wif import resolve_anthropic_base
|
||||
from litellm.llms.base_llm.anthropic_messages.transformation import BaseAnthropicMessagesConfig
|
||||
from litellm.types.llms.openai import AllMessageValues, ChatCompletionImageObject, ValidUserMessageContentTypes
|
||||
from litellm.types.utils import LlmProviders, TokenCountMessagesFormat, TokenCountResponse
|
||||
|
||||
|
|
@ -345,17 +346,61 @@ _COUNTED_MESSAGES: Final = TypeAdapter(tuple[Mapping[str, JsonValue], ...])
|
|||
_MAX_TOKENS_ABOVE_ANY_BUDGET: Final = 2**31 - 1
|
||||
|
||||
|
||||
def _with_provider_betas(headers: Mapping[str, str], provider: str) -> Mapping[str, str]:
|
||||
from litellm.anthropic_beta_headers_manager import filter_and_transform_beta_headers
|
||||
_STATUSES_NOT_CAUSED_BY_THE_REQUEST: Final = frozenset({401, 403, 429})
|
||||
|
||||
beta_header: Final = headers.get("anthropic-beta")
|
||||
if beta_header is None:
|
||||
return headers
|
||||
supported: Final = filter_and_transform_beta_headers(
|
||||
[beta.strip() for beta in beta_header.split(",") if beta.strip()], provider
|
||||
|
||||
def is_request_rejection(status_code: int) -> bool:
|
||||
return 400 <= status_code < 500 and status_code not in _STATUSES_NOT_CAUSED_BY_THE_REQUEST
|
||||
|
||||
|
||||
def _claude_count_options_for(
|
||||
messages_config: BaseAnthropicMessagesConfig,
|
||||
model: str,
|
||||
provider: LlmProviders,
|
||||
messages: Sequence[Mapping[str, object]],
|
||||
tools: Sequence[Mapping[str, object]] | None,
|
||||
sent: Mapping[str, JsonValue],
|
||||
anthropic_beta: str | None,
|
||||
deployment: Mapping[str, object] | None,
|
||||
) -> ClaudeCountOptions:
|
||||
from litellm.anthropic_beta_headers_manager import update_headers_with_filtered_beta
|
||||
from litellm.types.router import GenericLiteLLMParams
|
||||
|
||||
resolved_deployment: Final = deployment or {}
|
||||
deployment_params: Final = _INFERENCE_PARAMS.validate_python(resolved_deployment.get("litellm_params") or {})
|
||||
leading_system, conversation = split_leading_system_run(messages)
|
||||
sent_conversation: Final = copy.deepcopy([dict(message) for message in conversation])
|
||||
sent_params: Final = {
|
||||
**copy.deepcopy(sent),
|
||||
**({"tools": copy.deepcopy([dict(tool) for tool in tools])} if tools else {}),
|
||||
}
|
||||
beta_headers: Final = messages_config.headers_with_anthropic_betas(
|
||||
{"anthropic-beta": anthropic_beta} if anthropic_beta else {}, sent_params, sent_conversation
|
||||
)
|
||||
sent_headers: Final = (
|
||||
update_headers_with_filtered_beta(headers=dict(beta_headers), provider=provider.value)
|
||||
if messages_config.should_filter_anthropic_beta_headers()
|
||||
else dict(beta_headers)
|
||||
)
|
||||
request: Final = messages_config.transform_anthropic_messages_request( # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # the base config signature takes and returns bare dicts
|
||||
model=model,
|
||||
messages=sent_conversation,
|
||||
anthropic_messages_optional_request_params={**sent_params, "max_tokens": _MAX_TOKENS_ABOVE_ANY_BUDGET},
|
||||
litellm_params=GenericLiteLLMParams.model_validate(
|
||||
{
|
||||
"drop_params": deployment_params.get("drop_params"),
|
||||
"model_info": resolved_deployment.get("model_info"),
|
||||
}
|
||||
),
|
||||
headers=sent_headers,
|
||||
)
|
||||
transformed: Final = _ANTHROPIC_CHAT_REQUEST.validate_python(request)
|
||||
return ClaudeCountOptions(
|
||||
options=_count_options(transformed),
|
||||
betas=messages_config.sent_anthropic_betas(sent_headers, _INFERENCE_PARAMS.validate_python(request)),
|
||||
messages=(*_COUNTED_MESSAGES.validate_python(leading_system), *transformed["messages"]),
|
||||
tools=transformed.get("tools"),
|
||||
)
|
||||
other_headers: Final = {name: value for name, value in headers.items() if name != "anthropic-beta"}
|
||||
return {**other_headers, "anthropic-beta": ",".join(supported)} if supported else other_headers
|
||||
|
||||
|
||||
def claude_count_options(
|
||||
|
|
@ -372,50 +417,19 @@ def claude_count_options(
|
|||
return NO_CLAUDE_COUNT_OPTIONS
|
||||
from litellm.exceptions import BadRequestError
|
||||
from litellm.llms.anthropic.common_utils import AnthropicError
|
||||
from litellm.types.router import GenericLiteLLMParams
|
||||
from litellm.utils import ProviderConfigManager
|
||||
|
||||
messages_config: Final = ProviderConfigManager.get_provider_anthropic_messages_config(model, provider)
|
||||
if messages_config is None:
|
||||
return NO_CLAUDE_COUNT_OPTIONS
|
||||
resolved_deployment: Final = deployment or {}
|
||||
deployment_params: Final = _INFERENCE_PARAMS.validate_python(resolved_deployment.get("litellm_params") or {})
|
||||
leading_system, conversation = split_leading_system_run(messages)
|
||||
sent_conversation: Final = copy.deepcopy([dict(message) for message in conversation])
|
||||
sent_params: Final = {
|
||||
**copy.deepcopy(sent),
|
||||
**({"tools": copy.deepcopy([dict(tool) for tool in tools])} if tools else {}),
|
||||
}
|
||||
beta_headers: Final = messages_config.headers_with_anthropic_betas(
|
||||
{"anthropic-beta": anthropic_beta} if anthropic_beta else {}, sent_params, sent_conversation
|
||||
)
|
||||
sent_headers: Final = dict(
|
||||
_with_provider_betas(beta_headers, provider.value)
|
||||
if messages_config.should_filter_anthropic_beta_headers()
|
||||
else beta_headers
|
||||
)
|
||||
try:
|
||||
request: Final = messages_config.transform_anthropic_messages_request( # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # the base config signature takes and returns bare dicts
|
||||
model=model,
|
||||
messages=sent_conversation,
|
||||
anthropic_messages_optional_request_params={**sent_params, "max_tokens": _MAX_TOKENS_ABOVE_ANY_BUDGET},
|
||||
litellm_params=GenericLiteLLMParams.model_validate(
|
||||
{
|
||||
"drop_params": deployment_params.get("drop_params"),
|
||||
"model_info": resolved_deployment.get("model_info"),
|
||||
}
|
||||
),
|
||||
headers=sent_headers,
|
||||
return _claude_count_options_for(
|
||||
messages_config, model, provider, messages, tools, sent, anthropic_beta, deployment
|
||||
)
|
||||
except (AnthropicError, BadRequestError) as e:
|
||||
return ChatCountTranslationError(message=str(e.message))
|
||||
transformed: Final = _ANTHROPIC_CHAT_REQUEST.validate_python(request)
|
||||
return ClaudeCountOptions(
|
||||
options=_count_options(transformed),
|
||||
betas=messages_config.sent_anthropic_betas(sent_headers, _INFERENCE_PARAMS.validate_python(request)),
|
||||
messages=(*_COUNTED_MESSAGES.validate_python(leading_system), *transformed["messages"]),
|
||||
tools=transformed.get("tools"),
|
||||
)
|
||||
except (AttributeError, IndexError, KeyError, TypeError, ValueError) as e:
|
||||
return ChatCountTranslationError(message=f"Invalid request for token counting: {e}")
|
||||
|
||||
|
||||
async def anthropic_count_fields(
|
||||
|
|
|
|||
|
|
@ -19,6 +19,7 @@ from litellm.llms.anthropic.count_tokens.transformation import (
|
|||
ClaudeCountOptions,
|
||||
anthropic_count_fields,
|
||||
claude_count_options,
|
||||
is_request_rejection,
|
||||
)
|
||||
from litellm.llms.base_llm.base_utils import BaseTokenCounter
|
||||
from litellm.llms.bedrock.common_utils import BedrockError, get_bedrock_base_model
|
||||
|
|
@ -80,6 +81,10 @@ def _runtime_rejected_claude_model(outcome: CountTokensOutcome, resolved_model:
|
|||
)
|
||||
|
||||
|
||||
def _mantle_answered_for_runtime(fallback: CountTokensOutcome, forwarded_options: bool) -> bool:
|
||||
return not forwarded_options or isinstance(fallback, CountedTokens) or is_request_rejection(fallback.status_code)
|
||||
|
||||
|
||||
def _invoke_fields(claude_options: ClaudeCountOptions) -> Mapping[str, object]:
|
||||
return {
|
||||
**claude_options.options,
|
||||
|
|
@ -225,11 +230,16 @@ class BedrockTokenCounter(BaseTokenCounter):
|
|||
primary_tokenizer,
|
||||
)
|
||||
)
|
||||
outcome: Final = (
|
||||
fallback: Final = (
|
||||
await self._count_with(
|
||||
self._count_on_mantle(mantle_request, litellm_params, resolved_model), MANTLE_TOKENIZER_TYPE
|
||||
)
|
||||
if not use_mantle and _runtime_rejected_claude_model(primary, resolved_model)
|
||||
else None
|
||||
)
|
||||
outcome: Final = (
|
||||
fallback
|
||||
if fallback is not None and _mantle_answered_for_runtime(fallback, forwarded_options=bool(invoke_fields))
|
||||
else primary
|
||||
)
|
||||
match outcome:
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@ from pydantic import BaseModel, ConfigDict, ValidationError
|
|||
import litellm
|
||||
from litellm.anthropic_interface.exceptions.exception_mapping_utils import AnthropicExceptionMapping
|
||||
from litellm.litellm_core_utils.core_helpers import as_str_mapping
|
||||
from litellm.llms.anthropic.count_tokens.transformation import is_request_rejection
|
||||
from litellm.proxy._types import ProxyException
|
||||
from litellm.types.token_count import TokenCountProvider, TokenCountSourceFields, TokenCountTokenizer
|
||||
from litellm.types.utils import TokenCountResponse
|
||||
|
|
@ -74,7 +75,7 @@ class ProviderCountFailure:
|
|||
|
||||
@property
|
||||
def rejected_request(self) -> bool:
|
||||
return 400 <= self.status_code < 500 and self.status_code != 429
|
||||
return is_request_rejection(self.status_code)
|
||||
|
||||
def with_hint(self, hint: str) -> "ProviderCountFailure":
|
||||
if self.rejected_before_provider_call:
|
||||
|
|
|
|||
|
|
@ -14775,9 +14775,8 @@ _DEPLOYMENT_LITELLM_PARAMS: Final = TypeAdapter(Mapping[str, object])
|
|||
|
||||
|
||||
def _deployment_with_request_drop_params(
|
||||
deployment: dict[str, object] | None, # mutable-ok: BaseTokenCounter.count_tokens takes a dict deployment
|
||||
drop_params: object,
|
||||
) -> dict[str, object] | None: # mutable-ok: BaseTokenCounter.count_tokens takes a dict deployment
|
||||
deployment: Mapping[str, object] | None, drop_params: bool | None
|
||||
) -> Mapping[str, object] | None:
|
||||
if drop_params is None:
|
||||
return deployment
|
||||
resolved_deployment: Final = deployment or {}
|
||||
|
|
@ -14931,11 +14930,12 @@ def _retried_without_rejected_options(
|
|||
) -> Callable[[str, str | None], Awaitable[Union["TokenCountResponse", ProviderCountFailure, None]]]:
|
||||
if not request_options:
|
||||
return count_with
|
||||
return partial(_count_retrying_without_options, count_with)
|
||||
return partial(_count_retrying_without_options, count_with, tuple(request_options))
|
||||
|
||||
|
||||
async def _count_retrying_without_options(
|
||||
count_with: "partial[Awaitable[TokenCountResponse | ProviderCountFailure | None]]",
|
||||
forwarded_options: tuple[str, ...],
|
||||
model: str,
|
||||
region: str | None,
|
||||
) -> Union["TokenCountResponse", ProviderCountFailure, None]:
|
||||
|
|
@ -14943,10 +14943,10 @@ async def _count_retrying_without_options(
|
|||
if not (isinstance(counted, ProviderCountFailure) and counted.rejected_request):
|
||||
return counted
|
||||
verbose_proxy_logger.warning(
|
||||
"Token count for %s answered %s with the request's thinking, tool, and output options, "
|
||||
"so it counts again without them",
|
||||
"Token count for %s answered %s with the request's %s, so it counts again without them",
|
||||
model,
|
||||
counted.status_code,
|
||||
", ".join(forwarded_options),
|
||||
)
|
||||
return await count_with(model, region, request_options=MappingProxyType({}))
|
||||
|
||||
|
|
@ -15475,7 +15475,9 @@ async def _count_request_tokens(
|
|||
model_to_use = _model
|
||||
|
||||
sent_params: Final[Mapping[str, object]] = {**_router_default_litellm_params(llm_router), **request_litellm_params}
|
||||
count_deployment: Final = _deployment_with_request_drop_params(deployment, sent_params.get("drop_params"))
|
||||
count_deployment: Final = _deployment_with_request_drop_params(
|
||||
deployment, normalize_drop_params(sent_params.get("drop_params"))
|
||||
)
|
||||
conversation_empty: Final = not messages and not contents
|
||||
if prompt and not conversation_empty and not deployment_drops_params(count_deployment):
|
||||
raise ProxyException(
|
||||
|
|
|
|||
|
|
@ -916,21 +916,27 @@ async def _messages_inference_request(
|
|||
return sent.get()
|
||||
|
||||
|
||||
def _on_both_counters(
|
||||
*betas: str,
|
||||
) -> Mapping[type[AnthropicTokenCounter] | type[AzureAIAnthropicTokenCounter], frozenset[str]]:
|
||||
return MappingProxyType({AnthropicTokenCounter: frozenset(betas), AzureAIAnthropicTokenCounter: frozenset(betas)})
|
||||
|
||||
|
||||
def _betas(request: httpx.Request) -> frozenset[str]:
|
||||
return frozenset(beta for beta in request.headers.get("anthropic-beta", "").split(",") if beta)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.usefixtures("httpx_transport_clients")
|
||||
@pytest.mark.usefixtures("httpx_transport_clients", "local_beta_headers_config")
|
||||
@_COUNTERS
|
||||
@pytest.mark.parametrize(
|
||||
("request_options", "expected_betas"),
|
||||
(
|
||||
({"thinking": _ENABLED_THINKING}, frozenset()),
|
||||
({"tool_choice": {"type": "any"}}, frozenset()),
|
||||
({"thinking": _ENABLED_THINKING}, _on_both_counters()),
|
||||
({"tool_choice": {"type": "any"}}, _on_both_counters()),
|
||||
(
|
||||
{"output_config": {"effort": "low"}, "output_format": {"type": "json_schema", "schema": _ANSWER_SCHEMA}},
|
||||
frozenset((ANTHROPIC_BETA_HEADER_VALUES.STRUCTURED_OUTPUT_2025_09_25.value,)),
|
||||
_on_both_counters(ANTHROPIC_BETA_HEADER_VALUES.STRUCTURED_OUTPUT_2025_09_25.value),
|
||||
),
|
||||
(
|
||||
{
|
||||
|
|
@ -939,9 +945,17 @@ def _betas(request: httpx.Request) -> frozenset[str]:
|
|||
"edits": [{"type": "clear_tool_uses_20250919"}, {"type": "clear_thinking_20251015"}]
|
||||
},
|
||||
},
|
||||
frozenset((ANTHROPIC_BETA_HEADER_VALUES.CONTEXT_MANAGEMENT_2025_06_27.value,)),
|
||||
_on_both_counters(ANTHROPIC_BETA_HEADER_VALUES.CONTEXT_MANAGEMENT_2025_06_27.value),
|
||||
),
|
||||
(
|
||||
{"context_management": {"edits": [{"type": "compact_20260112"}]}},
|
||||
MappingProxyType(
|
||||
{
|
||||
AnthropicTokenCounter: frozenset((ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_01_12.value,)),
|
||||
AzureAIAnthropicTokenCounter: frozenset(),
|
||||
}
|
||||
),
|
||||
),
|
||||
({"context_management": {"edits": [{"type": "compact_20260112"}]}}, frozenset()),
|
||||
),
|
||||
ids=("thinking", "tool_choice", "output_config_and_output_format", "context_edits", "compact_edit"),
|
||||
)
|
||||
|
|
@ -949,7 +963,7 @@ async def test_counter_sends_the_options_and_context_edits_messages_inference_se
|
|||
counter_type: type[AnthropicTokenCounter] | type[AzureAIAnthropicTokenCounter],
|
||||
count_endpoint: str,
|
||||
request_options: dict[str, JsonValue],
|
||||
expected_betas: frozenset[str],
|
||||
expected_betas: Mapping[type[AnthropicTokenCounter] | type[AzureAIAnthropicTokenCounter], frozenset[str]],
|
||||
) -> None:
|
||||
inference: Final = await _messages_inference_request(
|
||||
counter_type, count_endpoint.removesuffix("/count_tokens"), request_options
|
||||
|
|
@ -977,7 +991,7 @@ async def test_counter_sends_the_options_and_context_edits_messages_inference_se
|
|||
assert {name: count_body.get(name) for name in ("thinking", "tool_choice", "context_management")} == {
|
||||
name: inference_body.get(name) for name in ("thinking", "tool_choice", "context_management")
|
||||
}
|
||||
assert expected_betas <= _betas(count) - {ANTHROPIC_TOKEN_COUNTING_BETA_VERSION} == _betas(inference)
|
||||
assert expected_betas[counter_type] == _betas(count) - {ANTHROPIC_TOKEN_COUNTING_BETA_VERSION} == _betas(inference)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
@ -1011,6 +1025,30 @@ async def test_counter_sends_the_thinking_display_and_caller_betas_messages_infe
|
|||
assert {ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER, caller_beta} <= counted_betas
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.usefixtures("httpx_transport_clients")
|
||||
@_COUNTERS
|
||||
async def test_counter_without_options_sends_only_the_token_counting_beta_and_the_bare_count_body(
|
||||
counter_type: type[AnthropicTokenCounter] | type[AzureAIAnthropicTokenCounter], count_endpoint: str
|
||||
) -> None:
|
||||
_, (count,) = await _counted_requests(
|
||||
counter_type,
|
||||
count_endpoint,
|
||||
[{"role": "user", "content": "Weather in SF?"}],
|
||||
[_WEATHER_TOOL],
|
||||
"anthropic",
|
||||
anthropic_beta=ANTHROPIC_BETA_HEADER_VALUES.WEB_FETCH_2025_09_10.value,
|
||||
model="claude-haiku-5-5",
|
||||
)
|
||||
|
||||
assert _betas(count) == {ANTHROPIC_TOKEN_COUNTING_BETA_VERSION}
|
||||
assert _JSON_BODY.validate_json(count.content) == {
|
||||
"model": "claude-haiku-5-5",
|
||||
"messages": [{"role": "user", "content": "Weather in SF?"}],
|
||||
"tools": [_WEATHER_TOOL],
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.usefixtures("httpx_transport_clients")
|
||||
@_COUNTERS
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
import base64
|
||||
import json
|
||||
from collections.abc import Mapping
|
||||
from collections.abc import Callable, Mapping
|
||||
from typing import Final
|
||||
|
||||
import httpx
|
||||
|
|
@ -489,6 +489,77 @@ async def test_mantle_count_sends_the_thinking_display_beta_mantle_inference_sen
|
|||
assert ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER in _sent_betas(bedrock.requests[0])
|
||||
|
||||
|
||||
def _mantle_answers(status_code: int, body: Mapping[str, JsonValue]) -> Callable[[httpx.Request], httpx.Response]:
|
||||
return lambda request: httpx.Response(status_code, json=dict(body), request=request)
|
||||
|
||||
|
||||
def _mantle_unreachable(error: type[httpx.TransportError]) -> Callable[[httpx.Request], httpx.Response]:
|
||||
def unreachable(request: httpx.Request) -> httpx.Response:
|
||||
raise error("bedrock-mantle is unreachable", request=request)
|
||||
|
||||
return unreachable
|
||||
|
||||
|
||||
_MANTLE_REJECTION: Final[dict[str, JsonValue]] = {
|
||||
"type": "error",
|
||||
"error": {"type": "invalid_request_error", "message": "thinking: unsupported"},
|
||||
}
|
||||
_MANTLE_FAILURE: Final[dict[str, JsonValue]] = {"type": "error", "error": {"type": "api_error", "message": "down"}}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("request_options", "mantle", "expected"),
|
||||
(
|
||||
(_DISPLAY_THINKING, _mantle_unreachable(httpx.ConnectError), (True, 400, "bedrock_api")),
|
||||
(_DISPLAY_THINKING, _mantle_unreachable(httpx.ReadTimeout), (True, 400, "bedrock_api")),
|
||||
(_DISPLAY_THINKING, _mantle_answers(503, _MANTLE_FAILURE), (True, 400, "bedrock_api")),
|
||||
(_DISPLAY_THINKING, _mantle_answers(429, _MANTLE_FAILURE), (True, 400, "bedrock_api")),
|
||||
(_DISPLAY_THINKING, _mantle_answers(403, _MANTLE_FAILURE), (True, 400, "bedrock_api")),
|
||||
(_DISPLAY_THINKING, _mantle_answers(400, _MANTLE_REJECTION), (True, 400, "bedrock_mantle_api")),
|
||||
(_DISPLAY_THINKING, _mantle_answers(422, _MANTLE_REJECTION), (True, 422, "bedrock_mantle_api")),
|
||||
(_DISPLAY_THINKING, _mantle_answers(200, {"input_tokens": 2177}), (False, None, "bedrock_mantle_api")),
|
||||
({}, _mantle_answers(503, _MANTLE_FAILURE), (True, 503, "bedrock_mantle_api")),
|
||||
),
|
||||
ids=(
|
||||
"mantle-unreachable",
|
||||
"mantle-timeout",
|
||||
"mantle-5xx",
|
||||
"mantle-rate-limit",
|
||||
"mantle-denied",
|
||||
"mantle-rejects-the-options",
|
||||
"mantle-rejects-the-body",
|
||||
"mantle-counts",
|
||||
"no-options-mantle-5xx",
|
||||
),
|
||||
)
|
||||
async def test_runtime_rejecting_the_options_is_answered_unless_mantle_counts_or_rejects_the_request(
|
||||
request_options: dict[str, JsonValue],
|
||||
mantle: Callable[[httpx.Request], httpx.Response],
|
||||
expected: tuple[bool, int | None, str],
|
||||
) -> None:
|
||||
def bedrock(request: httpx.Request) -> httpx.Response:
|
||||
if request.url.host == RUNTIME_HOST:
|
||||
return httpx.Response(400, json=UNSUPPORTED, request=request)
|
||||
return mantle(request)
|
||||
|
||||
result: Final = await BedrockTokenCounter(
|
||||
client=AsyncHTTPHandler(transport=httpx.MockTransport(bedrock))
|
||||
).count_tokens(
|
||||
model_to_use="us.anthropic.claude-opus-5-5",
|
||||
messages=MESSAGES,
|
||||
contents=None,
|
||||
deployment={
|
||||
"litellm_params": {**DEPLOYMENT["litellm_params"], "model": "bedrock/us.anthropic.claude-opus-5-5"}
|
||||
},
|
||||
request_model="claude-opus-5-5",
|
||||
request_options=request_options,
|
||||
)
|
||||
|
||||
assert result is not None
|
||||
assert (result.error, result.status_code, result.tokenizer_type) == expected
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_runtime_count_sends_the_tools_invoke_inference_sends() -> None:
|
||||
inference, bedrock = await _messages_inference_and_count(
|
||||
|
|
|
|||
|
|
@ -33,6 +33,7 @@ from litellm.llms.vertex_ai.common_utils import (
|
|||
)
|
||||
from litellm.llms.vertex_ai.count_tokens import handler as vertex_count_tokens_handler
|
||||
from litellm.llms.vertex_ai.vertex_ai_partner_models.anthropic.transformation import VertexAIAnthropicConfig
|
||||
from litellm.types.llms.anthropic_tool_search import get_tool_search_beta_header
|
||||
from litellm.types.utils import CountContextManagement, TokenCountMessagesFormat
|
||||
|
||||
|
||||
|
|
@ -1970,20 +1971,13 @@ _PARTNER_CLAUDE_REPLY: Final[dict[str, JsonValue]] = {
|
|||
}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_vertex_claude_count_sends_the_options_context_edits_and_beta_vertex_messages_inference_sends() -> None:
|
||||
async def _vertex_claude_inference_and_count(
|
||||
tools: list[dict[str, JsonValue]], request_options: Mapping[str, JsonValue]
|
||||
) -> tuple[httpx.Request, httpx.Request]:
|
||||
from litellm.llms.vertex_ai.vertex_ai_partner_models.count_tokens import handler
|
||||
from litellm.llms.vertex_ai.vertex_llm_base import VertexBase
|
||||
|
||||
context_edits: Final[dict[str, JsonValue]] = {
|
||||
"edits": [{"type": "clear_tool_uses_20250919"}, {"type": "clear_thinking_20251015"}]
|
||||
}
|
||||
options: Final[dict[str, JsonValue]] = {
|
||||
"thinking": {"type": "enabled", "budget_tokens": 2000},
|
||||
"tool_choice": {"type": "auto"},
|
||||
}
|
||||
messages: Final[list[dict[str, JsonValue]]] = [{"role": "user", "content": "Weather in SF?"}]
|
||||
tools: Final[list[dict[str, JsonValue]]] = [{"name": "get_weather", "input_schema": {"type": "object"}}]
|
||||
sent: Final[SimpleQueue[httpx.Request]] = SimpleQueue()
|
||||
|
||||
def vertex(request: httpx.Request) -> httpx.Response:
|
||||
|
|
@ -2004,11 +1998,10 @@ async def test_vertex_claude_count_sends_the_options_context_edits_and_beta_vert
|
|||
messages=messages,
|
||||
tools=tools,
|
||||
max_tokens=4096,
|
||||
context_management=context_edits,
|
||||
vertex_project="test-project",
|
||||
vertex_location="us-east5",
|
||||
client=client,
|
||||
**options,
|
||||
**request_options,
|
||||
)
|
||||
result: Final = await VertexAITokenCounter().count_tokens(
|
||||
model_to_use="claude-haiku-5-5",
|
||||
|
|
@ -2016,13 +2009,29 @@ async def test_vertex_claude_count_sends_the_options_context_edits_and_beta_vert
|
|||
contents=None,
|
||||
deployment={"litellm_params": {"vertex_project": "test-project", "vertex_location": "us-east5"}},
|
||||
tools=tools,
|
||||
request_options={**options, "context_management": context_edits},
|
||||
request_options=request_options,
|
||||
)
|
||||
|
||||
assert result is not None
|
||||
assert result.total_tokens == 37
|
||||
inference, count = tuple(sent.get() for _ in range(sent.qsize()))
|
||||
assert count.url.path.endswith("count-tokens:rawPredict")
|
||||
return inference, count
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_vertex_claude_count_sends_the_options_context_edits_and_beta_vertex_messages_inference_sends() -> None:
|
||||
inference, count = await _vertex_claude_inference_and_count(
|
||||
[{"name": "get_weather", "input_schema": {"type": "object"}}],
|
||||
{
|
||||
"thinking": {"type": "enabled", "budget_tokens": 2000},
|
||||
"tool_choice": {"type": "auto"},
|
||||
"context_management": {
|
||||
"edits": [{"type": "clear_tool_uses_20250919"}, {"type": "clear_thinking_20251015"}]
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
counted: Final = TypeAdapter(dict[str, JsonValue]).validate_json(count.content)
|
||||
inferred: Final = TypeAdapter(dict[str, JsonValue]).validate_json(inference.content)
|
||||
fields: Final = ("thinking", "tool_choice", "output_config", "context_management")
|
||||
|
|
@ -2030,6 +2039,20 @@ async def test_vertex_claude_count_sends_the_options_context_edits_and_beta_vert
|
|||
assert count.headers["anthropic-beta"] == inference.headers["anthropic-beta"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_vertex_claude_count_sends_the_tool_search_beta_vertex_messages_inference_sends() -> None:
|
||||
inference, count = await _vertex_claude_inference_and_count(
|
||||
[
|
||||
{"type": "tool_search_tool_regex_20251119", "name": "tool_search_tool_regex"},
|
||||
{"name": "get_weather", "input_schema": {"type": "object"}, "defer_loading": True},
|
||||
],
|
||||
{"thinking": {"type": "enabled", "budget_tokens": 2000}},
|
||||
)
|
||||
|
||||
assert count.headers["anthropic-beta"] == inference.headers["anthropic-beta"]
|
||||
assert get_tool_search_beta_header("vertex_ai") in count.headers["anthropic-beta"].split(",")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_vertex_claude_count_refuses_options_vertex_messages_inference_refuses_before_calling_vertex() -> None:
|
||||
result: Final = await VertexAITokenCounter().count_tokens(
|
||||
|
|
|
|||
|
|
@ -83,6 +83,8 @@ from litellm.proxy.proxy_server import (
|
|||
)
|
||||
from litellm.responses.utils import ResponsesAPIRequestUtils
|
||||
from litellm.rust_bridge.fork_guard import ForkedAfterNativeRuntimeStarted
|
||||
from litellm.types.llms.anthropic import ANTHROPIC_BETA_HEADER_VALUES
|
||||
from litellm.types.router import CustomRoutingStrategyBase
|
||||
from litellm.types.secret_managers.main import KeyManagementSettings, KeyManagementSystem
|
||||
from litellm.types.utils import (
|
||||
ADDRESSED_RESPONSE_ID_FIELD,
|
||||
|
|
@ -1739,7 +1741,7 @@ async def test_anthropic_count_tokens_on_a_claude_deployment_sends_the_options_i
|
|||
)
|
||||
|
||||
|
||||
_CALLER_BETA: Final = "web-fetch-2025-09-10"
|
||||
_CALLER_BETA: Final = ANTHROPIC_BETA_HEADER_VALUES.WEB_FETCH_2025_09_10.value
|
||||
|
||||
|
||||
async def _count_on_a_claude_deployment(
|
||||
|
|
@ -1811,9 +1813,9 @@ async def test_anthropic_count_tokens_counts_again_without_the_options_when_the_
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("status_code", (401, 403, 404))
|
||||
async def test_anthropic_count_tokens_answers_the_error_the_count_without_the_options_also_gets(
|
||||
monkeypatch: pytest.MonkeyPatch, status_code: int
|
||||
@pytest.mark.parametrize(("status_code", "counted_again"), ((401, False), (403, False), (404, True), (422, True)))
|
||||
async def test_anthropic_count_tokens_counts_again_only_when_the_claude_count_rejects_the_request(
|
||||
monkeypatch: pytest.MonkeyPatch, status_code: int, counted_again: bool
|
||||
) -> None:
|
||||
response, requests = await _count_on_a_claude_deployment(
|
||||
monkeypatch,
|
||||
|
|
@ -1824,7 +1826,30 @@ async def test_anthropic_count_tokens_answers_the_error_the_count_without_the_op
|
|||
|
||||
assert isinstance(response, JSONResponse)
|
||||
assert response.status_code == status_code
|
||||
assert [bool(_sent_option_paths(json.loads(sent.content))) for sent in requests] == [True, False]
|
||||
assert [bool(_sent_option_paths(json.loads(sent.content))) for sent in requests] == [
|
||||
True,
|
||||
*((False,) if counted_again else ()),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_anthropic_count_tokens_warns_with_every_option_it_counts_again_without(
|
||||
monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture
|
||||
) -> None:
|
||||
verbose_proxy_logger.addHandler(caplog.handler)
|
||||
try:
|
||||
with caplog.at_level(logging.WARNING, logger=verbose_proxy_logger.name):
|
||||
await _count_on_a_claude_deployment(
|
||||
monkeypatch, "anthropic/claude-haiku-5-5", _CLAUDE_COUNT_OPTIONS, _counts_only_without_options
|
||||
)
|
||||
finally:
|
||||
verbose_proxy_logger.removeHandler(caplog.handler)
|
||||
|
||||
warnings: Final = {
|
||||
record.getMessage() for record in caplog.records if "counts again without" in record.getMessage()
|
||||
}
|
||||
(warning,) = warnings
|
||||
assert all(option in warning for option in _CLAUDE_COUNT_OPTIONS)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
@ -1843,6 +1868,179 @@ async def test_anthropic_count_tokens_does_not_count_again_after_a_rate_limit(
|
|||
assert len(requests) == 1
|
||||
|
||||
|
||||
_BEDROCK_RUNTIME_HOST: Final = "bedrock-runtime.us-east-1.amazonaws.com"
|
||||
_BEDROCK_MANTLE_HOST: Final = "bedrock-mantle.us-east-1.api.aws"
|
||||
_BEDROCK_CLAUDE_DEPLOYMENT: Final[dict[str, str]] = {
|
||||
"model": "bedrock/us.anthropic.claude-count-probe-v1:0",
|
||||
"aws_access_key_id": "AKIATESTACCESSKEY",
|
||||
"aws_secret_access_key": "test-secret",
|
||||
"aws_region_name": "us-east-1",
|
||||
}
|
||||
|
||||
|
||||
async def _count_on_a_bedrock_claude_deployment(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
body: Mapping[str, JsonValue],
|
||||
runtime: Callable[[httpx.Request], httpx.Response],
|
||||
mantle: Callable[[httpx.Request], httpx.Response],
|
||||
) -> tuple[object, tuple[httpx.Request, ...]]:
|
||||
import litellm.proxy.proxy_server as proxy_server
|
||||
|
||||
monkeypatch.delenv("AWS_BEARER_TOKEN_BEDROCK", raising=False)
|
||||
monkeypatch.delenv("BEDROCK_MANTLE_API_BASE", raising=False)
|
||||
router: Final = Router(model_list=[{"model_name": "claude-agent", "litellm_params": _BEDROCK_CLAUDE_DEPLOYMENT}])
|
||||
request: Final = _json_request(
|
||||
{"model": "claude-agent", "messages": [{"role": "user", "content": "Weather in SF?"}], **body},
|
||||
"python-httpx/0.28.1",
|
||||
)
|
||||
monkeypatch.setattr(proxy_server, "llm_router", router)
|
||||
monkeypatch.setattr(proxy_server, "proxy_config", proxy_server.ProxyConfig())
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
|
||||
with respx.mock as bedrock:
|
||||
bedrock.route(host=_BEDROCK_RUNTIME_HOST).mock(side_effect=runtime)
|
||||
bedrock.route(host=_BEDROCK_MANTLE_HOST).mock(side_effect=mantle)
|
||||
response: Final = await anthropic_count_tokens(request, UserAPIKeyAuth())
|
||||
return response, tuple(call.request for call in bedrock.calls)
|
||||
|
||||
|
||||
def _runtime_counts_only_without_options(request: httpx.Request) -> httpx.Response:
|
||||
if "invokeModel" in json.loads(request.content)["input"]:
|
||||
return httpx.Response(400, json={"message": "thinking: unsupported"})
|
||||
return httpx.Response(200, json={"inputTokens": 9})
|
||||
|
||||
|
||||
def _mantle_unreachable(request: httpx.Request) -> httpx.Response:
|
||||
raise httpx.ConnectError("bedrock-mantle is unreachable", request=request)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_anthropic_count_tokens_on_bedrock_counts_a_malformed_tool_again_without_the_options(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
response, requests = await _count_on_a_bedrock_claude_deployment(
|
||||
monkeypatch,
|
||||
{
|
||||
"tools": [{"type": 5, "name": "x", "input_schema": {"type": "object"}}],
|
||||
"thinking": {"type": "enabled", "budget_tokens": 2000},
|
||||
},
|
||||
_runtime_counts_only_without_options,
|
||||
_mantle_unreachable,
|
||||
)
|
||||
|
||||
assert not isinstance(response, JSONResponse)
|
||||
assert response["input_tokens"] == 9
|
||||
assert [request.url.host for request in requests] == [_BEDROCK_RUNTIME_HOST]
|
||||
assert "invokeModel" not in json.loads(requests[0].content)["input"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_anthropic_count_tokens_on_bedrock_counts_again_without_the_options_when_mantle_is_unreachable(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
response, requests = await _count_on_a_bedrock_claude_deployment(
|
||||
monkeypatch,
|
||||
{"thinking": {"type": "enabled", "budget_tokens": 2000}},
|
||||
_runtime_counts_only_without_options,
|
||||
_mantle_unreachable,
|
||||
)
|
||||
|
||||
assert not isinstance(response, JSONResponse)
|
||||
assert response["input_tokens"] == 9
|
||||
hosts: Final = [request.url.host for request in requests]
|
||||
assert hosts[0] == hosts[-1] == _BEDROCK_RUNTIME_HOST
|
||||
assert _BEDROCK_MANTLE_HOST in hosts
|
||||
assert "invokeModel" not in json.loads(requests[-1].content)["input"]
|
||||
|
||||
|
||||
_KEEPS_CONTEXT_MANAGEMENT: Final = "sk-ant-keeps-context-management"
|
||||
_DROPS_CONTEXT_MANAGEMENT: Final = "sk-ant-drops-context-management"
|
||||
_CONTEXT_EDITS: Final[dict[str, JsonValue]] = {"edits": [{"type": "clear_tool_uses_20250919"}]}
|
||||
_CLAUDE_MESSAGE_REPLY: Final[dict[str, JsonValue]] = {
|
||||
"id": "msg_1",
|
||||
"type": "message",
|
||||
"role": "assistant",
|
||||
"model": "claude-haiku-5-5",
|
||||
"content": [{"type": "text", "text": "ok"}],
|
||||
"stop_reason": "end_turn",
|
||||
"usage": {"input_tokens": 3, "output_tokens": 1},
|
||||
}
|
||||
|
||||
|
||||
class _RoutesTo(CustomRoutingStrategyBase):
|
||||
def __init__(self, router: Router, api_key: str) -> None:
|
||||
self._router: Final = router
|
||||
self._api_key: Final = api_key
|
||||
|
||||
async def async_get_available_deployment(self, *args: object, **kwargs: object) -> Mapping[str, object]:
|
||||
return next(
|
||||
deployment
|
||||
for deployment in self._router.model_list
|
||||
if deployment["litellm_params"]["api_key"] == self._api_key
|
||||
)
|
||||
|
||||
|
||||
def _mixed_drop_group(routed_key: str) -> Router:
|
||||
router: Final = Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "claude-agent",
|
||||
"litellm_params": {"model": "anthropic/claude-haiku-5-5", "api_key": _KEEPS_CONTEXT_MANAGEMENT},
|
||||
},
|
||||
{
|
||||
"model_name": "claude-agent",
|
||||
"litellm_params": {
|
||||
"model": "anthropic/claude-haiku-5-5",
|
||||
"api_key": _DROPS_CONTEXT_MANAGEMENT,
|
||||
"additional_drop_params": ["context_management"],
|
||||
},
|
||||
},
|
||||
]
|
||||
)
|
||||
router.set_custom_routing_strategy(_RoutesTo(router, routed_key))
|
||||
return router
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("routed_key", (_KEEPS_CONTEXT_MANAGEMENT, _DROPS_CONTEXT_MANAGEMENT))
|
||||
async def test_anthropic_count_tokens_in_a_mixed_drop_group_sends_context_management_when_the_routed_inference_does(
|
||||
monkeypatch: pytest.MonkeyPatch, routed_key: str
|
||||
) -> None:
|
||||
import litellm.proxy.proxy_server as proxy_server
|
||||
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
monkeypatch.setattr(proxy_server, "llm_router", _mixed_drop_group(routed_key))
|
||||
monkeypatch.setattr(proxy_server, "proxy_config", proxy_server.ProxyConfig())
|
||||
messages: Final = [{"role": "user", "content": "Weather in SF?"}]
|
||||
|
||||
with respx.mock:
|
||||
inference_route: Final = respx.post("https://api.anthropic.com/v1/messages").mock(
|
||||
return_value=httpx.Response(200, json=_CLAUDE_MESSAGE_REPLY)
|
||||
)
|
||||
count_route: Final = respx.post("https://api.anthropic.com/v1/messages/count_tokens").mock(
|
||||
return_value=httpx.Response(200, json={"input_tokens": 7})
|
||||
)
|
||||
await _mixed_drop_group(routed_key).aanthropic_messages(
|
||||
model="claude-agent", messages=messages, max_tokens=16, context_management=_CONTEXT_EDITS
|
||||
)
|
||||
response: Final = await anthropic_count_tokens(
|
||||
_json_request(
|
||||
{"model": "claude-agent", "messages": messages, "context_management": _CONTEXT_EDITS},
|
||||
"python-httpx/0.28.1",
|
||||
),
|
||||
UserAPIKeyAuth(),
|
||||
)
|
||||
|
||||
assert not isinstance(response, JSONResponse)
|
||||
inference: Final = inference_route.calls.last.request
|
||||
count: Final = count_route.calls.last.request
|
||||
assert inference.headers["x-api-key"] == count.headers["x-api-key"] == routed_key
|
||||
assert ("context_management" in json.loads(count.content)) == (
|
||||
"context_management" in json.loads(inference.content)
|
||||
)
|
||||
assert ("context_management" in json.loads(count.content)) == (routed_key == _KEEPS_CONTEXT_MANAGEMENT)
|
||||
|
||||
|
||||
async def _claude_inference_body(model: str, options: Mapping[str, JsonValue]) -> dict[str, JsonValue]:
|
||||
with respx.mock:
|
||||
route: Final = respx.post("https://api.anthropic.com/v1/messages").mock(
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue