From 502ce4a8870aed0584cb537c2195f42ac1b21faf Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 6 Nov 2026 19:44:41 -0800 Subject: [PATCH] fix(count_tokens): count with the deployment's model, add the effort beta on Bedrock runtime counts, and match what Converse sends The count now transforms with the deployment's own inference model, falling back to the counted model, so a stand-in Mantle count sees the native output_config.format the deployment sends. Bedrock runtime CountTokens refuses output_config.effort without effort-2025-11-24, so the runtime count adds it while Mantle keeps inference's betas. A Converse deployment's count leaves out tool_choice.disable_parallel_tool_use, which the Converse bridge never sends. Only the transform's own errors on the caller's input become the 400 that counts again without the options, and the request's drop_params is normalized the way inference reads it --- .../anthropic/count_tokens/transformation.py | 69 ++++++-- .../count_tokens/bedrock_token_counter.py | 64 ++++++-- litellm/proxy/proxy_server.py | 4 +- .../test_bedrock_token_counter.py | 152 +++++++++++++++--- tests/unit/proxy/test_proxy_token_counter.py | 24 ++- 5 files changed, 256 insertions(+), 57 deletions(-) diff --git a/litellm/llms/anthropic/count_tokens/transformation.py b/litellm/llms/anthropic/count_tokens/transformation.py index b9427216637..e6fcf040bae 100644 --- a/litellm/llms/anthropic/count_tokens/transformation.py +++ b/litellm/llms/anthropic/count_tokens/transformation.py @@ -10,6 +10,7 @@ from dataclasses import dataclass from itertools import chain from types import MappingProxyType from typing import ( + TYPE_CHECKING, Final, Protocol, TypeAlias, @@ -32,6 +33,9 @@ from litellm.llms.base_llm.anthropic_messages.transformation import BaseAnthropi from litellm.types.llms.openai import AllMessageValues, ChatCompletionImageObject, ValidUserMessageContentTypes from litellm.types.utils import LlmProviders, TokenCountMessagesFormat, TokenCountResponse +if TYPE_CHECKING: + from litellm.types.router import GenericLiteLLMParams + _COUNT_REQUEST: Final = TypeAdapter(dict[str, JsonValue]) _SYSTEM_BLOCKS: Final = TypeAdapter(list[JsonValue]) _IMAGE_BLOCK: Final = TypeAdapter(ChatCompletionImageObject) @@ -353,21 +357,55 @@ def is_request_rejection(status_code: int) -> bool: return 400 <= status_code < 500 and status_code not in _STATUSES_NOT_CAUSED_BY_THE_REQUEST +def deployment_inference_model(deployment: Mapping[str, object] | None) -> str | None: + deployment_params: Final = _INFERENCE_PARAMS.validate_python((deployment or {}).get("litellm_params") or {}) + deployment_model: Final = deployment_params.get("model") + if not isinstance(deployment_model, str): + return None + from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider + + provider_hint: Final = deployment_params.get("custom_llm_provider") + api_base: Final = deployment_params.get("api_base") + inference_model, _, _, _ = get_llm_provider( + model=deployment_model, + custom_llm_provider=provider_hint if isinstance(provider_hint, str) else None, + api_base=api_base if isinstance(api_base, str) else None, + ) + return inference_model + + +def _messages_transform(model: str, provider: LlmProviders) -> tuple[str, BaseAnthropicMessagesConfig] | None: + from litellm.utils import ProviderConfigManager + + messages_config: Final = ProviderConfigManager.get_provider_anthropic_messages_config(model, provider) + return None if messages_config is None else (model, messages_config) + + +def _transform_litellm_params(deployment: Mapping[str, object] | None) -> "GenericLiteLLMParams": + from litellm.types.router import GenericLiteLLMParams + + resolved_deployment: Final = deployment or {} + deployment_params: Final = _INFERENCE_PARAMS.validate_python(resolved_deployment.get("litellm_params") or {}) + return GenericLiteLLMParams.model_validate( + { + "drop_params": deployment_params.get("drop_params"), + "model_info": resolved_deployment.get("model_info"), + } + ) + + def _claude_count_options_for( messages_config: BaseAnthropicMessagesConfig, model: str, + litellm_params: "GenericLiteLLMParams", provider: LlmProviders, messages: Sequence[Mapping[str, object]], tools: Sequence[Mapping[str, object]] | None, sent: Mapping[str, JsonValue], anthropic_beta: str | None, - deployment: Mapping[str, object] | None, ) -> ClaudeCountOptions: from litellm.anthropic_beta_headers_manager import update_headers_with_filtered_beta - from litellm.types.router import GenericLiteLLMParams - resolved_deployment: Final = deployment or {} - deployment_params: Final = _INFERENCE_PARAMS.validate_python(resolved_deployment.get("litellm_params") or {}) leading_system, conversation = split_leading_system_run(messages) sent_conversation: Final = copy.deepcopy([dict(message) for message in conversation]) sent_params: Final = { @@ -386,12 +424,7 @@ def _claude_count_options_for( model=model, messages=sent_conversation, anthropic_messages_optional_request_params={**sent_params, "max_tokens": _MAX_TOKENS_ABOVE_ANY_BUDGET}, - litellm_params=GenericLiteLLMParams.model_validate( - { - "drop_params": deployment_params.get("drop_params"), - "model_info": resolved_deployment.get("model_info"), - } - ), + litellm_params=litellm_params, headers=sent_headers, ) transformed: Final = _ANTHROPIC_CHAT_REQUEST.validate_python(request) @@ -415,20 +448,24 @@ def claude_count_options( sent: Final = {name: value for name, value in request_options.items() if value is not None} if not sent: return NO_CLAUDE_COUNT_OPTIONS + inference_model: Final = deployment_inference_model(deployment) + transform: Final = ( + None if inference_model is None else _messages_transform(inference_model, provider) + ) or _messages_transform(model, provider) + if transform is None: + return NO_CLAUDE_COUNT_OPTIONS + transform_model, messages_config = transform + litellm_params: Final = _transform_litellm_params(deployment) from litellm.exceptions import BadRequestError from litellm.llms.anthropic.common_utils import AnthropicError - from litellm.utils import ProviderConfigManager - messages_config: Final = ProviderConfigManager.get_provider_anthropic_messages_config(model, provider) - if messages_config is None: - return NO_CLAUDE_COUNT_OPTIONS try: return _claude_count_options_for( - messages_config, model, provider, messages, tools, sent, anthropic_beta, deployment + messages_config, transform_model, litellm_params, provider, messages, tools, sent, anthropic_beta ) except (AnthropicError, BadRequestError) as e: return ChatCountTranslationError(message=str(e.message)) - except (AttributeError, IndexError, KeyError, TypeError, ValueError) as e: + except (AttributeError, KeyError, TypeError, ValueError) as e: return ChatCountTranslationError(message=f"Invalid request for token counting: {e}") diff --git a/litellm/llms/bedrock/count_tokens/bedrock_token_counter.py b/litellm/llms/bedrock/count_tokens/bedrock_token_counter.py index 2846ad858fa..5889b3d3d2e 100644 --- a/litellm/llms/bedrock/count_tokens/bedrock_token_counter.py +++ b/litellm/llms/bedrock/count_tokens/bedrock_token_counter.py @@ -19,6 +19,7 @@ from litellm.llms.anthropic.count_tokens.transformation import ( ClaudeCountOptions, anthropic_count_fields, claude_count_options, + deployment_inference_model, is_request_rejection, ) from litellm.llms.base_llm.base_utils import BaseTokenCounter @@ -27,6 +28,7 @@ from litellm.llms.bedrock.count_tokens.handler import BedrockCountTokensHandler from litellm.llms.bedrock.count_tokens.mantle_handler import BedrockMantleCountTokensHandler from litellm.llms.bedrock_mantle.common_utils import is_mantle_claude_model from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler +from litellm.types.llms.anthropic import ANTHROPIC_EFFORT_BETA_HEADER from litellm.types.utils import ( CountContextManagement, LiteLLMPydanticObjectBase, @@ -85,10 +87,41 @@ def _mantle_answered_for_runtime(fallback: CountTokensOutcome, forwarded_options return not forwarded_options or isinstance(fallback, CountedTokens) or is_request_rejection(fallback.status_code) -def _invoke_fields(claude_options: ClaudeCountOptions) -> Mapping[str, object]: +def _invoke_fields(options: Mapping[str, JsonValue], betas: Sequence[str]) -> Mapping[str, object]: + return {**options, **({"anthropic_beta": list(betas)} if betas else {})} + + +def _sets_effort(options: Mapping[str, JsonValue]) -> bool: + output_config: Final = options.get("output_config") + return isinstance(output_config, dict) and "effort" in output_config + + +def _runtime_count_betas(claude_options: ClaudeCountOptions) -> tuple[str, ...]: + if not _sets_effort(claude_options.options) or ANTHROPIC_EFFORT_BETA_HEADER in claude_options.betas: + return claude_options.betas + return (*claude_options.betas, ANTHROPIC_EFFORT_BETA_HEADER) + + +def _runs_on_converse(inference_model: str) -> bool: + from litellm.llms.bedrock.common_utils import BedrockModelInfo + + return BedrockModelInfo.get_bedrock_provider_config_for_messages_api(inference_model) is None + + +def _options_converse_sends( + request_options: Mapping[str, JsonValue], inference_model: str | None +) -> Mapping[str, JsonValue]: + tool_choice: Final = request_options.get("tool_choice") + if ( + not isinstance(tool_choice, dict) + or "disable_parallel_tool_use" not in tool_choice + or inference_model is None + or not _runs_on_converse(inference_model) + ): + return request_options return { - **claude_options.options, - **({"anthropic_beta": list(claude_options.betas)} if claude_options.betas else {}), + **request_options, + "tool_choice": {key: value for key, value in tool_choice.items() if key != "disable_parallel_tool_use"}, } @@ -193,7 +226,7 @@ class BedrockTokenCounter(BaseTokenCounter): LlmProviders.BEDROCK_MANTLE if use_mantle else LlmProviders.BEDROCK, sent_messages, sent_tools, - request_options, + _options_converse_sends(request_options, deployment_inference_model(deployment)), anthropic_beta, deployment, ) @@ -202,17 +235,22 @@ class BedrockTokenCounter(BaseTokenCounter): ) if isinstance(claude_options, ChatCountTranslationError): return claude_options.response(request_model, model_to_use, primary_tokenizer) - invoke_fields: Final = _invoke_fields(claude_options) + invoke_fields: Final = _invoke_fields(claude_options.options, claude_options.betas) + counted_request: Final = _request_data( + model_to_use, + sent_messages if claude_options.messages is None else claude_options.messages, + sent_tools if claude_options.messages is None else claude_options.tools, + system if translated is None else translated.system, + ) runtime_request: Final = { - **_request_data( - model_to_use, - sent_messages if claude_options.messages is None else claude_options.messages, - sent_tools if claude_options.messages is None else claude_options.tools, - system if translated is None else translated.system, - ), - **invoke_fields, + **counted_request, + **_invoke_fields(claude_options.options, _runtime_count_betas(claude_options)), + } + mantle_request: Final = { + **counted_request, + **invoke_fields, + **({} if translated is None else translated.options), } - mantle_request: Final = runtime_request if translated is None else {**runtime_request, **translated.options} primary: Final = ( await self._count_with( diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 3f5b02ac63c..118b22859ab 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -14943,9 +14943,9 @@ async def _count_retrying_without_options( if not (isinstance(counted, ProviderCountFailure) and counted.rejected_request): return counted verbose_proxy_logger.warning( - "Token count for %s answered %s with the request's %s, so it counts again without them", + "Token count for %s %s with the request's %s, so it counts again without them and the caller's betas", model, - counted.status_code, + "could not be built" if counted.rejected_before_provider_call else f"answered {counted.status_code}", ", ".join(forwarded_options), ) return await count_with(model, region, request_options=MappingProxyType({})) diff --git a/tests/unit/llms/bedrock/count_tokens/test_bedrock_token_counter.py b/tests/unit/llms/bedrock/count_tokens/test_bedrock_token_counter.py index 61ee1729821..bf3627ff7f7 100644 --- a/tests/unit/llms/bedrock/count_tokens/test_bedrock_token_counter.py +++ b/tests/unit/llms/bedrock/count_tokens/test_bedrock_token_counter.py @@ -353,6 +353,14 @@ _THINKING_AND_TOOL_CHOICE: Final[dict[str, JsonValue]] = { _WEATHER_TOOL: Final[dict[str, JsonValue]] = {"name": "get_weather", "input_schema": {"type": "object"}} +_CITY_SCHEMA: Final[dict[str, JsonValue]] = { + "type": "object", + "properties": {"city": {"type": "string"}}, + "required": ["city"], +} +_CITY_FORMAT: Final[dict[str, JsonValue]] = {"output_config": {"format": {"type": "json_schema", "schema": _CITY_SCHEMA}}} + + _CONTEXT_EDITING_OPTIONS: Final[dict[str, JsonValue]] = {**_THINKING_AND_TOOL_CHOICE, "context_management": _CONTEXT_EDITS} @@ -362,9 +370,10 @@ async def _messages_inference_and_count( request_options: Mapping[str, JsonValue] = _CONTEXT_EDITING_OPTIONS, tool: Mapping[str, JsonValue] = _WEATHER_TOOL, runtime: tuple[int, Mapping[str, object]] = (200, {"inputTokens": 412}), + inference_reply: Mapping[str, object] = _CLAUDE_REPLY, ) -> tuple[_Bedrock, _Bedrock]: credentials: Final = DEPLOYMENT["litellm_params"] - inference: Final = _Bedrock(runtime=(200, _CLAUDE_REPLY), mantle=(200, _CLAUDE_REPLY)) + inference: Final = _Bedrock(runtime=(200, inference_reply), mantle=(200, inference_reply)) await litellm.anthropic_messages( model=deployment_model, messages=MESSAGES, @@ -389,11 +398,17 @@ async def _messages_inference_and_count( return inference, bedrock +def _betas_besides_effort(body: Mapping[str, JsonValue]) -> tuple[JsonValue, ...]: + betas: Final = body.get("anthropic_beta") or [] + assert isinstance(betas, list) + return tuple(beta for beta in betas if beta != ANTHROPIC_EFFORT_BETA_HEADER) + + @pytest.mark.asyncio @pytest.mark.parametrize( ("deployment_model", "model_to_use"), ( - ("bedrock/invoke/global.anthropic.claude-haiku-5-5", "global.anthropic.claude-haiku-5-5"), + ("bedrock/invoke/us.anthropic.claude-sonnet-4-6", "us.anthropic.claude-sonnet-4-6"), ("bedrock/invoke/us.anthropic.claude-sonnet-4-5-20250929-v1:0", "us.anthropic.claude-sonnet-4-5-20250929-v1:0"), ), ) @@ -405,8 +420,9 @@ async def test_runtime_count_sends_the_options_and_context_edits_invoke_inferenc assert bedrock.posted_hosts() == (RUNTIME_HOST,) counted: Final = _invoke_body(bedrock.requests[0]) sent: Final = _JSON_BODY.validate_json(inference.requests[0].content) - fields: Final = ("thinking", "tool_choice", "output_config", "context_management", "anthropic_beta") + fields: Final = ("thinking", "tool_choice", "output_config", "context_management") assert {name: counted.get(name) for name in fields} == {name: sent.get(name) for name in fields} + assert _betas_besides_effort(counted) == _betas_besides_effort(sent) thinking: Final = counted["thinking"] assert isinstance(thinking, dict) budget_tokens: Final = thinking.get("budget_tokens", 0) @@ -433,7 +449,7 @@ _DISPLAY_THINKING: Final[dict[str, JsonValue]] = {"thinking": {"type": "adaptive def _sent_betas(request: httpx.Request) -> tuple[str, ...]: - return tuple(sorted(request.headers["anthropic-beta"].split(","))) + return tuple(sorted(beta for beta in request.headers.get("anthropic-beta", "").split(",") if beta)) @pytest.mark.asyncio @@ -462,20 +478,109 @@ async def test_runtime_count_sends_the_betas_invoke_inference_sends( assert isinstance(counted_betas, list) and expected_beta in counted_betas +_EFFORT: Final[dict[str, JsonValue]] = {"output_config": {"effort": "low"}} +_CLEAR_THINKING_EDITS: Final[dict[str, JsonValue]] = { + "context_management": {"edits": [{"type": "clear_thinking_20251015"}]} +} + + @pytest.mark.asyncio -async def test_invoke_count_retried_on_mantle_sends_the_betas_invoke_inference_sends() -> None: +@pytest.mark.parametrize("request_options", (_DISPLAY_THINKING, _EFFORT), ids=("thinking_display", "effort")) +async def test_invoke_count_retried_on_mantle_sends_the_betas_invoke_inference_sends( + request_options: dict[str, JsonValue], +) -> None: inference, bedrock = await _messages_inference_and_count( "bedrock/invoke/us.anthropic.claude-opus-5-5", "us.anthropic.claude-opus-5-5", - _DISPLAY_THINKING, + request_options, runtime=(400, UNSUPPORTED), ) assert bedrock.posted_hosts() == (RUNTIME_HOST, MANTLE_HOST) - sent_betas: Final = _JSON_BODY.validate_json(inference.requests[0].content).get("anthropic_beta") + sent_betas: Final = _JSON_BODY.validate_json(inference.requests[0].content).get("anthropic_beta", []) assert isinstance(sent_betas, list) assert _sent_betas(bedrock.requests[1]) == tuple(sorted(sent_betas)) - assert ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER in sent_betas + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("deployment_model", "request_options"), + ( + ("bedrock/invoke/us.anthropic.claude-sonnet-4-6", _EFFORT), + ("bedrock/invoke/us.anthropic.claude-sonnet-4-6", _CLEAR_THINKING_EDITS), + ("bedrock/invoke/us.anthropic.claude-opus-4-5-20251101-v1:0", _EFFORT), + ), + ids=("effort", "clear_thinking_sets_effort", "inference_sends_the_effort_beta"), +) +async def test_runtime_count_of_an_effort_sends_the_effort_beta_once_beside_the_betas_invoke_inference_sends( + deployment_model: str, request_options: dict[str, JsonValue] +) -> None: + inference, bedrock = await _messages_inference_and_count( + deployment_model, deployment_model.removeprefix("bedrock/invoke/"), request_options + ) + + assert bedrock.posted_hosts() == (RUNTIME_HOST,) + counted: Final = _invoke_body(bedrock.requests[0]) + sent: Final = _JSON_BODY.validate_json(inference.requests[0].content) + counted_config: Final = counted.get("output_config") + assert isinstance(counted_config, dict) and counted_config.get("effort") == "low" + counted_betas: Final = counted.get("anthropic_beta") + sent_betas: Final = sent.get("anthropic_beta", []) + assert isinstance(counted_betas, list) and isinstance(sent_betas, list) + assert counted_betas.count(ANTHROPIC_EFFORT_BETA_HEADER) == 1 + assert set(counted_betas) - {ANTHROPIC_EFFORT_BETA_HEADER} == set(sent_betas) - {ANTHROPIC_EFFORT_BETA_HEADER} + + +_CONVERSE_REPLY: Final = { + "output": {"message": {"role": "assistant", "content": [{"text": "ok"}]}}, + "stopReason": "end_turn", + "usage": {"inputTokens": 3, "outputTokens": 1, "totalTokens": 4}, + "metrics": {"latencyMs": 1}, +} +_NO_PARALLEL_TOOL_USE: Final[dict[str, JsonValue]] = { + "tool_choice": {"type": "auto", "disable_parallel_tool_use": True} +} + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("deployment_model", "inference_reply"), + ( + ("bedrock/converse/us.anthropic.claude-sonnet-4-6", _CONVERSE_REPLY), + ("bedrock/converse/us.anthropic.claude-opus-4-6-v1", _CONVERSE_REPLY), + ("bedrock/invoke/us.anthropic.claude-sonnet-4-6", _CLAUDE_REPLY), + ), + ids=("converse", "converse_parallel_tool_use_config_model", "invoke"), +) +async def test_runtime_count_sends_disable_parallel_tool_use_only_when_messages_inference_sends_it( + deployment_model: str, inference_reply: Mapping[str, object] +) -> None: + inference, bedrock = await _messages_inference_and_count( + deployment_model, + deployment_model.split("/")[-1], + _NO_PARALLEL_TOOL_USE, + inference_reply=inference_reply, + ) + + assert bedrock.posted_hosts() == (RUNTIME_HOST,) + counted_tool_choice: Final = _invoke_body(bedrock.requests[0]).get("tool_choice") + assert isinstance(counted_tool_choice, dict) and counted_tool_choice.get("type") == "auto" + assert ("disable_parallel_tool_use" in counted_tool_choice) == ( + b"disable_parallel_tool_use" in inference.requests[0].content + ) + + +@pytest.mark.asyncio +async def test_count_on_a_stand_in_model_counts_the_messages_and_output_config_the_deployment_model_sends() -> None: + inference, bedrock = await _messages_inference_and_count( + "bedrock/us.anthropic.claude-haiku-5-5", "bedrock_mantle/us.anthropic.claude-opus-5-5", _CITY_FORMAT + ) + + assert bedrock.posted_hosts() == (MANTLE_HOST,) + counted: Final = _JSON_BODY.validate_json(bedrock.requests[0].content) + sent: Final = _JSON_BODY.validate_json(inference.requests[0].content) + fields: Final = ("messages", "output_config") + assert {name: counted.get(name) for name in fields} == {name: sent.get(name) for name in fields} @pytest.mark.asyncio @@ -573,14 +678,6 @@ async def test_runtime_count_sends_the_tools_invoke_inference_sends() -> None: assert _invoke_body(bedrock.requests[0])["tools"] == _JSON_BODY.validate_json(inference.requests[0].content)["tools"] -_CITY_SCHEMA: Final[dict[str, JsonValue]] = { - "type": "object", - "properties": {"city": {"type": "string"}}, - "required": ["city"], -} -_CITY_FORMAT: Final[dict[str, JsonValue]] = {"output_config": {"format": {"type": "json_schema", "schema": _CITY_SCHEMA}}} - - def _texts(message: JsonValue) -> tuple[str, ...]: content: Final = message.get("content") if isinstance(message, dict) else None if not isinstance(content, list): @@ -634,17 +731,34 @@ async def test_runtime_count_counts_the_messages_bedrock_invoke_inference_sends( assert (counted == MESSAGES) is not schema_inlined +_THINKING: Final[dict[str, JsonValue]] = {"thinking": {"type": "enabled", "budget_tokens": 2000}} + + @pytest.mark.asyncio -async def test_claude_count_refuses_options_bedrock_messages_inference_refuses_without_calling_bedrock() -> None: +@pytest.mark.parametrize( + ("messages", "tools", "request_options"), + ( + (MESSAGES, None, {"output_config": {"effort": "fastest"}}), + (MESSAGES, [{"type": 5, "name": "x", "input_schema": {"type": "object"}}], _THINKING), + (MESSAGES, ["get_weather"], _THINKING), + ([{"role": "user", "content": 5}], None, _THINKING), + ([{"role": "user", "content": [{"text": "hi"}]}], None, _THINKING), + ), + ids=("option_refused", "tool_type_not_a_string", "tool_not_a_mapping", "content_not_iterable", "block_without_type"), +) +async def test_claude_count_refuses_options_bedrock_messages_inference_refuses_without_calling_bedrock( + messages: list[dict[str, JsonValue]], tools: list[JsonValue] | None, request_options: dict[str, JsonValue] +) -> None: bedrock: Final = _Bedrock(runtime=(200, {"inputTokens": 412}), mantle=(200, {"input_tokens": 2177})) result: Final = await _counter(bedrock).count_tokens( model_to_use="us.anthropic.claude-sonnet-4-5-20250929-v1:0", - messages=MESSAGES, + messages=messages, contents=None, deployment=DEPLOYMENT, request_model="claude-sonnet-4-5", - request_options={"output_config": {"effort": "fastest"}}, + tools=tools, + request_options=request_options, ) assert result is not None diff --git a/tests/unit/proxy/test_proxy_token_counter.py b/tests/unit/proxy/test_proxy_token_counter.py index ae272cef1a9..6fa522331a4 100644 --- a/tests/unit/proxy/test_proxy_token_counter.py +++ b/tests/unit/proxy/test_proxy_token_counter.py @@ -1915,15 +1915,21 @@ def _mantle_unreachable(request: httpx.Request) -> httpx.Response: @pytest.mark.asyncio -async def test_anthropic_count_tokens_on_bedrock_counts_a_malformed_tool_again_without_the_options( - monkeypatch: pytest.MonkeyPatch, +@pytest.mark.parametrize( + "malformed", + ( + {"tools": [{"type": 5, "name": "x", "input_schema": {"type": "object"}}]}, + {"messages": [{"role": "user", "content": 5}]}, + {"messages": [{"role": "user", "content": [{"text": "Weather in SF?"}]}]}, + ), + ids=("tool_type_not_a_string", "content_not_iterable", "block_without_type"), +) +async def test_anthropic_count_tokens_on_bedrock_counts_a_malformed_request_again_without_the_options( + monkeypatch: pytest.MonkeyPatch, malformed: dict[str, JsonValue] ) -> None: response, requests = await _count_on_a_bedrock_claude_deployment( monkeypatch, - { - "tools": [{"type": 5, "name": "x", "input_schema": {"type": "object"}}], - "thinking": {"type": "enabled", "budget_tokens": 2000}, - }, + {**malformed, "thinking": {"type": "enabled", "budget_tokens": 2000}}, _runtime_counts_only_without_options, _mantle_unreachable, ) @@ -2070,7 +2076,11 @@ async def _claude_inference_body(model: str, options: Mapping[str, JsonValue]) - @pytest.mark.asyncio @pytest.mark.parametrize( ("request_drop_params", "user_agent"), - (({"drop_params": True}, "python-httpx/0.28.1"), ({}, "claude-cli/2.1.0 (external, cli)")), + ( + ({"drop_params": True}, "python-httpx/0.28.1"), + ({"drop_params": " true "}, "python-httpx/0.28.1"), + ({}, "claude-cli/2.1.0 (external, cli)"), + ), ) async def test_anthropic_count_tokens_on_a_claude_deployment_sends_the_thinking_drop_params_makes_inference_send( monkeypatch: pytest.MonkeyPatch, request_drop_params: dict[str, JsonValue], user_agent: str