fix(count_tokens): filter a Bedrock deployment's Mantle count betas the way its invoke inference does

A bedrock/ deployment whose count goes to Mantle (the cost map sends opus-5-5 counts there) built its count options with the Mantle config, which forwards every caller beta, while its invoke inference drops the betas the bedrock row marks unsupported and keeps only the model's keep list. The count now uses the deployment's own Bedrock config, so the Mantle count sends the same anthropic-beta its inference sends. A Mantle count for a bedrock_mantle deployment, or with no deployment model, still uses the Mantle config

DeploymentBetaFilter.filtered now returns a filtered copy and leaves the caller's headers alone, and deployment_inference_model reuses _deployment_api_base

Tests now cover the count-again-without-options flag on the Anthropic, Azure AI, Vertex and Bedrock counters, a count that forwards only the caller's betas, a Foundry host that comes from ANTHROPIC_API_BASE, a direct Anthropic deployment that sends betas its row marks unsupported, and a runtime count with no deployment
This commit is contained in:
mateo-berri 2026-11-07 01:35:31 -08:00
parent 4bdba8bbd5
commit 77f0f3104a
7 changed files with 267 additions and 85 deletions

View file

@ -481,9 +481,9 @@ class DeploymentBetaFilter:
api_base: str | None
unlisted: UnlistedBetas
def filtered(self, headers: dict[str, str], keep: frozenset[str]) -> dict[str, str]:
def filtered(self, headers: Mapping[str, str], keep: frozenset[str]) -> dict[str, str]:
return update_headers_with_filtered_beta(
headers=headers, provider=self.provider, api_base=self.api_base, unlisted=self.unlisted, keep=keep
headers=dict(headers), provider=self.provider, api_base=self.api_base, unlisted=self.unlisted, keep=keep
)

View file

@ -369,6 +369,11 @@ def is_request_rejection(status_code: int) -> bool:
return 400 <= status_code < 500 and status_code not in _STATUSES_NOT_CAUSED_BY_THE_REQUEST
def _deployment_api_base(deployment: Mapping[str, object] | None) -> str | None:
api_base: Final = _INFERENCE_PARAMS.validate_python((deployment or {}).get("litellm_params") or {}).get("api_base")
return api_base if isinstance(api_base, str) else None
def deployment_inference_model(deployment: Mapping[str, object] | None) -> str | None:
deployment_params: Final = _INFERENCE_PARAMS.validate_python((deployment or {}).get("litellm_params") or {})
deployment_model: Final = deployment_params.get("model")
@ -377,20 +382,14 @@ def deployment_inference_model(deployment: Mapping[str, object] | None) -> str |
from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
provider_hint: Final = deployment_params.get("custom_llm_provider")
api_base: Final = deployment_params.get("api_base")
inference_model, _, _, _ = get_llm_provider(
model=deployment_model,
custom_llm_provider=provider_hint if isinstance(provider_hint, str) else None,
api_base=api_base if isinstance(api_base, str) else None,
api_base=_deployment_api_base(deployment),
)
return inference_model
def _deployment_api_base(deployment: Mapping[str, object] | None) -> str | None:
api_base: Final = _INFERENCE_PARAMS.validate_python((deployment or {}).get("litellm_params") or {}).get("api_base")
return api_base if isinstance(api_base, str) else None
def _messages_transform(model: str, provider: LlmProviders) -> tuple[str, BaseAnthropicMessagesConfig] | None:
from litellm.utils import ProviderConfigManager
@ -434,7 +433,7 @@ def _claude_count_options_for(
dict(beta_headers)
if beta_filter is None
else beta_filter.filtered(
dict(beta_headers), keep=messages_config.model_specific_anthropic_betas(model, litellm_params)
beta_headers, keep=messages_config.model_specific_anthropic_betas(model, litellm_params)
)
)
request: Final = messages_config.transform_anthropic_messages_request( # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # the base config signature takes and returns bare dicts

View file

@ -68,18 +68,34 @@ class CountTokensFailure:
CountTokensOutcome = CountedTokens | CountTokensFailure
def _claude_count_translator(litellm_params: Mapping[str, object]) -> AnthropicConfig:
def _deployment_route(litellm_params: Mapping[str, object]) -> tuple[str, str] | None:
deployment_model: Final = litellm_params.get("model")
if not isinstance(deployment_model, str):
return AnthropicConfig()
return None
provider_hint: Final = litellm_params.get("custom_llm_provider")
model, provider, _, _ = litellm.get_llm_provider(
model=deployment_model, custom_llm_provider=provider_hint if isinstance(provider_hint, str) else None
)
return model, provider
def _claude_count_translator(litellm_params: Mapping[str, object]) -> AnthropicConfig:
route: Final = _deployment_route(litellm_params)
if route is None:
return AnthropicConfig()
model, provider = route
chat_config: Final = litellm.ProviderConfigManager.get_provider_chat_config(model, LlmProviders(provider))
return chat_config if isinstance(chat_config, AnthropicConfig) else AnthropicConfig()
def _claude_options_provider(litellm_params: Mapping[str, object], use_mantle: bool) -> LlmProviders:
if not use_mantle:
return LlmProviders.BEDROCK
route: Final = _deployment_route(litellm_params)
deployment_provider: Final = None if route is None else route[1]
return LlmProviders.BEDROCK if deployment_provider == LlmProviders.BEDROCK.value else LlmProviders.BEDROCK_MANTLE
def _runtime_rejected_claude_model(outcome: CountTokensOutcome, resolved_model: str) -> bool:
return (
isinstance(outcome, CountTokensFailure)
@ -264,7 +280,7 @@ class BedrockTokenCounter(BaseTokenCounter):
claude_options: Final = (
claude_count_options(
counted_model,
LlmProviders.BEDROCK_MANTLE if use_mantle else LlmProviders.BEDROCK,
_claude_options_provider(litellm_params, use_mantle),
sent_messages,
sent_tools,
_options_converse_sends(request_options, deployment_inference_model(deployment)),

View file

@ -567,7 +567,7 @@ async def _counted_requests(
request_options: Mapping[str, JsonValue] = MappingProxyType({}),
anthropic_beta: str | None = None,
model: str = "claude-opus-5-5",
api_base: str = "https://gateway.example",
api_base: str | None = "https://gateway.example",
) -> tuple[TokenCountResponse | None, tuple[httpx.Request, ...]]:
sent: Final[SimpleQueue[httpx.Request]] = SimpleQueue()
@ -581,7 +581,12 @@ async def _counted_requests(
model_to_use=model,
messages=messages,
contents=None,
deployment={"litellm_params": {"api_key": "sk-ant-api03-test-key", "api_base": api_base}},
deployment={
"litellm_params": {
"api_key": "sk-ant-api03-test-key",
**({} if api_base is None else {"api_base": api_base}),
}
},
request_model=model,
tools=tools,
messages_format=messages_format,
@ -973,7 +978,7 @@ async def _messages_inference_request(
request_options: Mapping[str, JsonValue],
model: str = "claude-haiku-5-5",
extra_headers: Mapping[str, str] = MappingProxyType({}),
api_base: str = "https://gateway.example",
api_base: str | None = "https://gateway.example",
) -> httpx.Request:
sent: Final[SimpleQueue[httpx.Request]] = SimpleQueue()
@ -1107,23 +1112,16 @@ _FOUNDRY_BASE: Final = "https://my-resource.services.ai.azure.com/anthropic"
_UNLISTED_BETA: Final = "unlisted-beta-2099-01-01"
@pytest.mark.asyncio
@pytest.mark.usefixtures("httpx_transport_clients", "local_beta_headers_config")
@pytest.mark.parametrize(
("counter_type", "expected_betas"),
((AnthropicTokenCounter, frozenset({_UNLISTED_BETA})), (AzureAIAnthropicTokenCounter, frozenset())),
ids=("anthropic-deployment-forwards-unlisted", "azure-ai-deployment-drops-unlisted"),
)
async def test_counter_on_a_foundry_host_drops_the_caller_betas_messages_inference_drops_there(
counter_type: type[AnthropicTokenCounter] | type[AzureAIAnthropicTokenCounter], expected_betas: frozenset[str]
) -> None:
async def _foundry_count_and_inference_betas(
counter_type: type[AnthropicTokenCounter] | type[AzureAIAnthropicTokenCounter], api_base: str | None
) -> tuple[frozenset[str], frozenset[str]]:
caller_betas: Final = f"mcp-servers-2025-12-04,{_UNLISTED_BETA}"
inference: Final = await _messages_inference_request(
counter_type,
f"{_FOUNDRY_BASE}/v1/messages",
{"thinking": _ENABLED_THINKING},
extra_headers={"anthropic-beta": caller_betas},
api_base=_FOUNDRY_BASE,
api_base=api_base,
)
_, (count,) = await _counted_requests(
counter_type,
@ -1134,11 +1132,36 @@ async def test_counter_on_a_foundry_host_drops_the_caller_betas_messages_inferen
request_options={"thinking": _ENABLED_THINKING},
anthropic_beta=caller_betas,
model="claude-haiku-5-5",
api_base=_FOUNDRY_BASE,
api_base=api_base,
)
return _betas(count) - {ANTHROPIC_TOKEN_COUNTING_BETA_VERSION}, _betas(inference)
counted_betas: Final = _betas(count) - {ANTHROPIC_TOKEN_COUNTING_BETA_VERSION}
assert counted_betas == _betas(inference) == expected_betas
@pytest.mark.asyncio
@pytest.mark.usefixtures("httpx_transport_clients", "local_beta_headers_config")
@pytest.mark.parametrize(
("counter_type", "expected_betas"),
((AnthropicTokenCounter, frozenset({_UNLISTED_BETA})), (AzureAIAnthropicTokenCounter, frozenset())),
ids=("anthropic-deployment-forwards-unlisted", "azure-ai-deployment-drops-unlisted"),
)
async def test_counter_on_a_foundry_host_drops_the_caller_betas_messages_inference_drops_there(
counter_type: type[AnthropicTokenCounter] | type[AzureAIAnthropicTokenCounter], expected_betas: frozenset[str]
) -> None:
counted_betas, inference_betas = await _foundry_count_and_inference_betas(counter_type, _FOUNDRY_BASE)
assert counted_betas == inference_betas == expected_betas
@pytest.mark.asyncio
@pytest.mark.usefixtures("httpx_transport_clients", "local_beta_headers_config")
async def test_anthropic_counter_on_a_foundry_host_from_the_environment_drops_the_betas_inference_drops_there(
monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.setenv("ANTHROPIC_API_BASE", _FOUNDRY_BASE)
counted_betas, inference_betas = await _foundry_count_and_inference_betas(AnthropicTokenCounter, None)
assert counted_betas == inference_betas == frozenset({_UNLISTED_BETA})
@pytest.mark.asyncio

View file

@ -665,11 +665,8 @@ async def test_runtime_count_of_an_output_format_sends_only_the_betas_invoke_inf
_PER_TURN_CONTROL: Final = ANTHROPIC_BETA_HEADER_VALUES.PER_TURN_CONTROL_2026_07_01.value
@pytest.mark.asyncio
@pytest.mark.usefixtures("local_beta_headers_config")
@pytest.mark.parametrize(
_PER_TURN_CONTROL_DEPLOYMENT_MODEL: Final = "bedrock/invoke/us.anthropic.claude-opus-5-5"
_CALLER_BETA_CASES: Final = pytest.mark.parametrize(
("caller_beta", "model_info", "kept"),
(
(_PER_TURN_CONTROL, {"supports_per_turn_control": True}, True),
@ -678,39 +675,99 @@ _PER_TURN_CONTROL: Final = ANTHROPIC_BETA_HEADER_VALUES.PER_TURN_CONTROL_2026_07
),
ids=("deployment-supports-per-turn-control", "deployment-lacks-per-turn-control", "unlisted-beta"),
)
async def test_runtime_count_keeps_and_drops_the_caller_betas_invoke_inference_keeps_and_drops(
caller_beta: str, model_info: dict[str, JsonValue], kept: bool
) -> None:
deployment_model: Final = "bedrock/invoke/us.anthropic.claude-opus-5-5"
credentials: Final = DEPLOYMENT["litellm_params"]
async def _invoke_inference_betas(caller_beta: str, model_info: dict[str, JsonValue]) -> tuple[JsonValue, ...]:
inference: Final = _Bedrock(runtime=(200, _CLAUDE_REPLY), mantle=(200, _CLAUDE_REPLY))
await litellm.anthropic_messages(
model=deployment_model,
model=_PER_TURN_CONTROL_DEPLOYMENT_MODEL,
messages=MESSAGES,
max_tokens=4096,
extra_headers={"anthropic-beta": caller_beta},
model_info=model_info,
client=AsyncHTTPHandler(transport=httpx.MockTransport(inference)),
**_THINKING,
**credentials,
**DEPLOYMENT["litellm_params"],
)
return _betas_besides_effort(_JSON_BODY.validate_json(inference.requests[0].content))
def _per_turn_control_deployment(model_info: dict[str, JsonValue]) -> dict[str, object]:
return {
"litellm_params": {**DEPLOYMENT["litellm_params"], "model": _PER_TURN_CONTROL_DEPLOYMENT_MODEL},
"model_info": model_info,
}
async def _count_with_caller_beta(
model_to_use: str, caller_beta: str, deployment: dict[str, object] | None
) -> _Bedrock:
bedrock: Final = _Bedrock(runtime=(200, {"inputTokens": 412}), mantle=(200, {"input_tokens": 2177}))
result: Final = await _counter(bedrock).count_tokens(
model_to_use="us.anthropic.claude-opus-5-5",
model_to_use=model_to_use,
messages=MESSAGES,
contents=None,
deployment={"litellm_params": {**credentials, "model": deployment_model}, "model_info": model_info},
deployment=deployment,
request_model="claude",
request_options=_THINKING,
anthropic_beta=caller_beta,
)
assert result is not None and result.error is False
return bedrock
@pytest.mark.asyncio
@pytest.mark.usefixtures("local_beta_headers_config")
@_CALLER_BETA_CASES
async def test_runtime_count_keeps_and_drops_the_caller_betas_invoke_inference_keeps_and_drops(
caller_beta: str, model_info: dict[str, JsonValue], kept: bool
) -> None:
sent: Final = await _invoke_inference_betas(caller_beta, model_info)
bedrock: Final = await _count_with_caller_beta(
"us.anthropic.claude-opus-5-5", caller_beta, _per_turn_control_deployment(model_info)
)
assert bedrock.posted_hosts() == (RUNTIME_HOST,)
counted: Final = _betas_besides_effort(_invoke_body(bedrock.requests[0]))
assert counted == _betas_besides_effort(_JSON_BODY.validate_json(inference.requests[0].content))
assert counted == sent
assert (caller_beta in counted) is kept
@pytest.mark.asyncio
@pytest.mark.usefixtures("local_beta_headers_config")
@_CALLER_BETA_CASES
async def test_mantle_count_for_a_bedrock_deployment_keeps_and_drops_the_caller_betas_invoke_inference_keeps_and_drops(
caller_beta: str, model_info: dict[str, JsonValue], kept: bool
) -> None:
sent: Final = await _invoke_inference_betas(caller_beta, model_info)
bedrock: Final = await _count_with_caller_beta(
"bedrock_mantle/anthropic.claude-opus-5-5", caller_beta, _per_turn_control_deployment(model_info)
)
assert bedrock.posted_hosts() == (MANTLE_HOST,)
counted: Final = _sent_betas(bedrock.requests[0])
assert frozenset(counted) == frozenset(sent)
assert (caller_beta in counted) is kept
@pytest.mark.asyncio
@pytest.mark.usefixtures("local_beta_headers_config")
async def test_runtime_count_without_a_deployment_drops_the_caller_betas_invoke_inference_drops(
monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.setenv("AWS_ACCESS_KEY_ID", "AKIATESTACCESSKEY")
monkeypatch.setenv("AWS_SECRET_ACCESS_KEY", "test-secret")
monkeypatch.setenv("AWS_REGION_NAME", "us-east-1")
interleaved_thinking: Final = "interleaved-thinking-2025-05-14"
sent: Final = await _invoke_inference_betas(interleaved_thinking, {})
bedrock: Final = await _count_with_caller_beta("us.anthropic.claude-opus-5-5", interleaved_thinking, None)
assert bedrock.posted_hosts() == (RUNTIME_HOST,)
counted: Final = _betas_besides_effort(_invoke_body(bedrock.requests[0]))
assert counted == sent
assert interleaved_thinking not in counted
_CONVERSE_REPLY: Final = {
"output": {"message": {"role": "assistant", "content": [{"text": "ok"}]}},
"stopReason": "end_turn",

View file

@ -1959,45 +1959,6 @@ async def test_anthropic_count_tokens_does_not_count_again_after_a_rate_limit(
assert len(requests) == 1
@pytest.mark.asyncio
@pytest.mark.parametrize(
("deployment_model", "count_host", "provider_calls"),
(
("gemini/gemini-3.8-flash", "generativelanguage.googleapis.com", 1),
("anthropic/claude-haiku-5-5", "api.anthropic.com", 2),
),
ids=("counter-without-claude-options", "claude-counter-forwarding-the-options"),
)
async def test_anthropic_count_tokens_counts_again_without_the_options_only_when_the_counter_forwarded_them(
monkeypatch: pytest.MonkeyPatch, deployment_model: str, count_host: str, provider_calls: int
) -> None:
from litellm.proxy import proxy_server
request: Final = _json_request(
{
"model": "claude-agent",
"messages": [{"role": "user", "content": "Weather in SF?"}],
"thinking": {"type": "enabled", "budget_tokens": 2000},
},
"python-httpx/0.28.1",
)
monkeypatch.setattr(proxy_server, "llm_router", _claude_router(deployment_model, {}))
monkeypatch.setattr(proxy_server, "proxy_config", proxy_server.ProxyConfig())
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
with respx.mock:
route: Final = respx.route(host=count_host).mock(
return_value=httpx.Response(
400, json={"type": "error", "error": {"type": "invalid_request_error", "message": "rejected"}}
)
)
response: Final = await anthropic_count_tokens(request, UserAPIKeyAuth())
assert isinstance(response, JSONResponse)
assert response.status_code == 400
assert route.call_count == provider_calls
_BEDROCK_RUNTIME_HOST: Final = "bedrock-runtime.us-east-1.amazonaws.com"
_BEDROCK_MANTLE_HOST: Final = "bedrock-mantle.us-east-1.api.aws"
_BEDROCK_CLAUDE_DEPLOYMENT: Final[dict[str, str]] = {
@ -2006,6 +1967,88 @@ _BEDROCK_CLAUDE_DEPLOYMENT: Final[dict[str, str]] = {
"aws_secret_access_key": "test-secret",
"aws_region_name": "us-east-1",
}
_THINKING_BODY: Final[dict[str, JsonValue]] = {"thinking": {"type": "enabled", "budget_tokens": 2000}}
_EFFORT_BODY: Final[dict[str, JsonValue]] = {"output_config": {"effort": "low"}}
_DROPS_THE_EFFORT: Final[dict[str, JsonValue]] = {"additional_drop_params": ["output_config.effort"]}
_ANTHROPIC_DEPLOYMENT: Final[dict[str, JsonValue]] = {"model": "anthropic/claude-haiku-5-5", "api_key": "sk-ant-test"}
_AZURE_AI_DEPLOYMENT: Final[dict[str, JsonValue]] = {
"model": "azure_ai/claude-haiku-5-5",
"api_base": "https://my-resource.services.ai.azure.com/anthropic",
"api_key": "sk-azure-test",
}
_VERTEX_DEPLOYMENT: Final[dict[str, JsonValue]] = {
"model": "vertex_ai/claude-haiku-5-5",
"vertex_project": "test-project",
"vertex_location": "us-east5",
}
@pytest.mark.asyncio
@pytest.mark.parametrize(
("deployment_params", "body", "caller_beta", "provider_calls"),
(
({"model": "gemini/gemini-3.8-flash", "api_key": "sk-gemini-test"}, _THINKING_BODY, None, 1),
(_ANTHROPIC_DEPLOYMENT, _THINKING_BODY, None, 2),
({**_ANTHROPIC_DEPLOYMENT, **_DROPS_THE_EFFORT}, _EFFORT_BODY, None, 1),
({**_ANTHROPIC_DEPLOYMENT, **_DROPS_THE_EFFORT}, _EFFORT_BODY, _CALLER_BETA, 2),
(_AZURE_AI_DEPLOYMENT, _THINKING_BODY, None, 2),
({**_AZURE_AI_DEPLOYMENT, **_DROPS_THE_EFFORT}, _EFFORT_BODY, None, 1),
(_VERTEX_DEPLOYMENT, _THINKING_BODY, None, 2),
({**_VERTEX_DEPLOYMENT, **_DROPS_THE_EFFORT}, _EFFORT_BODY, None, 1),
({**_BEDROCK_CLAUDE_DEPLOYMENT, **_DROPS_THE_EFFORT}, _EFFORT_BODY, None, 2),
),
ids=(
"counter-without-claude-options",
"anthropic-forwarding-the-options",
"anthropic-forwarding-nothing",
"anthropic-forwarding-only-the-caller-beta",
"azure-ai-forwarding-the-options",
"azure-ai-forwarding-nothing",
"vertex-forwarding-the-options",
"vertex-forwarding-nothing",
"bedrock-forwarding-nothing-counts-on-runtime-then-mantle-once",
),
)
async def test_anthropic_count_tokens_counts_again_without_the_options_only_when_the_counter_forwarded_them(
monkeypatch: pytest.MonkeyPatch,
deployment_params: dict[str, JsonValue],
body: dict[str, JsonValue],
caller_beta: str | None,
provider_calls: int,
) -> None:
from litellm.proxy import proxy_server
for name in ("ANTHROPIC_API_BASE", "ANTHROPIC_BASE_URL", "AWS_BEARER_TOKEN_BEDROCK", "BEDROCK_MANTLE_API_BASE"):
monkeypatch.delenv(name, raising=False)
monkeypatch.setattr(
"google.auth.default", lambda *_, **__: (MagicMock(token="fake-token", valid=True, expiry=None), "test-project")
)
request: Final = _json_request(
{"model": "claude-agent", "messages": [{"role": "user", "content": "Weather in SF?"}], **body},
"python-httpx/0.28.1",
{} if caller_beta is None else {"anthropic-beta": caller_beta},
)
router: Final = Router(model_list=[{"model_name": "claude-agent", "litellm_params": deployment_params}])
monkeypatch.setattr(proxy_server, "llm_router", router)
monkeypatch.setattr(proxy_server, "proxy_config", proxy_server.ProxyConfig())
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
with respx.mock:
route: Final = respx.route().mock(
return_value=httpx.Response(
400,
json={
"type": "error",
"message": "rejected",
"error": {"type": "invalid_request_error", "message": "rejected"},
},
)
)
response: Final = await anthropic_count_tokens(request, UserAPIKeyAuth())
assert isinstance(response, JSONResponse)
assert response.status_code == 400
assert route.call_count == provider_calls
async def _count_on_a_bedrock_claude_deployment(

View file

@ -10,9 +10,11 @@ This test validates:
import json
import os
from queue import SimpleQueue
from typing import Dict, Final, List, get_args
from unittest.mock import AsyncMock, MagicMock, patch
import httpx
import pytest
import respx
from anthropic.types.anthropic_beta_param import AnthropicBetaParam
@ -775,3 +777,45 @@ def test_each_row_forwards_exactly_its_verified_preserved_thinking_cells(bundled
headers={"anthropic-beta": ",".join(PRESERVED_THINKING_BETAS)}, provider=provider
)
assert sorted(result["anthropic-beta"].split(",")) == sorted(forwarded), (provider, result)
_MESSAGE_REPLY: Final = {
"id": "msg_123",
"type": "message",
"role": "assistant",
"content": [{"type": "text", "text": "Hello"}],
"model": "claude-haiku-5-5",
"stop_reason": "end_turn",
"usage": {"input_tokens": 10, "output_tokens": 20},
}
@pytest.mark.asyncio
async def test_direct_anthropic_deployment_sends_the_caller_betas_its_row_marks_unsupported(
bundled_config: None, monkeypatch: pytest.MonkeyPatch
) -> None:
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
monkeypatch.delenv("ANTHROPIC_API_BASE", raising=False)
monkeypatch.delenv("ANTHROPIC_BASE_URL", raising=False)
row: Final[dict[str, str | None]] = GetAnthropicBetaHeadersConfig.load_local_beta_headers_config()["anthropic"]
unsupported: Final = min(beta for beta, mapped in row.items() if mapped is None and "-" in beta)
sent: Final[SimpleQueue[httpx.Request]] = SimpleQueue()
def reply(request: httpx.Request) -> httpx.Response:
sent.put(request)
return httpx.Response(200, json=_MESSAGE_REPLY)
await litellm.anthropic_messages(
model="anthropic/claude-haiku-5-5",
messages=[{"role": "user", "content": "Hi"}],
max_tokens=16,
api_key="sk-ant-test",
extra_headers={"anthropic-beta": unsupported},
client=AsyncHTTPHandler(transport=httpx.MockTransport(reply)),
)
request: Final = sent.get_nowait()
assert sent.empty()
assert request.url.host == "api.anthropic.com"
assert unsupported in request.headers["anthropic-beta"].split(",")