diff --git a/litellm/anthropic_beta_headers_manager.py b/litellm/anthropic_beta_headers_manager.py index f7a8e92df29..8e16682c93a 100644 --- a/litellm/anthropic_beta_headers_manager.py +++ b/litellm/anthropic_beta_headers_manager.py @@ -481,9 +481,9 @@ class DeploymentBetaFilter: api_base: str | None unlisted: UnlistedBetas - def filtered(self, headers: dict[str, str], keep: frozenset[str]) -> dict[str, str]: + def filtered(self, headers: Mapping[str, str], keep: frozenset[str]) -> dict[str, str]: return update_headers_with_filtered_beta( - headers=headers, provider=self.provider, api_base=self.api_base, unlisted=self.unlisted, keep=keep + headers=dict(headers), provider=self.provider, api_base=self.api_base, unlisted=self.unlisted, keep=keep ) diff --git a/litellm/llms/anthropic/count_tokens/transformation.py b/litellm/llms/anthropic/count_tokens/transformation.py index 26defadfda1..299c4e6bdfd 100644 --- a/litellm/llms/anthropic/count_tokens/transformation.py +++ b/litellm/llms/anthropic/count_tokens/transformation.py @@ -369,6 +369,11 @@ def is_request_rejection(status_code: int) -> bool: return 400 <= status_code < 500 and status_code not in _STATUSES_NOT_CAUSED_BY_THE_REQUEST +def _deployment_api_base(deployment: Mapping[str, object] | None) -> str | None: + api_base: Final = _INFERENCE_PARAMS.validate_python((deployment or {}).get("litellm_params") or {}).get("api_base") + return api_base if isinstance(api_base, str) else None + + def deployment_inference_model(deployment: Mapping[str, object] | None) -> str | None: deployment_params: Final = _INFERENCE_PARAMS.validate_python((deployment or {}).get("litellm_params") or {}) deployment_model: Final = deployment_params.get("model") @@ -377,20 +382,14 @@ def deployment_inference_model(deployment: Mapping[str, object] | None) -> str | from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider provider_hint: Final = deployment_params.get("custom_llm_provider") - api_base: Final = deployment_params.get("api_base") inference_model, _, _, _ = get_llm_provider( model=deployment_model, custom_llm_provider=provider_hint if isinstance(provider_hint, str) else None, - api_base=api_base if isinstance(api_base, str) else None, + api_base=_deployment_api_base(deployment), ) return inference_model -def _deployment_api_base(deployment: Mapping[str, object] | None) -> str | None: - api_base: Final = _INFERENCE_PARAMS.validate_python((deployment or {}).get("litellm_params") or {}).get("api_base") - return api_base if isinstance(api_base, str) else None - - def _messages_transform(model: str, provider: LlmProviders) -> tuple[str, BaseAnthropicMessagesConfig] | None: from litellm.utils import ProviderConfigManager @@ -434,7 +433,7 @@ def _claude_count_options_for( dict(beta_headers) if beta_filter is None else beta_filter.filtered( - dict(beta_headers), keep=messages_config.model_specific_anthropic_betas(model, litellm_params) + beta_headers, keep=messages_config.model_specific_anthropic_betas(model, litellm_params) ) ) request: Final = messages_config.transform_anthropic_messages_request( # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # the base config signature takes and returns bare dicts diff --git a/litellm/llms/bedrock/count_tokens/bedrock_token_counter.py b/litellm/llms/bedrock/count_tokens/bedrock_token_counter.py index fa1cbe9ead9..24c70465cfa 100644 --- a/litellm/llms/bedrock/count_tokens/bedrock_token_counter.py +++ b/litellm/llms/bedrock/count_tokens/bedrock_token_counter.py @@ -68,18 +68,34 @@ class CountTokensFailure: CountTokensOutcome = CountedTokens | CountTokensFailure -def _claude_count_translator(litellm_params: Mapping[str, object]) -> AnthropicConfig: +def _deployment_route(litellm_params: Mapping[str, object]) -> tuple[str, str] | None: deployment_model: Final = litellm_params.get("model") if not isinstance(deployment_model, str): - return AnthropicConfig() + return None provider_hint: Final = litellm_params.get("custom_llm_provider") model, provider, _, _ = litellm.get_llm_provider( model=deployment_model, custom_llm_provider=provider_hint if isinstance(provider_hint, str) else None ) + return model, provider + + +def _claude_count_translator(litellm_params: Mapping[str, object]) -> AnthropicConfig: + route: Final = _deployment_route(litellm_params) + if route is None: + return AnthropicConfig() + model, provider = route chat_config: Final = litellm.ProviderConfigManager.get_provider_chat_config(model, LlmProviders(provider)) return chat_config if isinstance(chat_config, AnthropicConfig) else AnthropicConfig() +def _claude_options_provider(litellm_params: Mapping[str, object], use_mantle: bool) -> LlmProviders: + if not use_mantle: + return LlmProviders.BEDROCK + route: Final = _deployment_route(litellm_params) + deployment_provider: Final = None if route is None else route[1] + return LlmProviders.BEDROCK if deployment_provider == LlmProviders.BEDROCK.value else LlmProviders.BEDROCK_MANTLE + + def _runtime_rejected_claude_model(outcome: CountTokensOutcome, resolved_model: str) -> bool: return ( isinstance(outcome, CountTokensFailure) @@ -264,7 +280,7 @@ class BedrockTokenCounter(BaseTokenCounter): claude_options: Final = ( claude_count_options( counted_model, - LlmProviders.BEDROCK_MANTLE if use_mantle else LlmProviders.BEDROCK, + _claude_options_provider(litellm_params, use_mantle), sent_messages, sent_tools, _options_converse_sends(request_options, deployment_inference_model(deployment)), diff --git a/tests/unit/llms/anthropic/test_anthropic_count_tokens_transformation.py b/tests/unit/llms/anthropic/test_anthropic_count_tokens_transformation.py index 1f731faa724..a9381599a7a 100644 --- a/tests/unit/llms/anthropic/test_anthropic_count_tokens_transformation.py +++ b/tests/unit/llms/anthropic/test_anthropic_count_tokens_transformation.py @@ -567,7 +567,7 @@ async def _counted_requests( request_options: Mapping[str, JsonValue] = MappingProxyType({}), anthropic_beta: str | None = None, model: str = "claude-opus-5-5", - api_base: str = "https://gateway.example", + api_base: str | None = "https://gateway.example", ) -> tuple[TokenCountResponse | None, tuple[httpx.Request, ...]]: sent: Final[SimpleQueue[httpx.Request]] = SimpleQueue() @@ -581,7 +581,12 @@ async def _counted_requests( model_to_use=model, messages=messages, contents=None, - deployment={"litellm_params": {"api_key": "sk-ant-api03-test-key", "api_base": api_base}}, + deployment={ + "litellm_params": { + "api_key": "sk-ant-api03-test-key", + **({} if api_base is None else {"api_base": api_base}), + } + }, request_model=model, tools=tools, messages_format=messages_format, @@ -973,7 +978,7 @@ async def _messages_inference_request( request_options: Mapping[str, JsonValue], model: str = "claude-haiku-5-5", extra_headers: Mapping[str, str] = MappingProxyType({}), - api_base: str = "https://gateway.example", + api_base: str | None = "https://gateway.example", ) -> httpx.Request: sent: Final[SimpleQueue[httpx.Request]] = SimpleQueue() @@ -1107,23 +1112,16 @@ _FOUNDRY_BASE: Final = "https://my-resource.services.ai.azure.com/anthropic" _UNLISTED_BETA: Final = "unlisted-beta-2099-01-01" -@pytest.mark.asyncio -@pytest.mark.usefixtures("httpx_transport_clients", "local_beta_headers_config") -@pytest.mark.parametrize( - ("counter_type", "expected_betas"), - ((AnthropicTokenCounter, frozenset({_UNLISTED_BETA})), (AzureAIAnthropicTokenCounter, frozenset())), - ids=("anthropic-deployment-forwards-unlisted", "azure-ai-deployment-drops-unlisted"), -) -async def test_counter_on_a_foundry_host_drops_the_caller_betas_messages_inference_drops_there( - counter_type: type[AnthropicTokenCounter] | type[AzureAIAnthropicTokenCounter], expected_betas: frozenset[str] -) -> None: +async def _foundry_count_and_inference_betas( + counter_type: type[AnthropicTokenCounter] | type[AzureAIAnthropicTokenCounter], api_base: str | None +) -> tuple[frozenset[str], frozenset[str]]: caller_betas: Final = f"mcp-servers-2025-12-04,{_UNLISTED_BETA}" inference: Final = await _messages_inference_request( counter_type, f"{_FOUNDRY_BASE}/v1/messages", {"thinking": _ENABLED_THINKING}, extra_headers={"anthropic-beta": caller_betas}, - api_base=_FOUNDRY_BASE, + api_base=api_base, ) _, (count,) = await _counted_requests( counter_type, @@ -1134,11 +1132,36 @@ async def test_counter_on_a_foundry_host_drops_the_caller_betas_messages_inferen request_options={"thinking": _ENABLED_THINKING}, anthropic_beta=caller_betas, model="claude-haiku-5-5", - api_base=_FOUNDRY_BASE, + api_base=api_base, ) + return _betas(count) - {ANTHROPIC_TOKEN_COUNTING_BETA_VERSION}, _betas(inference) - counted_betas: Final = _betas(count) - {ANTHROPIC_TOKEN_COUNTING_BETA_VERSION} - assert counted_betas == _betas(inference) == expected_betas + +@pytest.mark.asyncio +@pytest.mark.usefixtures("httpx_transport_clients", "local_beta_headers_config") +@pytest.mark.parametrize( + ("counter_type", "expected_betas"), + ((AnthropicTokenCounter, frozenset({_UNLISTED_BETA})), (AzureAIAnthropicTokenCounter, frozenset())), + ids=("anthropic-deployment-forwards-unlisted", "azure-ai-deployment-drops-unlisted"), +) +async def test_counter_on_a_foundry_host_drops_the_caller_betas_messages_inference_drops_there( + counter_type: type[AnthropicTokenCounter] | type[AzureAIAnthropicTokenCounter], expected_betas: frozenset[str] +) -> None: + counted_betas, inference_betas = await _foundry_count_and_inference_betas(counter_type, _FOUNDRY_BASE) + + assert counted_betas == inference_betas == expected_betas + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("httpx_transport_clients", "local_beta_headers_config") +async def test_anthropic_counter_on_a_foundry_host_from_the_environment_drops_the_betas_inference_drops_there( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("ANTHROPIC_API_BASE", _FOUNDRY_BASE) + + counted_betas, inference_betas = await _foundry_count_and_inference_betas(AnthropicTokenCounter, None) + + assert counted_betas == inference_betas == frozenset({_UNLISTED_BETA}) @pytest.mark.asyncio diff --git a/tests/unit/llms/bedrock/count_tokens/test_bedrock_token_counter.py b/tests/unit/llms/bedrock/count_tokens/test_bedrock_token_counter.py index 94188c260fd..afb3a285f2c 100644 --- a/tests/unit/llms/bedrock/count_tokens/test_bedrock_token_counter.py +++ b/tests/unit/llms/bedrock/count_tokens/test_bedrock_token_counter.py @@ -665,11 +665,8 @@ async def test_runtime_count_of_an_output_format_sends_only_the_betas_invoke_inf _PER_TURN_CONTROL: Final = ANTHROPIC_BETA_HEADER_VALUES.PER_TURN_CONTROL_2026_07_01.value - - -@pytest.mark.asyncio -@pytest.mark.usefixtures("local_beta_headers_config") -@pytest.mark.parametrize( +_PER_TURN_CONTROL_DEPLOYMENT_MODEL: Final = "bedrock/invoke/us.anthropic.claude-opus-5-5" +_CALLER_BETA_CASES: Final = pytest.mark.parametrize( ("caller_beta", "model_info", "kept"), ( (_PER_TURN_CONTROL, {"supports_per_turn_control": True}, True), @@ -678,39 +675,99 @@ _PER_TURN_CONTROL: Final = ANTHROPIC_BETA_HEADER_VALUES.PER_TURN_CONTROL_2026_07 ), ids=("deployment-supports-per-turn-control", "deployment-lacks-per-turn-control", "unlisted-beta"), ) -async def test_runtime_count_keeps_and_drops_the_caller_betas_invoke_inference_keeps_and_drops( - caller_beta: str, model_info: dict[str, JsonValue], kept: bool -) -> None: - deployment_model: Final = "bedrock/invoke/us.anthropic.claude-opus-5-5" - credentials: Final = DEPLOYMENT["litellm_params"] + + +async def _invoke_inference_betas(caller_beta: str, model_info: dict[str, JsonValue]) -> tuple[JsonValue, ...]: inference: Final = _Bedrock(runtime=(200, _CLAUDE_REPLY), mantle=(200, _CLAUDE_REPLY)) await litellm.anthropic_messages( - model=deployment_model, + model=_PER_TURN_CONTROL_DEPLOYMENT_MODEL, messages=MESSAGES, max_tokens=4096, extra_headers={"anthropic-beta": caller_beta}, model_info=model_info, client=AsyncHTTPHandler(transport=httpx.MockTransport(inference)), **_THINKING, - **credentials, + **DEPLOYMENT["litellm_params"], ) + return _betas_besides_effort(_JSON_BODY.validate_json(inference.requests[0].content)) + + +def _per_turn_control_deployment(model_info: dict[str, JsonValue]) -> dict[str, object]: + return { + "litellm_params": {**DEPLOYMENT["litellm_params"], "model": _PER_TURN_CONTROL_DEPLOYMENT_MODEL}, + "model_info": model_info, + } + + +async def _count_with_caller_beta( + model_to_use: str, caller_beta: str, deployment: dict[str, object] | None +) -> _Bedrock: bedrock: Final = _Bedrock(runtime=(200, {"inputTokens": 412}), mantle=(200, {"input_tokens": 2177})) result: Final = await _counter(bedrock).count_tokens( - model_to_use="us.anthropic.claude-opus-5-5", + model_to_use=model_to_use, messages=MESSAGES, contents=None, - deployment={"litellm_params": {**credentials, "model": deployment_model}, "model_info": model_info}, + deployment=deployment, request_model="claude", request_options=_THINKING, anthropic_beta=caller_beta, ) - assert result is not None and result.error is False + return bedrock + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("local_beta_headers_config") +@_CALLER_BETA_CASES +async def test_runtime_count_keeps_and_drops_the_caller_betas_invoke_inference_keeps_and_drops( + caller_beta: str, model_info: dict[str, JsonValue], kept: bool +) -> None: + sent: Final = await _invoke_inference_betas(caller_beta, model_info) + bedrock: Final = await _count_with_caller_beta( + "us.anthropic.claude-opus-5-5", caller_beta, _per_turn_control_deployment(model_info) + ) + + assert bedrock.posted_hosts() == (RUNTIME_HOST,) counted: Final = _betas_besides_effort(_invoke_body(bedrock.requests[0])) - assert counted == _betas_besides_effort(_JSON_BODY.validate_json(inference.requests[0].content)) + assert counted == sent assert (caller_beta in counted) is kept +@pytest.mark.asyncio +@pytest.mark.usefixtures("local_beta_headers_config") +@_CALLER_BETA_CASES +async def test_mantle_count_for_a_bedrock_deployment_keeps_and_drops_the_caller_betas_invoke_inference_keeps_and_drops( + caller_beta: str, model_info: dict[str, JsonValue], kept: bool +) -> None: + sent: Final = await _invoke_inference_betas(caller_beta, model_info) + bedrock: Final = await _count_with_caller_beta( + "bedrock_mantle/anthropic.claude-opus-5-5", caller_beta, _per_turn_control_deployment(model_info) + ) + + assert bedrock.posted_hosts() == (MANTLE_HOST,) + counted: Final = _sent_betas(bedrock.requests[0]) + assert frozenset(counted) == frozenset(sent) + assert (caller_beta in counted) is kept + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("local_beta_headers_config") +async def test_runtime_count_without_a_deployment_drops_the_caller_betas_invoke_inference_drops( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("AWS_ACCESS_KEY_ID", "AKIATESTACCESSKEY") + monkeypatch.setenv("AWS_SECRET_ACCESS_KEY", "test-secret") + monkeypatch.setenv("AWS_REGION_NAME", "us-east-1") + interleaved_thinking: Final = "interleaved-thinking-2025-05-14" + sent: Final = await _invoke_inference_betas(interleaved_thinking, {}) + bedrock: Final = await _count_with_caller_beta("us.anthropic.claude-opus-5-5", interleaved_thinking, None) + + assert bedrock.posted_hosts() == (RUNTIME_HOST,) + counted: Final = _betas_besides_effort(_invoke_body(bedrock.requests[0])) + assert counted == sent + assert interleaved_thinking not in counted + + _CONVERSE_REPLY: Final = { "output": {"message": {"role": "assistant", "content": [{"text": "ok"}]}}, "stopReason": "end_turn", diff --git a/tests/unit/proxy/test_proxy_token_counter.py b/tests/unit/proxy/test_proxy_token_counter.py index 45199f48cdd..3f20179b12b 100644 --- a/tests/unit/proxy/test_proxy_token_counter.py +++ b/tests/unit/proxy/test_proxy_token_counter.py @@ -1959,45 +1959,6 @@ async def test_anthropic_count_tokens_does_not_count_again_after_a_rate_limit( assert len(requests) == 1 -@pytest.mark.asyncio -@pytest.mark.parametrize( - ("deployment_model", "count_host", "provider_calls"), - ( - ("gemini/gemini-3.8-flash", "generativelanguage.googleapis.com", 1), - ("anthropic/claude-haiku-5-5", "api.anthropic.com", 2), - ), - ids=("counter-without-claude-options", "claude-counter-forwarding-the-options"), -) -async def test_anthropic_count_tokens_counts_again_without_the_options_only_when_the_counter_forwarded_them( - monkeypatch: pytest.MonkeyPatch, deployment_model: str, count_host: str, provider_calls: int -) -> None: - from litellm.proxy import proxy_server - - request: Final = _json_request( - { - "model": "claude-agent", - "messages": [{"role": "user", "content": "Weather in SF?"}], - "thinking": {"type": "enabled", "budget_tokens": 2000}, - }, - "python-httpx/0.28.1", - ) - monkeypatch.setattr(proxy_server, "llm_router", _claude_router(deployment_model, {})) - monkeypatch.setattr(proxy_server, "proxy_config", proxy_server.ProxyConfig()) - monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) - - with respx.mock: - route: Final = respx.route(host=count_host).mock( - return_value=httpx.Response( - 400, json={"type": "error", "error": {"type": "invalid_request_error", "message": "rejected"}} - ) - ) - response: Final = await anthropic_count_tokens(request, UserAPIKeyAuth()) - - assert isinstance(response, JSONResponse) - assert response.status_code == 400 - assert route.call_count == provider_calls - - _BEDROCK_RUNTIME_HOST: Final = "bedrock-runtime.us-east-1.amazonaws.com" _BEDROCK_MANTLE_HOST: Final = "bedrock-mantle.us-east-1.api.aws" _BEDROCK_CLAUDE_DEPLOYMENT: Final[dict[str, str]] = { @@ -2006,6 +1967,88 @@ _BEDROCK_CLAUDE_DEPLOYMENT: Final[dict[str, str]] = { "aws_secret_access_key": "test-secret", "aws_region_name": "us-east-1", } +_THINKING_BODY: Final[dict[str, JsonValue]] = {"thinking": {"type": "enabled", "budget_tokens": 2000}} +_EFFORT_BODY: Final[dict[str, JsonValue]] = {"output_config": {"effort": "low"}} +_DROPS_THE_EFFORT: Final[dict[str, JsonValue]] = {"additional_drop_params": ["output_config.effort"]} +_ANTHROPIC_DEPLOYMENT: Final[dict[str, JsonValue]] = {"model": "anthropic/claude-haiku-5-5", "api_key": "sk-ant-test"} +_AZURE_AI_DEPLOYMENT: Final[dict[str, JsonValue]] = { + "model": "azure_ai/claude-haiku-5-5", + "api_base": "https://my-resource.services.ai.azure.com/anthropic", + "api_key": "sk-azure-test", +} +_VERTEX_DEPLOYMENT: Final[dict[str, JsonValue]] = { + "model": "vertex_ai/claude-haiku-5-5", + "vertex_project": "test-project", + "vertex_location": "us-east5", +} + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("deployment_params", "body", "caller_beta", "provider_calls"), + ( + ({"model": "gemini/gemini-3.8-flash", "api_key": "sk-gemini-test"}, _THINKING_BODY, None, 1), + (_ANTHROPIC_DEPLOYMENT, _THINKING_BODY, None, 2), + ({**_ANTHROPIC_DEPLOYMENT, **_DROPS_THE_EFFORT}, _EFFORT_BODY, None, 1), + ({**_ANTHROPIC_DEPLOYMENT, **_DROPS_THE_EFFORT}, _EFFORT_BODY, _CALLER_BETA, 2), + (_AZURE_AI_DEPLOYMENT, _THINKING_BODY, None, 2), + ({**_AZURE_AI_DEPLOYMENT, **_DROPS_THE_EFFORT}, _EFFORT_BODY, None, 1), + (_VERTEX_DEPLOYMENT, _THINKING_BODY, None, 2), + ({**_VERTEX_DEPLOYMENT, **_DROPS_THE_EFFORT}, _EFFORT_BODY, None, 1), + ({**_BEDROCK_CLAUDE_DEPLOYMENT, **_DROPS_THE_EFFORT}, _EFFORT_BODY, None, 2), + ), + ids=( + "counter-without-claude-options", + "anthropic-forwarding-the-options", + "anthropic-forwarding-nothing", + "anthropic-forwarding-only-the-caller-beta", + "azure-ai-forwarding-the-options", + "azure-ai-forwarding-nothing", + "vertex-forwarding-the-options", + "vertex-forwarding-nothing", + "bedrock-forwarding-nothing-counts-on-runtime-then-mantle-once", + ), +) +async def test_anthropic_count_tokens_counts_again_without_the_options_only_when_the_counter_forwarded_them( + monkeypatch: pytest.MonkeyPatch, + deployment_params: dict[str, JsonValue], + body: dict[str, JsonValue], + caller_beta: str | None, + provider_calls: int, +) -> None: + from litellm.proxy import proxy_server + + for name in ("ANTHROPIC_API_BASE", "ANTHROPIC_BASE_URL", "AWS_BEARER_TOKEN_BEDROCK", "BEDROCK_MANTLE_API_BASE"): + monkeypatch.delenv(name, raising=False) + monkeypatch.setattr( + "google.auth.default", lambda *_, **__: (MagicMock(token="fake-token", valid=True, expiry=None), "test-project") + ) + request: Final = _json_request( + {"model": "claude-agent", "messages": [{"role": "user", "content": "Weather in SF?"}], **body}, + "python-httpx/0.28.1", + {} if caller_beta is None else {"anthropic-beta": caller_beta}, + ) + router: Final = Router(model_list=[{"model_name": "claude-agent", "litellm_params": deployment_params}]) + monkeypatch.setattr(proxy_server, "llm_router", router) + monkeypatch.setattr(proxy_server, "proxy_config", proxy_server.ProxyConfig()) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + + with respx.mock: + route: Final = respx.route().mock( + return_value=httpx.Response( + 400, + json={ + "type": "error", + "message": "rejected", + "error": {"type": "invalid_request_error", "message": "rejected"}, + }, + ) + ) + response: Final = await anthropic_count_tokens(request, UserAPIKeyAuth()) + + assert isinstance(response, JSONResponse) + assert response.status_code == 400 + assert route.call_count == provider_calls async def _count_on_a_bedrock_claude_deployment( diff --git a/tests/unit/test_anthropic_beta_headers_filtering.py b/tests/unit/test_anthropic_beta_headers_filtering.py index a9bc741f800..d55618aa9ea 100644 --- a/tests/unit/test_anthropic_beta_headers_filtering.py +++ b/tests/unit/test_anthropic_beta_headers_filtering.py @@ -10,9 +10,11 @@ This test validates: import json import os +from queue import SimpleQueue from typing import Dict, Final, List, get_args from unittest.mock import AsyncMock, MagicMock, patch +import httpx import pytest import respx from anthropic.types.anthropic_beta_param import AnthropicBetaParam @@ -775,3 +777,45 @@ def test_each_row_forwards_exactly_its_verified_preserved_thinking_cells(bundled headers={"anthropic-beta": ",".join(PRESERVED_THINKING_BETAS)}, provider=provider ) assert sorted(result["anthropic-beta"].split(",")) == sorted(forwarded), (provider, result) + + +_MESSAGE_REPLY: Final = { + "id": "msg_123", + "type": "message", + "role": "assistant", + "content": [{"type": "text", "text": "Hello"}], + "model": "claude-haiku-5-5", + "stop_reason": "end_turn", + "usage": {"input_tokens": 10, "output_tokens": 20}, +} + + +@pytest.mark.asyncio +async def test_direct_anthropic_deployment_sends_the_caller_betas_its_row_marks_unsupported( + bundled_config: None, monkeypatch: pytest.MonkeyPatch +) -> None: + from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler + + monkeypatch.delenv("ANTHROPIC_API_BASE", raising=False) + monkeypatch.delenv("ANTHROPIC_BASE_URL", raising=False) + row: Final[dict[str, str | None]] = GetAnthropicBetaHeadersConfig.load_local_beta_headers_config()["anthropic"] + unsupported: Final = min(beta for beta, mapped in row.items() if mapped is None and "-" in beta) + sent: Final[SimpleQueue[httpx.Request]] = SimpleQueue() + + def reply(request: httpx.Request) -> httpx.Response: + sent.put(request) + return httpx.Response(200, json=_MESSAGE_REPLY) + + await litellm.anthropic_messages( + model="anthropic/claude-haiku-5-5", + messages=[{"role": "user", "content": "Hi"}], + max_tokens=16, + api_key="sk-ant-test", + extra_headers={"anthropic-beta": unsupported}, + client=AsyncHTTPHandler(transport=httpx.MockTransport(reply)), + ) + + request: Final = sent.get_nowait() + assert sent.empty() + assert request.url.host == "api.anthropic.com" + assert unsupported in request.headers["anthropic-beta"].split(",")