diff --git a/litellm-rust/crates/llms/src/anthropic/messages/transformation.rs b/litellm-rust/crates/llms/src/anthropic/messages/transformation.rs index 6657cbf262f..2351df0fac1 100644 --- a/litellm-rust/crates/llms/src/anthropic/messages/transformation.rs +++ b/litellm-rust/crates/llms/src/anthropic/messages/transformation.rs @@ -1,7 +1,8 @@ use litellm_auth::CredentialPlacement; use litellm_llms_types::{ formats::messages::{ - ContextEdit, ContextManagement, Message, MessagesOptionalParams, MessagesRequest, Speed, + ContentBlock, ContentBlockType, ContextEdit, ContextManagement, Message, + MessagesOptionalParams, MessagesRequest, Speed, ThinkingConfig, ThinkingDisplay, }, providers::anthropic::{AnthropicBeta, BetaProvider, BetaSet, DEFAULT_HEADERS}, recognized::Recognized, @@ -203,6 +204,10 @@ fn feature_betas(request: &MessagesRequest) -> BetaSet { .then_some(AnthropicBeta::FastMode20260201), messages_carry_output_config(&request.messages) .then_some(AnthropicBeta::PerTurnControl20260701), + uses_thinking_display_updates(params) + .then_some(AnthropicBeta::ThinkingDisplayUpdates20260818), + messages_carry_tool_changes(&request.messages) + .then_some(AnthropicBeta::MidConversationToolChanges20260701), has_advisor_tool(tools).then_some(AnthropicBeta::AdvisorTool20260301), is_tool_search_used(tools).then_some(AnthropicBeta::AdvancedToolUse20251120), ] @@ -253,6 +258,36 @@ fn messages_carry_output_config(messages: &[Message]) -> bool { .any(|message| message.extra.contains_key("output_config")) } +fn uses_thinking_display_updates(params: &MessagesOptionalParams) -> bool { + let display = match params.thinking.as_ref().and_then(Recognized::known) { + Some(ThinkingConfig::Enabled(thinking)) => thinking.display.as_ref(), + Some(ThinkingConfig::Adaptive(thinking)) => thinking.display.as_ref(), + _ => None, + }; + display.and_then(Recognized::known) == Some(&ThinkingDisplay::Updates) +} + +fn is_tool_reference_change(block: &ContentBlock) -> bool { + matches!( + &block.block_type, + Some(ContentBlockType::Other(block_type)) + if block_type == "tool_addition" || block_type == "tool_removal" + ) && block + .extra + .get("tool") + .and_then(|tool| tool.get("type")) + .and_then(Value::as_str) + == Some("tool_reference") +} + +fn messages_carry_tool_changes(messages: &[Message]) -> bool { + messages + .iter() + .filter(|message| message.role == "system") + .flat_map(Message::blocks) + .any(is_tool_reference_change) +} + fn unsupported_param(model: &str, param: &str, value: &str, hint: &str) -> Error { Error::InvalidRequest(crate::ErrorDetail::UnsupportedParameter { model: model.into(), @@ -1071,6 +1106,41 @@ mod tests { json!({"safeguards": [{"type": "dangerous_tool_use", "classifier_context": {"v": 1, "permission_mode": "auto"}}]}), &[] )] + #[case::adaptive_display_updates( + json!({"thinking": {"type": "adaptive", "display": "updates"}}), + &["thinking-display-updates-2026-08-18"] + )] + #[case::enabled_display_updates( + json!({"thinking": {"type": "enabled", "budget_tokens": 2048, "display": "updates"}}), + &["thinking-display-updates-2026-08-18"] + )] + #[case::summarized_display(json!({"thinking": {"type": "adaptive", "display": "summarized"}}), &[])] + #[case::disabled_thinking(json!({"thinking": {"type": "disabled"}}), &[])] + #[case::system_tool_addition( + json!({"messages": [ + {"role": "user", "content": "Use the ping tool"}, + {"role": "system", "content": [{"type": "tool_addition", "tool": {"type": "tool_reference", "name": "ping"}}]}, + ]}), + &["mid-conversation-tool-changes-2026-07-01"] + )] + #[case::system_tool_removal( + json!({"messages": [ + {"role": "system", "content": [{"type": "tool_removal", "tool": {"type": "tool_reference", "name": "ping"}}]}, + ]}), + &["mid-conversation-tool-changes-2026-07-01"] + )] + #[case::user_tool_addition( + json!({"messages": [ + {"role": "user", "content": [{"type": "tool_addition", "tool": {"type": "tool_reference", "name": "ping"}}]}, + ]}), + &[] + )] + #[case::tool_addition_without_a_tool_reference( + json!({"messages": [ + {"role": "system", "content": [{"type": "tool_addition", "tool": {"type": "tool_definition"}}]}, + ]}), + &[] + )] fn feature_betas_follow_the_request(#[case] fields: Value, #[case] expected: &[&str]) { assert_eq!(feature_betas(&request(fields)), betas(expected)); } @@ -1093,6 +1163,33 @@ mod tests { ); } + #[rstest] + #[case::vertex_ai( + BetaProvider::VertexAi, + &["mid-conversation-tool-changes-2026-07-01", "thinking-display-updates-2026-08-18"] + )] + #[case::azure_ai( + BetaProvider::AzureAi, + &["mid-conversation-tool-changes-2026-07-01", "thinking-display-updates-2026-08-18"] + )] + #[case::bedrock_converse(BetaProvider::BedrockConverse, &["thinking-display-updates-2026-08-18"])] + #[case::databricks(BetaProvider::Databricks, &[])] + fn tool_change_and_display_betas_reach_the_providers_that_accept_them( + #[case] provider: BetaProvider, + #[case] expected: &[&str], + ) { + let fields = json!({ + "thinking": {"type": "adaptive", "display": "updates"}, + "messages": [ + {"role": "system", "content": [{"type": "tool_addition", "tool": {"type": "tool_reference", "name": "ping"}}]}, + ], + }); + assert_eq!( + provider_feature_betas(&request(fields), provider), + betas(expected) + ); + } + #[rstest] fn hosts_without_a_beta_map_column_get_safeguards_without_a_derived_beta() { let client = headers(&[("anthropic-beta", "claude-code-20250219")]); diff --git a/litellm-rust/crates/llms/src/vertex_ai/messages/transformation.rs b/litellm-rust/crates/llms/src/vertex_ai/messages/transformation.rs index f4e8ab3bcca..775cf005d4c 100644 --- a/litellm-rust/crates/llms/src/vertex_ai/messages/transformation.rs +++ b/litellm-rust/crates/llms/src/vertex_ai/messages/transformation.rs @@ -662,6 +662,17 @@ mod tests { &["compact-2026-01-12", "context-management-2025-06-27"] )] #[case::compaction(json!({"compaction": {"type": "summarize"}}), &["compact-2026-09-04"])] + #[case::thinking_display_updates( + json!({"thinking": {"type": "adaptive", "display": "updates"}}), + &["thinking-display-updates-2026-08-18"] + )] + #[case::tool_addition( + json!({"messages": [ + {"role": "user", "content": "Use the ping tool"}, + {"role": "system", "content": [{"type": "tool_addition", "tool": {"type": "tool_reference", "name": "ping"}}]} + ]}), + &["mid-conversation-tool-changes-2026-07-01"] + )] #[case::advisor_tool_is_not_a_vertex_beta( json!({"tools": [{"type": "advisor_20260301", "name": "advisor"}]}), &[] diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index e2eed3cf6d5..7b0377ec4df 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -551,6 +551,7 @@ def _resolve_mantle_region_for_cost( _PER_CALL_LITELLM_PARAM_DICTS: Final = frozenset({"metadata", "litellm_metadata"}) _LITELLM_PARAMS: Final = TypeAdapter(dict[str, object]) +_SERVED_ANTHROPIC_BODY: Final = TypeAdapter(dict[str, object]) def _provider_response_id(source: object) -> str | None: @@ -4693,13 +4694,11 @@ class Logging(LiteLLMLoggingBaseClass): verbose_logger.warning("LiteLLM: the anthropic_messages stream assembled no response, logging an empty one") return litellm.ModelResponse(model=self.model) else: - from litellm.types.llms.anthropic import AnthropicResponse - - pydantic_result: Final = AnthropicResponse.model_validate(result) - import httpx - + served_body: Final = _SERVED_ANTHROPIC_BODY.validate_python( + result.model_dump() if isinstance(result, BaseModel) else result + ) result = litellm.AnthropicConfig().transform_parsed_response( - completion_response=pydantic_result.model_dump(), + completion_response=served_body, raw_response=httpx.Response( status_code=200, headers={}, diff --git a/litellm/llms/anthropic/common_utils.py b/litellm/llms/anthropic/common_utils.py index 7cba8bd47f4..fc3314baeff 100644 --- a/litellm/llms/anthropic/common_utils.py +++ b/litellm/llms/anthropic/common_utils.py @@ -44,6 +44,7 @@ from litellm.llms.anthropic.wif import ( from litellm.llms.base_llm.base_utils import BaseLLMModelInfo, BaseTokenCounter from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.types.llms.anthropic import ( + ANTHROPIC_BETA_HEADER_VALUES, ANTHROPIC_HOSTED_TOOLS, ANTHROPIC_MID_CONVERSATION_OUTPUT_CONFIG_BETA_HEADER, ANTHROPIC_MID_CONVERSATION_TOOL_CHANGES_BETA_HEADER, @@ -150,6 +151,27 @@ def requires_native_compaction_beta( ) +def count_tokens_feature_betas( + custom_llm_provider: str, + optional_params: Mapping[str, object], + messages: Sequence[object], +) -> tuple[str, ...]: + return tuple( + beta + for beta, used in ( + ( + ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_09_04.value, + requires_native_compaction_beta(custom_llm_provider, optional_params, messages), + ), + ( + ANTHROPIC_MID_CONVERSATION_TOOL_CHANGES_BETA_HEADER, + AnthropicModelInfo().is_mid_conversation_tool_change_used(messages), + ), + ) + if used + ) + + def supports_anthropic_cache_control(model: str, custom_llm_provider: str | None) -> bool: from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider from litellm.utils import supports_prompt_caching @@ -1313,8 +1335,6 @@ class AnthropicModelInfo(BaseLLMModelInfo): if is_vertex_request is True: # Vertex AI requires web search beta header for web search to work if web_search_tool_used: - from litellm.types.llms.anthropic import ANTHROPIC_BETA_HEADER_VALUES - headers["anthropic-beta"] = ANTHROPIC_BETA_HEADER_VALUES.WEB_SEARCH_2025_03_05.value elif len(all_betas) > 0: headers["anthropic-beta"] = ",".join(all_betas) diff --git a/litellm/llms/azure_ai/anthropic/count_tokens/transformation.py b/litellm/llms/azure_ai/anthropic/count_tokens/transformation.py index 0e42de201d8..0c661514123 100644 --- a/litellm/llms/azure_ai/anthropic/count_tokens/transformation.py +++ b/litellm/llms/azure_ai/anthropic/count_tokens/transformation.py @@ -8,12 +8,11 @@ from collections.abc import Sequence from typing import Final from litellm.constants import ANTHROPIC_TOKEN_COUNTING_BETA_VERSION -from litellm.llms.anthropic.common_utils import merge_anthropic_beta_headers, requires_native_compaction_beta +from litellm.llms.anthropic.common_utils import count_tokens_feature_betas, merge_anthropic_beta_headers from litellm.llms.anthropic.count_tokens.transformation import ( AnthropicCountTokensConfig, ) from litellm.llms.azure.common_utils import BaseAzureLLM -from litellm.types.llms.anthropic import ANTHROPIC_BETA_HEADER_VALUES from litellm.types.router import GenericLiteLLMParams @@ -49,10 +48,7 @@ class AzureAIAnthropicCountTokensConfig(AnthropicCountTokensConfig): "Content-Type": "application/json", "anthropic-version": "2023-06-01", "anthropic-beta": merge_anthropic_beta_headers( - ANTHROPIC_TOKEN_COUNTING_BETA_VERSION, - ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_09_04.value - if requires_native_compaction_beta("azure_ai", {}, messages) - else None, + ANTHROPIC_TOKEN_COUNTING_BETA_VERSION, count_tokens_feature_betas("azure_ai", {}, messages) ), **({"x-api-key": api_key} if api_key else {}), } diff --git a/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/transformation.py b/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/transformation.py index e7aea7d0057..5fa24bd1ec1 100644 --- a/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/transformation.py +++ b/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/transformation.py @@ -7,6 +7,7 @@ import httpx from pydantic import TypeAdapter, ValidationError import litellm +from litellm.litellm_core_utils.core_helpers import drops_unsupported_params from litellm.litellm_core_utils.prompt_templates.image_handling import RemoteMedia, inline_remote_image_urls from litellm.llms.base_llm.chat.transformation import LiteLLMLoggingObj from litellm.types.llms.anthropic import ANTHROPIC_BETA_HEADER_VALUES @@ -176,7 +177,7 @@ class VertexAIAnthropicConfig(AnthropicConfig): if requires_native_compaction_beta("vertex_ai", params, _REQUEST_MESSAGES.validate_python(data["messages"])) else frozenset[str]() ) - drop_params: Final = litellm.drop_params is True or litellm_params.get("drop_params") is True + drop_params: Final = drops_unsupported_params(litellm_params.get("drop_params")) extra_headers: Final = _NON_DEFAULT_PARAMS.validate_python(params.get("extra_headers") or {}) data.pop("extra_headers", None) injected_betas: Final = frozenset(beta_set) | _client_betas( diff --git a/litellm/llms/vertex_ai/vertex_ai_partner_models/count_tokens/handler.py b/litellm/llms/vertex_ai/vertex_ai_partner_models/count_tokens/handler.py index 103c51eb71a..94d2e130c0f 100644 --- a/litellm/llms/vertex_ai/vertex_ai_partner_models/count_tokens/handler.py +++ b/litellm/llms/vertex_ai/vertex_ai_partner_models/count_tokens/handler.py @@ -11,11 +11,10 @@ from typing import Any, Final from pydantic import TypeAdapter -from litellm.llms.anthropic.common_utils import requires_native_compaction_beta +from litellm.llms.anthropic.common_utils import count_tokens_feature_betas from litellm.llms.custom_httpx.http_handler import get_async_httpx_client from litellm.llms.vertex_ai.common_utils import get_vertex_base_url from litellm.llms.vertex_ai.vertex_llm_base import VertexBase -from litellm.types.llms.anthropic import ANTHROPIC_BETA_HEADER_VALUES _COUNT_TOKENS_PUBLISHERS: Final = ( ("claude", "anthropic"), @@ -33,11 +32,10 @@ def count_tokens_publisher(model: str) -> str | None: def count_tokens_headers(access_token: str, request_data: Mapping[str, object]) -> Mapping[str, str]: authorization: Final = {"Authorization": f"Bearer {access_token}"} - if not requires_native_compaction_beta( + betas: Final = count_tokens_feature_betas( "vertex_ai", request_data, _REQUEST_MESSAGES.validate_python(request_data["messages"]) - ): - return authorization - return {**authorization, "anthropic-beta": ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_09_04.value} + ) + return {**authorization, "anthropic-beta": ",".join(betas)} if betas else authorization class VertexAIPartnerModelsTokenCounter(VertexBase): diff --git a/litellm/types/llms/anthropic.py b/litellm/types/llms/anthropic.py index e13df3aaa31..c3cf532db09 100644 --- a/litellm/types/llms/anthropic.py +++ b/litellm/types/llms/anthropic.py @@ -783,7 +783,7 @@ class AnthropicThinkingParam(TypedDict, total=False): block_binding: ReadOnly[AnthropicThinkingBlockBinding] -LEGACY_THINKING_DISPLAY_UPDATES_PROVIDERS: Final = frozenset(("azure_ai", "vertex_ai")) +LEGACY_THINKING_DISPLAY_UPDATES_PROVIDERS: Final = frozenset(("anthropic", "azure_ai", "vertex_ai")) class ANTHROPIC_HOSTED_TOOLS(str, Enum): diff --git a/tests/unit/litellm_core_utils/test_litellm_logging.py b/tests/unit/litellm_core_utils/test_litellm_logging.py index 15ea822147c..873ef0d0baf 100644 --- a/tests/unit/litellm_core_utils/test_litellm_logging.py +++ b/tests/unit/litellm_core_utils/test_litellm_logging.py @@ -6541,11 +6541,64 @@ def test_handle_anthropic_messages_response_logging_reparses_a_served_compaction assert isinstance(logged, ModelResponse) assert logged.id == "msg_compacted" - assert logged.usage.prompt_tokens == 410 # type: ignore[attr-defined] - assert logged.usage.completion_tokens == 95 # type: ignore[attr-defined] - assert logged.choices[0].message.provider_specific_fields["compaction_blocks"] == [ # type: ignore[union-attr] - {"content": None, **compaction_block} - ] + assert logged.usage.prompt_tokens == 410 + assert logged.usage.completion_tokens == 95 + assert logged.choices[0].message.provider_specific_fields["compaction_blocks"] == [compaction_block] + + +_WEB_SEARCH_REPLY: Final = ( + {"type": "server_tool_use", "id": "srvtoolu_1", "name": "web_search", "input": {"query": "weather"}}, + { + "type": "web_search_tool_result", + "tool_use_id": "srvtoolu_1", + "content": [{"type": "web_search_result", "url": "https://example.com", "title": "Weather"}], + }, + {"type": "text", "text": "It is sunny."}, +) + + +@pytest.mark.parametrize( + ("content", "stop_reason"), + [ + (list(_WEB_SEARCH_REPLY), "end_turn"), + (list(_WEB_SEARCH_REPLY), "pause_turn"), + ([{"type": "text", "text": "The context window is full."}], "model_context_window_exceeded"), + ( + [ + {"type": "mcp_tool_use", "id": "mcptoolu_1", "name": "lookup", "server_name": "docs", "input": {}}, + {"type": "mcp_tool_result", "tool_use_id": "mcptoolu_1", "is_error": False, "content": [{"type": "text", "text": "ok"}]}, + {"type": "text", "text": "Found it."}, + ], + "end_turn", + ), + ([{"type": "container_upload", "file_id": "file_1"}, {"type": "text", "text": "Uploaded."}], "end_turn"), + ], + ids=["web_search", "pause_turn", "model_context_window_exceeded", "mcp_tool_use", "container_upload"], +) +def test_handle_anthropic_messages_response_logging_bills_a_served_server_tool_response( + content: list[dict[str, object]], stop_reason: str +): + logging_obj = _anthropic_messages_logging_obj() + logging_obj.optional_params = {} + + logged = logging_obj._handle_anthropic_messages_response_logging( + result={ + "id": "msg_server_tools", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-5-5", + "content": content, + "stop_reason": stop_reason, + "stop_sequence": None, + "usage": {"input_tokens": 1200, "output_tokens": 80}, + } + ) + + assert isinstance(logged, ModelResponse) + assert logged.id == "msg_server_tools" + assert logged.usage.prompt_tokens == 1200 + assert logged.usage.completion_tokens == 80 + assert logged.choices[0].message.content == content[-1]["text"] def test_handle_anthropic_messages_response_logging_passes_model_response_through(): diff --git a/tests/unit/llms/anthropic/test_anthropic_common_utils.py b/tests/unit/llms/anthropic/test_anthropic_common_utils.py index 6f84e03fb62..0692a762bf8 100644 --- a/tests/unit/llms/anthropic/test_anthropic_common_utils.py +++ b/tests/unit/llms/anthropic/test_anthropic_common_utils.py @@ -4242,7 +4242,8 @@ def test_thinking_display_beta_requires_active_thinking(thinking: object, expect ("azure_ai", "omitted", {"type": "adaptive", "display": "omitted"}), ("azure_ai", "updates", {"type": "adaptive", "display": "updates"}), ("vertex_ai", "updates", {"type": "adaptive", "display": "updates"}), - ("anthropic", "updates", {"type": "adaptive"}), + ("anthropic", "updates", {"type": "adaptive", "display": "updates"}), + ("databricks", "updates", {"type": "adaptive"}), ), ) def test_shared_legacy_thinking_translation_preserves_supported_display( diff --git a/tests/unit/llms/azure_ai/anthropic/count_tokens/test_token_counter.py b/tests/unit/llms/azure_ai/anthropic/count_tokens/test_token_counter.py index 0194e6736d5..7807975be24 100644 --- a/tests/unit/llms/azure_ai/anthropic/count_tokens/test_token_counter.py +++ b/tests/unit/llms/azure_ai/anthropic/count_tokens/test_token_counter.py @@ -248,3 +248,23 @@ async def test_count_sends_the_compact_beta_only_when_the_history_replays_a_comp assert result is not None assert result.total_tokens == 376 assert route.calls.last.request.headers["anthropic-beta"] == expected_beta + + +@pytest.mark.asyncio +async def test_count_sends_the_tool_change_beta_when_the_history_adds_a_tool(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + with respx.mock: + route: Final = respx.post(COUNT_URL).mock(return_value=httpx.Response(200, json={"input_tokens": 541})) + result: Final = await AzureAIAnthropicTokenCounter().count_tokens( + model_to_use="claude-sonnet-5-5", + messages=[ + {"role": "user", "content": "Use the ping tool once."}, + {"role": "system", "content": [{"type": "tool_addition", "tool": {"type": "tool_reference", "name": "ping"}}]}, + ], + contents=None, + deployment={"litellm_params": {"api_key": "azure-key", "api_base": API_BASE}}, + request_model="azure_ai/claude-sonnet-5-5", + ) + assert result is not None + assert result.total_tokens == 541 + assert route.calls.last.request.headers["anthropic-beta"] == "mid-conversation-tool-changes-2026-07-01,token-counting-2024-11-01" diff --git a/tests/unit/llms/vertex_ai/vertex_ai_partner_models/count_tokens/test_handler.py b/tests/unit/llms/vertex_ai/vertex_ai_partner_models/count_tokens/test_handler.py index b23461823f8..c03f08ab610 100644 --- a/tests/unit/llms/vertex_ai/vertex_ai_partner_models/count_tokens/test_handler.py +++ b/tests/unit/llms/vertex_ai/vertex_ai_partner_models/count_tokens/test_handler.py @@ -32,3 +32,29 @@ def test_count_tokens_on_a_compaction_request_sends_the_compact_beta(request_dat ) def test_count_tokens_without_compaction_sends_no_beta(request_data: dict[str, object]) -> None: assert count_tokens_headers("token", request_data) == {"Authorization": "Bearer token"} + + +_TOOL_ADDITION: Final = { + "role": "system", + "content": [{"type": "tool_addition", "tool": {"type": "tool_reference", "name": "ping"}}], +} + + +@pytest.mark.parametrize( + ("request_data", "expected_beta"), + [ + ({"messages": [*_HISTORY, _TOOL_ADDITION]}, "mid-conversation-tool-changes-2026-07-01"), + ( + {"messages": [*_HISTORY, _TOOL_ADDITION], "compaction": {"type": "summarize"}}, + "compact-2026-09-04,mid-conversation-tool-changes-2026-07-01", + ), + ], + ids=["tool_addition", "tool_addition_with_compaction"], +) +def test_count_tokens_on_a_tool_change_request_sends_the_tool_change_beta( + request_data: dict[str, object], expected_beta: str +) -> None: + assert count_tokens_headers("token", request_data) == { + "Authorization": "Bearer token", + "anthropic-beta": expected_beta, + }