fix(vertex_ai,azure_ai,rust): add tool change betas on count tokens and Rust messages, bill Rust server tool replies
Some checks failed
LiteLLM Rust / rust-lint (push) Has been cancelled
LiteLLM Rust / rust-test (push) Has been cancelled
LiteLLM Rust / rust-wheel (push) Has been cancelled
Terraform Modules / fmt, validate, test (aws) (push) Has been cancelled
Terraform Modules / fmt, validate, test (gcp) (push) Has been cancelled
Terraform Provider / gofmt, vet, build, test (push) Has been cancelled
Terraform Provider / Provider endpoints vs proxy OpenAPI schema (push) Has been cancelled

Vertex and Azure count tokens now add mid-conversation-tool-changes-2026-07-01 when the history carries a tool_addition or tool_removal block, through one shared helper with the compaction beta.

The Rust /v1/messages route adds thinking-display-updates-2026-08-18 and mid-conversation-tool-changes-2026-07-01 when a request uses them, on the providers that accept each beta.

Rust-route logging parses the served body with the Anthropic response parser, so web_search, pause_turn, mcp_tool_use and container_upload replies get a spend row.

Legacy enabled thinking keeps display updates on anthropic too, and the Vertex transform reads drop_params through drops_unsupported_params.
This commit is contained in:
mateo-berri 2026-10-20 12:14:32 -07:00
parent fbbcd52439
commit b649f33f17
12 changed files with 251 additions and 29 deletions

View file

@ -1,7 +1,8 @@
use litellm_auth::CredentialPlacement;
use litellm_llms_types::{
formats::messages::{
ContextEdit, ContextManagement, Message, MessagesOptionalParams, MessagesRequest, Speed,
ContentBlock, ContentBlockType, ContextEdit, ContextManagement, Message,
MessagesOptionalParams, MessagesRequest, Speed, ThinkingConfig, ThinkingDisplay,
},
providers::anthropic::{AnthropicBeta, BetaProvider, BetaSet, DEFAULT_HEADERS},
recognized::Recognized,
@ -203,6 +204,10 @@ fn feature_betas(request: &MessagesRequest) -> BetaSet {
.then_some(AnthropicBeta::FastMode20260201),
messages_carry_output_config(&request.messages)
.then_some(AnthropicBeta::PerTurnControl20260701),
uses_thinking_display_updates(params)
.then_some(AnthropicBeta::ThinkingDisplayUpdates20260818),
messages_carry_tool_changes(&request.messages)
.then_some(AnthropicBeta::MidConversationToolChanges20260701),
has_advisor_tool(tools).then_some(AnthropicBeta::AdvisorTool20260301),
is_tool_search_used(tools).then_some(AnthropicBeta::AdvancedToolUse20251120),
]
@ -253,6 +258,36 @@ fn messages_carry_output_config(messages: &[Message]) -> bool {
.any(|message| message.extra.contains_key("output_config"))
}
fn uses_thinking_display_updates(params: &MessagesOptionalParams) -> bool {
let display = match params.thinking.as_ref().and_then(Recognized::known) {
Some(ThinkingConfig::Enabled(thinking)) => thinking.display.as_ref(),
Some(ThinkingConfig::Adaptive(thinking)) => thinking.display.as_ref(),
_ => None,
};
display.and_then(Recognized::known) == Some(&ThinkingDisplay::Updates)
}
fn is_tool_reference_change(block: &ContentBlock) -> bool {
matches!(
&block.block_type,
Some(ContentBlockType::Other(block_type))
if block_type == "tool_addition" || block_type == "tool_removal"
) && block
.extra
.get("tool")
.and_then(|tool| tool.get("type"))
.and_then(Value::as_str)
== Some("tool_reference")
}
fn messages_carry_tool_changes(messages: &[Message]) -> bool {
messages
.iter()
.filter(|message| message.role == "system")
.flat_map(Message::blocks)
.any(is_tool_reference_change)
}
fn unsupported_param(model: &str, param: &str, value: &str, hint: &str) -> Error {
Error::InvalidRequest(crate::ErrorDetail::UnsupportedParameter {
model: model.into(),
@ -1071,6 +1106,41 @@ mod tests {
json!({"safeguards": [{"type": "dangerous_tool_use", "classifier_context": {"v": 1, "permission_mode": "auto"}}]}),
&[]
)]
#[case::adaptive_display_updates(
json!({"thinking": {"type": "adaptive", "display": "updates"}}),
&["thinking-display-updates-2026-08-18"]
)]
#[case::enabled_display_updates(
json!({"thinking": {"type": "enabled", "budget_tokens": 2048, "display": "updates"}}),
&["thinking-display-updates-2026-08-18"]
)]
#[case::summarized_display(json!({"thinking": {"type": "adaptive", "display": "summarized"}}), &[])]
#[case::disabled_thinking(json!({"thinking": {"type": "disabled"}}), &[])]
#[case::system_tool_addition(
json!({"messages": [
{"role": "user", "content": "Use the ping tool"},
{"role": "system", "content": [{"type": "tool_addition", "tool": {"type": "tool_reference", "name": "ping"}}]},
]}),
&["mid-conversation-tool-changes-2026-07-01"]
)]
#[case::system_tool_removal(
json!({"messages": [
{"role": "system", "content": [{"type": "tool_removal", "tool": {"type": "tool_reference", "name": "ping"}}]},
]}),
&["mid-conversation-tool-changes-2026-07-01"]
)]
#[case::user_tool_addition(
json!({"messages": [
{"role": "user", "content": [{"type": "tool_addition", "tool": {"type": "tool_reference", "name": "ping"}}]},
]}),
&[]
)]
#[case::tool_addition_without_a_tool_reference(
json!({"messages": [
{"role": "system", "content": [{"type": "tool_addition", "tool": {"type": "tool_definition"}}]},
]}),
&[]
)]
fn feature_betas_follow_the_request(#[case] fields: Value, #[case] expected: &[&str]) {
assert_eq!(feature_betas(&request(fields)), betas(expected));
}
@ -1093,6 +1163,33 @@ mod tests {
);
}
#[rstest]
#[case::vertex_ai(
BetaProvider::VertexAi,
&["mid-conversation-tool-changes-2026-07-01", "thinking-display-updates-2026-08-18"]
)]
#[case::azure_ai(
BetaProvider::AzureAi,
&["mid-conversation-tool-changes-2026-07-01", "thinking-display-updates-2026-08-18"]
)]
#[case::bedrock_converse(BetaProvider::BedrockConverse, &["thinking-display-updates-2026-08-18"])]
#[case::databricks(BetaProvider::Databricks, &[])]
fn tool_change_and_display_betas_reach_the_providers_that_accept_them(
#[case] provider: BetaProvider,
#[case] expected: &[&str],
) {
let fields = json!({
"thinking": {"type": "adaptive", "display": "updates"},
"messages": [
{"role": "system", "content": [{"type": "tool_addition", "tool": {"type": "tool_reference", "name": "ping"}}]},
],
});
assert_eq!(
provider_feature_betas(&request(fields), provider),
betas(expected)
);
}
#[rstest]
fn hosts_without_a_beta_map_column_get_safeguards_without_a_derived_beta() {
let client = headers(&[("anthropic-beta", "claude-code-20250219")]);

View file

@ -662,6 +662,17 @@ mod tests {
&["compact-2026-01-12", "context-management-2025-06-27"]
)]
#[case::compaction(json!({"compaction": {"type": "summarize"}}), &["compact-2026-09-04"])]
#[case::thinking_display_updates(
json!({"thinking": {"type": "adaptive", "display": "updates"}}),
&["thinking-display-updates-2026-08-18"]
)]
#[case::tool_addition(
json!({"messages": [
{"role": "user", "content": "Use the ping tool"},
{"role": "system", "content": [{"type": "tool_addition", "tool": {"type": "tool_reference", "name": "ping"}}]}
]}),
&["mid-conversation-tool-changes-2026-07-01"]
)]
#[case::advisor_tool_is_not_a_vertex_beta(
json!({"tools": [{"type": "advisor_20260301", "name": "advisor"}]}),
&[]

View file

@ -551,6 +551,7 @@ def _resolve_mantle_region_for_cost(
_PER_CALL_LITELLM_PARAM_DICTS: Final = frozenset({"metadata", "litellm_metadata"})
_LITELLM_PARAMS: Final = TypeAdapter(dict[str, object])
_SERVED_ANTHROPIC_BODY: Final = TypeAdapter(dict[str, object])
def _provider_response_id(source: object) -> str | None:
@ -4693,13 +4694,11 @@ class Logging(LiteLLMLoggingBaseClass):
verbose_logger.warning("LiteLLM: the anthropic_messages stream assembled no response, logging an empty one")
return litellm.ModelResponse(model=self.model)
else:
from litellm.types.llms.anthropic import AnthropicResponse
pydantic_result: Final = AnthropicResponse.model_validate(result)
import httpx
served_body: Final = _SERVED_ANTHROPIC_BODY.validate_python(
result.model_dump() if isinstance(result, BaseModel) else result
)
result = litellm.AnthropicConfig().transform_parsed_response(
completion_response=pydantic_result.model_dump(),
completion_response=served_body,
raw_response=httpx.Response(
status_code=200,
headers={},

View file

@ -44,6 +44,7 @@ from litellm.llms.anthropic.wif import (
from litellm.llms.base_llm.base_utils import BaseLLMModelInfo, BaseTokenCounter
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.types.llms.anthropic import (
ANTHROPIC_BETA_HEADER_VALUES,
ANTHROPIC_HOSTED_TOOLS,
ANTHROPIC_MID_CONVERSATION_OUTPUT_CONFIG_BETA_HEADER,
ANTHROPIC_MID_CONVERSATION_TOOL_CHANGES_BETA_HEADER,
@ -150,6 +151,27 @@ def requires_native_compaction_beta(
)
def count_tokens_feature_betas(
custom_llm_provider: str,
optional_params: Mapping[str, object],
messages: Sequence[object],
) -> tuple[str, ...]:
return tuple(
beta
for beta, used in (
(
ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_09_04.value,
requires_native_compaction_beta(custom_llm_provider, optional_params, messages),
),
(
ANTHROPIC_MID_CONVERSATION_TOOL_CHANGES_BETA_HEADER,
AnthropicModelInfo().is_mid_conversation_tool_change_used(messages),
),
)
if used
)
def supports_anthropic_cache_control(model: str, custom_llm_provider: str | None) -> bool:
from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
from litellm.utils import supports_prompt_caching
@ -1313,8 +1335,6 @@ class AnthropicModelInfo(BaseLLMModelInfo):
if is_vertex_request is True:
# Vertex AI requires web search beta header for web search to work
if web_search_tool_used:
from litellm.types.llms.anthropic import ANTHROPIC_BETA_HEADER_VALUES
headers["anthropic-beta"] = ANTHROPIC_BETA_HEADER_VALUES.WEB_SEARCH_2025_03_05.value
elif len(all_betas) > 0:
headers["anthropic-beta"] = ",".join(all_betas)

View file

@ -8,12 +8,11 @@ from collections.abc import Sequence
from typing import Final
from litellm.constants import ANTHROPIC_TOKEN_COUNTING_BETA_VERSION
from litellm.llms.anthropic.common_utils import merge_anthropic_beta_headers, requires_native_compaction_beta
from litellm.llms.anthropic.common_utils import count_tokens_feature_betas, merge_anthropic_beta_headers
from litellm.llms.anthropic.count_tokens.transformation import (
AnthropicCountTokensConfig,
)
from litellm.llms.azure.common_utils import BaseAzureLLM
from litellm.types.llms.anthropic import ANTHROPIC_BETA_HEADER_VALUES
from litellm.types.router import GenericLiteLLMParams
@ -49,10 +48,7 @@ class AzureAIAnthropicCountTokensConfig(AnthropicCountTokensConfig):
"Content-Type": "application/json",
"anthropic-version": "2023-06-01",
"anthropic-beta": merge_anthropic_beta_headers(
ANTHROPIC_TOKEN_COUNTING_BETA_VERSION,
ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_09_04.value
if requires_native_compaction_beta("azure_ai", {}, messages)
else None,
ANTHROPIC_TOKEN_COUNTING_BETA_VERSION, count_tokens_feature_betas("azure_ai", {}, messages)
),
**({"x-api-key": api_key} if api_key else {}),
}

View file

@ -7,6 +7,7 @@ import httpx
from pydantic import TypeAdapter, ValidationError
import litellm
from litellm.litellm_core_utils.core_helpers import drops_unsupported_params
from litellm.litellm_core_utils.prompt_templates.image_handling import RemoteMedia, inline_remote_image_urls
from litellm.llms.base_llm.chat.transformation import LiteLLMLoggingObj
from litellm.types.llms.anthropic import ANTHROPIC_BETA_HEADER_VALUES
@ -176,7 +177,7 @@ class VertexAIAnthropicConfig(AnthropicConfig):
if requires_native_compaction_beta("vertex_ai", params, _REQUEST_MESSAGES.validate_python(data["messages"]))
else frozenset[str]()
)
drop_params: Final = litellm.drop_params is True or litellm_params.get("drop_params") is True
drop_params: Final = drops_unsupported_params(litellm_params.get("drop_params"))
extra_headers: Final = _NON_DEFAULT_PARAMS.validate_python(params.get("extra_headers") or {})
data.pop("extra_headers", None)
injected_betas: Final = frozenset(beta_set) | _client_betas(

View file

@ -11,11 +11,10 @@ from typing import Any, Final
from pydantic import TypeAdapter
from litellm.llms.anthropic.common_utils import requires_native_compaction_beta
from litellm.llms.anthropic.common_utils import count_tokens_feature_betas
from litellm.llms.custom_httpx.http_handler import get_async_httpx_client
from litellm.llms.vertex_ai.common_utils import get_vertex_base_url
from litellm.llms.vertex_ai.vertex_llm_base import VertexBase
from litellm.types.llms.anthropic import ANTHROPIC_BETA_HEADER_VALUES
_COUNT_TOKENS_PUBLISHERS: Final = (
("claude", "anthropic"),
@ -33,11 +32,10 @@ def count_tokens_publisher(model: str) -> str | None:
def count_tokens_headers(access_token: str, request_data: Mapping[str, object]) -> Mapping[str, str]:
authorization: Final = {"Authorization": f"Bearer {access_token}"}
if not requires_native_compaction_beta(
betas: Final = count_tokens_feature_betas(
"vertex_ai", request_data, _REQUEST_MESSAGES.validate_python(request_data["messages"])
):
return authorization
return {**authorization, "anthropic-beta": ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_09_04.value}
)
return {**authorization, "anthropic-beta": ",".join(betas)} if betas else authorization
class VertexAIPartnerModelsTokenCounter(VertexBase):

View file

@ -783,7 +783,7 @@ class AnthropicThinkingParam(TypedDict, total=False):
block_binding: ReadOnly[AnthropicThinkingBlockBinding]
LEGACY_THINKING_DISPLAY_UPDATES_PROVIDERS: Final = frozenset(("azure_ai", "vertex_ai"))
LEGACY_THINKING_DISPLAY_UPDATES_PROVIDERS: Final = frozenset(("anthropic", "azure_ai", "vertex_ai"))
class ANTHROPIC_HOSTED_TOOLS(str, Enum):

View file

@ -6541,11 +6541,64 @@ def test_handle_anthropic_messages_response_logging_reparses_a_served_compaction
assert isinstance(logged, ModelResponse)
assert logged.id == "msg_compacted"
assert logged.usage.prompt_tokens == 410 # type: ignore[attr-defined]
assert logged.usage.completion_tokens == 95 # type: ignore[attr-defined]
assert logged.choices[0].message.provider_specific_fields["compaction_blocks"] == [ # type: ignore[union-attr]
{"content": None, **compaction_block}
]
assert logged.usage.prompt_tokens == 410
assert logged.usage.completion_tokens == 95
assert logged.choices[0].message.provider_specific_fields["compaction_blocks"] == [compaction_block]
_WEB_SEARCH_REPLY: Final = (
{"type": "server_tool_use", "id": "srvtoolu_1", "name": "web_search", "input": {"query": "weather"}},
{
"type": "web_search_tool_result",
"tool_use_id": "srvtoolu_1",
"content": [{"type": "web_search_result", "url": "https://example.com", "title": "Weather"}],
},
{"type": "text", "text": "It is sunny."},
)
@pytest.mark.parametrize(
("content", "stop_reason"),
[
(list(_WEB_SEARCH_REPLY), "end_turn"),
(list(_WEB_SEARCH_REPLY), "pause_turn"),
([{"type": "text", "text": "The context window is full."}], "model_context_window_exceeded"),
(
[
{"type": "mcp_tool_use", "id": "mcptoolu_1", "name": "lookup", "server_name": "docs", "input": {}},
{"type": "mcp_tool_result", "tool_use_id": "mcptoolu_1", "is_error": False, "content": [{"type": "text", "text": "ok"}]},
{"type": "text", "text": "Found it."},
],
"end_turn",
),
([{"type": "container_upload", "file_id": "file_1"}, {"type": "text", "text": "Uploaded."}], "end_turn"),
],
ids=["web_search", "pause_turn", "model_context_window_exceeded", "mcp_tool_use", "container_upload"],
)
def test_handle_anthropic_messages_response_logging_bills_a_served_server_tool_response(
content: list[dict[str, object]], stop_reason: str
):
logging_obj = _anthropic_messages_logging_obj()
logging_obj.optional_params = {}
logged = logging_obj._handle_anthropic_messages_response_logging(
result={
"id": "msg_server_tools",
"type": "message",
"role": "assistant",
"model": "claude-sonnet-5-5",
"content": content,
"stop_reason": stop_reason,
"stop_sequence": None,
"usage": {"input_tokens": 1200, "output_tokens": 80},
}
)
assert isinstance(logged, ModelResponse)
assert logged.id == "msg_server_tools"
assert logged.usage.prompt_tokens == 1200
assert logged.usage.completion_tokens == 80
assert logged.choices[0].message.content == content[-1]["text"]
def test_handle_anthropic_messages_response_logging_passes_model_response_through():

View file

@ -4242,7 +4242,8 @@ def test_thinking_display_beta_requires_active_thinking(thinking: object, expect
("azure_ai", "omitted", {"type": "adaptive", "display": "omitted"}),
("azure_ai", "updates", {"type": "adaptive", "display": "updates"}),
("vertex_ai", "updates", {"type": "adaptive", "display": "updates"}),
("anthropic", "updates", {"type": "adaptive"}),
("anthropic", "updates", {"type": "adaptive", "display": "updates"}),
("databricks", "updates", {"type": "adaptive"}),
),
)
def test_shared_legacy_thinking_translation_preserves_supported_display(

View file

@ -248,3 +248,23 @@ async def test_count_sends_the_compact_beta_only_when_the_history_replays_a_comp
assert result is not None
assert result.total_tokens == 376
assert route.calls.last.request.headers["anthropic-beta"] == expected_beta
@pytest.mark.asyncio
async def test_count_sends_the_tool_change_beta_when_the_history_adds_a_tool(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
with respx.mock:
route: Final = respx.post(COUNT_URL).mock(return_value=httpx.Response(200, json={"input_tokens": 541}))
result: Final = await AzureAIAnthropicTokenCounter().count_tokens(
model_to_use="claude-sonnet-5-5",
messages=[
{"role": "user", "content": "Use the ping tool once."},
{"role": "system", "content": [{"type": "tool_addition", "tool": {"type": "tool_reference", "name": "ping"}}]},
],
contents=None,
deployment={"litellm_params": {"api_key": "azure-key", "api_base": API_BASE}},
request_model="azure_ai/claude-sonnet-5-5",
)
assert result is not None
assert result.total_tokens == 541
assert route.calls.last.request.headers["anthropic-beta"] == "mid-conversation-tool-changes-2026-07-01,token-counting-2024-11-01"

View file

@ -32,3 +32,29 @@ def test_count_tokens_on_a_compaction_request_sends_the_compact_beta(request_dat
)
def test_count_tokens_without_compaction_sends_no_beta(request_data: dict[str, object]) -> None:
assert count_tokens_headers("token", request_data) == {"Authorization": "Bearer token"}
_TOOL_ADDITION: Final = {
"role": "system",
"content": [{"type": "tool_addition", "tool": {"type": "tool_reference", "name": "ping"}}],
}
@pytest.mark.parametrize(
("request_data", "expected_beta"),
[
({"messages": [*_HISTORY, _TOOL_ADDITION]}, "mid-conversation-tool-changes-2026-07-01"),
(
{"messages": [*_HISTORY, _TOOL_ADDITION], "compaction": {"type": "summarize"}},
"compact-2026-09-04,mid-conversation-tool-changes-2026-07-01",
),
],
ids=["tool_addition", "tool_addition_with_compaction"],
)
def test_count_tokens_on_a_tool_change_request_sends_the_tool_change_beta(
request_data: dict[str, object], expected_beta: str
) -> None:
assert count_tokens_headers("token", request_data) == {
"Authorization": "Bearer token",
"anthropic-beta": expected_beta,
}