mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-17 09:34:48 +00:00
fix(vertex_ai,azure_ai,rust): add tool change betas on count tokens and Rust messages, bill Rust server tool replies
Some checks failed
LiteLLM Rust / rust-lint (push) Has been cancelled
LiteLLM Rust / rust-test (push) Has been cancelled
LiteLLM Rust / rust-wheel (push) Has been cancelled
Terraform Modules / fmt, validate, test (aws) (push) Has been cancelled
Terraform Modules / fmt, validate, test (gcp) (push) Has been cancelled
Terraform Provider / gofmt, vet, build, test (push) Has been cancelled
Terraform Provider / Provider endpoints vs proxy OpenAPI schema (push) Has been cancelled
Some checks failed
LiteLLM Rust / rust-lint (push) Has been cancelled
LiteLLM Rust / rust-test (push) Has been cancelled
LiteLLM Rust / rust-wheel (push) Has been cancelled
Terraform Modules / fmt, validate, test (aws) (push) Has been cancelled
Terraform Modules / fmt, validate, test (gcp) (push) Has been cancelled
Terraform Provider / gofmt, vet, build, test (push) Has been cancelled
Terraform Provider / Provider endpoints vs proxy OpenAPI schema (push) Has been cancelled
Vertex and Azure count tokens now add mid-conversation-tool-changes-2026-07-01 when the history carries a tool_addition or tool_removal block, through one shared helper with the compaction beta. The Rust /v1/messages route adds thinking-display-updates-2026-08-18 and mid-conversation-tool-changes-2026-07-01 when a request uses them, on the providers that accept each beta. Rust-route logging parses the served body with the Anthropic response parser, so web_search, pause_turn, mcp_tool_use and container_upload replies get a spend row. Legacy enabled thinking keeps display updates on anthropic too, and the Vertex transform reads drop_params through drops_unsupported_params.
This commit is contained in:
parent
fbbcd52439
commit
b649f33f17
12 changed files with 251 additions and 29 deletions
|
|
@ -1,7 +1,8 @@
|
|||
use litellm_auth::CredentialPlacement;
|
||||
use litellm_llms_types::{
|
||||
formats::messages::{
|
||||
ContextEdit, ContextManagement, Message, MessagesOptionalParams, MessagesRequest, Speed,
|
||||
ContentBlock, ContentBlockType, ContextEdit, ContextManagement, Message,
|
||||
MessagesOptionalParams, MessagesRequest, Speed, ThinkingConfig, ThinkingDisplay,
|
||||
},
|
||||
providers::anthropic::{AnthropicBeta, BetaProvider, BetaSet, DEFAULT_HEADERS},
|
||||
recognized::Recognized,
|
||||
|
|
@ -203,6 +204,10 @@ fn feature_betas(request: &MessagesRequest) -> BetaSet {
|
|||
.then_some(AnthropicBeta::FastMode20260201),
|
||||
messages_carry_output_config(&request.messages)
|
||||
.then_some(AnthropicBeta::PerTurnControl20260701),
|
||||
uses_thinking_display_updates(params)
|
||||
.then_some(AnthropicBeta::ThinkingDisplayUpdates20260818),
|
||||
messages_carry_tool_changes(&request.messages)
|
||||
.then_some(AnthropicBeta::MidConversationToolChanges20260701),
|
||||
has_advisor_tool(tools).then_some(AnthropicBeta::AdvisorTool20260301),
|
||||
is_tool_search_used(tools).then_some(AnthropicBeta::AdvancedToolUse20251120),
|
||||
]
|
||||
|
|
@ -253,6 +258,36 @@ fn messages_carry_output_config(messages: &[Message]) -> bool {
|
|||
.any(|message| message.extra.contains_key("output_config"))
|
||||
}
|
||||
|
||||
fn uses_thinking_display_updates(params: &MessagesOptionalParams) -> bool {
|
||||
let display = match params.thinking.as_ref().and_then(Recognized::known) {
|
||||
Some(ThinkingConfig::Enabled(thinking)) => thinking.display.as_ref(),
|
||||
Some(ThinkingConfig::Adaptive(thinking)) => thinking.display.as_ref(),
|
||||
_ => None,
|
||||
};
|
||||
display.and_then(Recognized::known) == Some(&ThinkingDisplay::Updates)
|
||||
}
|
||||
|
||||
fn is_tool_reference_change(block: &ContentBlock) -> bool {
|
||||
matches!(
|
||||
&block.block_type,
|
||||
Some(ContentBlockType::Other(block_type))
|
||||
if block_type == "tool_addition" || block_type == "tool_removal"
|
||||
) && block
|
||||
.extra
|
||||
.get("tool")
|
||||
.and_then(|tool| tool.get("type"))
|
||||
.and_then(Value::as_str)
|
||||
== Some("tool_reference")
|
||||
}
|
||||
|
||||
fn messages_carry_tool_changes(messages: &[Message]) -> bool {
|
||||
messages
|
||||
.iter()
|
||||
.filter(|message| message.role == "system")
|
||||
.flat_map(Message::blocks)
|
||||
.any(is_tool_reference_change)
|
||||
}
|
||||
|
||||
fn unsupported_param(model: &str, param: &str, value: &str, hint: &str) -> Error {
|
||||
Error::InvalidRequest(crate::ErrorDetail::UnsupportedParameter {
|
||||
model: model.into(),
|
||||
|
|
@ -1071,6 +1106,41 @@ mod tests {
|
|||
json!({"safeguards": [{"type": "dangerous_tool_use", "classifier_context": {"v": 1, "permission_mode": "auto"}}]}),
|
||||
&[]
|
||||
)]
|
||||
#[case::adaptive_display_updates(
|
||||
json!({"thinking": {"type": "adaptive", "display": "updates"}}),
|
||||
&["thinking-display-updates-2026-08-18"]
|
||||
)]
|
||||
#[case::enabled_display_updates(
|
||||
json!({"thinking": {"type": "enabled", "budget_tokens": 2048, "display": "updates"}}),
|
||||
&["thinking-display-updates-2026-08-18"]
|
||||
)]
|
||||
#[case::summarized_display(json!({"thinking": {"type": "adaptive", "display": "summarized"}}), &[])]
|
||||
#[case::disabled_thinking(json!({"thinking": {"type": "disabled"}}), &[])]
|
||||
#[case::system_tool_addition(
|
||||
json!({"messages": [
|
||||
{"role": "user", "content": "Use the ping tool"},
|
||||
{"role": "system", "content": [{"type": "tool_addition", "tool": {"type": "tool_reference", "name": "ping"}}]},
|
||||
]}),
|
||||
&["mid-conversation-tool-changes-2026-07-01"]
|
||||
)]
|
||||
#[case::system_tool_removal(
|
||||
json!({"messages": [
|
||||
{"role": "system", "content": [{"type": "tool_removal", "tool": {"type": "tool_reference", "name": "ping"}}]},
|
||||
]}),
|
||||
&["mid-conversation-tool-changes-2026-07-01"]
|
||||
)]
|
||||
#[case::user_tool_addition(
|
||||
json!({"messages": [
|
||||
{"role": "user", "content": [{"type": "tool_addition", "tool": {"type": "tool_reference", "name": "ping"}}]},
|
||||
]}),
|
||||
&[]
|
||||
)]
|
||||
#[case::tool_addition_without_a_tool_reference(
|
||||
json!({"messages": [
|
||||
{"role": "system", "content": [{"type": "tool_addition", "tool": {"type": "tool_definition"}}]},
|
||||
]}),
|
||||
&[]
|
||||
)]
|
||||
fn feature_betas_follow_the_request(#[case] fields: Value, #[case] expected: &[&str]) {
|
||||
assert_eq!(feature_betas(&request(fields)), betas(expected));
|
||||
}
|
||||
|
|
@ -1093,6 +1163,33 @@ mod tests {
|
|||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::vertex_ai(
|
||||
BetaProvider::VertexAi,
|
||||
&["mid-conversation-tool-changes-2026-07-01", "thinking-display-updates-2026-08-18"]
|
||||
)]
|
||||
#[case::azure_ai(
|
||||
BetaProvider::AzureAi,
|
||||
&["mid-conversation-tool-changes-2026-07-01", "thinking-display-updates-2026-08-18"]
|
||||
)]
|
||||
#[case::bedrock_converse(BetaProvider::BedrockConverse, &["thinking-display-updates-2026-08-18"])]
|
||||
#[case::databricks(BetaProvider::Databricks, &[])]
|
||||
fn tool_change_and_display_betas_reach_the_providers_that_accept_them(
|
||||
#[case] provider: BetaProvider,
|
||||
#[case] expected: &[&str],
|
||||
) {
|
||||
let fields = json!({
|
||||
"thinking": {"type": "adaptive", "display": "updates"},
|
||||
"messages": [
|
||||
{"role": "system", "content": [{"type": "tool_addition", "tool": {"type": "tool_reference", "name": "ping"}}]},
|
||||
],
|
||||
});
|
||||
assert_eq!(
|
||||
provider_feature_betas(&request(fields), provider),
|
||||
betas(expected)
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn hosts_without_a_beta_map_column_get_safeguards_without_a_derived_beta() {
|
||||
let client = headers(&[("anthropic-beta", "claude-code-20250219")]);
|
||||
|
|
|
|||
|
|
@ -662,6 +662,17 @@ mod tests {
|
|||
&["compact-2026-01-12", "context-management-2025-06-27"]
|
||||
)]
|
||||
#[case::compaction(json!({"compaction": {"type": "summarize"}}), &["compact-2026-09-04"])]
|
||||
#[case::thinking_display_updates(
|
||||
json!({"thinking": {"type": "adaptive", "display": "updates"}}),
|
||||
&["thinking-display-updates-2026-08-18"]
|
||||
)]
|
||||
#[case::tool_addition(
|
||||
json!({"messages": [
|
||||
{"role": "user", "content": "Use the ping tool"},
|
||||
{"role": "system", "content": [{"type": "tool_addition", "tool": {"type": "tool_reference", "name": "ping"}}]}
|
||||
]}),
|
||||
&["mid-conversation-tool-changes-2026-07-01"]
|
||||
)]
|
||||
#[case::advisor_tool_is_not_a_vertex_beta(
|
||||
json!({"tools": [{"type": "advisor_20260301", "name": "advisor"}]}),
|
||||
&[]
|
||||
|
|
|
|||
|
|
@ -551,6 +551,7 @@ def _resolve_mantle_region_for_cost(
|
|||
|
||||
_PER_CALL_LITELLM_PARAM_DICTS: Final = frozenset({"metadata", "litellm_metadata"})
|
||||
_LITELLM_PARAMS: Final = TypeAdapter(dict[str, object])
|
||||
_SERVED_ANTHROPIC_BODY: Final = TypeAdapter(dict[str, object])
|
||||
|
||||
|
||||
def _provider_response_id(source: object) -> str | None:
|
||||
|
|
@ -4693,13 +4694,11 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
verbose_logger.warning("LiteLLM: the anthropic_messages stream assembled no response, logging an empty one")
|
||||
return litellm.ModelResponse(model=self.model)
|
||||
else:
|
||||
from litellm.types.llms.anthropic import AnthropicResponse
|
||||
|
||||
pydantic_result: Final = AnthropicResponse.model_validate(result)
|
||||
import httpx
|
||||
|
||||
served_body: Final = _SERVED_ANTHROPIC_BODY.validate_python(
|
||||
result.model_dump() if isinstance(result, BaseModel) else result
|
||||
)
|
||||
result = litellm.AnthropicConfig().transform_parsed_response(
|
||||
completion_response=pydantic_result.model_dump(),
|
||||
completion_response=served_body,
|
||||
raw_response=httpx.Response(
|
||||
status_code=200,
|
||||
headers={},
|
||||
|
|
|
|||
|
|
@ -44,6 +44,7 @@ from litellm.llms.anthropic.wif import (
|
|||
from litellm.llms.base_llm.base_utils import BaseLLMModelInfo, BaseTokenCounter
|
||||
from litellm.llms.base_llm.chat.transformation import BaseLLMException
|
||||
from litellm.types.llms.anthropic import (
|
||||
ANTHROPIC_BETA_HEADER_VALUES,
|
||||
ANTHROPIC_HOSTED_TOOLS,
|
||||
ANTHROPIC_MID_CONVERSATION_OUTPUT_CONFIG_BETA_HEADER,
|
||||
ANTHROPIC_MID_CONVERSATION_TOOL_CHANGES_BETA_HEADER,
|
||||
|
|
@ -150,6 +151,27 @@ def requires_native_compaction_beta(
|
|||
)
|
||||
|
||||
|
||||
def count_tokens_feature_betas(
|
||||
custom_llm_provider: str,
|
||||
optional_params: Mapping[str, object],
|
||||
messages: Sequence[object],
|
||||
) -> tuple[str, ...]:
|
||||
return tuple(
|
||||
beta
|
||||
for beta, used in (
|
||||
(
|
||||
ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_09_04.value,
|
||||
requires_native_compaction_beta(custom_llm_provider, optional_params, messages),
|
||||
),
|
||||
(
|
||||
ANTHROPIC_MID_CONVERSATION_TOOL_CHANGES_BETA_HEADER,
|
||||
AnthropicModelInfo().is_mid_conversation_tool_change_used(messages),
|
||||
),
|
||||
)
|
||||
if used
|
||||
)
|
||||
|
||||
|
||||
def supports_anthropic_cache_control(model: str, custom_llm_provider: str | None) -> bool:
|
||||
from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
|
||||
from litellm.utils import supports_prompt_caching
|
||||
|
|
@ -1313,8 +1335,6 @@ class AnthropicModelInfo(BaseLLMModelInfo):
|
|||
if is_vertex_request is True:
|
||||
# Vertex AI requires web search beta header for web search to work
|
||||
if web_search_tool_used:
|
||||
from litellm.types.llms.anthropic import ANTHROPIC_BETA_HEADER_VALUES
|
||||
|
||||
headers["anthropic-beta"] = ANTHROPIC_BETA_HEADER_VALUES.WEB_SEARCH_2025_03_05.value
|
||||
elif len(all_betas) > 0:
|
||||
headers["anthropic-beta"] = ",".join(all_betas)
|
||||
|
|
|
|||
|
|
@ -8,12 +8,11 @@ from collections.abc import Sequence
|
|||
from typing import Final
|
||||
|
||||
from litellm.constants import ANTHROPIC_TOKEN_COUNTING_BETA_VERSION
|
||||
from litellm.llms.anthropic.common_utils import merge_anthropic_beta_headers, requires_native_compaction_beta
|
||||
from litellm.llms.anthropic.common_utils import count_tokens_feature_betas, merge_anthropic_beta_headers
|
||||
from litellm.llms.anthropic.count_tokens.transformation import (
|
||||
AnthropicCountTokensConfig,
|
||||
)
|
||||
from litellm.llms.azure.common_utils import BaseAzureLLM
|
||||
from litellm.types.llms.anthropic import ANTHROPIC_BETA_HEADER_VALUES
|
||||
from litellm.types.router import GenericLiteLLMParams
|
||||
|
||||
|
||||
|
|
@ -49,10 +48,7 @@ class AzureAIAnthropicCountTokensConfig(AnthropicCountTokensConfig):
|
|||
"Content-Type": "application/json",
|
||||
"anthropic-version": "2023-06-01",
|
||||
"anthropic-beta": merge_anthropic_beta_headers(
|
||||
ANTHROPIC_TOKEN_COUNTING_BETA_VERSION,
|
||||
ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_09_04.value
|
||||
if requires_native_compaction_beta("azure_ai", {}, messages)
|
||||
else None,
|
||||
ANTHROPIC_TOKEN_COUNTING_BETA_VERSION, count_tokens_feature_betas("azure_ai", {}, messages)
|
||||
),
|
||||
**({"x-api-key": api_key} if api_key else {}),
|
||||
}
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ import httpx
|
|||
from pydantic import TypeAdapter, ValidationError
|
||||
|
||||
import litellm
|
||||
from litellm.litellm_core_utils.core_helpers import drops_unsupported_params
|
||||
from litellm.litellm_core_utils.prompt_templates.image_handling import RemoteMedia, inline_remote_image_urls
|
||||
from litellm.llms.base_llm.chat.transformation import LiteLLMLoggingObj
|
||||
from litellm.types.llms.anthropic import ANTHROPIC_BETA_HEADER_VALUES
|
||||
|
|
@ -176,7 +177,7 @@ class VertexAIAnthropicConfig(AnthropicConfig):
|
|||
if requires_native_compaction_beta("vertex_ai", params, _REQUEST_MESSAGES.validate_python(data["messages"]))
|
||||
else frozenset[str]()
|
||||
)
|
||||
drop_params: Final = litellm.drop_params is True or litellm_params.get("drop_params") is True
|
||||
drop_params: Final = drops_unsupported_params(litellm_params.get("drop_params"))
|
||||
extra_headers: Final = _NON_DEFAULT_PARAMS.validate_python(params.get("extra_headers") or {})
|
||||
data.pop("extra_headers", None)
|
||||
injected_betas: Final = frozenset(beta_set) | _client_betas(
|
||||
|
|
|
|||
|
|
@ -11,11 +11,10 @@ from typing import Any, Final
|
|||
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
from litellm.llms.anthropic.common_utils import requires_native_compaction_beta
|
||||
from litellm.llms.anthropic.common_utils import count_tokens_feature_betas
|
||||
from litellm.llms.custom_httpx.http_handler import get_async_httpx_client
|
||||
from litellm.llms.vertex_ai.common_utils import get_vertex_base_url
|
||||
from litellm.llms.vertex_ai.vertex_llm_base import VertexBase
|
||||
from litellm.types.llms.anthropic import ANTHROPIC_BETA_HEADER_VALUES
|
||||
|
||||
_COUNT_TOKENS_PUBLISHERS: Final = (
|
||||
("claude", "anthropic"),
|
||||
|
|
@ -33,11 +32,10 @@ def count_tokens_publisher(model: str) -> str | None:
|
|||
|
||||
def count_tokens_headers(access_token: str, request_data: Mapping[str, object]) -> Mapping[str, str]:
|
||||
authorization: Final = {"Authorization": f"Bearer {access_token}"}
|
||||
if not requires_native_compaction_beta(
|
||||
betas: Final = count_tokens_feature_betas(
|
||||
"vertex_ai", request_data, _REQUEST_MESSAGES.validate_python(request_data["messages"])
|
||||
):
|
||||
return authorization
|
||||
return {**authorization, "anthropic-beta": ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_09_04.value}
|
||||
)
|
||||
return {**authorization, "anthropic-beta": ",".join(betas)} if betas else authorization
|
||||
|
||||
|
||||
class VertexAIPartnerModelsTokenCounter(VertexBase):
|
||||
|
|
|
|||
|
|
@ -783,7 +783,7 @@ class AnthropicThinkingParam(TypedDict, total=False):
|
|||
block_binding: ReadOnly[AnthropicThinkingBlockBinding]
|
||||
|
||||
|
||||
LEGACY_THINKING_DISPLAY_UPDATES_PROVIDERS: Final = frozenset(("azure_ai", "vertex_ai"))
|
||||
LEGACY_THINKING_DISPLAY_UPDATES_PROVIDERS: Final = frozenset(("anthropic", "azure_ai", "vertex_ai"))
|
||||
|
||||
|
||||
class ANTHROPIC_HOSTED_TOOLS(str, Enum):
|
||||
|
|
|
|||
|
|
@ -6541,11 +6541,64 @@ def test_handle_anthropic_messages_response_logging_reparses_a_served_compaction
|
|||
|
||||
assert isinstance(logged, ModelResponse)
|
||||
assert logged.id == "msg_compacted"
|
||||
assert logged.usage.prompt_tokens == 410 # type: ignore[attr-defined]
|
||||
assert logged.usage.completion_tokens == 95 # type: ignore[attr-defined]
|
||||
assert logged.choices[0].message.provider_specific_fields["compaction_blocks"] == [ # type: ignore[union-attr]
|
||||
{"content": None, **compaction_block}
|
||||
]
|
||||
assert logged.usage.prompt_tokens == 410
|
||||
assert logged.usage.completion_tokens == 95
|
||||
assert logged.choices[0].message.provider_specific_fields["compaction_blocks"] == [compaction_block]
|
||||
|
||||
|
||||
_WEB_SEARCH_REPLY: Final = (
|
||||
{"type": "server_tool_use", "id": "srvtoolu_1", "name": "web_search", "input": {"query": "weather"}},
|
||||
{
|
||||
"type": "web_search_tool_result",
|
||||
"tool_use_id": "srvtoolu_1",
|
||||
"content": [{"type": "web_search_result", "url": "https://example.com", "title": "Weather"}],
|
||||
},
|
||||
{"type": "text", "text": "It is sunny."},
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("content", "stop_reason"),
|
||||
[
|
||||
(list(_WEB_SEARCH_REPLY), "end_turn"),
|
||||
(list(_WEB_SEARCH_REPLY), "pause_turn"),
|
||||
([{"type": "text", "text": "The context window is full."}], "model_context_window_exceeded"),
|
||||
(
|
||||
[
|
||||
{"type": "mcp_tool_use", "id": "mcptoolu_1", "name": "lookup", "server_name": "docs", "input": {}},
|
||||
{"type": "mcp_tool_result", "tool_use_id": "mcptoolu_1", "is_error": False, "content": [{"type": "text", "text": "ok"}]},
|
||||
{"type": "text", "text": "Found it."},
|
||||
],
|
||||
"end_turn",
|
||||
),
|
||||
([{"type": "container_upload", "file_id": "file_1"}, {"type": "text", "text": "Uploaded."}], "end_turn"),
|
||||
],
|
||||
ids=["web_search", "pause_turn", "model_context_window_exceeded", "mcp_tool_use", "container_upload"],
|
||||
)
|
||||
def test_handle_anthropic_messages_response_logging_bills_a_served_server_tool_response(
|
||||
content: list[dict[str, object]], stop_reason: str
|
||||
):
|
||||
logging_obj = _anthropic_messages_logging_obj()
|
||||
logging_obj.optional_params = {}
|
||||
|
||||
logged = logging_obj._handle_anthropic_messages_response_logging(
|
||||
result={
|
||||
"id": "msg_server_tools",
|
||||
"type": "message",
|
||||
"role": "assistant",
|
||||
"model": "claude-sonnet-5-5",
|
||||
"content": content,
|
||||
"stop_reason": stop_reason,
|
||||
"stop_sequence": None,
|
||||
"usage": {"input_tokens": 1200, "output_tokens": 80},
|
||||
}
|
||||
)
|
||||
|
||||
assert isinstance(logged, ModelResponse)
|
||||
assert logged.id == "msg_server_tools"
|
||||
assert logged.usage.prompt_tokens == 1200
|
||||
assert logged.usage.completion_tokens == 80
|
||||
assert logged.choices[0].message.content == content[-1]["text"]
|
||||
|
||||
|
||||
def test_handle_anthropic_messages_response_logging_passes_model_response_through():
|
||||
|
|
|
|||
|
|
@ -4242,7 +4242,8 @@ def test_thinking_display_beta_requires_active_thinking(thinking: object, expect
|
|||
("azure_ai", "omitted", {"type": "adaptive", "display": "omitted"}),
|
||||
("azure_ai", "updates", {"type": "adaptive", "display": "updates"}),
|
||||
("vertex_ai", "updates", {"type": "adaptive", "display": "updates"}),
|
||||
("anthropic", "updates", {"type": "adaptive"}),
|
||||
("anthropic", "updates", {"type": "adaptive", "display": "updates"}),
|
||||
("databricks", "updates", {"type": "adaptive"}),
|
||||
),
|
||||
)
|
||||
def test_shared_legacy_thinking_translation_preserves_supported_display(
|
||||
|
|
|
|||
|
|
@ -248,3 +248,23 @@ async def test_count_sends_the_compact_beta_only_when_the_history_replays_a_comp
|
|||
assert result is not None
|
||||
assert result.total_tokens == 376
|
||||
assert route.calls.last.request.headers["anthropic-beta"] == expected_beta
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_count_sends_the_tool_change_beta_when_the_history_adds_a_tool(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
with respx.mock:
|
||||
route: Final = respx.post(COUNT_URL).mock(return_value=httpx.Response(200, json={"input_tokens": 541}))
|
||||
result: Final = await AzureAIAnthropicTokenCounter().count_tokens(
|
||||
model_to_use="claude-sonnet-5-5",
|
||||
messages=[
|
||||
{"role": "user", "content": "Use the ping tool once."},
|
||||
{"role": "system", "content": [{"type": "tool_addition", "tool": {"type": "tool_reference", "name": "ping"}}]},
|
||||
],
|
||||
contents=None,
|
||||
deployment={"litellm_params": {"api_key": "azure-key", "api_base": API_BASE}},
|
||||
request_model="azure_ai/claude-sonnet-5-5",
|
||||
)
|
||||
assert result is not None
|
||||
assert result.total_tokens == 541
|
||||
assert route.calls.last.request.headers["anthropic-beta"] == "mid-conversation-tool-changes-2026-07-01,token-counting-2024-11-01"
|
||||
|
|
|
|||
|
|
@ -32,3 +32,29 @@ def test_count_tokens_on_a_compaction_request_sends_the_compact_beta(request_dat
|
|||
)
|
||||
def test_count_tokens_without_compaction_sends_no_beta(request_data: dict[str, object]) -> None:
|
||||
assert count_tokens_headers("token", request_data) == {"Authorization": "Bearer token"}
|
||||
|
||||
|
||||
_TOOL_ADDITION: Final = {
|
||||
"role": "system",
|
||||
"content": [{"type": "tool_addition", "tool": {"type": "tool_reference", "name": "ping"}}],
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("request_data", "expected_beta"),
|
||||
[
|
||||
({"messages": [*_HISTORY, _TOOL_ADDITION]}, "mid-conversation-tool-changes-2026-07-01"),
|
||||
(
|
||||
{"messages": [*_HISTORY, _TOOL_ADDITION], "compaction": {"type": "summarize"}},
|
||||
"compact-2026-09-04,mid-conversation-tool-changes-2026-07-01",
|
||||
),
|
||||
],
|
||||
ids=["tool_addition", "tool_addition_with_compaction"],
|
||||
)
|
||||
def test_count_tokens_on_a_tool_change_request_sends_the_tool_change_beta(
|
||||
request_data: dict[str, object], expected_beta: str
|
||||
) -> None:
|
||||
assert count_tokens_headers("token", request_data) == {
|
||||
"Authorization": "Bearer token",
|
||||
"anthropic-beta": expected_beta,
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue