mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-17 09:34:48 +00:00
Merge 685eaf8fdf into d9aae590e3
This commit is contained in:
commit
f112bb6de1
27 changed files with 1314 additions and 79 deletions
|
|
@ -199,7 +199,7 @@ impl AnthropicBeta {
|
|||
&[BetaProvider::BedrockConverse, BetaProvider::Databricks]
|
||||
}
|
||||
Self::ContextManagement20250627 => &[BetaProvider::BedrockConverse],
|
||||
Self::AdvisorTool20260301 | Self::Compact20260904 => &[
|
||||
Self::AdvisorTool20260301 => &[
|
||||
BetaProvider::AzureAi,
|
||||
BetaProvider::BedrockConverse,
|
||||
BetaProvider::Bedrock,
|
||||
|
|
@ -207,6 +207,12 @@ impl AnthropicBeta {
|
|||
BetaProvider::VertexAi,
|
||||
BetaProvider::Databricks,
|
||||
],
|
||||
Self::Compact20260904 => &[
|
||||
BetaProvider::BedrockConverse,
|
||||
BetaProvider::Bedrock,
|
||||
BetaProvider::BedrockMantle,
|
||||
BetaProvider::Databricks,
|
||||
],
|
||||
Self::Bash20241022
|
||||
| Self::Bash20250124
|
||||
| Self::McpServers20251204
|
||||
|
|
@ -255,14 +261,18 @@ impl AnthropicBeta {
|
|||
Self::InterleavedThinking20250514 | Self::WebSearch20250305 => {
|
||||
&[BetaProvider::BedrockConverse, BetaProvider::Bedrock]
|
||||
}
|
||||
Self::MidConversationOutputConfig20260701
|
||||
| Self::MidConversationToolChanges20260701 => &[
|
||||
Self::MidConversationOutputConfig20260701 => &[
|
||||
BetaProvider::AzureAi,
|
||||
BetaProvider::BedrockConverse,
|
||||
BetaProvider::BedrockMantle,
|
||||
BetaProvider::VertexAi,
|
||||
BetaProvider::Databricks,
|
||||
],
|
||||
Self::MidConversationToolChanges20260701 => &[
|
||||
BetaProvider::BedrockConverse,
|
||||
BetaProvider::BedrockMantle,
|
||||
BetaProvider::Databricks,
|
||||
],
|
||||
Self::Output128k20250219 | Self::TokenEfficientTools20250219 => &[
|
||||
BetaProvider::AzureAi,
|
||||
BetaProvider::BedrockConverse,
|
||||
|
|
@ -483,7 +493,7 @@ mod tests {
|
|||
let config = beta_headers_config();
|
||||
let config_columns: Vec<String> = config
|
||||
.keys()
|
||||
.filter(|column| column.as_str() != "description")
|
||||
.filter(|column| column.as_str() != "description" && !column.ends_with("_header_only"))
|
||||
.cloned()
|
||||
.collect();
|
||||
let beta_provider_columns: Vec<String> = BetaProvider::VARIANTS
|
||||
|
|
@ -502,12 +512,16 @@ mod tests {
|
|||
let config = beta_headers_config();
|
||||
for provider in BetaProvider::VARIANTS {
|
||||
for beta in &AnthropicBeta::KNOWN {
|
||||
let expected = config
|
||||
.get(provider.as_ref())
|
||||
.and_then(serde_json::Value::as_object)
|
||||
.and_then(|column| column.get(beta.as_str()))
|
||||
.and_then(serde_json::Value::as_str)
|
||||
.map(str::to_string);
|
||||
let cell = |column: &str| {
|
||||
config
|
||||
.get(column)
|
||||
.and_then(serde_json::Value::as_object)
|
||||
.and_then(|values| values.get(beta.as_str()))
|
||||
.and_then(serde_json::Value::as_str)
|
||||
.map(str::to_string)
|
||||
};
|
||||
let expected =
|
||||
cell(&format!("{provider}_header_only")).or_else(|| cell(provider.as_ref()));
|
||||
let actual = beta.on(*provider).map(|name| name.to_string());
|
||||
assert_eq!(actual, expected, "provider {provider}, beta {beta}");
|
||||
if matches!(beta, AnthropicBeta::AdvancedToolUse20251120)
|
||||
|
|
|
|||
|
|
@ -1,7 +1,8 @@
|
|||
use litellm_auth::CredentialPlacement;
|
||||
use litellm_llms_types::{
|
||||
formats::messages::{
|
||||
ContextEdit, ContextManagement, Message, MessagesOptionalParams, MessagesRequest, Speed,
|
||||
ContentBlock, ContentBlockType, ContextEdit, ContextManagement, Message,
|
||||
MessagesOptionalParams, MessagesRequest, Speed, ThinkingConfig, ThinkingDisplay,
|
||||
},
|
||||
providers::anthropic::{AnthropicBeta, BetaProvider, BetaSet, DEFAULT_HEADERS},
|
||||
recognized::Recognized,
|
||||
|
|
@ -203,6 +204,10 @@ fn feature_betas(request: &MessagesRequest) -> BetaSet {
|
|||
.then_some(AnthropicBeta::FastMode20260201),
|
||||
messages_carry_output_config(&request.messages)
|
||||
.then_some(AnthropicBeta::PerTurnControl20260701),
|
||||
uses_thinking_display_updates(params)
|
||||
.then_some(AnthropicBeta::ThinkingDisplayUpdates20260818),
|
||||
messages_carry_tool_changes(&request.messages)
|
||||
.then_some(AnthropicBeta::MidConversationToolChanges20260701),
|
||||
has_advisor_tool(tools).then_some(AnthropicBeta::AdvisorTool20260301),
|
||||
is_tool_search_used(tools).then_some(AnthropicBeta::AdvancedToolUse20251120),
|
||||
]
|
||||
|
|
@ -253,6 +258,36 @@ fn messages_carry_output_config(messages: &[Message]) -> bool {
|
|||
.any(|message| message.extra.contains_key("output_config"))
|
||||
}
|
||||
|
||||
fn uses_thinking_display_updates(params: &MessagesOptionalParams) -> bool {
|
||||
let display = match params.thinking.as_ref().and_then(Recognized::known) {
|
||||
Some(ThinkingConfig::Enabled(thinking)) => thinking.display.as_ref(),
|
||||
Some(ThinkingConfig::Adaptive(thinking)) => thinking.display.as_ref(),
|
||||
_ => None,
|
||||
};
|
||||
display.and_then(Recognized::known) == Some(&ThinkingDisplay::Updates)
|
||||
}
|
||||
|
||||
fn is_tool_reference_change(block: &ContentBlock) -> bool {
|
||||
matches!(
|
||||
&block.block_type,
|
||||
Some(ContentBlockType::Other(block_type))
|
||||
if block_type == "tool_addition" || block_type == "tool_removal"
|
||||
) && block
|
||||
.extra
|
||||
.get("tool")
|
||||
.and_then(|tool| tool.get("type"))
|
||||
.and_then(Value::as_str)
|
||||
== Some("tool_reference")
|
||||
}
|
||||
|
||||
fn messages_carry_tool_changes(messages: &[Message]) -> bool {
|
||||
messages
|
||||
.iter()
|
||||
.filter(|message| message.role == "system")
|
||||
.flat_map(Message::blocks)
|
||||
.any(is_tool_reference_change)
|
||||
}
|
||||
|
||||
fn unsupported_param(model: &str, param: &str, value: &str, hint: &str) -> Error {
|
||||
Error::InvalidRequest(crate::ErrorDetail::UnsupportedParameter {
|
||||
model: model.into(),
|
||||
|
|
@ -1071,6 +1106,41 @@ mod tests {
|
|||
json!({"safeguards": [{"type": "dangerous_tool_use", "classifier_context": {"v": 1, "permission_mode": "auto"}}]}),
|
||||
&[]
|
||||
)]
|
||||
#[case::adaptive_display_updates(
|
||||
json!({"thinking": {"type": "adaptive", "display": "updates"}}),
|
||||
&["thinking-display-updates-2026-08-18"]
|
||||
)]
|
||||
#[case::enabled_display_updates(
|
||||
json!({"thinking": {"type": "enabled", "budget_tokens": 2048, "display": "updates"}}),
|
||||
&["thinking-display-updates-2026-08-18"]
|
||||
)]
|
||||
#[case::summarized_display(json!({"thinking": {"type": "adaptive", "display": "summarized"}}), &[])]
|
||||
#[case::disabled_thinking(json!({"thinking": {"type": "disabled"}}), &[])]
|
||||
#[case::system_tool_addition(
|
||||
json!({"messages": [
|
||||
{"role": "user", "content": "Use the ping tool"},
|
||||
{"role": "system", "content": [{"type": "tool_addition", "tool": {"type": "tool_reference", "name": "ping"}}]},
|
||||
]}),
|
||||
&["mid-conversation-tool-changes-2026-07-01"]
|
||||
)]
|
||||
#[case::system_tool_removal(
|
||||
json!({"messages": [
|
||||
{"role": "system", "content": [{"type": "tool_removal", "tool": {"type": "tool_reference", "name": "ping"}}]},
|
||||
]}),
|
||||
&["mid-conversation-tool-changes-2026-07-01"]
|
||||
)]
|
||||
#[case::user_tool_addition(
|
||||
json!({"messages": [
|
||||
{"role": "user", "content": [{"type": "tool_addition", "tool": {"type": "tool_reference", "name": "ping"}}]},
|
||||
]}),
|
||||
&[]
|
||||
)]
|
||||
#[case::tool_addition_without_a_tool_reference(
|
||||
json!({"messages": [
|
||||
{"role": "system", "content": [{"type": "tool_addition", "tool": {"type": "tool_definition"}}]},
|
||||
]}),
|
||||
&[]
|
||||
)]
|
||||
fn feature_betas_follow_the_request(#[case] fields: Value, #[case] expected: &[&str]) {
|
||||
assert_eq!(feature_betas(&request(fields)), betas(expected));
|
||||
}
|
||||
|
|
@ -1093,6 +1163,33 @@ mod tests {
|
|||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::vertex_ai(
|
||||
BetaProvider::VertexAi,
|
||||
&["mid-conversation-tool-changes-2026-07-01", "thinking-display-updates-2026-08-18"]
|
||||
)]
|
||||
#[case::azure_ai(
|
||||
BetaProvider::AzureAi,
|
||||
&["mid-conversation-tool-changes-2026-07-01", "thinking-display-updates-2026-08-18"]
|
||||
)]
|
||||
#[case::bedrock_converse(BetaProvider::BedrockConverse, &["thinking-display-updates-2026-08-18"])]
|
||||
#[case::databricks(BetaProvider::Databricks, &[])]
|
||||
fn tool_change_and_display_betas_reach_the_providers_that_accept_them(
|
||||
#[case] provider: BetaProvider,
|
||||
#[case] expected: &[&str],
|
||||
) {
|
||||
let fields = json!({
|
||||
"thinking": {"type": "adaptive", "display": "updates"},
|
||||
"messages": [
|
||||
{"role": "system", "content": [{"type": "tool_addition", "tool": {"type": "tool_reference", "name": "ping"}}]},
|
||||
],
|
||||
});
|
||||
assert_eq!(
|
||||
provider_feature_betas(&request(fields), provider),
|
||||
betas(expected)
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn hosts_without_a_beta_map_column_get_safeguards_without_a_derived_beta() {
|
||||
let client = headers(&[("anthropic-beta", "claude-code-20250219")]);
|
||||
|
|
|
|||
|
|
@ -545,6 +545,11 @@ mod tests {
|
|||
&[("anthropic-beta", "web-search-2025-03-05")],
|
||||
&[("x-api-key", "k"), ("anthropic-beta", "structured-outputs-2025-11-13,web-search-2025-03-05")]
|
||||
)]
|
||||
#[case::compaction_needs_the_compact_2026_09_04_beta(
|
||||
json!({"compaction": {"type": "summarize"}}),
|
||||
&[],
|
||||
&[("x-api-key", "k"), ("anthropic-beta", "compact-2026-09-04")]
|
||||
)]
|
||||
#[case::no_feature_needs_a_beta(json!({}), &[], &[("x-api-key", "k")])]
|
||||
#[case::safeguards_need_the_dangerous_tool_use_beta(
|
||||
json!({"safeguards": [{"type": "dangerous_tool_use"}]}),
|
||||
|
|
|
|||
|
|
@ -661,6 +661,18 @@ mod tests {
|
|||
]}}),
|
||||
&["compact-2026-01-12", "context-management-2025-06-27"]
|
||||
)]
|
||||
#[case::compaction(json!({"compaction": {"type": "summarize"}}), &["compact-2026-09-04"])]
|
||||
#[case::thinking_display_updates(
|
||||
json!({"thinking": {"type": "adaptive", "display": "updates"}}),
|
||||
&["thinking-display-updates-2026-08-18"]
|
||||
)]
|
||||
#[case::tool_addition(
|
||||
json!({"messages": [
|
||||
{"role": "user", "content": "Use the ping tool"},
|
||||
{"role": "system", "content": [{"type": "tool_addition", "tool": {"type": "tool_reference", "name": "ping"}}]}
|
||||
]}),
|
||||
&["mid-conversation-tool-changes-2026-07-01"]
|
||||
)]
|
||||
#[case::advisor_tool_is_not_a_vertex_beta(
|
||||
json!({"tools": [{"type": "advisor_20260301", "name": "advisor"}]}),
|
||||
&[]
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
{
|
||||
"description": "Mapping of Anthropic beta headers for each provider. Keys are input header names, values are provider-specific header names (or null if unsupported). Only headers present in mapping keys with non-null values can be forwarded.",
|
||||
"description": "Mapping of Anthropic beta headers for each provider. Keys are input header names, values are provider-specific header names (or null if unsupported). Only headers present in mapping keys with non-null values can be forwarded. A <provider>_header_only column lists betas that provider accepts only as the anthropic-beta HTTP header: they are forwarded there and dropped from the anthropic_beta body field. Proxies that predate these columns ignore them and read the provider column alone.",
|
||||
"anthropic": {
|
||||
"advisor-tool-2026-03-01": "advisor-tool-2026-03-01",
|
||||
"advanced-tool-use-2025-11-20": "advanced-tool-use-2025-11-20",
|
||||
|
|
@ -46,6 +46,7 @@
|
|||
"bash_20250124": null,
|
||||
"code-execution-2025-08-25": "code-execution-2025-08-25",
|
||||
"compact-2026-01-12": null,
|
||||
"compact-2026-09-04": "compact-2026-09-04",
|
||||
"computer-use-2025-01-24": "computer-use-2025-01-24",
|
||||
"computer-use-2025-11-24": "computer-use-2025-11-24",
|
||||
"context-1m-2025-08-07": "context-1m-2025-08-07",
|
||||
|
|
@ -59,6 +60,7 @@
|
|||
"mcp-client-2025-11-20": "mcp-client-2025-11-20",
|
||||
"mcp-client-2025-04-04": "mcp-client-2025-04-04",
|
||||
"mcp-servers-2025-12-04": null,
|
||||
"mid-conversation-tool-changes-2026-07-01": "mid-conversation-tool-changes-2026-07-01",
|
||||
"output-128k-2025-02-19": null,
|
||||
"structured-output-2024-03-01": null,
|
||||
"per-turn-control-2026-07-01": "per-turn-control-2026-07-01",
|
||||
|
|
@ -202,6 +204,7 @@
|
|||
"mcp-client-2025-11-20": null,
|
||||
"mcp-client-2025-04-04": null,
|
||||
"mcp-servers-2025-12-04": null,
|
||||
"mid-conversation-tool-changes-2026-07-01": "mid-conversation-tool-changes-2026-07-01",
|
||||
"output-128k-2025-02-19": null,
|
||||
"structured-output-2024-03-01": null,
|
||||
"per-turn-control-2026-07-01": "per-turn-control-2026-07-01",
|
||||
|
|
@ -218,6 +221,10 @@
|
|||
"web-search-2025-03-05": "web-search-2025-03-05",
|
||||
"inline-tools-2026-09-15": "inline-tools-2026-09-15"
|
||||
},
|
||||
"vertex_ai_header_only": {
|
||||
"compact-2026-09-04": "compact-2026-09-04",
|
||||
"inline-tools-2026-09-15": "inline-tools-2026-09-15"
|
||||
},
|
||||
"databricks": {
|
||||
"advisor-tool-2026-03-01": null,
|
||||
"advanced-tool-use-2025-11-20": null,
|
||||
|
|
|
|||
|
|
@ -30,7 +30,7 @@ from collections.abc import Mapping
|
|||
from concurrent.futures import ThreadPoolExecutor
|
||||
from functools import cache
|
||||
from importlib.resources import files
|
||||
from typing import Final, TypeAlias
|
||||
from typing import Final, Literal, TypeAlias
|
||||
|
||||
import httpx
|
||||
from pydantic import TypeAdapter
|
||||
|
|
@ -47,6 +47,9 @@ BetaHeadersConfig: TypeAlias = Mapping[str, str | Mapping[str, str | None]]
|
|||
_BETA_HEADERS_CONFIG_ADAPTER: Final = TypeAdapter(BetaHeadersConfig)
|
||||
_DECODED_JSON: Final = TypeAdapter(object)
|
||||
|
||||
BetaPlacement: TypeAlias = Literal["header", "body"]
|
||||
_HEADER_ONLY_COLUMN_SUFFIX: Final = "_header_only"
|
||||
|
||||
|
||||
class GetAnthropicBetaHeadersConfig:
|
||||
"""
|
||||
|
|
@ -241,9 +244,35 @@ def get_provider_name(provider: str) -> str:
|
|||
return aliases.get(provider, provider)
|
||||
|
||||
|
||||
def _beta_column(name: str) -> Mapping[str, str | None]:
|
||||
column: Final[Mapping[str, str | None]] = _load_beta_headers_config().get(name, {})
|
||||
return column
|
||||
|
||||
|
||||
def _header_only_column(canonical_provider: str) -> Mapping[str, str | None]:
|
||||
return _beta_column(f"{canonical_provider}{_HEADER_ONLY_COLUMN_SUFFIX}")
|
||||
|
||||
|
||||
def _provider_beta_mapping(provider: str, placement: BetaPlacement) -> Mapping[str, str | None]:
|
||||
canonical_provider: Final = get_provider_name(provider)
|
||||
column: Final = _beta_column(canonical_provider)
|
||||
header_only: Final = _header_only_column(canonical_provider)
|
||||
match placement:
|
||||
case "header":
|
||||
return {**column, **header_only}
|
||||
case "body":
|
||||
return {beta: name for beta, name in column.items() if beta not in header_only}
|
||||
|
||||
|
||||
def get_header_only_betas(provider: str) -> frozenset[str]:
|
||||
header_only: Final = _header_only_column(get_provider_name(provider))
|
||||
return frozenset(beta for beta, name in header_only.items() if name is not None)
|
||||
|
||||
|
||||
def filter_and_transform_beta_headers(
|
||||
beta_headers: list[str],
|
||||
provider: str,
|
||||
placement: BetaPlacement = "header",
|
||||
) -> list[str]:
|
||||
"""
|
||||
Filter and transform beta headers based on provider's mapping configuration.
|
||||
|
|
@ -256,6 +285,7 @@ def filter_and_transform_beta_headers(
|
|||
Args:
|
||||
beta_headers: List of Anthropic beta header values
|
||||
provider: Provider name (e.g., "anthropic", "bedrock", "vertex_ai")
|
||||
placement: Whether the betas travel as the anthropic-beta header or the anthropic_beta body field
|
||||
|
||||
Returns:
|
||||
List of filtered and transformed beta headers for the provider
|
||||
|
|
@ -263,11 +293,7 @@ def filter_and_transform_beta_headers(
|
|||
if not beta_headers:
|
||||
return []
|
||||
|
||||
config: Final = _load_beta_headers_config()
|
||||
provider = get_provider_name(provider)
|
||||
|
||||
# Get the header mapping for this provider
|
||||
provider_mapping: Final = config.get(provider, {})
|
||||
provider_mapping: Final = _provider_beta_mapping(provider, placement)
|
||||
|
||||
filtered_headers: Final[set[str]] = set()
|
||||
|
||||
|
|
@ -419,6 +445,7 @@ def update_request_with_filtered_beta(
|
|||
filtered_body_betas: Final = filter_and_transform_beta_headers(
|
||||
beta_headers=existing_body_betas,
|
||||
provider=provider,
|
||||
placement="body",
|
||||
)
|
||||
|
||||
if filtered_body_betas:
|
||||
|
|
|
|||
|
|
@ -563,6 +563,7 @@ def _resolve_mantle_region_for_cost(
|
|||
|
||||
_PER_CALL_LITELLM_PARAM_DICTS: Final = frozenset({"metadata", "litellm_metadata"})
|
||||
_LITELLM_PARAMS: Final = TypeAdapter(dict[str, object])
|
||||
_SERVED_ANTHROPIC_BODY: Final = TypeAdapter(dict[str, object])
|
||||
_REDACTED_RESULT: Final = TypeAdapter(object)
|
||||
|
||||
|
||||
|
|
@ -4935,13 +4936,15 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
verbose_logger.warning("LiteLLM: the anthropic_messages stream assembled no response, logging an empty one")
|
||||
return litellm.ModelResponse(model=self.model)
|
||||
else:
|
||||
from litellm.types.llms.anthropic import AnthropicResponse
|
||||
|
||||
pydantic_result: Final = AnthropicResponse.model_validate({"stop_sequence": None, **(served_body or {})})
|
||||
import httpx
|
||||
|
||||
anthropic_body: Final = (
|
||||
served_body
|
||||
if served_body is not None
|
||||
else _SERVED_ANTHROPIC_BODY.validate_python(
|
||||
result.model_dump() if isinstance(result, BaseModel) else result
|
||||
)
|
||||
)
|
||||
result = litellm.AnthropicConfig().transform_parsed_response(
|
||||
completion_response=pydantic_result.model_dump(),
|
||||
completion_response={"stop_sequence": None, **anthropic_body},
|
||||
raw_response=httpx.Response(
|
||||
status_code=200,
|
||||
headers={},
|
||||
|
|
|
|||
|
|
@ -249,6 +249,7 @@ def _redacted_provider_specific_fields(fields: Mapping[str, object]) -> Mapping[
|
|||
"reasoningContentBlocks",
|
||||
"citations",
|
||||
"compaction_blocks",
|
||||
"compaction_start",
|
||||
"compaction_delta",
|
||||
)
|
||||
if key in fields
|
||||
|
|
|
|||
|
|
@ -45,6 +45,7 @@ from litellm.llms.anthropic.wif import (
|
|||
from litellm.llms.base_llm.base_utils import BaseLLMModelInfo, BaseTokenCounter
|
||||
from litellm.llms.base_llm.chat.transformation import BaseLLMException
|
||||
from litellm.types.llms.anthropic import (
|
||||
ANTHROPIC_BETA_HEADER_VALUES,
|
||||
ANTHROPIC_HOSTED_TOOLS,
|
||||
ANTHROPIC_MID_CONVERSATION_OUTPUT_CONFIG_BETA_HEADER,
|
||||
ANTHROPIC_MID_CONVERSATION_TOOL_CHANGES_BETA_HEADER,
|
||||
|
|
@ -53,6 +54,7 @@ from litellm.types.llms.anthropic import (
|
|||
ANTHROPIC_SERVER_TOOL_USE_ID_PREFIX,
|
||||
ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER,
|
||||
ANTHROPIC_TOOL_SEARCH_TOOL_TYPES,
|
||||
LEGACY_THINKING_DISPLAY_UPDATES_PROVIDERS,
|
||||
AllAnthropicToolsValues,
|
||||
AnthropicMcpServerTool,
|
||||
AnthropicMessagesToolChoice,
|
||||
|
|
@ -120,6 +122,8 @@ _DECODED_JSON: Final = TypeAdapter(object)
|
|||
|
||||
_CLAUDE_CODE_USER_AGENT_PREFIXES: Final = ("claude-cli/", "claude-code/")
|
||||
|
||||
NATIVE_COMPACTION_PROVIDERS: Final = frozenset({"anthropic", "azure_ai", "vertex_ai"})
|
||||
|
||||
|
||||
def is_anthropic_messages_url(url: str) -> bool:
|
||||
"""Check whether a URL addresses Anthropic's Messages API."""
|
||||
|
|
@ -132,7 +136,7 @@ def requires_native_compaction_beta(
|
|||
optional_params: Mapping[str, object],
|
||||
messages: Sequence[object],
|
||||
) -> bool:
|
||||
return custom_llm_provider == "anthropic" and (
|
||||
return custom_llm_provider in NATIVE_COMPACTION_PROVIDERS and (
|
||||
optional_params.get("compaction") is not None
|
||||
or any(
|
||||
isinstance(block, Mapping)
|
||||
|
|
@ -148,6 +152,27 @@ def requires_native_compaction_beta(
|
|||
)
|
||||
|
||||
|
||||
def count_tokens_feature_betas(
|
||||
custom_llm_provider: str,
|
||||
optional_params: Mapping[str, object],
|
||||
messages: Sequence[object],
|
||||
) -> tuple[str, ...]:
|
||||
return tuple(
|
||||
beta
|
||||
for beta, used in (
|
||||
(
|
||||
ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_09_04.value,
|
||||
requires_native_compaction_beta(custom_llm_provider, optional_params, messages),
|
||||
),
|
||||
(
|
||||
ANTHROPIC_MID_CONVERSATION_TOOL_CHANGES_BETA_HEADER,
|
||||
AnthropicModelInfo().is_mid_conversation_tool_change_used(messages),
|
||||
),
|
||||
)
|
||||
if used
|
||||
)
|
||||
|
||||
|
||||
def supports_anthropic_cache_control(model: str, custom_llm_provider: str | None) -> bool:
|
||||
from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
|
||||
from litellm.utils import supports_prompt_caching
|
||||
|
|
@ -547,10 +572,12 @@ class AnthropicModelInfo(BaseLLMModelInfo):
|
|||
file_ids: Final = get_file_ids_from_messages(messages)
|
||||
return len(file_ids) > 0
|
||||
|
||||
def is_thinking_display_updates_used(self, thinking: AnthropicThinkingParam | None) -> bool:
|
||||
if not isinstance(thinking, dict):
|
||||
return False
|
||||
return thinking.get("type") in ("adaptive", "enabled") and thinking.get("display") == "updates"
|
||||
def is_thinking_display_updates_used(self, thinking: object) -> bool:
|
||||
match thinking:
|
||||
case {"type": "adaptive" | "enabled", "display": "updates"}:
|
||||
return True
|
||||
case _:
|
||||
return False
|
||||
|
||||
def is_mid_conversation_tool_change_used(self, messages: Sequence[object]) -> bool:
|
||||
for message in messages:
|
||||
|
|
@ -1002,7 +1029,10 @@ class AnthropicModelInfo(BaseLLMModelInfo):
|
|||
|
||||
budget_tokens: Final = _legacy_budget_tokens(thinking.get("budget_tokens", 0), drop_params=drop_params)
|
||||
display: Final = thinking.get("display")
|
||||
supported_display: Final = {"display": display} if display in ("summarized", "omitted") else {}
|
||||
keeps_display: Final = display in ("summarized", "omitted") or (
|
||||
display == "updates" and custom_llm_provider in LEGACY_THINKING_DISPLAY_UPDATES_PROVIDERS
|
||||
)
|
||||
supported_display: Final = {"display": display} if keeps_display else {}
|
||||
carried_keys: Final = carried_thinking_keys(
|
||||
thinking, rebuilt_keys=frozenset(("type", "budget_tokens", "display")), drop_params=drop_params
|
||||
)
|
||||
|
|
@ -1306,8 +1336,6 @@ class AnthropicModelInfo(BaseLLMModelInfo):
|
|||
if is_vertex_request is True:
|
||||
# Vertex AI requires web search beta header for web search to work
|
||||
if web_search_tool_used:
|
||||
from litellm.types.llms.anthropic import ANTHROPIC_BETA_HEADER_VALUES
|
||||
|
||||
headers["anthropic-beta"] = ANTHROPIC_BETA_HEADER_VALUES.WEB_SEARCH_2025_03_05.value
|
||||
elif len(all_betas) > 0:
|
||||
headers["anthropic-beta"] = ",".join(all_betas)
|
||||
|
|
|
|||
|
|
@ -29,6 +29,7 @@ from litellm.types.llms.anthropic_tool_search import get_tool_search_beta_header
|
|||
from litellm.types.router import GenericLiteLLMParams
|
||||
|
||||
from ...common_utils import (
|
||||
NATIVE_COMPACTION_PROVIDERS,
|
||||
AnthropicError,
|
||||
AnthropicModelInfo,
|
||||
merge_anthropic_beta_headers,
|
||||
|
|
@ -106,7 +107,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
|
|||
"tool_choice",
|
||||
"thinking",
|
||||
"context_management",
|
||||
*(("compaction",) if self._resolved_provider == "anthropic" else ()),
|
||||
*(("compaction",) if self._resolved_provider in NATIVE_COMPACTION_PROVIDERS else ()),
|
||||
"output_format",
|
||||
"inference_geo",
|
||||
"speed",
|
||||
|
|
|
|||
|
|
@ -82,6 +82,7 @@ class AzureAIAnthropicCountTokensHandler(AzureAIAnthropicCountTokensConfig):
|
|||
headers: Final = self.get_required_headers(
|
||||
api_key=api_key,
|
||||
litellm_params=litellm_params,
|
||||
messages=messages,
|
||||
)
|
||||
if not {"x-api-key", "Authorization"} & headers.keys():
|
||||
raise AnthropicError(
|
||||
|
|
|
|||
|
|
@ -4,11 +4,13 @@ Azure AI Anthropic CountTokens API transformation logic.
|
|||
Extends the base Anthropic CountTokens transformation with Azure authentication.
|
||||
"""
|
||||
|
||||
from collections.abc import Sequence
|
||||
from typing import Final
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
from litellm.constants import ANTHROPIC_TOKEN_COUNTING_BETA_VERSION
|
||||
from litellm.litellm_core_utils.url_utils import join_api_base
|
||||
from litellm.llms.anthropic.common_utils import count_tokens_feature_betas, merge_anthropic_beta_headers
|
||||
from litellm.llms.anthropic.count_tokens.transformation import (
|
||||
AnthropicCountTokensConfig,
|
||||
)
|
||||
|
|
@ -28,6 +30,7 @@ class AzureAIAnthropicCountTokensConfig(AnthropicCountTokensConfig):
|
|||
self,
|
||||
api_key: str | None,
|
||||
litellm_params: dict[str, object] | None = None,
|
||||
messages: Sequence[object] = (),
|
||||
) -> dict[str, str]:
|
||||
"""
|
||||
Get the required headers for the Azure AI Anthropic CountTokens API.
|
||||
|
|
@ -38,6 +41,7 @@ class AzureAIAnthropicCountTokensConfig(AnthropicCountTokensConfig):
|
|||
Args:
|
||||
api_key: The Azure AI API key
|
||||
litellm_params: Optional LiteLLM parameters for additional auth config
|
||||
messages: The request messages, read for betas the counted features need
|
||||
|
||||
Returns:
|
||||
Dictionary of required headers with both x-api-key and Azure authentication
|
||||
|
|
@ -46,7 +50,9 @@ class AzureAIAnthropicCountTokensConfig(AnthropicCountTokensConfig):
|
|||
headers: Final = {
|
||||
"Content-Type": "application/json",
|
||||
"anthropic-version": "2023-06-01",
|
||||
"anthropic-beta": ANTHROPIC_TOKEN_COUNTING_BETA_VERSION,
|
||||
"anthropic-beta": merge_anthropic_beta_headers(
|
||||
ANTHROPIC_TOKEN_COUNTING_BETA_VERSION, count_tokens_feature_betas("azure_ai", {}, messages)
|
||||
),
|
||||
**({"x-api-key": api_key} if api_key else {}),
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -2,14 +2,19 @@
|
|||
Azure Anthropic transformation config - extends AnthropicConfig with Azure authentication
|
||||
"""
|
||||
|
||||
from collections.abc import Mapping
|
||||
from typing import Final
|
||||
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
from litellm.litellm_core_utils.url_utils import join_api_base
|
||||
from litellm.llms.anthropic.chat.transformation import AnthropicConfig
|
||||
from litellm.llms.azure.common_utils import BaseAzureLLM
|
||||
from litellm.types.llms.openai import AllMessageValues
|
||||
from litellm.types.router import GenericLiteLLMParams
|
||||
|
||||
_OPTIONAL_PARAMS: Final = TypeAdapter(Mapping[str, object])
|
||||
|
||||
|
||||
def azure_anthropic_messages_url(api_base: str) -> str:
|
||||
base: Final = api_base.rstrip("/")
|
||||
|
|
@ -107,9 +112,16 @@ class AzureAnthropicConfig(AnthropicConfig):
|
|||
is_vertex_request=optional_params.get("is_vertex_request", False),
|
||||
user_anthropic_beta_headers=user_anthropic_beta_headers,
|
||||
mcp_server_used=mcp_server_used,
|
||||
is_thinking_display_updates_used=self.is_thinking_display_updates_used(
|
||||
_OPTIONAL_PARAMS.validate_python(optional_params).get("thinking")
|
||||
),
|
||||
)
|
||||
# Merge headers - Azure auth (api-key or Authorization) takes precedence
|
||||
headers = {**anthropic_headers, **headers}
|
||||
litellm_beta: Final = _OPTIONAL_PARAMS.validate_python(anthropic_headers).get("anthropic-beta")
|
||||
litellm_betas: Final[Mapping[str, str]] = (
|
||||
{"anthropic-beta": litellm_beta} if isinstance(litellm_beta, str) else {}
|
||||
)
|
||||
headers = {**anthropic_headers, **headers, **litellm_betas}
|
||||
|
||||
# Ensure anthropic-version header is set
|
||||
if "anthropic-version" not in headers:
|
||||
|
|
|
|||
|
|
@ -1,7 +1,9 @@
|
|||
from collections.abc import Mapping, Sequence
|
||||
from typing import Final
|
||||
|
||||
from litellm.llms.anthropic.common_utils import AnthropicModelInfo
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
from litellm.llms.anthropic.common_utils import AnthropicModelInfo, requires_native_compaction_beta
|
||||
from litellm.llms.anthropic.messages.native.transformation import (
|
||||
AnthropicMessagesConfig,
|
||||
messages_carry_output_config,
|
||||
|
|
@ -9,6 +11,8 @@ from litellm.llms.anthropic.messages.native.transformation import (
|
|||
from litellm.types.llms.anthropic import (
|
||||
ANTHROPIC_BETA_HEADER_VALUES,
|
||||
ANTHROPIC_HOSTED_TOOLS,
|
||||
ANTHROPIC_MID_CONVERSATION_TOOL_CHANGES_BETA_HEADER,
|
||||
ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER,
|
||||
)
|
||||
from litellm.types.llms.anthropic_tool_search import get_tool_search_beta_header
|
||||
from litellm.types.llms.vertex_ai import VertexPartnerProvider
|
||||
|
|
@ -17,6 +21,8 @@ from litellm.types.router import GenericLiteLLMParams
|
|||
from ....vertex_llm_base import VertexBase
|
||||
from ..output_params_utils import sanitize_vertex_anthropic_output_params
|
||||
|
||||
_OPTIONAL_PARAMS: Final = TypeAdapter(Mapping[str, object])
|
||||
|
||||
|
||||
class VertexAIPartnerModelsAnthropicMessagesConfig(AnthropicMessagesConfig, VertexBase):
|
||||
@property
|
||||
|
|
@ -174,8 +180,23 @@ class VertexAIPartnerModelsAnthropicMessagesConfig(AnthropicMessagesConfig, Vert
|
|||
if messages_carry_output_config(messages):
|
||||
beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.PER_TURN_CONTROL_2026_07_01.value)
|
||||
|
||||
if beta_values:
|
||||
request_headers["anthropic-beta"] = ",".join(beta_values)
|
||||
typed_params: Final = _OPTIONAL_PARAMS.validate_python(optional_params)
|
||||
if requires_native_compaction_beta("vertex_ai", typed_params, messages):
|
||||
beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_09_04.value)
|
||||
|
||||
thinking_display_betas: Final = (
|
||||
(ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER,)
|
||||
if anthropic_model_info.is_thinking_display_updates_used(typed_params.get("thinking"))
|
||||
else ()
|
||||
)
|
||||
tool_change_betas: Final = (
|
||||
(ANTHROPIC_MID_CONVERSATION_TOOL_CHANGES_BETA_HEADER,)
|
||||
if anthropic_model_info.is_mid_conversation_tool_change_used(messages)
|
||||
else ()
|
||||
)
|
||||
all_beta_values: Final = beta_values.union(thinking_display_betas, tool_change_betas)
|
||||
if all_beta_values:
|
||||
request_headers["anthropic-beta"] = ",".join(sorted(all_beta_values))
|
||||
|
||||
return request_headers, api_base
|
||||
|
||||
|
|
|
|||
|
|
@ -2,19 +2,23 @@
|
|||
## Handler file for calling claude-3 on vertex ai
|
||||
import base64
|
||||
import json
|
||||
from collections.abc import Mapping
|
||||
from collections.abc import Mapping, Sequence
|
||||
from typing import TYPE_CHECKING, Final
|
||||
|
||||
import httpx
|
||||
from pydantic import TypeAdapter
|
||||
from pydantic import TypeAdapter, ValidationError
|
||||
|
||||
import litellm
|
||||
from litellm.anthropic_beta_headers_manager import get_header_only_betas
|
||||
from litellm.litellm_core_utils.core_helpers import drops_unsupported_params
|
||||
from litellm.litellm_core_utils.prompt_templates.image_handling import RemoteMedia, inline_remote_image_urls
|
||||
from litellm.llms.base_llm.chat.transformation import LiteLLMLoggingObj
|
||||
from litellm.types.llms.anthropic import ANTHROPIC_BETA_HEADER_VALUES
|
||||
from litellm.types.llms.openai import AllMessageValues
|
||||
from litellm.types.utils import ModelResponse
|
||||
|
||||
from ....anthropic.chat.transformation import AnthropicConfig
|
||||
from ....anthropic.common_utils import AnthropicError, requires_native_compaction_beta
|
||||
from .output_params_utils import sanitize_vertex_anthropic_output_params
|
||||
|
||||
if TYPE_CHECKING:
|
||||
|
|
@ -36,6 +40,32 @@ def _vertex_labels_header(labels: object, header_names: Mapping[str, object]) ->
|
|||
return base64.b64encode(json.dumps(string_labels).encode()).decode() if string_labels else None
|
||||
|
||||
|
||||
_BETA_VALUE: Final[TypeAdapter[str | tuple[str, ...] | None]] = TypeAdapter(str | tuple[str, ...] | None)
|
||||
|
||||
_REQUEST_MESSAGES: Final = TypeAdapter(Sequence[object])
|
||||
|
||||
|
||||
def _beta_names(value: object) -> frozenset[str] | None:
|
||||
try:
|
||||
parsed: Final = _BETA_VALUE.validate_python(value)
|
||||
except ValidationError:
|
||||
return None
|
||||
names: Final = parsed.split(",") if isinstance(parsed, str) else parsed or ()
|
||||
return frozenset(name.strip() for name in names if name.strip())
|
||||
|
||||
|
||||
def _client_betas(value: object, field: str, drop_params: bool) -> frozenset[str]:
|
||||
names: Final = _beta_names(value)
|
||||
if names is not None:
|
||||
return names
|
||||
if drop_params:
|
||||
return frozenset()
|
||||
raise AnthropicError(
|
||||
status_code=400,
|
||||
message=f"{field} must be a string or a list of strings, got {type(value).__name__}",
|
||||
)
|
||||
|
||||
|
||||
class VertexAIError(Exception):
|
||||
def __init__(self, status_code, message):
|
||||
self.status_code = status_code
|
||||
|
|
@ -145,6 +175,9 @@ class VertexAIAnthropicConfig(AnthropicConfig):
|
|||
file_id_used=self.is_file_id_used(messages),
|
||||
mcp_server_used=self.is_mcp_server_used(optional_params.get("mcp_servers")),
|
||||
custom_llm_provider="vertex_ai",
|
||||
is_thinking_display_updates_used=self.is_thinking_display_updates_used(
|
||||
_NON_DEFAULT_PARAMS.validate_python(data).get("thinking")
|
||||
),
|
||||
)
|
||||
|
||||
beta_set: Final = set(auto_betas)
|
||||
|
|
@ -156,21 +189,42 @@ class VertexAIAnthropicConfig(AnthropicConfig):
|
|||
if context_management:
|
||||
self._add_context_management_beta_headers(beta_set, context_management)
|
||||
|
||||
extra_headers: Final = optional_params.get("extra_headers") or {}
|
||||
anthropic_beta_value: Final = extra_headers.get("anthropic-beta", "")
|
||||
if isinstance(anthropic_beta_value, str) and anthropic_beta_value:
|
||||
for beta in anthropic_beta_value.split(","):
|
||||
beta = beta.strip()
|
||||
if beta:
|
||||
beta_set.add(beta)
|
||||
elif isinstance(anthropic_beta_value, list):
|
||||
beta_set.update(anthropic_beta_value)
|
||||
|
||||
request_params: Final = _NON_DEFAULT_PARAMS.validate_python(optional_params)
|
||||
compaction_betas: Final = (
|
||||
frozenset({ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_09_04.value})
|
||||
if requires_native_compaction_beta(
|
||||
"vertex_ai", request_params, _REQUEST_MESSAGES.validate_python(data["messages"])
|
||||
)
|
||||
else frozenset[str]()
|
||||
)
|
||||
drop_params: Final = drops_unsupported_params(litellm_params.get("drop_params"))
|
||||
extra_headers: Final = _NON_DEFAULT_PARAMS.validate_python(request_params.get("extra_headers") or {})
|
||||
data.pop("extra_headers", None)
|
||||
injected_betas: Final = frozenset(beta_set) | _client_betas(
|
||||
extra_headers.get("anthropic-beta"), "The anthropic-beta header", drop_params
|
||||
)
|
||||
client_body_betas: Final = _client_betas(
|
||||
_NON_DEFAULT_PARAMS.validate_python(data).get("anthropic_beta"), "anthropic_beta", drop_params
|
||||
)
|
||||
|
||||
if beta_set:
|
||||
data["anthropic_beta"] = list(beta_set)
|
||||
headers["anthropic-beta"] = ",".join(beta_set)
|
||||
header_only_betas: Final = get_header_only_betas("vertex_ai")
|
||||
body_betas: Final = (injected_betas | client_body_betas) - header_only_betas
|
||||
if body_betas:
|
||||
data["anthropic_beta"] = sorted(body_betas)
|
||||
elif "anthropic_beta" in data:
|
||||
del data["anthropic_beta"]
|
||||
header_betas: Final = (
|
||||
injected_betas
|
||||
| compaction_betas
|
||||
| (client_body_betas & header_only_betas)
|
||||
| _client_betas(
|
||||
_NON_DEFAULT_PARAMS.validate_python(headers).get("anthropic-beta"),
|
||||
"The anthropic-beta header",
|
||||
drop_params,
|
||||
)
|
||||
)
|
||||
if header_betas:
|
||||
headers["anthropic-beta"] = ",".join(sorted(header_betas))
|
||||
|
||||
return data
|
||||
|
||||
|
|
|
|||
|
|
@ -6,9 +6,13 @@ Unlike Gemini models which use Google's token counting API, partner models use
|
|||
their respective publisher-specific count-tokens endpoints.
|
||||
"""
|
||||
|
||||
from collections.abc import Mapping, Sequence
|
||||
from typing import Any, Final
|
||||
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
from litellm.litellm_core_utils.url_utils import join_api_base
|
||||
from litellm.llms.anthropic.common_utils import count_tokens_feature_betas
|
||||
from litellm.llms.custom_httpx.http_handler import get_async_httpx_client
|
||||
from litellm.llms.vertex_ai.common_utils import get_vertex_base_url
|
||||
from litellm.llms.vertex_ai.vertex_llm_base import VertexBase
|
||||
|
|
@ -20,10 +24,21 @@ _COUNT_TOKENS_PUBLISHERS: Final = (
|
|||
)
|
||||
|
||||
|
||||
_REQUEST_MESSAGES: Final = TypeAdapter(Sequence[object])
|
||||
|
||||
|
||||
def count_tokens_publisher(model: str) -> str | None:
|
||||
return next((publisher for marker, publisher in _COUNT_TOKENS_PUBLISHERS if marker in model), None)
|
||||
|
||||
|
||||
def count_tokens_headers(access_token: str, request_data: Mapping[str, object]) -> Mapping[str, str]:
|
||||
authorization: Final = {"Authorization": f"Bearer {access_token}"}
|
||||
betas: Final = count_tokens_feature_betas(
|
||||
"vertex_ai", request_data, _REQUEST_MESSAGES.validate_python(request_data["messages"])
|
||||
)
|
||||
return {**authorization, "anthropic-beta": ",".join(betas)} if betas else authorization
|
||||
|
||||
|
||||
class VertexAIPartnerModelsTokenCounter(VertexBase):
|
||||
"""
|
||||
Token counter for Vertex AI Partner Models.
|
||||
|
|
@ -156,8 +171,7 @@ class VertexAIPartnerModelsTokenCounter(VertexBase):
|
|||
api_base=litellm_params.get("api_base"),
|
||||
)
|
||||
|
||||
# Prepare headers
|
||||
headers: Final = {"Authorization": f"Bearer {access_token}"}
|
||||
headers: Final = dict(count_tokens_headers(access_token, request_data))
|
||||
|
||||
# Get async HTTP client
|
||||
from litellm import LlmProviders
|
||||
|
|
|
|||
|
|
@ -713,6 +713,14 @@ class AnthropicResponseContentBlockRedactedThinking(LiteLLMBaseModel):
|
|||
data: str
|
||||
|
||||
|
||||
class AnthropicResponseContentBlockCompaction(LiteLLMBaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
|
||||
type: Literal["compaction"]
|
||||
content: str | None = None
|
||||
signature: str | None = None
|
||||
|
||||
|
||||
class AnthropicResponseUsageBlock(LiteLLMBaseModel):
|
||||
model_config = ConfigDict(extra="allow")
|
||||
|
||||
|
|
@ -726,7 +734,7 @@ class AnthropicOutputTokensDetails(LiteLLMBaseModel):
|
|||
thinking_tokens: int | None = None
|
||||
|
||||
|
||||
AnthropicFinishReason = Literal["end_turn", "max_tokens", "stop_sequence", "tool_use", "refusal"]
|
||||
AnthropicFinishReason = Literal["end_turn", "max_tokens", "stop_sequence", "tool_use", "refusal", "compaction"]
|
||||
|
||||
|
||||
class AnthropicResponse(LiteLLMBaseModel):
|
||||
|
|
@ -744,6 +752,7 @@ class AnthropicResponse(LiteLLMBaseModel):
|
|||
| AnthropicResponseContentBlockToolUse
|
||||
| AnthropicResponseContentBlockThinking
|
||||
| AnthropicResponseContentBlockRedactedThinking
|
||||
| AnthropicResponseContentBlockCompaction
|
||||
]
|
||||
"""Content generated by the model."""
|
||||
|
||||
|
|
@ -789,6 +798,9 @@ class AnthropicThinkingParam(TypedDict, total=False):
|
|||
block_binding: ReadOnly[AnthropicThinkingBlockBinding]
|
||||
|
||||
|
||||
LEGACY_THINKING_DISPLAY_UPDATES_PROVIDERS: Final = frozenset(("anthropic", "azure_ai", "vertex_ai"))
|
||||
|
||||
|
||||
class ANTHROPIC_HOSTED_TOOLS(str, Enum):
|
||||
WEB_SEARCH = "web_search"
|
||||
BASH = "bash"
|
||||
|
|
@ -809,6 +821,7 @@ class ANTHROPIC_BETA_HEADER_VALUES(str, Enum):
|
|||
CONTEXT_MANAGEMENT_2025_06_27 = "context-management-2025-06-27"
|
||||
COMPACT_2026_01_12 = "compact-2026-01-12"
|
||||
COMPACT_2026_09_04 = "compact-2026-09-04"
|
||||
INLINE_TOOLS_2026_09_15 = "inline-tools-2026-09-15"
|
||||
STRUCTURED_OUTPUT_2025_09_25 = "structured-outputs-2025-11-13"
|
||||
ADVANCED_TOOL_USE_2025_11_20 = "advanced-tool-use-2025-11-20"
|
||||
FAST_MODE_2026_02_01 = "fast-mode-2026-02-01"
|
||||
|
|
|
|||
|
|
@ -6706,6 +6706,95 @@ def test_handle_anthropic_messages_response_logging_reparses_the_served_body_wit
|
|||
assert logged.usage.completion_tokens == 7 # type: ignore[attr-defined]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"compaction_block",
|
||||
[
|
||||
{"type": "compaction", "content": "Summary of the chat so far.", "signature": "sig-abc"},
|
||||
{"type": "compaction", "signature": "sig-abc"},
|
||||
],
|
||||
ids=["with_summary", "without_content"],
|
||||
)
|
||||
def test_handle_anthropic_messages_response_logging_reparses_a_served_compaction_response(
|
||||
compaction_block: dict[str, str],
|
||||
):
|
||||
logging_obj = _anthropic_messages_logging_obj()
|
||||
logging_obj.optional_params = {}
|
||||
|
||||
logged = logging_obj._handle_anthropic_messages_response_logging(
|
||||
result={
|
||||
"id": "msg_compacted",
|
||||
"type": "message",
|
||||
"role": "assistant",
|
||||
"model": "claude-sonnet-5-5",
|
||||
"content": [compaction_block],
|
||||
"stop_reason": "compaction",
|
||||
"stop_sequence": None,
|
||||
"usage": {"input_tokens": 410, "output_tokens": 95},
|
||||
}
|
||||
)
|
||||
|
||||
assert isinstance(logged, ModelResponse)
|
||||
assert logged.id == "msg_compacted"
|
||||
assert logged.usage.prompt_tokens == 410
|
||||
assert logged.usage.completion_tokens == 95
|
||||
assert logged.choices[0].message.provider_specific_fields["compaction_blocks"] == [compaction_block]
|
||||
|
||||
|
||||
_WEB_SEARCH_REPLY: Final = (
|
||||
{"type": "server_tool_use", "id": "srvtoolu_1", "name": "web_search", "input": {"query": "weather"}},
|
||||
{
|
||||
"type": "web_search_tool_result",
|
||||
"tool_use_id": "srvtoolu_1",
|
||||
"content": [{"type": "web_search_result", "url": "https://example.com", "title": "Weather"}],
|
||||
},
|
||||
{"type": "text", "text": "It is sunny."},
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("content", "stop_reason"),
|
||||
[
|
||||
(list(_WEB_SEARCH_REPLY), "end_turn"),
|
||||
(list(_WEB_SEARCH_REPLY), "pause_turn"),
|
||||
([{"type": "text", "text": "The context window is full."}], "model_context_window_exceeded"),
|
||||
(
|
||||
[
|
||||
{"type": "mcp_tool_use", "id": "mcptoolu_1", "name": "lookup", "server_name": "docs", "input": {}},
|
||||
{"type": "mcp_tool_result", "tool_use_id": "mcptoolu_1", "is_error": False, "content": [{"type": "text", "text": "ok"}]},
|
||||
{"type": "text", "text": "Found it."},
|
||||
],
|
||||
"end_turn",
|
||||
),
|
||||
([{"type": "container_upload", "file_id": "file_1"}, {"type": "text", "text": "Uploaded."}], "end_turn"),
|
||||
],
|
||||
ids=["web_search", "pause_turn", "model_context_window_exceeded", "mcp_tool_use", "container_upload"],
|
||||
)
|
||||
def test_handle_anthropic_messages_response_logging_bills_a_served_server_tool_response(
|
||||
content: list[dict[str, object]], stop_reason: str
|
||||
):
|
||||
logging_obj = _anthropic_messages_logging_obj()
|
||||
logging_obj.optional_params = {}
|
||||
|
||||
logged = logging_obj._handle_anthropic_messages_response_logging(
|
||||
result={
|
||||
"id": "msg_server_tools",
|
||||
"type": "message",
|
||||
"role": "assistant",
|
||||
"model": "claude-sonnet-5-5",
|
||||
"content": content,
|
||||
"stop_reason": stop_reason,
|
||||
"stop_sequence": None,
|
||||
"usage": {"input_tokens": 1200, "output_tokens": 80},
|
||||
}
|
||||
)
|
||||
|
||||
assert isinstance(logged, ModelResponse)
|
||||
assert logged.id == "msg_server_tools"
|
||||
assert logged.usage.prompt_tokens == 1200
|
||||
assert logged.usage.completion_tokens == 80
|
||||
assert logged.choices[0].message.content == content[-1]["text"]
|
||||
|
||||
|
||||
def test_handle_anthropic_messages_response_logging_passes_model_response_through():
|
||||
"""Anthropic-native path already yields a ModelResponse; it must be returned unchanged."""
|
||||
logging_obj = _anthropic_messages_logging_obj()
|
||||
|
|
|
|||
|
|
@ -57,9 +57,10 @@ def _provider_fields() -> dict[str, object]:
|
|||
"thinking_blocks": [{"type": "thinking", "thinking": SECRET}],
|
||||
"reasoningContentBlocks": [{"reasoningText": {"text": SECRET}}],
|
||||
"citations": [{"cited_text": SECRET}],
|
||||
"thought_signatures": [SIGNATURE],
|
||||
"compaction_blocks": [{"type": "compaction", "content": SECRET}],
|
||||
"compaction_blocks": [{"type": "compaction", "content": SECRET, "signature": "compaction-signature"}],
|
||||
"compaction_start": {"type": "compaction", "content": SECRET},
|
||||
"compaction_delta": {"type": "compaction_delta", "content": SECRET},
|
||||
"thought_signatures": [SIGNATURE],
|
||||
}
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -4236,15 +4236,18 @@ def test_thinking_display_beta_requires_active_thinking(thinking: object, expect
|
|||
|
||||
@pytest.mark.usefixtures("local_model_cost_map")
|
||||
@pytest.mark.parametrize(
|
||||
("display", "expected_thinking"),
|
||||
("provider", "display", "expected_thinking"),
|
||||
(
|
||||
("summarized", {"type": "adaptive", "display": "summarized"}),
|
||||
("omitted", {"type": "adaptive", "display": "omitted"}),
|
||||
("updates", {"type": "adaptive"}),
|
||||
("azure_ai", "summarized", {"type": "adaptive", "display": "summarized"}),
|
||||
("azure_ai", "omitted", {"type": "adaptive", "display": "omitted"}),
|
||||
("azure_ai", "updates", {"type": "adaptive", "display": "updates"}),
|
||||
("vertex_ai", "updates", {"type": "adaptive", "display": "updates"}),
|
||||
("anthropic", "updates", {"type": "adaptive", "display": "updates"}),
|
||||
("databricks", "updates", {"type": "adaptive"}),
|
||||
),
|
||||
)
|
||||
def test_shared_legacy_thinking_translation_preserves_supported_display(
|
||||
display: str, expected_thinking: dict[str, str]
|
||||
provider: str, display: str, expected_thinking: dict[str, str]
|
||||
) -> None:
|
||||
from litellm.llms.anthropic.common_utils import AnthropicModelInfo
|
||||
|
||||
|
|
@ -4255,7 +4258,7 @@ def test_shared_legacy_thinking_translation_preserves_supported_display(
|
|||
AnthropicModelInfo.translate_legacy_thinking_for_adaptive_model(
|
||||
model="claude-opus-5",
|
||||
optional_params=optional_params,
|
||||
custom_llm_provider="azure_ai",
|
||||
custom_llm_provider=provider,
|
||||
drop_params=False,
|
||||
)
|
||||
|
||||
|
|
@ -4522,3 +4525,44 @@ def test_shared_legacy_thinking_translation_drops_a_non_numeric_budget_under_dro
|
|||
)
|
||||
|
||||
assert optional_params == {"thinking": {"type": "adaptive"}}
|
||||
|
||||
|
||||
_SIGNED_COMPACTION_BLOCK: Final = {"type": "compaction", "content": "summary so far", "signature": "sig"}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"provider,expected",
|
||||
[("anthropic", True), ("vertex_ai", True), ("azure_ai", True), ("bedrock", False), ("bedrock_converse", False)],
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"optional_params,messages",
|
||||
[
|
||||
({"compaction": {"type": "summarize"}}, [{"role": "user", "content": "hi"}]),
|
||||
({}, [{"role": "assistant", "content": [_SIGNED_COMPACTION_BLOCK]}, {"role": "user", "content": "go on"}]),
|
||||
],
|
||||
ids=["compaction_param", "signed_compaction_block"],
|
||||
)
|
||||
def test_requires_native_compaction_beta_per_provider(
|
||||
provider: str, expected: bool, optional_params: dict[str, object], messages: list[object]
|
||||
) -> None:
|
||||
from litellm.llms.anthropic.common_utils import requires_native_compaction_beta
|
||||
|
||||
assert requires_native_compaction_beta(provider, optional_params, messages) is expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize("provider", ["anthropic", "azure_ai", "vertex_ai"])
|
||||
@pytest.mark.parametrize(
|
||||
"optional_params,messages",
|
||||
[
|
||||
({"max_tokens": 64}, [{"role": "user", "content": "hi"}]),
|
||||
({}, [{"role": "assistant", "content": [{"type": "compaction", "content": "unsigned"}]}]),
|
||||
({"compaction": None}, [{"role": "user", "content": "hi"}]),
|
||||
],
|
||||
ids=["no_signal", "unsigned_block", "compaction_none"],
|
||||
)
|
||||
def test_requires_native_compaction_beta_is_false_without_a_signal(
|
||||
provider: str, optional_params: dict[str, object], messages: list[object]
|
||||
) -> None:
|
||||
from litellm.llms.anthropic.common_utils import requires_native_compaction_beta
|
||||
|
||||
assert requires_native_compaction_beta(provider, optional_params, messages) is False
|
||||
|
|
|
|||
|
|
@ -215,3 +215,56 @@ async def test_count_uses_the_shared_api_base_chat_would_use(monkeypatch: pytest
|
|||
assert result is not None
|
||||
assert result.total_tokens == 6
|
||||
assert route.called
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("assistant_content", "expected_beta"),
|
||||
[
|
||||
(
|
||||
[{"type": "compaction", "content": "Summary of the chat so far.", "signature": "sig-abc"}],
|
||||
"compact-2026-09-04,token-counting-2024-11-01",
|
||||
),
|
||||
("Use Recipe and Step tables.", "token-counting-2024-11-01"),
|
||||
],
|
||||
ids=("signed_compaction_block", "plain_turn"),
|
||||
)
|
||||
async def test_count_sends_the_compact_beta_only_when_the_history_replays_a_compaction_block(
|
||||
monkeypatch: pytest.MonkeyPatch, assistant_content: object, expected_beta: str
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
with respx.mock:
|
||||
route: Final = respx.post(COUNT_URL).mock(return_value=httpx.Response(200, json={"input_tokens": 376}))
|
||||
result: Final = await AzureAIAnthropicTokenCounter().count_tokens(
|
||||
model_to_use="claude-sonnet-5-5",
|
||||
messages=[
|
||||
{"role": "assistant", "content": assistant_content},
|
||||
{"role": "user", "content": "What tables did we pick?"},
|
||||
],
|
||||
contents=None,
|
||||
deployment={"litellm_params": {"api_key": "azure-key", "api_base": API_BASE}},
|
||||
request_model="azure_ai/claude-sonnet-5-5",
|
||||
)
|
||||
assert result is not None
|
||||
assert result.total_tokens == 376
|
||||
assert route.calls.last.request.headers["anthropic-beta"] == expected_beta
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_count_sends_the_tool_change_beta_when_the_history_adds_a_tool(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
with respx.mock:
|
||||
route: Final = respx.post(COUNT_URL).mock(return_value=httpx.Response(200, json={"input_tokens": 541}))
|
||||
result: Final = await AzureAIAnthropicTokenCounter().count_tokens(
|
||||
model_to_use="claude-sonnet-5-5",
|
||||
messages=[
|
||||
{"role": "user", "content": "Use the ping tool once."},
|
||||
{"role": "system", "content": [{"type": "tool_addition", "tool": {"type": "tool_reference", "name": "ping"}}]},
|
||||
],
|
||||
contents=None,
|
||||
deployment={"litellm_params": {"api_key": "azure-key", "api_base": API_BASE}},
|
||||
request_model="azure_ai/claude-sonnet-5-5",
|
||||
)
|
||||
assert result is not None
|
||||
assert result.total_tokens == 541
|
||||
assert route.calls.last.request.headers["anthropic-beta"] == "mid-conversation-tool-changes-2026-07-01,token-counting-2024-11-01"
|
||||
|
|
|
|||
|
|
@ -607,3 +607,91 @@ async def test_outgoing_azure_request_keeps_an_object_output_format_under_drop_p
|
|||
body: Final = await _azure_request_body_for_output_format(output_format, drop_params=True)
|
||||
|
||||
assert body["output_format"] == output_format
|
||||
|
||||
|
||||
_SIGNED_COMPACTION_BLOCK: Final = {"type": "compaction", "content": "Summary of the chat so far.", "signature": "sig"}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.usefixtures("local_beta_headers_config")
|
||||
@pytest.mark.parametrize(
|
||||
("messages", "params"),
|
||||
[
|
||||
([{"role": "user", "content": "Summarize the chat so far."}], {"compaction": {"type": "summarize"}}),
|
||||
(
|
||||
[{"role": "assistant", "content": [_SIGNED_COMPACTION_BLOCK]}, {"role": "user", "content": "Go on"}],
|
||||
{},
|
||||
),
|
||||
],
|
||||
ids=["compaction_param", "signed_compaction_block"],
|
||||
)
|
||||
async def test_outgoing_azure_compaction_request_carries_the_compact_2026_09_04_beta(
|
||||
messages: list[object], params: dict[str, object]
|
||||
) -> None:
|
||||
"""Foundry answers 400 "compaction: this parameter requires anthropic-beta: compact-2026-09-04" for the param
|
||||
and 400 "Input tag 'compaction' ... does not match" for a replayed block without the header, and 200 with it
|
||||
(live, claude-sonnet-5-5 on Azure AI Foundry, 2026-10-20)."""
|
||||
received: Final[list[httpx.Request]] = []
|
||||
|
||||
def foundry_answers(request: httpx.Request) -> httpx.Response:
|
||||
received.append(request)
|
||||
return _foundry_answer(request)
|
||||
|
||||
upstream: Final = AsyncHTTPHandler(transport=httpx.MockTransport(foundry_answers))
|
||||
try:
|
||||
await litellm.anthropic.messages.acreate(
|
||||
max_tokens=256,
|
||||
messages=messages,
|
||||
model="azure_ai/claude-sonnet-5-5",
|
||||
api_key="test-api-key",
|
||||
api_base="https://test-resource.services.ai.azure.com",
|
||||
client=upstream,
|
||||
**params,
|
||||
)
|
||||
finally:
|
||||
await upstream.close()
|
||||
|
||||
assert len(received) == 1, received
|
||||
assert received[0].headers["anthropic-beta"].split(",").count("compact-2026-09-04") == 1, dict(received[0].headers)
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("local_model_cost_map")
|
||||
@pytest.mark.parametrize("feature", ("thinking", "output_config", "tool_addition", "tool_removal", "none"))
|
||||
@pytest.mark.parametrize("explicit_beta", (False, True))
|
||||
def test_messages_feature_beta_headers(feature: str, explicit_beta: bool) -> None:
|
||||
from typing import Final
|
||||
|
||||
from litellm.llms.azure_ai.anthropic.messages_transformation import AzureAnthropicMessagesConfig
|
||||
from litellm.types.llms.anthropic import (
|
||||
ANTHROPIC_BETA_HEADER_VALUES,
|
||||
ANTHROPIC_MID_CONVERSATION_TOOL_CHANGES_BETA_HEADER,
|
||||
ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER,
|
||||
)
|
||||
|
||||
beta: Final = (
|
||||
ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER
|
||||
if feature == "thinking"
|
||||
else ANTHROPIC_BETA_HEADER_VALUES.PER_TURN_CONTROL_2026_07_01.value
|
||||
if feature == "output_config"
|
||||
else ANTHROPIC_MID_CONVERSATION_TOOL_CHANGES_BETA_HEADER
|
||||
)
|
||||
message: Final = (
|
||||
{"role": "system", "content": [], "output_config": {"effort": "high"}}
|
||||
if feature == "output_config"
|
||||
else {"role": "system", "content": [{"type": feature, "tool": {"type": "tool_reference", "name": "ping"}}]}
|
||||
if feature in ("tool_addition", "tool_removal")
|
||||
else {"role": "user", "content": "Reply with OK"}
|
||||
)
|
||||
headers, _ = AzureAnthropicMessagesConfig().validate_anthropic_messages_environment(
|
||||
headers={"anthropic-beta": f"existing-beta,{beta}" if explicit_beta else "existing-beta"},
|
||||
model="claude-fable-5-1",
|
||||
messages=[{"role": "user", "content": "Hello"}, message],
|
||||
optional_params={"thinking": {"type": "adaptive", "display": "updates"}} if feature == "thinking" else {},
|
||||
litellm_params={"vertex_ai_project": "test-project", "vertex_ai_location": "us-east5"},
|
||||
api_key="test-key",
|
||||
api_base="https://example.com/anthropic",
|
||||
)
|
||||
|
||||
betas: Final = headers["anthropic-beta"].split(",")
|
||||
assert betas.count(beta) == int(feature != "none" or explicit_beta)
|
||||
assert betas.count("existing-beta") == 1
|
||||
|
|
|
|||
|
|
@ -5,10 +5,12 @@ sys.path.insert(
|
|||
0, os.path.abspath(os.path.join(os.path.dirname(__file__), "../../../../.."))
|
||||
)
|
||||
|
||||
from typing import Final
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from litellm.anthropic_beta_headers_manager import update_headers_with_filtered_beta
|
||||
from litellm.llms.azure_ai.anthropic.transformation import AzureAnthropicConfig
|
||||
from litellm.types.router import GenericLiteLLMParams
|
||||
|
||||
|
|
@ -484,3 +486,85 @@ def test_chat_flagged_model_keeps_mid_conversation_system_role_in_place(local_mo
|
|||
"role": "system",
|
||||
"content": [{"type": "text", "text": "<system-reminder>Answer with exactly one word.</system-reminder>"}],
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("optional_params", "messages", "expected"),
|
||||
[
|
||||
({"compaction": {"type": "summarize"}}, [{"role": "user", "content": "Hello"}], True),
|
||||
(
|
||||
{},
|
||||
[
|
||||
{"role": "assistant", "content": [{"type": "compaction", "content": "Summary.", "signature": "sig"}]},
|
||||
{"role": "user", "content": "Go on"},
|
||||
],
|
||||
True,
|
||||
),
|
||||
({}, [{"role": "user", "content": "Hello"}], False),
|
||||
],
|
||||
ids=["compaction_param", "signed_compaction_block", "no_compaction"],
|
||||
)
|
||||
def test_azure_chat_compaction_sends_the_compact_2026_09_04_beta(
|
||||
local_beta_headers_config: None, optional_params: dict[str, object], messages: list[object], expected: bool
|
||||
) -> None:
|
||||
headers: Final = AzureAnthropicConfig().update_headers_with_optional_anthropic_beta(
|
||||
headers={}, optional_params=optional_params, messages=messages
|
||||
)
|
||||
|
||||
filtered: Final = update_headers_with_filtered_beta(headers=headers, provider="azure_ai")
|
||||
|
||||
assert ("compact-2026-09-04" in filtered.get("anthropic-beta", "").split(",")) is expected
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("local_model_cost_map")
|
||||
@pytest.mark.parametrize("thinking_type", ("adaptive", "enabled"))
|
||||
@pytest.mark.parametrize("display", ("summarized", "omitted", "updates"))
|
||||
def test_chat_preserves_thinking_display_and_beta(thinking_type: str, display: str) -> None:
|
||||
from typing import Final
|
||||
|
||||
from litellm.llms.azure_ai.anthropic.transformation import AzureAnthropicConfig
|
||||
from litellm.types.llms.anthropic import ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER
|
||||
|
||||
config: Final = AzureAnthropicConfig()
|
||||
optional_params: Final = config.map_openai_params(
|
||||
non_default_params={"thinking": {"type": thinking_type, "display": display, "budget_tokens": 2048}},
|
||||
optional_params={},
|
||||
model="claude-fable-5-1",
|
||||
drop_params=False,
|
||||
)
|
||||
headers: Final = config.validate_environment(
|
||||
headers={"anthropic-beta": "existing-beta"},
|
||||
model="claude-fable-5-1",
|
||||
messages=[{"role": "user", "content": "Reply with OK"}],
|
||||
optional_params=optional_params,
|
||||
litellm_params={},
|
||||
api_key="test-key",
|
||||
)
|
||||
result: Final = config.transform_request(
|
||||
model="claude-fable-5-1",
|
||||
messages=[{"role": "user", "content": "Reply with OK"}],
|
||||
optional_params=optional_params,
|
||||
litellm_params={},
|
||||
headers=headers,
|
||||
)
|
||||
|
||||
assert result["thinking"]["type"] == "adaptive"
|
||||
assert result["thinking"]["display"] == display
|
||||
assert (ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER in headers["anthropic-beta"].split(",")) == (display == "updates")
|
||||
assert "existing-beta" in headers["anthropic-beta"].split(",")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("thinking", ("enabled", {"type": "bogus"}, {"type": "adaptive", "display": "future-mode"}))
|
||||
def test_beta_detection_preserves_unrecognized_thinking(thinking: object) -> None:
|
||||
from typing import Final
|
||||
|
||||
from litellm.types.llms.anthropic import ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER
|
||||
|
||||
params: Final = {"thinking": thinking}
|
||||
headers: Final = AzureAnthropicConfig().validate_environment(
|
||||
headers={}, model="claude-fable-5-1", messages=[], optional_params=params,
|
||||
litellm_params={}, api_key="test-key",
|
||||
)
|
||||
|
||||
assert params["thinking"] == thinking
|
||||
assert ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER not in headers.get("anthropic-beta", "").split(",")
|
||||
|
|
|
|||
|
|
@ -6,15 +6,19 @@ from datetime import datetime, timedelta
|
|||
from queue import SimpleQueue
|
||||
from threading import get_ident
|
||||
from typing import Final
|
||||
from unittest.mock import MagicMock, patch
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
import requests
|
||||
import responses
|
||||
|
||||
import litellm
|
||||
from litellm.anthropic_beta_headers_manager import update_headers_with_filtered_beta
|
||||
from litellm.llms.vertex_ai.vertex_ai_partner_models.anthropic.messages.transformation import (
|
||||
VertexAIPartnerModelsAnthropicMessagesConfig,
|
||||
)
|
||||
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
|
||||
from litellm.llms.vertex_ai.vertex_ai_partner_models.main import VertexAIPartnerModels
|
||||
from litellm.types.llms.anthropic import ANTHROPIC_BETA_HEADER_VALUES, ANTHROPIC_HOSTED_TOOLS
|
||||
from litellm.types.router import GenericLiteLLMParams
|
||||
|
|
@ -208,7 +212,9 @@ def test_no_safeguards_leaves_dangerous_tool_use_beta_header_out():
|
|||
assert "dangerous-tool-use-2026-09-03" not in updated_headers.get("anthropic-beta", "")
|
||||
|
||||
|
||||
def _validate_vertex_headers(client_headers, messages):
|
||||
def _validate_vertex_headers(
|
||||
client_headers: dict[str, str], messages: list[object], optional_params: dict[str, object] | None = None
|
||||
) -> dict[str, str]:
|
||||
config = VertexAIPartnerModelsAnthropicMessagesConfig()
|
||||
litellm_params = {
|
||||
"vertex_ai_project": "test-project",
|
||||
|
|
@ -224,7 +230,7 @@ def _validate_vertex_headers(client_headers, messages):
|
|||
headers=client_headers,
|
||||
model="claude-opus-5-5",
|
||||
messages=messages,
|
||||
optional_params={"max_tokens": 64},
|
||||
optional_params=optional_params or {"max_tokens": 64},
|
||||
litellm_params=litellm_params,
|
||||
api_base=None,
|
||||
)
|
||||
|
|
@ -903,3 +909,168 @@ def test_vertex_claude_4_8_plus_cost_map_entries_carry_mid_conversation_system_f
|
|||
and info.get("supports_mid_conversation_system") is not True
|
||||
]
|
||||
assert missing == []
|
||||
|
||||
|
||||
_SIGNED_COMPACTION_BLOCK: Final = {"type": "compaction", "content": "summary so far", "signature": "sig"}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"client_headers",
|
||||
[{"anthropic-beta": "compact-2026-09-04"}, {}],
|
||||
ids=["client_sends_beta", "client_omits_beta"],
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"optional_params,messages",
|
||||
[
|
||||
({"max_tokens": 64, "compaction": {"type": "summarize"}}, [{"role": "user", "content": "Hello"}]),
|
||||
(
|
||||
{"max_tokens": 64},
|
||||
[{"role": "assistant", "content": [_SIGNED_COMPACTION_BLOCK]}, {"role": "user", "content": "Go on"}],
|
||||
),
|
||||
],
|
||||
ids=["compaction_param", "signed_compaction_block"],
|
||||
)
|
||||
def test_native_compaction_reaches_vertex_with_compact_2026_09_04_beta(
|
||||
local_beta_headers_config: None,
|
||||
client_headers: dict[str, str],
|
||||
optional_params: dict[str, object],
|
||||
messages: list[object],
|
||||
) -> None:
|
||||
"""Vertex serves on-demand compaction only behind compact-2026-09-04 (Google Cloud: beta on
|
||||
https://platform.claude.com/docs/en/build-with-claude/compaction-on-demand, read 2026-10-05)
|
||||
and answers 400 "compaction: Extra inputs are not permitted" without it, so the beta rides
|
||||
along with the `compaction` param or a signed compaction block exactly once, whether or not
|
||||
the client sent it, and survives the Vertex beta filter."""
|
||||
headers: Final = _validate_vertex_headers(client_headers, messages, optional_params)
|
||||
|
||||
filtered: Final = update_headers_with_filtered_beta(headers=headers, provider="vertex_ai")
|
||||
|
||||
assert filtered["anthropic-beta"].split(",").count("compact-2026-09-04") == 1
|
||||
|
||||
|
||||
def test_no_compaction_signal_leaves_compact_2026_09_04_beta_out() -> None:
|
||||
headers: Final = _validate_vertex_headers({}, [{"role": "user", "content": "Hello"}])
|
||||
|
||||
assert "compact-2026-09-04" not in headers.get("anthropic-beta", "")
|
||||
|
||||
|
||||
_VERTEX_MESSAGE_RESPONSE: Final = {
|
||||
"id": "msg_vrtx_test",
|
||||
"type": "message",
|
||||
"role": "assistant",
|
||||
"model": "claude-sonnet-4-6",
|
||||
"content": [{"type": "text", "text": "pong"}],
|
||||
"stop_reason": "end_turn",
|
||||
"stop_sequence": None,
|
||||
"usage": {"input_tokens": 5, "output_tokens": 1},
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_vertex_messages_request_sends_compact_2026_09_04_beta_on_the_wire(local_beta_headers_config: None) -> None:
|
||||
"""A /v1/messages compaction request to a Vertex Claude deployment reaches Vertex with the beta in the
|
||||
anthropic-beta request header and never in the body, the shape Vertex accepts (live, 2026-10-05)."""
|
||||
mock_response: Final = MagicMock()
|
||||
mock_response.status_code = 200
|
||||
mock_response.text = json.dumps(_VERTEX_MESSAGE_RESPONSE)
|
||||
mock_response.headers = httpx.Headers({})
|
||||
mock_response.json.return_value = _VERTEX_MESSAGE_RESPONSE
|
||||
|
||||
client: Final = MagicMock(spec=AsyncHTTPHandler)
|
||||
client.post = AsyncMock(return_value=mock_response)
|
||||
|
||||
with patch(
|
||||
"litellm.llms.vertex_ai.vertex_llm_base.VertexBase._ensure_access_token_async",
|
||||
AsyncMock(return_value=("token", "test-project")),
|
||||
):
|
||||
await litellm.anthropic.messages.acreate(
|
||||
model="vertex_ai/claude-sonnet-4-6",
|
||||
max_tokens=64,
|
||||
messages=[{"role": "user", "content": "Reply with the single word: pong"}],
|
||||
compaction={"type": "summarize"},
|
||||
vertex_project="test-project",
|
||||
vertex_location="us-east5",
|
||||
client=client,
|
||||
)
|
||||
|
||||
sent: Final = client.post.call_args.kwargs
|
||||
sent_body: Final = sent["json"] if "json" in sent else json.loads(sent["data"])
|
||||
assert sent["headers"]["anthropic-beta"].split(",").count("compact-2026-09-04") == 1
|
||||
assert sent_body["compaction"] == {"type": "summarize"}
|
||||
assert "anthropic_beta" not in sent_body
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("local_model_cost_map", "local_beta_headers_config")
|
||||
@pytest.mark.parametrize("feature", ("thinking", "output_config", "tool_addition", "tool_removal", "none"))
|
||||
@pytest.mark.parametrize("explicit_beta", (False, True))
|
||||
def test_messages_feature_beta_headers(feature: str, explicit_beta: bool) -> None:
|
||||
from typing import Final
|
||||
from unittest.mock import patch
|
||||
|
||||
from google.oauth2.credentials import Credentials
|
||||
from litellm.anthropic_beta_headers_manager import update_headers_with_filtered_beta
|
||||
from litellm.llms.vertex_ai.vertex_ai_partner_models.anthropic.messages.transformation import VertexAIPartnerModelsAnthropicMessagesConfig
|
||||
from litellm.types.llms.anthropic import (
|
||||
ANTHROPIC_BETA_HEADER_VALUES,
|
||||
ANTHROPIC_MID_CONVERSATION_TOOL_CHANGES_BETA_HEADER,
|
||||
ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER,
|
||||
)
|
||||
|
||||
beta: Final = (
|
||||
ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER
|
||||
if feature == "thinking"
|
||||
else ANTHROPIC_BETA_HEADER_VALUES.PER_TURN_CONTROL_2026_07_01.value
|
||||
if feature == "output_config"
|
||||
else ANTHROPIC_MID_CONVERSATION_TOOL_CHANGES_BETA_HEADER
|
||||
)
|
||||
message: Final = (
|
||||
{"role": "system", "content": [], "output_config": {"effort": "high"}}
|
||||
if feature == "output_config"
|
||||
else {"role": "system", "content": [{"type": feature, "tool": {"type": "tool_reference", "name": "ping"}}]}
|
||||
if feature in ("tool_addition", "tool_removal")
|
||||
else {"role": "user", "content": "Reply with OK"}
|
||||
)
|
||||
with patch(
|
||||
"google.auth.default",
|
||||
return_value=(MagicMock(spec=Credentials, token="test-token", expired=False), "test-project"),
|
||||
):
|
||||
headers, _ = VertexAIPartnerModelsAnthropicMessagesConfig().validate_anthropic_messages_environment(
|
||||
headers={"anthropic-beta": f"existing-beta,{beta}" if explicit_beta else "existing-beta"},
|
||||
model="claude-fable-5-1",
|
||||
messages=[{"role": "user", "content": "Hello"}, message],
|
||||
optional_params={"thinking": {"type": "adaptive", "display": "updates"}} if feature == "thinking" else {},
|
||||
litellm_params={"vertex_ai_project": "test-project", "vertex_ai_location": "us-east5"},
|
||||
api_key="test-key",
|
||||
api_base="https://example.com/anthropic",
|
||||
)
|
||||
|
||||
betas: Final = headers["anthropic-beta"].split(",")
|
||||
assert betas.count(beta) == int(feature != "none" or explicit_beta)
|
||||
assert betas.count("existing-beta") == 1
|
||||
|
||||
if feature in ("tool_addition", "tool_removal"):
|
||||
filtered_headers: Final = update_headers_with_filtered_beta(headers=headers, provider="vertex_ai")
|
||||
assert filtered_headers.get("anthropic-beta", "").split(",").count(beta) == 1
|
||||
|
||||
|
||||
@pytest.mark.parametrize("thinking", ("enabled", {"type": "bogus"}, {"type": "adaptive", "display": "future-mode"}))
|
||||
def test_beta_detection_preserves_unrecognized_thinking(thinking: object) -> None:
|
||||
from typing import Final
|
||||
from unittest.mock import patch
|
||||
|
||||
from google.oauth2.credentials import Credentials
|
||||
from litellm.types.llms.anthropic import ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER
|
||||
|
||||
params: Final = {"thinking": thinking}
|
||||
with patch(
|
||||
"google.auth.default",
|
||||
return_value=(MagicMock(spec=Credentials, token="test-token", expired=False), "test-project"),
|
||||
):
|
||||
headers, _ = VertexAIPartnerModelsAnthropicMessagesConfig().validate_anthropic_messages_environment(
|
||||
headers={}, model="claude-fable-5-1", messages=[], optional_params=params,
|
||||
litellm_params={"vertex_ai_project": "test-project", "vertex_ai_location": "us-east5"},
|
||||
api_key="test-key", api_base="https://example.com/anthropic",
|
||||
)
|
||||
|
||||
assert params["thinking"] == thinking
|
||||
assert ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER not in headers.get("anthropic-beta", "").split(",")
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ import copy
|
|||
import json
|
||||
from collections.abc import Mapping
|
||||
from typing import Final
|
||||
from unittest.mock import MagicMock
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
|
|
@ -12,10 +12,12 @@ import respx
|
|||
from google.oauth2.credentials import Credentials
|
||||
|
||||
import litellm
|
||||
from litellm.llms.custom_httpx.http_handler import HTTPHandler
|
||||
from litellm.anthropic_beta_headers_manager import (
|
||||
update_headers_with_filtered_beta,
|
||||
update_request_with_filtered_beta,
|
||||
)
|
||||
from litellm.llms.anthropic.common_utils import AnthropicError
|
||||
from litellm.llms.vertex_ai.vertex_ai_partner_models.anthropic.transformation import (
|
||||
VertexAIAnthropicConfig,
|
||||
)
|
||||
|
|
@ -456,8 +458,10 @@ def test_vertex_ai_anthropic_extra_headers_beta_propagation():
|
|||
|
||||
def test_vertex_ai_anthropic_inline_tools_beta_survives_the_chat_beta_filter(local_beta_headers_config: None) -> None:
|
||||
"""The chat path runs the Vertex beta filter over both the header and the `anthropic_beta` body field right
|
||||
before the request goes out, so inline-tools-2026-09-15 must survive both (source and date in
|
||||
tests/unit/test_anthropic_beta_headers_filtering.py)."""
|
||||
before the request goes out, so inline-tools-2026-09-15 must survive it as the header (source and date in
|
||||
tests/unit/test_anthropic_beta_headers_filtering.py) and stay out of the body field, which Vertex rejects
|
||||
with 400 "Unexpected value(s) `inline-tools-2026-09-15` for the `anthropic_beta` parameter" (live, global,
|
||||
claude-sonnet-5-5, 2026-10-20)."""
|
||||
config: Final = VertexAIAnthropicConfig()
|
||||
headers: Final[dict[str, str]] = {}
|
||||
optional_params: Final[dict[str, object]] = {
|
||||
|
|
@ -478,7 +482,7 @@ def test_vertex_ai_anthropic_inline_tools_beta_survives_the_chat_beta_filter(loc
|
|||
)
|
||||
|
||||
assert filtered_headers["anthropic-beta"].split(",").count("inline-tools-2026-09-15") == 1
|
||||
assert "inline-tools-2026-09-15" in filtered_request["anthropic_beta"]
|
||||
assert "inline-tools-2026-09-15" not in filtered_request.get("anthropic_beta", [])
|
||||
|
||||
|
||||
def test_vertex_ai_anthropic_extra_headers_beta_merged_with_auto_betas():
|
||||
|
|
@ -1213,3 +1217,293 @@ def test_refusal_stop_details_kept_through_vertex_anthropic_transform():
|
|||
provider_fields: Final = result.choices[0].message.provider_specific_fields
|
||||
assert provider_fields["stop_reason"] == "refusal"
|
||||
assert provider_fields["stop_details"] == ANTHROPIC_DOCS_REFUSAL_MESSAGE["stop_details"]
|
||||
|
||||
|
||||
_SIGNED_COMPACTION_BLOCK: Final = {"type": "compaction", "content": "summary so far", "signature": "sig"}
|
||||
_CLEAR_TOOL_USES: Final = {"edits": [{"type": "clear_tool_uses_20250919"}]}
|
||||
|
||||
|
||||
def _vertex_request(optional_params: dict[str, object], headers: dict[str, str], litellm_params: dict[str, object] | None = None) -> dict:
|
||||
return VertexAIAnthropicConfig().transform_request(
|
||||
model="claude-opus-4-6",
|
||||
messages=[{"role": "user", "content": "Hello"}],
|
||||
optional_params={"max_tokens": 100, "is_vertex_request": True, **optional_params},
|
||||
litellm_params=litellm_params or {},
|
||||
headers=headers,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"optional_params,messages",
|
||||
[
|
||||
(
|
||||
{"max_tokens": 100, "is_vertex_request": True, "compaction": {"type": "summarize"}},
|
||||
[{"role": "user", "content": "Hello"}],
|
||||
),
|
||||
(
|
||||
{"max_tokens": 100, "is_vertex_request": True},
|
||||
[
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": "",
|
||||
"provider_specific_fields": {"compaction_blocks": [_SIGNED_COMPACTION_BLOCK]},
|
||||
},
|
||||
{"role": "user", "content": "Go on"},
|
||||
],
|
||||
),
|
||||
],
|
||||
ids=["compaction_param", "signed_compaction_block_sent_back"],
|
||||
)
|
||||
def test_vertex_ai_anthropic_native_compaction_adds_compact_2026_09_04_beta(
|
||||
local_beta_headers_config: None, optional_params: dict[str, object], messages: list[dict[str, object]]
|
||||
) -> None:
|
||||
"""The chat path skips the Anthropic beta injection for Vertex requests, so the Vertex config
|
||||
adds compact-2026-09-04 itself (Google Cloud: beta on
|
||||
https://platform.claude.com/docs/en/build-with-claude/compaction-on-demand, read 2026-10-05)
|
||||
as the HTTP header only, since Vertex answered 400 "Unexpected value(s) `compact-2026-09-04`
|
||||
for the `anthropic_beta` parameter" with the name in the body field (us-east5,
|
||||
claude-sonnet-4-6, 2026-10-05), and the header must survive the Vertex beta filter."""
|
||||
headers: Final[dict[str, str]] = {}
|
||||
|
||||
result: Final = VertexAIAnthropicConfig().transform_request(
|
||||
model="claude-opus-4-6",
|
||||
messages=messages,
|
||||
optional_params=optional_params,
|
||||
litellm_params={},
|
||||
headers=headers,
|
||||
)
|
||||
filtered_headers, filtered_body = update_request_with_filtered_beta(
|
||||
headers=headers, request_data=result, provider="vertex_ai"
|
||||
)
|
||||
|
||||
assert filtered_headers["anthropic-beta"].split(",").count("compact-2026-09-04") == 1
|
||||
assert "compact-2026-09-04" not in filtered_body.get("anthropic_beta", [])
|
||||
|
||||
|
||||
def test_vertex_ai_anthropic_no_compaction_signal_leaves_compact_2026_09_04_beta_out() -> None:
|
||||
headers: Final[dict[str, str]] = {}
|
||||
|
||||
result: Final = _vertex_request({}, headers)
|
||||
|
||||
assert "compact-2026-09-04" not in result.get("anthropic_beta", [])
|
||||
assert "compact-2026-09-04" not in headers.get("anthropic-beta", "")
|
||||
|
||||
|
||||
def test_vertex_ai_anthropic_client_sent_compact_2026_09_04_stays_out_of_the_body_field(
|
||||
local_beta_headers_config: None,
|
||||
) -> None:
|
||||
headers: Final[dict[str, str]] = {}
|
||||
|
||||
result: Final = _vertex_request(
|
||||
{
|
||||
"compaction": {"type": "summarize"},
|
||||
"extra_headers": {"anthropic-beta": "compact-2026-09-04,interleaved-thinking-2025-05-14"},
|
||||
},
|
||||
headers,
|
||||
)
|
||||
|
||||
assert headers["anthropic-beta"].split(",").count("compact-2026-09-04") == 1
|
||||
assert result["anthropic_beta"] == ["interleaved-thinking-2025-05-14"]
|
||||
|
||||
|
||||
def test_vertex_ai_anthropic_compaction_keeps_the_betas_the_client_sent_as_headers(
|
||||
local_beta_headers_config: None,
|
||||
) -> None:
|
||||
headers: Final = {"anthropic-beta": "context-1m-2025-08-07"}
|
||||
|
||||
result: Final = _vertex_request({"compaction": {"type": "summarize"}}, headers)
|
||||
|
||||
assert sorted(headers["anthropic-beta"].split(",")) == ["compact-2026-09-04", "context-1m-2025-08-07"]
|
||||
assert "anthropic_beta" not in result
|
||||
|
||||
|
||||
@pytest.mark.parametrize("header_only_beta", ["compact-2026-09-04", "inline-tools-2026-09-15"])
|
||||
def test_vertex_ai_anthropic_client_sent_body_header_only_beta_moves_to_the_header(
|
||||
local_beta_headers_config: None, header_only_beta: str
|
||||
) -> None:
|
||||
"""Vertex rejects these betas inside the body `anthropic_beta` field with 400 "Unexpected value(s)" and
|
||||
accepts them as the header (live, global, claude-sonnet-5-5, 2026-10-20), so a client that opts in through that field still
|
||||
gets the beta as a header and keeps its other betas in the body."""
|
||||
headers: Final[dict[str, str]] = {}
|
||||
|
||||
result: Final = _vertex_request(
|
||||
{"anthropic_beta": [header_only_beta, "interleaved-thinking-2025-05-14"]}, headers
|
||||
)
|
||||
|
||||
assert headers["anthropic-beta"] == header_only_beta
|
||||
assert result["anthropic_beta"] == ["interleaved-thinking-2025-05-14"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"client_beta,body,header",
|
||||
[
|
||||
(
|
||||
"context-1m-2025-08-07",
|
||||
["context-1m-2025-08-07", "context-management-2025-06-27"],
|
||||
"context-management-2025-06-27",
|
||||
),
|
||||
(
|
||||
"inline-tools-2026-09-15",
|
||||
["context-management-2025-06-27"],
|
||||
"context-management-2025-06-27,inline-tools-2026-09-15",
|
||||
),
|
||||
],
|
||||
ids=["body_beta", "header_only_beta"],
|
||||
)
|
||||
def test_vertex_ai_anthropic_litellm_betas_merge_with_the_client_body_betas(
|
||||
local_beta_headers_config: None, client_beta: str, body: list[str], header: str
|
||||
) -> None:
|
||||
headers: Final[dict[str, str]] = {}
|
||||
|
||||
result: Final = _vertex_request({"context_management": _CLEAR_TOOL_USES, "anthropic_beta": [client_beta]}, headers)
|
||||
|
||||
assert result["anthropic_beta"] == body
|
||||
assert headers["anthropic-beta"] == header
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"anthropic_beta",
|
||||
[["interleaved-thinking-2025-05-14"], ("interleaved-thinking-2025-05-14",)],
|
||||
ids=["list", "sdk_tuple"],
|
||||
)
|
||||
def test_vertex_ai_anthropic_client_sent_body_betas_stay_out_of_the_header(anthropic_beta: object) -> None:
|
||||
headers: Final[dict[str, str]] = {}
|
||||
|
||||
result: Final = _vertex_request({"anthropic_beta": anthropic_beta}, headers)
|
||||
|
||||
assert result["anthropic_beta"] == ["interleaved-thinking-2025-05-14"]
|
||||
assert "anthropic-beta" not in headers
|
||||
|
||||
|
||||
_MALFORMED_BETAS: Final = pytest.mark.parametrize(
|
||||
"malformed", [5, [1, 2], {"beta": "x"}], ids=["int", "list_of_ints", "dict"]
|
||||
)
|
||||
_MALFORMED_BETA_SOURCES: Final = pytest.mark.parametrize(
|
||||
"source,field",
|
||||
[("anthropic_beta", "anthropic_beta"), ("extra_headers", "The anthropic-beta header")],
|
||||
ids=["body_field", "extra_header"],
|
||||
)
|
||||
|
||||
|
||||
def _malformed_beta_params(source: str, malformed: object) -> dict[str, object]:
|
||||
if source == "extra_headers":
|
||||
return {"extra_headers": {"anthropic-beta": malformed}}
|
||||
return {"anthropic_beta": malformed}
|
||||
|
||||
|
||||
@_MALFORMED_BETAS
|
||||
@_MALFORMED_BETA_SOURCES
|
||||
def test_vertex_ai_anthropic_malformed_anthropic_beta_answers_400_naming_its_field(
|
||||
malformed: object, source: str, field: str
|
||||
) -> None:
|
||||
with pytest.raises(AnthropicError) as excinfo:
|
||||
_vertex_request(_malformed_beta_params(source, malformed), {})
|
||||
|
||||
assert excinfo.value.status_code == 400
|
||||
assert excinfo.value.message.startswith(f"{field} must be")
|
||||
|
||||
|
||||
@_MALFORMED_BETAS
|
||||
@_MALFORMED_BETA_SOURCES
|
||||
@pytest.mark.parametrize("auto_beta", [False, True], ids=["alone", "next_to_litellm_beta"])
|
||||
def test_vertex_ai_anthropic_malformed_anthropic_beta_is_dropped_under_drop_params(
|
||||
local_beta_headers_config: None, malformed: object, source: str, field: str, auto_beta: bool
|
||||
) -> None:
|
||||
headers: Final[dict[str, str]] = {}
|
||||
params: Final = {**_malformed_beta_params(source, malformed), **({"context_management": _CLEAR_TOOL_USES} if auto_beta else {})}
|
||||
|
||||
result: Final = _vertex_request(params, headers, litellm_params={"drop_params": True})
|
||||
|
||||
assert result.get("anthropic_beta") == (["context-management-2025-06-27"] if auto_beta else None)
|
||||
assert headers.get("anthropic-beta") == ("context-management-2025-06-27" if auto_beta else None)
|
||||
|
||||
|
||||
_VERTEX_MESSAGE_RESPONSE: Final = {
|
||||
"id": "msg_vrtx_test",
|
||||
"type": "message",
|
||||
"role": "assistant",
|
||||
"model": "claude-sonnet-4-6",
|
||||
"content": [{"type": "text", "text": "pong"}],
|
||||
"stop_reason": "end_turn",
|
||||
"stop_sequence": None,
|
||||
"usage": {"input_tokens": 5, "output_tokens": 1},
|
||||
}
|
||||
|
||||
|
||||
def test_vertex_ai_anthropic_completion_sends_compact_2026_09_04_beta_on_the_wire(local_beta_headers_config: None) -> None:
|
||||
"""A chat completion with `compaction` on a Vertex Claude deployment reaches Vertex with the beta in the
|
||||
anthropic-beta request header and never in the body field, the shape Vertex accepts (live, 2026-10-05)."""
|
||||
mock_response: Final = MagicMock()
|
||||
mock_response.status_code = 200
|
||||
mock_response.text = json.dumps(_VERTEX_MESSAGE_RESPONSE)
|
||||
mock_response.headers = {}
|
||||
mock_response.json.return_value = _VERTEX_MESSAGE_RESPONSE
|
||||
client: Final = MagicMock(spec=HTTPHandler)
|
||||
client.post.return_value = mock_response
|
||||
|
||||
with patch(
|
||||
"litellm.llms.vertex_ai.vertex_llm_base.VertexBase._ensure_access_token",
|
||||
return_value=("token", "test-project"),
|
||||
):
|
||||
litellm.completion(
|
||||
model="vertex_ai/claude-sonnet-4-6",
|
||||
messages=[{"role": "user", "content": "Reply with the single word: pong"}],
|
||||
compaction={"type": "summarize"},
|
||||
vertex_project="test-project",
|
||||
vertex_location="us-east5",
|
||||
client=client,
|
||||
)
|
||||
|
||||
sent: Final = client.post.call_args.kwargs
|
||||
sent_body: Final = sent["json"] if "json" in sent else json.loads(sent["data"])
|
||||
assert sent["headers"]["anthropic-beta"].split(",").count("compact-2026-09-04") == 1
|
||||
assert sent_body["compaction"] == {"type": "summarize"}
|
||||
assert "anthropic_beta" not in sent_body
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("local_model_cost_map")
|
||||
@pytest.mark.parametrize("thinking_type", ("adaptive", "enabled"))
|
||||
@pytest.mark.parametrize("display", ("summarized", "omitted", "updates"))
|
||||
def test_chat_preserves_thinking_display_and_beta(thinking_type: str, display: str) -> None:
|
||||
from typing import Final
|
||||
|
||||
from litellm.llms.vertex_ai.vertex_ai_partner_models.anthropic.transformation import VertexAIAnthropicConfig
|
||||
from litellm.types.llms.anthropic import ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER
|
||||
|
||||
config: Final = VertexAIAnthropicConfig()
|
||||
optional_params: Final = config.map_openai_params(
|
||||
non_default_params={"thinking": {"type": thinking_type, "display": display, "budget_tokens": 2048}},
|
||||
optional_params={},
|
||||
model="claude-fable-5-1",
|
||||
drop_params=False,
|
||||
)
|
||||
optional_params["extra_headers"] = {"anthropic-beta": "existing-beta"}
|
||||
headers: Final = {}
|
||||
result: Final = config.transform_request(
|
||||
model="claude-fable-5-1",
|
||||
messages=[{"role": "user", "content": "Reply with OK"}],
|
||||
optional_params=optional_params,
|
||||
litellm_params={},
|
||||
headers=headers,
|
||||
)
|
||||
|
||||
assert result["thinking"]["type"] == "adaptive"
|
||||
assert result["thinking"]["display"] == display
|
||||
assert (ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER in headers["anthropic-beta"].split(",")) == (display == "updates")
|
||||
assert "existing-beta" in headers["anthropic-beta"].split(",")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("thinking", ("enabled", {"type": "bogus"}, {"type": "adaptive", "display": "future-mode"}))
|
||||
def test_beta_detection_preserves_unrecognized_thinking(thinking: object) -> None:
|
||||
from typing import Final
|
||||
|
||||
from litellm.types.llms.anthropic import ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER
|
||||
|
||||
headers: Final = {}
|
||||
result: Final = VertexAIAnthropicConfig().transform_request(
|
||||
model="claude-fable-5-1", messages=[{"role": "user", "content": "Hello"}],
|
||||
optional_params={"thinking": thinking}, litellm_params={}, headers=headers,
|
||||
)
|
||||
|
||||
assert result["thinking"] == thinking
|
||||
assert ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER not in headers.get("anthropic-beta", "").split(",")
|
||||
|
|
|
|||
|
|
@ -0,0 +1,60 @@
|
|||
from typing import Final
|
||||
|
||||
import pytest
|
||||
|
||||
from litellm.llms.vertex_ai.vertex_ai_partner_models.count_tokens.handler import count_tokens_headers
|
||||
|
||||
_HISTORY: Final = ({"role": "user", "content": "Hello"},)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"request_data",
|
||||
[
|
||||
{"messages": [{"role": "assistant", "content": [{"type": "compaction", "content": "so far", "signature": "sig"}]}]},
|
||||
{"messages": list(_HISTORY), "compaction": {"type": "summarize"}},
|
||||
],
|
||||
ids=["signed_compaction_block", "compaction_param"],
|
||||
)
|
||||
def test_count_tokens_on_a_compaction_request_sends_the_compact_beta(request_data: dict[str, object]) -> None:
|
||||
assert count_tokens_headers("token", request_data) == {
|
||||
"Authorization": "Bearer token",
|
||||
"anthropic-beta": "compact-2026-09-04",
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"request_data",
|
||||
[
|
||||
{"messages": list(_HISTORY)},
|
||||
{"messages": [{"role": "assistant", "content": [{"type": "compaction", "content": "so far", "signature": ""}]}]},
|
||||
],
|
||||
ids=["plain", "unsigned_compaction_block"],
|
||||
)
|
||||
def test_count_tokens_without_compaction_sends_no_beta(request_data: dict[str, object]) -> None:
|
||||
assert count_tokens_headers("token", request_data) == {"Authorization": "Bearer token"}
|
||||
|
||||
|
||||
_TOOL_ADDITION: Final = {
|
||||
"role": "system",
|
||||
"content": [{"type": "tool_addition", "tool": {"type": "tool_reference", "name": "ping"}}],
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("request_data", "expected_beta"),
|
||||
[
|
||||
({"messages": [*_HISTORY, _TOOL_ADDITION]}, "mid-conversation-tool-changes-2026-07-01"),
|
||||
(
|
||||
{"messages": [*_HISTORY, _TOOL_ADDITION], "compaction": {"type": "summarize"}},
|
||||
"compact-2026-09-04,mid-conversation-tool-changes-2026-07-01",
|
||||
),
|
||||
],
|
||||
ids=["tool_addition", "tool_addition_with_compaction"],
|
||||
)
|
||||
def test_count_tokens_on_a_tool_change_request_sends_the_tool_change_beta(
|
||||
request_data: dict[str, object], expected_beta: str
|
||||
) -> None:
|
||||
assert count_tokens_headers("token", request_data) == {
|
||||
"Authorization": "Bearer token",
|
||||
"anthropic-beta": expected_beta,
|
||||
}
|
||||
|
|
@ -85,7 +85,8 @@ class TestAnthropicBetaHeadersFiltering:
|
|||
filtered = filter_and_transform_beta_headers(
|
||||
beta_headers=all_headers, provider=provider
|
||||
)
|
||||
assert ("compact-2026-09-04" in filtered) is (provider == "anthropic")
|
||||
assert ("compact-2026-09-04" in filtered) is (provider in ("anthropic", "azure_ai", "vertex_ai"))
|
||||
assert ("mid-conversation-tool-changes-2026-07-01" in filtered) is (provider != "bedrock_converse")
|
||||
|
||||
for header in unsupported_headers:
|
||||
assert (
|
||||
|
|
@ -406,15 +407,49 @@ class TestAnthropicBetaHeadersFiltering:
|
|||
test_case["expected"] in filtered
|
||||
), f"Header '{test_case['input']}' should be mapped to '{test_case['expected']}' for {test_case['provider']}, but got: {filtered}"
|
||||
|
||||
def test_filter_and_transform_beta_headers_vertex_ai_keeps_compact(self):
|
||||
"""Vertex AI supports compact context edits, so the compact beta header
|
||||
must be forwarded instead of stripped (it was previously mapped to null,
|
||||
which broke compact_20260112 context edits over /v1/messages)."""
|
||||
filtered = filter_and_transform_beta_headers(
|
||||
beta_headers=["compact-2026-01-12"], provider="vertex_ai"
|
||||
@pytest.mark.parametrize("beta", ["compact-2026-09-04", "inline-tools-2026-09-15"])
|
||||
def test_vertex_ai_header_only_beta_is_forwarded_as_the_header_and_dropped_from_the_body(
|
||||
self, beta: str
|
||||
) -> None:
|
||||
headers, body = update_request_with_filtered_beta(
|
||||
headers={"anthropic-beta": f"{beta},context-1m-2025-08-07"},
|
||||
request_data={"anthropic_beta": [beta, "context-1m-2025-08-07"]},
|
||||
provider="vertex_ai",
|
||||
)
|
||||
|
||||
assert filtered == ["compact-2026-01-12"]
|
||||
assert set(headers["anthropic-beta"].split(",")) == {beta, "context-1m-2025-08-07"}
|
||||
assert body["anthropic_beta"] == ["context-1m-2025-08-07"]
|
||||
|
||||
def test_header_only_betas_stay_out_of_the_provider_columns_older_proxies_read(self) -> None:
|
||||
"""Proxies released before header-only columns existed fetch this map from main, read only the
|
||||
provider column, and copy its betas into the body field too, where Vertex answers 400. So a
|
||||
header-only beta may sit in its provider column only if older proxies already forwarded it."""
|
||||
header_only_columns: Final = {
|
||||
column.removesuffix("_header_only"): betas
|
||||
for column, betas in self.config.items()
|
||||
if column.endswith("_header_only")
|
||||
}
|
||||
forwarded_by_older_proxies: Final = {
|
||||
provider: {beta for beta in betas if self.config[provider].get(beta) is not None}
|
||||
for provider, betas in header_only_columns.items()
|
||||
}
|
||||
|
||||
assert "vertex_ai" in header_only_columns
|
||||
assert {provider: betas for provider, betas in forwarded_by_older_proxies.items() if betas} == {
|
||||
"vertex_ai": {"inline-tools-2026-09-15"}
|
||||
}
|
||||
|
||||
def test_filter_and_transform_beta_headers_vertex_ai_keeps_compact(self):
|
||||
"""Vertex AI serves both compaction betas (threshold compaction behind compact-2026-01-12
|
||||
and on-demand compaction behind compact-2026-09-04, Google Cloud: beta on
|
||||
https://platform.claude.com/docs/en/build-with-claude/compaction-on-demand, read 2026-10-05),
|
||||
so the Vertex filter forwards both instead of stripping them: a dropped header makes
|
||||
Vertex answer 400 "compaction: Extra inputs are not permitted"."""
|
||||
filtered = filter_and_transform_beta_headers(
|
||||
beta_headers=["compact-2026-01-12", "compact-2026-09-04"], provider="vertex_ai"
|
||||
)
|
||||
|
||||
assert filtered == ["compact-2026-01-12", "compact-2026-09-04"]
|
||||
|
||||
@pytest.mark.parametrize("provider", ["bedrock_converse", "bedrock"])
|
||||
def test_fine_grained_tool_streaming_forwarded_for_bedrock(self, provider):
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue