scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-11-08 08:08:37 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 80

Merge remote-tracking branch 'origin/main' into litellm_internal_copy_40078
Some checks failed
Cost map pins recheck / rerun-cost-map-guard (push) Has been cancelled
Details
LiteLLM Rust DB / db-queries (push) Has been cancelled
Details
LiteLLM Rust / rust-lint (push) Has been cancelled
Details
LiteLLM Rust / rust-test (push) Has been cancelled
Details
LiteLLM Rust / rust-wheel (push) Has been cancelled
Details

Browse source
This commit is contained in:
mateo-berri 2026-10-30 12:07:00 -07:00
commit 3aa4e0b7c0
347 changed files with 25906 additions and 2250 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
.github/workflows/test-unit.yml vendored
View file

@ -227,6 +227,7 @@ jobs:
tests/unit/llms/test_oss_decision.py
tests/unit/llms/test_polling_url_origin_match.py
tests/unit/llms/test_predibase_transformation.py
tests/unit/llms/test_subscription_accounts.py
tests/unit/llms/a2a
tests/unit/llms/aiml
tests/unit/llms/aiohttp_openai

1
backend/routes/allowlist.py
View file

@ -55,6 +55,7 @@ BACKEND_PATH_PREFIXES: tuple[str, ...] = (
"/cache_settings",
"/coordination_redis/",
"/cost_tracking",
"/subscription_accounts/",
"/cost_optimization/",
"/cost/",
"/credentials",

21
litellm-proxy-extras/litellm_proxy_extras/migrations/20261028000000_subscription_accounts/migration.sql Normal file
View file

@ -0,0 +1,21 @@
SET LOCAL lock_timeout = '10s';
-- CreateTable
CREATE TABLE IF NOT EXISTS "LiteLLM_SubscriptionAccountTable" (
"subscription_account_id" TEXT NOT NULL,
"custom_llm_provider" TEXT NOT NULL,
"account_id" TEXT NOT NULL,
"label" TEXT,
"monthly_fee" DOUBLE PRECISION NOT NULL,
"currency" TEXT NOT NULL,
"billing_period_start" TEXT NOT NULL,
"created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
"created_by" TEXT NOT NULL,
"updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
"updated_by" TEXT NOT NULL,
CONSTRAINT "LiteLLM_SubscriptionAccountTable_pkey" PRIMARY KEY ("subscription_account_id")
);
-- CreateIndex
CREATE UNIQUE INDEX IF NOT EXISTS "LiteLLM_SubscriptionAccountTable_provider_account_key" ON "LiteLLM_SubscriptionAccountTable"("custom_llm_provider", "account_id");

16
litellm-proxy-extras/litellm_proxy_extras/schema.prisma
View file

@ -2131,3 +2131,19 @@ model LiteLLM_AccountingDelivery {
@@id([event_id, target, version])
@@index([target, state, next_attempt_at])
}
model LiteLLM_SubscriptionAccountTable {
subscription_account_id String @id @default(uuid())
custom_llm_provider String
account_id String
label String?
monthly_fee Float
currency String
billing_period_start String
created_at DateTime @default(now()) @map("created_at")
created_by String
updated_at DateTime @default(now()) @updatedAt @map("updated_at")
updated_by String
@@unique([custom_llm_provider, account_id], map: "LiteLLM_SubscriptionAccountTable_provider_account_key")
}

34
litellm-rust/crates/llms-types/src/providers/anthropic/beta.rs
View file

@ -199,7 +199,7 @@ impl AnthropicBeta {
&[BetaProvider::BedrockConverse, BetaProvider::Databricks]
}
Self::ContextManagement20250627 => &[BetaProvider::BedrockConverse],
Self::AdvisorTool20260301 | Self::Compact20260904 => &[
Self::AdvisorTool20260301 => &[
BetaProvider::AzureAi,
BetaProvider::BedrockConverse,
BetaProvider::Bedrock,
@ -207,6 +207,12 @@ impl AnthropicBeta {
BetaProvider::VertexAi,
BetaProvider::Databricks,
],
Self::Compact20260904 => &[
BetaProvider::BedrockConverse,
BetaProvider::Bedrock,
BetaProvider::BedrockMantle,
BetaProvider::Databricks,
],
Self::Bash20241022
| Self::Bash20250124
| Self::McpServers20251204
@ -256,14 +262,18 @@ impl AnthropicBeta {
Self::InterleavedThinking20250514 | Self::WebSearch20250305 => {
&[BetaProvider::BedrockConverse, BetaProvider::Bedrock]
}
Self::MidConversationOutputConfig20260701
| Self::MidConversationToolChanges20260701 => &[
Self::MidConversationOutputConfig20260701 => &[
BetaProvider::AzureAi,
BetaProvider::BedrockConverse,
BetaProvider::BedrockMantle,
BetaProvider::VertexAi,
BetaProvider::Databricks,
],
Self::MidConversationToolChanges20260701 => &[
BetaProvider::BedrockConverse,
BetaProvider::BedrockMantle,
BetaProvider::Databricks,
],
Self::Output128k20250219 | Self::TokenEfficientTools20250219 => &[
BetaProvider::AzureAi,
BetaProvider::BedrockConverse,
@ -483,7 +493,7 @@ mod tests {
let config = beta_headers_config();
let config_columns: Vec<String> = config
.keys()
.filter(|column| column.as_str() != "description")
.filter(|column| column.as_str() != "description" && !column.ends_with("_header_only"))
.cloned()
.collect();
let beta_provider_columns: Vec<String> = BetaProvider::VARIANTS
@ -502,12 +512,16 @@ mod tests {
let config = beta_headers_config();
for provider in BetaProvider::VARIANTS {
for beta in &AnthropicBeta::KNOWN {
let expected = config
.get(provider.as_ref())
.and_then(serde_json::Value::as_object)
.and_then(|column| column.get(beta.as_str()))
.and_then(serde_json::Value::as_str)
.map(str::to_string);
let cell = |column: &str| {
config
.get(column)
.and_then(serde_json::Value::as_object)
.and_then(|values| values.get(beta.as_str()))
.and_then(serde_json::Value::as_str)
.map(str::to_string)
};
let expected =
cell(&format!("{provider}_header_only")).or_else(|| cell(provider.as_ref()));
let actual = beta.on(*provider).map(|name| name.to_string());
assert_eq!(actual, expected, "provider {provider}, beta {beta}");
if matches!(beta, AnthropicBeta::AdvancedToolUse20251120)

99
litellm-rust/crates/llms/src/anthropic/messages/transformation.rs
View file

@ -1,7 +1,8 @@
use litellm_auth::CredentialPlacement;
use litellm_llms_types::{
formats::messages::{
ContextEdit, ContextManagement, Message, MessagesOptionalParams, MessagesRequest, Speed,
ContentBlock, ContentBlockType, ContextEdit, ContextManagement, Message,
MessagesOptionalParams, MessagesRequest, Speed, ThinkingConfig, ThinkingDisplay,
},
providers::anthropic::{AnthropicBeta, BetaProvider, BetaSet, DEFAULT_HEADERS},
recognized::Recognized,
@ -203,6 +204,10 @@ fn feature_betas(request: &MessagesRequest) -> BetaSet {
.then_some(AnthropicBeta::FastMode20260201),
messages_carry_output_config(&request.messages)
.then_some(AnthropicBeta::PerTurnControl20260701),
uses_thinking_display_updates(params)
.then_some(AnthropicBeta::ThinkingDisplayUpdates20260818),
messages_carry_tool_changes(&request.messages)
.then_some(AnthropicBeta::MidConversationToolChanges20260701),
has_advisor_tool(tools).then_some(AnthropicBeta::AdvisorTool20260301),
is_tool_search_used(tools).then_some(AnthropicBeta::AdvancedToolUse20251120),
]
@ -253,6 +258,36 @@ fn messages_carry_output_config(messages: &[Message]) -> bool {
.any(|message| message.extra.contains_key("output_config"))
}
fn uses_thinking_display_updates(params: &MessagesOptionalParams) -> bool {
let display = match params.thinking.as_ref().and_then(Recognized::known) {
Some(ThinkingConfig::Enabled(thinking)) => thinking.display.as_ref(),
Some(ThinkingConfig::Adaptive(thinking)) => thinking.display.as_ref(),
_ => None,
};
display.and_then(Recognized::known) == Some(&ThinkingDisplay::Updates)
}
fn is_tool_reference_change(block: &ContentBlock) -> bool {
matches!(
&block.block_type,
Some(ContentBlockType::Other(block_type))
if block_type == "tool_addition" || block_type == "tool_removal"
) && block
.extra
.get("tool")
.and_then(|tool| tool.get("type"))
.and_then(Value::as_str)
== Some("tool_reference")
}
fn messages_carry_tool_changes(messages: &[Message]) -> bool {
messages
.iter()
.filter(|message| message.role == "system")
.flat_map(Message::blocks)
.any(is_tool_reference_change)
}
fn unsupported_param(model: &str, param: &str, value: &str, hint: &str) -> Error {
Error::InvalidRequest(crate::ErrorDetail::UnsupportedParameter {
model: model.into(),
@ -1071,6 +1106,41 @@ mod tests {
json!({"safeguards": [{"type": "dangerous_tool_use", "classifier_context": {"v": 1, "permission_mode": "auto"}}]}),
&[]
)]
#[case::adaptive_display_updates(
json!({"thinking": {"type": "adaptive", "display": "updates"}}),
&["thinking-display-updates-2026-08-18"]
)]
#[case::enabled_display_updates(
json!({"thinking": {"type": "enabled", "budget_tokens": 2048, "display": "updates"}}),
&["thinking-display-updates-2026-08-18"]
)]
#[case::summarized_display(json!({"thinking": {"type": "adaptive", "display": "summarized"}}), &[])]
#[case::disabled_thinking(json!({"thinking": {"type": "disabled"}}), &[])]
#[case::system_tool_addition(
json!({"messages": [
{"role": "user", "content": "Use the ping tool"},
{"role": "system", "content": [{"type": "tool_addition", "tool": {"type": "tool_reference", "name": "ping"}}]},
]}),
&["mid-conversation-tool-changes-2026-07-01"]
)]
#[case::system_tool_removal(
json!({"messages": [
{"role": "system", "content": [{"type": "tool_removal", "tool": {"type": "tool_reference", "name": "ping"}}]},
]}),
&["mid-conversation-tool-changes-2026-07-01"]
)]
#[case::user_tool_addition(
json!({"messages": [
{"role": "user", "content": [{"type": "tool_addition", "tool": {"type": "tool_reference", "name": "ping"}}]},
]}),
&[]
)]
#[case::tool_addition_without_a_tool_reference(
json!({"messages": [
{"role": "system", "content": [{"type": "tool_addition", "tool": {"type": "tool_definition"}}]},
]}),
&[]
)]
fn feature_betas_follow_the_request(#[case] fields: Value, #[case] expected: &[&str]) {
assert_eq!(feature_betas(&request(fields)), betas(expected));
}
@ -1093,6 +1163,33 @@ mod tests {
);
}
#[rstest]
#[case::vertex_ai(
BetaProvider::VertexAi,
&["mid-conversation-tool-changes-2026-07-01", "thinking-display-updates-2026-08-18"]
)]
#[case::azure_ai(
BetaProvider::AzureAi,
&["mid-conversation-tool-changes-2026-07-01", "thinking-display-updates-2026-08-18"]
)]
#[case::bedrock_converse(BetaProvider::BedrockConverse, &["thinking-display-updates-2026-08-18"])]
#[case::databricks(BetaProvider::Databricks, &["thinking-display-updates-2026-08-18"])]
fn tool_change_and_display_betas_reach_the_providers_that_accept_them(
#[case] provider: BetaProvider,
#[case] expected: &[&str],
) {
let fields = json!({
"thinking": {"type": "adaptive", "display": "updates"},
"messages": [
{"role": "system", "content": [{"type": "tool_addition", "tool": {"type": "tool_reference", "name": "ping"}}]},
],
});
assert_eq!(
provider_feature_betas(&request(fields), provider),
betas(expected)
);
}
#[rstest]
fn hosts_without_a_beta_map_column_get_safeguards_without_a_derived_beta() {
let client = headers(&[("anthropic-beta", "claude-code-20250219")]);

5
litellm-rust/crates/llms/src/azure_ai/messages/transformation.rs
View file

@ -545,6 +545,11 @@ mod tests {
&[("anthropic-beta", "web-search-2025-03-05")],
&[("x-api-key", "k"), ("anthropic-beta", "structured-outputs-2025-11-13,web-search-2025-03-05")]
)]
#[case::compaction_needs_the_compact_2026_09_04_beta(
json!({"compaction": {"type": "summarize"}}),
&[],
&[("x-api-key", "k"), ("anthropic-beta", "compact-2026-09-04")]
)]
#[case::no_feature_needs_a_beta(json!({}), &[], &[("x-api-key", "k")])]
#[case::safeguards_need_the_dangerous_tool_use_beta(
json!({"safeguards": [{"type": "dangerous_tool_use"}]}),

12
litellm-rust/crates/llms/src/vertex_ai/messages/transformation.rs
View file

@ -798,6 +798,18 @@ mod tests {
]}}),
&["compact-2026-01-12", "context-management-2025-06-27"]
)]
#[case::compaction(json!({"compaction": {"type": "summarize"}}), &["compact-2026-09-04"])]
#[case::thinking_display_updates(
json!({"thinking": {"type": "adaptive", "display": "updates"}}),
&["thinking-display-updates-2026-08-18"]
)]
#[case::tool_addition(
json!({"messages": [
{"role": "user", "content": "Use the ping tool"},
{"role": "system", "content": [{"type": "tool_addition", "tool": {"type": "tool_reference", "name": "ping"}}]}
]}),
&["mid-conversation-tool-changes-2026-07-01"]
)]
#[case::advisor_tool_is_not_a_vertex_beta(
json!({"tools": [{"type": "advisor_20260301", "name": "advisor"}]}),
&[]

24
litellm/__init__.py
View file

@ -7,7 +7,6 @@ _apply_process_defaults()
### INIT VARIABLES #########################
import threading
import os
import datetime
from collections.abc import Mapping, Sequence
from typing import (
@ -550,6 +549,7 @@ _key_management_system: Optional["KeyManagementSystem"] = None
output_parse_pii: bool = False
#############################################
from litellm.litellm_core_utils.get_model_cost_map import get_model_cost_map, mark_litellm_import_complete
from litellm.litellm_core_utils.model_retirement import is_hidden_as_retired
model_cost = get_model_cost_map(url=model_cost_map_url)
cost_discount_config: Dict[str, float] = {} # Provider-specific cost discounts {"vertex_ai": 0.05} = 5% discount
@ -756,16 +756,6 @@ def is_openai_finetune_model(key: str) -> bool:
return key.startswith("ft:") and not key.count(":") > 1
def _is_past_deprecation_date(value: Mapping[str, object]) -> bool:
deprecation_date: Final = value.get("deprecation_date")
if not isinstance(deprecation_date, str):
return False
try:
return datetime.date.fromisoformat(deprecation_date) <= datetime.date.today()
except ValueError:
return False
def _populate_provider_model_sets(model_cost_map: Mapping[str, Mapping[str, object]]) -> None:
for key, value in tuple(model_cost_map.items()):
if value.get("litellm_provider") == "openai" and not is_openai_finetune_model(key):
@ -872,8 +862,13 @@ def _populate_provider_model_sets(model_cost_map: Mapping[str, Mapping[str, obje
# ignore the 'up-to', '-to-' model names -> not real models. just for cost tracking based on model params.
if "-to-" not in key and "fireworks-ai-default" not in key:
fireworks_ai_models.add(key)
elif value.get("litellm_provider") == "together_ai":
if is_hidden_as_retired(value):
together_ai_models.discard(key)
elif not key.startswith("together-ai-"):
together_ai_models.add(key)
elif value.get("litellm_provider") == "baseten":
if _is_past_deprecation_date(value):
if is_hidden_as_retired(value):
baseten_models.discard(key)
else:
baseten_models.add(key)
@ -1046,7 +1041,10 @@ def _populate_provider_model_sets(model_cost_map: Mapping[str, Mapping[str, obje
elif value.get("litellm_provider") == "coralbricks":
coralbricks_models.add(key)
elif value.get("litellm_provider") == "aiand":
aiand_models.add(key)
if is_hidden_as_retired(value):
aiand_models.discard(key)
else:
aiand_models.add(key)
elif value.get("litellm_provider") == "reducto":
reducto_models.add(key)
elif value.get("litellm_provider") == "bedrock_mantle":

9
litellm/anthropic_beta_headers_config.json
View file

@ -1,5 +1,5 @@
{
"description": "Mapping of Anthropic beta headers for each provider. Keys are input header names, values are provider-specific header names (or null if unsupported). Only headers present in mapping keys with non-null values can be forwarded.",
"description": "Mapping of Anthropic beta headers for each provider. Keys are input header names, values are provider-specific header names (or null if unsupported). Only headers present in mapping keys with non-null values can be forwarded. A <provider>_header_only column lists betas that provider accepts only as the anthropic-beta HTTP header: they are forwarded there and dropped from the anthropic_beta body field. Proxies that predate these columns ignore them and read the provider column alone.",
"anthropic": {
"advisor-tool-2026-03-01": "advisor-tool-2026-03-01",
"advanced-tool-use-2025-11-20": "advanced-tool-use-2025-11-20",
@ -46,6 +46,7 @@
"bash_20250124": null,
"code-execution-2025-08-25": "code-execution-2025-08-25",
"compact-2026-01-12": null,
"compact-2026-09-04": "compact-2026-09-04",
"computer-use-2025-01-24": "computer-use-2025-01-24",
"computer-use-2025-11-24": "computer-use-2025-11-24",
"context-1m-2025-08-07": "context-1m-2025-08-07",
@ -59,6 +60,7 @@
"mcp-client-2025-11-20": "mcp-client-2025-11-20",
"mcp-client-2025-04-04": "mcp-client-2025-04-04",
"mcp-servers-2025-12-04": null,
"mid-conversation-tool-changes-2026-07-01": "mid-conversation-tool-changes-2026-07-01",
"output-128k-2025-02-19": null,
"structured-output-2024-03-01": null,
"per-turn-control-2026-07-01": "per-turn-control-2026-07-01",
@ -202,6 +204,7 @@
"mcp-client-2025-11-20": null,
"mcp-client-2025-04-04": null,
"mcp-servers-2025-12-04": null,
"mid-conversation-tool-changes-2026-07-01": "mid-conversation-tool-changes-2026-07-01",
"output-128k-2025-02-19": null,
"structured-output-2024-03-01": null,
"per-turn-control-2026-07-01": "per-turn-control-2026-07-01",
@ -218,6 +221,10 @@
"web-search-2025-03-05": "web-search-2025-03-05",
"inline-tools-2026-09-15": "inline-tools-2026-09-15"
},
"vertex_ai_header_only": {
"compact-2026-09-04": "compact-2026-09-04",
"inline-tools-2026-09-15": "inline-tools-2026-09-15"
},
"databricks": {
"advisor-tool-2026-03-01": null,
"advanced-tool-use-2025-11-20": null,

39
litellm/anthropic_beta_headers_manager.py
View file

@ -30,7 +30,7 @@ from collections.abc import Mapping
from concurrent.futures import ThreadPoolExecutor
from functools import cache
from importlib.resources import files
from typing import Final, TypeAlias
from typing import Final, Literal, TypeAlias
import httpx
from pydantic import TypeAdapter
@ -47,6 +47,9 @@ BetaHeadersConfig: TypeAlias = Mapping[str, str | Mapping[str, str | None]]
_BETA_HEADERS_CONFIG_ADAPTER: Final = TypeAdapter(BetaHeadersConfig)
_DECODED_JSON: Final = TypeAdapter(object)
BetaPlacement: TypeAlias = Literal["header", "body"]
_HEADER_ONLY_COLUMN_SUFFIX: Final = "_header_only"
class GetAnthropicBetaHeadersConfig:
"""
@ -241,6 +244,31 @@ def get_provider_name(provider: str) -> str:
return aliases.get(provider, provider)
def _beta_column(name: str) -> Mapping[str, str | None]:
column: Final[Mapping[str, str | None]] = _load_beta_headers_config().get(name, {})
return column
def _header_only_column(canonical_provider: str) -> Mapping[str, str | None]:
return _beta_column(f"{canonical_provider}{_HEADER_ONLY_COLUMN_SUFFIX}")
def _provider_beta_mapping(provider: str, placement: BetaPlacement) -> Mapping[str, str | None]:
canonical_provider: Final = get_provider_name(provider)
column: Final = _beta_column(canonical_provider)
header_only: Final = _header_only_column(canonical_provider)
match placement:
case "header":
return {**column, **header_only}
case "body":
return {beta: name for beta, name in column.items() if beta not in header_only}
def get_header_only_betas(provider: str) -> frozenset[str]:
header_only: Final = _header_only_column(get_provider_name(provider))
return frozenset(beta for beta, name in header_only.items() if name is not None)
def has_beta_headers_config(provider: str) -> bool:
config: Final = _bundled_beta_headers_config()
aliases: Final = config.get("provider_aliases", {})
@ -251,6 +279,7 @@ def has_beta_headers_config(provider: str) -> bool:
def filter_and_transform_beta_headers(
beta_headers: list[str],
provider: str,
placement: BetaPlacement = "header",
) -> list[str]:
"""
Filter and transform beta headers based on provider's mapping configuration.
@ -263,6 +292,7 @@ def filter_and_transform_beta_headers(
Args:
beta_headers: List of Anthropic beta header values
provider: Provider name (e.g., "anthropic", "bedrock", "vertex_ai")
placement: Whether the betas travel as the anthropic-beta header or the anthropic_beta body field
Returns:
List of filtered and transformed beta headers for the provider
@ -270,11 +300,7 @@ def filter_and_transform_beta_headers(
if not beta_headers:
return []
config: Final = _load_beta_headers_config()
provider = get_provider_name(provider)
# Get the header mapping for this provider
provider_mapping: Final = config.get(provider, {})
provider_mapping: Final = _provider_beta_mapping(provider, placement)
filtered_headers: Final[set[str]] = set()
@ -426,6 +452,7 @@ def update_request_with_filtered_beta(
filtered_body_betas: Final = filter_and_transform_beta_headers(
beta_headers=existing_body_betas,
provider=provider,
placement="body",
)
if filtered_body_betas:

6
litellm/constants.py
View file

@ -68,6 +68,7 @@ RUNTIME_UPDATABLE_ROUTER_SETTINGS: Final[frozenset[str]] = frozenset(
"retry_after",
"fallbacks",
"context_window_fallbacks",
"content_policy_fallbacks",
"retry_policy",
"model_group_retry_policy",
"model_group_alias",
@ -420,6 +421,8 @@ REALTIME_CREDENTIAL_RESOLUTION_TIMEOUT_SECONDS: Final = get_env_float(
"REALTIME_CREDENTIAL_RESOLUTION_TIMEOUT_SECONDS", 20.0
)
BEDROCK_REALTIME_CLOSE_TIMEOUT_SECONDS: Final = get_env_float("BEDROCK_REALTIME_CLOSE_TIMEOUT_SECONDS", 5.0)
REALTIME_LAZY_SETUP_DEADLINE_SECONDS: Final = 240.0
REALTIME_LAZY_SETUP_CONTENT_GRACE_SECONDS: Final = 1.0
# RFC 6455 caps the close frame payload at 125 bytes, 2 of which carry the status code
WEBSOCKET_CLOSE_REASON_MAX_BYTES: Final = 123
@ -897,6 +900,7 @@ PROVIDERS_THAT_AUTHENTICATE_ON_PROVIDER_INFO: Final = frozenset(
}
)
SUBSCRIPTION_BACKED_PROVIDERS: Final = frozenset({"chatgpt"})
A2A_CHAT_COMPLETION_BRIDGE_PROVIDERS: Final = frozenset({"databricks_agent"})
LITELLM_EMBEDDING_PROVIDERS_SUPPORTING_INPUT_ARRAY_OF_TOKENS: Final = [
@ -1274,7 +1278,7 @@ empower_models: Final = set(
]
)
together_ai_models: Final[set] = set(
together_ai_models: Final[set[str]] = set(
[
# llama llms - chat
"togethercomputer/llama-2-70b-chat",

43
litellm/integrations/anthropic_cache_control_hook.py
View file

@ -23,6 +23,7 @@ from litellm._logging import verbose_logger
from litellm.integrations.custom_logger import CustomLogger
from litellm.integrations.custom_prompt_management import CustomPromptManagement
from litellm.integrations.prompt_management_base import PromptManagementClient
from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap
from litellm.litellm_core_utils.prompt_templates.common_utils import (
cache_control_as_prompt_cache_breakpoint,
with_prompt_cache_breakpoint,
@ -115,12 +116,12 @@ def configured_injection_points(value: object) -> Sequence[CacheControlInjection
return tuple(cast(CacheControlInjectionPoint, entry) for entry in value if isinstance(entry, dict))
def _served_provider_for_model(model: str) -> str | None:
def _served_provider_for_model(model: str, api_base: object = None) -> str | None:
from litellm.exceptions import BadRequestError
from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
try:
_, provider, _, _ = get_llm_provider(model=model)
_, provider, _, _ = get_llm_provider(model=model, api_base=api_base if isinstance(api_base, str) else None)
except BadRequestError:
return None
return provider
@ -175,7 +176,10 @@ def _hosted_entry_flag(entry: Mapping[str, object], resolve_provider: Callable[[
def _hosted_openai_dialect_flag(
model: str, custom_llm_provider: str | None, resolve_provider: Callable[[str], str | None]
model: str,
custom_llm_provider: str | None,
resolve_provider: Callable[[str], str | None],
model_cost: Mapping[str, Mapping[str, object]] | None = None,
) -> bool | None:
"""
Explicit opt-in for an OpenAI-shaped deployment served by another provider.
@ -194,14 +198,19 @@ def _hosted_openai_dialect_flag(
"""
import litellm
exact_entry: Final = litellm.model_cost.get(model)
exact_provider: Final = exact_entry.get("litellm_provider") if isinstance(exact_entry, dict) else None
cost_map: Final = (
cast(Mapping[str, Mapping[str, object]], litellm.model_cost) # cast-ok: the live map holds catalog entries
if model_cost is None
else model_cost
)
exact_entry: Final = cost_map.get(model)
exact_provider: Final = exact_entry.get("litellm_provider") if isinstance(exact_entry, Mapping) else None
if custom_llm_provider is None and (exact_provider == "openai" or ("/" not in model and exact_provider is None)):
return None
provider: Final = custom_llm_provider or resolve_provider(model)
if provider is None or provider == "openai":
return None
if isinstance(exact_entry, dict) and exact_provider == provider:
if isinstance(exact_entry, Mapping) and exact_provider == provider:
return _hosted_entry_flag(exact_entry, lambda: provider)
from litellm.utils import get_potential_model_names
@ -214,8 +223,8 @@ def _hosted_openai_dialect_flag(
names["stripped_model_name"],
names["provider_prefixed_model_name"],
)
entries: Final = (litellm.model_cost.get(candidate) for candidate in candidates)
flags: Final = (_hosted_entry_flag(entry, lambda: provider) for entry in entries if isinstance(entry, dict))
entries: Final = (cost_map.get(candidate) for candidate in candidates)
flags: Final = (_hosted_entry_flag(entry, lambda: provider) for entry in entries if isinstance(entry, Mapping))
return next((flag for flag in flags if flag is not None), None)
@ -527,7 +536,7 @@ class AnthropicCacheControlHook(CustomPromptManagement):
if model is None:
return False
hosted_flag: Final = _hosted_openai_dialect_flag(
model, custom_llm_provider, AnthropicCacheControlHook._resolve_provider
model, custom_llm_provider, lambda name: AnthropicCacheControlHook._resolve_provider(name, api_base)
)
if hosted_flag is not None:
return hosted_flag
@ -538,8 +547,20 @@ class AnthropicCacheControlHook(CustomPromptManagement):
return prompt_cache_options is not None or targets_openai_api(api_base)
@staticmethod
def _resolve_provider(model: str) -> str | None:
return _served_provider_for_model(model)
def catalog_targets_openai_prompt_cache_breakpoint(model: str, custom_llm_provider: str) -> bool:
return (
_hosted_openai_dialect_flag(
model,
custom_llm_provider,
AnthropicCacheControlHook._resolve_provider,
GetModelCostMap.loaded_model_cost_map(),
)
is True
)
@staticmethod
def _resolve_provider(model: str, api_base: object = None) -> str | None:
return _served_provider_for_model(model, api_base)
@staticmethod
def count_request_cache_breakpoints(

32
litellm/integrations/custom_guardrail.py
View file

@ -9,6 +9,7 @@ from types import MappingProxyType
from typing import TYPE_CHECKING, Any, ClassVar, Final, Literal, Optional, cast, get_args
import httpx
from pydantic import TypeAdapter
from litellm._internal_context import with_service_target
from litellm._logging import verbose_logger
@ -218,6 +219,15 @@ def _request_is_streaming(data: object, event_type: GuardrailEventHooks | None =
)
_REQUEST_FIELDS: Final = TypeAdapter(Mapping[str, object])
def _guardrails_listed_in(metadata: object) -> list[str | dict[str, DynamicGuardrailParams]]:
if not isinstance(metadata, dict):
return []
return metadata.get("guardrails") or []
class CustomGuardrail(CustomLogger):
# If True, during_call runs async_moderation_hook instead of the unified apply_guardrail path.
use_native_during_call_hook: ClassVar[bool] = False
@ -791,28 +801,22 @@ class CustomGuardrail(CustomLogger):
return True
raise
def get_guardrail_from_metadata(self, data: dict) -> list[str] | list[dict[str, DynamicGuardrailParams]]:
def get_guardrail_from_metadata(self, data: dict) -> list[str | dict[str, DynamicGuardrailParams]]:
"""
Returns the guardrail(s) to be run from the metadata or root
Returns the guardrail(s) to be run from the root, else from both metadata fields together
"""
if "guardrails" in data:
return data["guardrails"]
# Check both metadata locations. For regular endpoints move_guardrails_to_metadata
# writes to "metadata"; for thread/assistant endpoints it writes to
# "litellm_metadata". We check the one that actually contains the "guardrails"
# key so that a non-empty litellm_metadata without guardrails does not shadow
# the merged list stored in metadata (which would cause team guardrails to be
# silently skipped while default_on=True policy guardrails still fire).
for meta_key in ("metadata", "litellm_metadata"):
meta = data.get(meta_key) or {}
if isinstance(meta, dict) and "guardrails" in meta:
return meta.get("guardrails") or []
return []
request_fields: Final = _REQUEST_FIELDS.validate_python(data)
return [
*_guardrails_listed_in(request_fields.get("metadata")),
*_guardrails_listed_in(request_fields.get("litellm_metadata")),
]
def _guardrail_is_in_requested_guardrails(
self,
requested_guardrails: list[str] | list[dict[str, DynamicGuardrailParams]],
requested_guardrails: Sequence[str | dict[str, DynamicGuardrailParams]],
) -> bool:
for _guardrail in requested_guardrails:
if isinstance(_guardrail, dict):

1
litellm/integrations/otel/model/spans.py
View file

@ -361,6 +361,7 @@ _PRISMA_MODELS: Final[frozenset[str]] = frozenset(
"LiteLLM_LensReview",
"LiteLLM_LensIngestionKey",
"LiteLLM_LensWorker",
"LiteLLM_SubscriptionAccountTable",
"LiteLLM_LensSignalConfig",
"LiteLLM_LensTraceSignal",
)

53
litellm/integrations/prometheus.py
View file

@ -82,6 +82,7 @@ from litellm.types.proxy.carried_budget_state import (
UserBudgetSnapshot,
)
from litellm.types.utils import (
CallTypes,
StandardLoggingGuardrailInformation,
StandardLoggingPayload,
StandardLoggingZeroCostDiagnostic,
@ -279,6 +280,11 @@ def _logged_fields(value: object) -> Mapping[str, object]:
return _NO_LOGGED_FIELDS
_GOOGLE_PASS_THROUGH_PROVIDERS: Final = frozenset(
(litellm.LlmProviders.GEMINI.value, litellm.LlmProviders.VERTEX_AI.value)
)
def _is_llm_provider(name: str) -> bool:
routable: Final = (
name in litellm.provider_list or JSONProviderRegistry.exists(name) or is_registered_custom_provider(name)
@ -2942,6 +2948,22 @@ class PrometheusLogger(CustomLogger):
return False
@staticmethod
def _google_pass_through_provider(request_data: Mapping[str, object]) -> str | None:
declared_provider: Final = request_data.get("custom_llm_provider")
if request_data.get("call_type") != CallTypes.pass_through.value or not isinstance(declared_provider, str):
return None
return declared_provider if declared_provider in _GOOGLE_PASS_THROUGH_PROVIDERS else None
@staticmethod
def _failed_request_requested_model(request_data: Mapping[str, object]) -> object:
requested_model: Final = request_data.get("model", "")
if PrometheusLogger._google_pass_through_provider(request_data) is None:
return requested_model
request_metadata: Final = _logged_fields(_logged_fields(request_data.get("litellm_params")).get("metadata"))
model_group: Final = request_metadata.get("model_group")
return model_group if isinstance(model_group, str) and model_group else requested_model
@staticmethod
def _extract_api_provider_from_request_data(request_data: dict) -> str | None:
"""
@ -2950,22 +2972,27 @@ class PrometheusLogger(CustomLogger):
A request can fail before a deployment is resolved, so the provider is
not always known. Prefer the resolved ``custom_llm_provider`` on
``litellm_params``, then any provider recovered onto a partial
``standard_logging_object`` (e.g. a stream that broke mid-flight), and
finally infer it from the requested model name (e.g. ``gpt-4o-mini`` ->
``openai``) since the proxy's failure ``request_data`` usually carries
only the client-supplied model. Return ``None`` when it cannot be
determined so the label emits empty rather than a guess.
``standard_logging_object`` (e.g. a stream that broke mid-flight). On a
Gemini or Vertex pass-through failure only, next take the provider the
failure payload names (``gemini`` or ``vertex_ai``), so every other
pass-through route keeps its earlier label. Finally infer it from the
requested model name (e.g. ``gpt-4o-mini`` -> ``openai``) since the
proxy's failure ``request_data`` usually carries only the client-supplied
model. Return ``None`` when it cannot be determined so the label emits
empty rather than a guess.
"""
litellm_params: Final = request_data.get("litellm_params") or {}
provider = litellm_params.get("custom_llm_provider")
if provider:
return provider
standard_logging_object: Final = request_data.get("standard_logging_object") or {}
provider = standard_logging_object.get("custom_llm_provider")
if provider:
return provider
resolved_provider: Final = litellm_params.get("custom_llm_provider") or standard_logging_object.get(
"custom_llm_provider"
)
if resolved_provider:
return resolved_provider
google_provider: Final = PrometheusLogger._google_pass_through_provider(request_data)
if google_provider is not None:
return google_provider
model: Final = litellm_params.get("model") or request_data.get("model")
if not model:
if not isinstance(model, str) or not model:
return None
try:
return litellm.get_llm_provider(model=model)[1] or None
@ -3052,7 +3079,7 @@ class PrometheusLogger(CustomLogger):
team_alias=user_api_key_dict.team_alias,
org_id=user_api_key_dict.org_id,
org_alias=user_api_key_dict.organization_alias,
requested_model=_bounded_requested_model_label(request_data.get("model", "")),
requested_model=_bounded_requested_model_label(self._failed_request_requested_model(request_data)),
status_code=str(status_code),
exception_status=str(status_code),
exception_class=self._get_exception_class_name(original_exception),

15
litellm/integrations/websearch_interception/handler.py
View file

@ -398,14 +398,17 @@ class WebSearchInterceptionLogger(CustomLogger):
(t for t in tools if is_anthropic_native_web_search_tool(t)),
None,
)
emit_native_blocks: Final = native_tool is not None or (
kwargs is not None and bool(kwargs.get(WEBSEARCH_EMIT_NATIVE_BLOCKS_KEY))
)
outcome: Final = await self._short_circuit_search_outcome(query, kwargs=kwargs)
search_result_text: Final = WebSearchTransformation.search_outcome_text(outcome)
content: Final[list[dict[str, object]]] = []
if native_tool is not None:
if emit_native_blocks:
tool_use_id: Final = f"srvtoolu_{uuid.uuid4().hex}"
tool_name: Final = native_tool.get("name") or "web_search"
tool_name: Final = (native_tool.get("name") if native_tool is not None else None) or "web_search"
content.append(
{
"type": "server_tool_use",
@ -415,10 +418,10 @@ class WebSearchInterceptionLogger(CustomLogger):
}
)
content.append(
WebSearchTransformation.build_web_search_outcome_block(tool_use_id=tool_use_id, outcome=outcome)
WebSearchTransformation.build_web_search_outcome_block(
tool_use_id=tool_use_id, outcome=outcome, with_snippets=False
)
)
# Keep the text block so non-native short-circuit callers (Claude Code,
# github_copilot, etc.) see the same payload they always have.
content.append({"type": "text", "text": search_result_text})
response: Final[dict[str, object]] = {
@ -435,7 +438,7 @@ class WebSearchInterceptionLogger(CustomLogger):
verbose_logger.debug(
"WebSearchInterception: Short-circuit search completed, returning synthetic response (%s chars, native_blocks=%s)",
len(search_result_text),
native_tool is not None,
emit_native_blocks,
)
return response

10
litellm/integrations/websearch_interception/transformation.py
View file

@ -412,6 +412,7 @@ class WebSearchTransformation:
def build_web_search_tool_result_block(
tool_use_id: str,
search_response: SearchResponse | None,
with_snippets: bool = True,
) -> dict[str, object]:
"""
Build an Anthropic-native ``web_search_tool_result`` content block.
@ -445,6 +446,8 @@ class WebSearchTransformation:
``litellm.asearch()``. If None or empty, the block is still
emitted with an empty result list (signals "search ran, no
results" rather than "search did not run").
with_snippets: False when a text block in the same reply already
carries the snippets.
"""
items: Final[list[dict[str, object]]] = []
if search_response is not None:
@ -460,7 +463,7 @@ class WebSearchTransformation:
"title": title,
"page_age": page_age,
"encrypted_content": "",
"snippet": getattr(r, "snippet", "") or "",
**({"snippet": getattr(r, "snippet", "") or ""} if with_snippets else {}),
}
)
return {
@ -481,12 +484,15 @@ class WebSearchTransformation:
}
@staticmethod
def build_web_search_outcome_block(tool_use_id: str, outcome: SearchOutcome) -> dict[str, object]:
def build_web_search_outcome_block(
tool_use_id: str, outcome: SearchOutcome, with_snippets: bool = True
) -> dict[str, object]:
match outcome:
case SearchSucceeded(response=response):
return WebSearchTransformation.build_web_search_tool_result_block(
tool_use_id=tool_use_id,
search_response=response,
with_snippets=with_snippets,
)
case SearchFailed(error_code=error_code):
return WebSearchTransformation.build_web_search_tool_result_error_block(

28
litellm/litellm_core_utils/core_helpers.py
View file

@ -9,12 +9,13 @@ from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Final, Literal, Protocol, TypeVar, cast, runtime_checkable
import httpx
from pydantic import TypeAdapter, ValidationError
from pydantic import BaseModel, TypeAdapter, ValidationError
from typing_extensions import TypeIs # noqa: TID251 # narrows a duck-typed value to the Protocol its caller reads
import litellm
from litellm._logging import verbose_logger
from litellm.constants import KUBERNETES_POD_ROUTING_KEY
from litellm.litellm_core_utils.hidden_params import set_extra_attribute
from litellm.litellm_core_utils.llm_response_utils.get_headers import get_provider_request_id
from litellm.types.llms.openai import AllMessageValues, OpenAIChatCompletionFinishReason
@ -710,9 +711,9 @@ def preserve_upstream_non_openai_attributes(
_assign_attributes(model_response, original_chunk.model_dump(include=frozenset(candidate_keys)).items())
def _assign_attributes(target: object, values: Iterable[tuple[str, object]]) -> None:
def _assign_attributes(target: BaseModel, values: Iterable[tuple[str, object]]) -> None:
for key, value in values:
setattr(target, key, value)
set_extra_attribute(target, key, value)
def safe_deep_copy(data):
@ -982,25 +983,36 @@ def _usage_field(usage: object, name: str) -> object:
return getattr(usage, name, None)
def _cost_number(value: object) -> float | None:
if isinstance(value, bool) or not isinstance(value, int | float):
def finite_cost(value: object) -> float | None:
if isinstance(value, bool) or not isinstance(value, int | float) or not math.isfinite(value):
return None
return float(value)
def single_log_line(value: object) -> str:
return str(value).replace("\r", "").replace("\n", "")
def finite_reported_cost(value: object) -> float | None:
cost: Final = finite_cost(value)
if cost is None and isinstance(value, float):
verbose_logger.warning("Ignoring a non-finite provider-reported cost %s", value)
return cost
def _reported_cost_value(value: object) -> float | None:
"""A reported cost is a number, or a breakdown whose total lives under ``total_cost``."""
raw: Final[object] = value
if isinstance(value, Mapping):
return _cost_number(_usage_field(raw, "total_cost"))
return _cost_number(value)
return finite_reported_cost(_usage_field(raw, "total_cost"))
return finite_reported_cost(value)
def response_cost_in_hidden_params(response: _CarriesHiddenParams) -> float | None:
additional_headers: Final[object] = response.hidden_params.get("additional_headers")
if not isinstance(additional_headers, Mapping):
return None
return _cost_number(_USAGE_BLOCK_ADAPTER.validate_python(additional_headers).get(RESPONSE_COST_HEADER))
return finite_reported_cost(_USAGE_BLOCK_ADAPTER.validate_python(additional_headers).get(RESPONSE_COST_HEADER))
def override_stamped_response_cost(response: _CarriesHiddenParams, cost: float) -> None:

32
litellm/litellm_core_utils/exception_mapping_utils.py
View file

@ -18,6 +18,7 @@ from litellm.litellm_core_utils.bug_report import (
)
from litellm.litellm_core_utils.core_helpers import as_str_mapping, attribute_or_none
from litellm.litellm_core_utils.secret_redaction import redact_string as redact_secret_string
from litellm.llms.bedrock.common_utils import bedrock_model_reaches_converse_by_default
from litellm.llms.bigmodel.common_utils import raise_bigmodel_business_error
from litellm.types.utils import LlmProviders
@ -938,13 +939,22 @@ _BEDROCK_MANTLE_CONTEXT_WINDOW_GENERIC_MESSAGE: Final = (
)
def _bedrock_mantle_context_window_response(original_exception: _ProviderHTTPException) -> httpx.Response:
def _bedrock_response_with_status(original_exception: _ProviderHTTPException, status_code: int) -> httpx.Response:
original_response: Final = getattr(original_exception, "response", None)
if not isinstance(original_response, httpx.Response) or getattr(original_response, "_request", None) is None:
return httpx.Response(status_code=400, request=httpx.Request(method="POST", url="https://api.openai.com/v1/"))
if original_response.status_code == 400:
return httpx.Response(
status_code=status_code, request=httpx.Request(method="POST", url="https://api.openai.com/v1/")
)
if original_response.status_code == status_code:
return original_response
return httpx.Response(status_code=400, headers=original_response.headers, request=original_response.request)
return httpx.Response(status_code=status_code, headers=original_response.headers, request=original_response.request)
def _came_from_bedrock_converse(original_exception: _ProviderHTTPException) -> bool:
original_response: Final = getattr(original_exception, "response", None)
if not isinstance(original_response, httpx.Response) or getattr(original_response, "_request", None) is None:
return False
return original_response.request.url.path.endswith(("/converse", "/converse-stream"))
def _get_bedrock_mantle_context_window_message(error_str: str) -> str | None:
@ -980,7 +990,7 @@ def _map_bedrock_exception(
message=mantle_context_window_message,
model=model,
llm_provider=custom_llm_provider,
response=_bedrock_mantle_context_window_response(original_exception),
response=_bedrock_response_with_status(original_exception, 400),
)
if (
"too many tokens" in error_str
@ -1003,6 +1013,18 @@ def _map_bedrock_exception(
llm_provider="bedrock",
response=getattr(original_exception, "response", None),
)
elif (
"The provided model identifier is invalid" in error_str
and bool(model)
and _came_from_bedrock_converse(original_exception)
and bedrock_model_reaches_converse_by_default(model)
):
raise NotFoundError(
message=f"BedrockException - {error_str}",
model=model,
llm_provider="bedrock",
response=_bedrock_response_with_status(original_exception, 404),
)
elif "Malformed input request" in error_str:
raise BadRequestError(
message=f"BedrockException - {error_str}",

1
litellm/litellm_core_utils/get_litellm_params.py
View file

@ -85,6 +85,7 @@ OPTIONAL_KWARGS_KEYS: Final = (
"github_copilot_user_session",
"fireworks_forward_user_id",
"responses_api_version",
"allowed_openai_params",
PROVIDER_AFFINITY_HEADER_KWARG_KEY,
}
)

35
litellm/litellm_core_utils/hidden_params.py
View file

@ -1,8 +1,9 @@
from collections.abc import Iterator, Mapping, MutableMapping
from functools import cache, lru_cache
from types import MappingProxyType
from typing import Final, Protocol, cast
from pydantic import BaseModel
from pydantic import BaseModel, ConfigDict, create_model
from litellm.types.llms.base import HiddenParams
@ -60,6 +61,38 @@ def stored_attribute(obj: object, name: str) -> object:
return extras.get(name)
@cache
def _assigned_extras_join_fields_set() -> bool:
probe: Final = create_model("_ExtraAssignmentProbe", __config__=ConfigDict(extra="allow"))()
setattr(probe, "assigned", None)
return "assigned" in probe.model_fields_set
@lru_cache(maxsize=1024)
def _writes_plain_extra(model_class: type[BaseModel], name: str) -> bool:
config: Final = model_class.model_config
return (
model_class.__setattr__ is BaseModel.__setattr__
and config.get("extra") == "allow"
and not config.get("frozen")
and not config.get("validate_assignment")
and not name.startswith("_")
and name not in model_class.__pydantic_fields__
and name not in model_class.__class_vars__
and not hasattr(model_class, name)
and _assigned_extras_join_fields_set()
)
def set_extra_attribute(model: BaseModel, name: str, value: object) -> None:
extras: Final = model.__pydantic_extra__
if extras is None or not _writes_plain_extra(type(model), name):
setattr(model, name, value)
return
extras[name] = value
model.__pydantic_fields_set__.add(name)
def _get_hidden_params_storage(obj: object) -> object | None:
if isinstance(obj, dict):
return obj.get(_HIDDEN_PARAMS_ATTR)

6
litellm/litellm_core_utils/internal_call_metadata.py
View file

@ -35,6 +35,12 @@ from litellm.types.utils import (
BUDGET_RESERVATION_METADATA_KEYS: Final = frozenset({"user_api_key_budget_reservation"})
ITPM_RESERVED_KEY: Final = "_litellm_itpm_reserved"
OTPM_RESERVED_KEY: Final = "_litellm_otpm_reserved"
ITPM_CACHE_KEY: Final = "_litellm_itpm_cache_key"
OTPM_CACHE_KEY: Final = "_litellm_otpm_cache_key"
IO_TOKEN_RESERVATION_KEYS: Final = frozenset({ITPM_RESERVED_KEY, OTPM_RESERVED_KEY, ITPM_CACHE_KEY, OTPM_CACHE_KEY})
MODEL_ACCESS_GROUP_METADATA_KEY: Final = "user_api_key_matched_model_access_groups"
"""Where auth records the model access groups that authorized the request, for the spend writer.

62
litellm/litellm_core_utils/litellm_logging.py
View file

@ -76,6 +76,7 @@ from litellm.litellm_core_utils.classifier_logging import (
is_classifier_call,
)
from litellm.litellm_core_utils.core_helpers import (
finite_cost,
get_provider_response_headers_from_hidden_params,
is_expected_client_error,
native_messages_reported_cost,
@ -84,6 +85,7 @@ from litellm.litellm_core_utils.core_helpers import (
proxy_stamped_used_client_oauth_token,
reconstruct_model_name,
set_response_cost_in_hidden_params,
single_log_line,
)
from litellm.litellm_core_utils.error_normalization import normalize_error
from litellm.litellm_core_utils.get_litellm_params import get_litellm_params
@ -567,7 +569,13 @@ def _resolve_mantle_region_for_cost(
_PER_CALL_LITELLM_PARAM_DICTS: Final = frozenset({"metadata", "litellm_metadata"})
_LITELLM_PARAMS: Final = TypeAdapter(dict[str, object])
_SERVED_ANTHROPIC_BODY: Final = TypeAdapter(dict[str, object])
_REDACTED_RESULT: Final = TypeAdapter(object)
_GUARDRAIL_INFORMATION_KEY: Final = "standard_logging_guardrail_information"
def _without_guardrail_information(metadata: Mapping[str, object]) -> Mapping[str, object]:
return {key: value for key, value in metadata.items() if key != _GUARDRAIL_INFORMATION_KEY}
def _provider_response_id(source: object) -> str | None:
@ -2088,7 +2096,7 @@ class Logging(LiteLLMLoggingBaseClass):
litellm_model_name=litellm_model_name,
router_model_id=router_model_id or hidden_params.get("model_id"),
)
return hidden_params["response_cost"]
return self._recorded_response_cost(hidden_params["response_cost"])
elif router_model_id is None and "model_id" in hidden_params: # use model_id if not already set
router_model_id = hidden_params["model_id"]
@ -2188,7 +2196,7 @@ class Logging(LiteLLMLoggingBaseClass):
litellm_model_name=litellm_model_name,
router_model_id=router_model_id,
)
return total_response_cost
return self._recorded_response_cost(total_response_cost)
except Exception as e: # error calculating cost
debug_info = StandardLoggingModelCostFailureDebugInformation(
error_str=str(e),
@ -2214,6 +2222,15 @@ class Logging(LiteLLMLoggingBaseClass):
_response_cost_calculator = response_cost_calculator
def _recorded_response_cost(self, cost: float | None) -> float | None:
if cost is None or finite_cost(cost) is not None:
return cost
verbose_logger.warning(
"Recording a non-finite response_cost as 0 for litellm_call_id=%s", single_log_line(self.litellm_call_id)
)
self.cost_breakdown = None
return 0.0
def uncached_price(self, result: object) -> UncachedPrice:
previous_cost_breakdown: Final = self.cost_breakdown
previous_billed_token_rates: Final = self.billed_token_rates
@ -2566,6 +2583,23 @@ class Logging(LiteLLMLoggingBaseClass):
attempt.model_call_details = dict(self.model_call_details)
return attempt
def guardrail_scoped_copy(self) -> "Logging":
request_params: Final = _LITELLM_PARAMS.validate_python(
self.litellm_params or {} # pyright: ignore[reportUnknownMemberType] # Logging.litellm_params is untyped
)
litellm_params: Final = {
**request_params,
**{
key: _without_guardrail_information(_METADATA_MAPPING.validate_python(value))
for key, value in request_params.items()
if key in _PER_CALL_LITELLM_PARAM_DICTS and isinstance(value, dict)
},
}
scoped: Final = copy.copy(self)
scoped.litellm_params = litellm_params
scoped.model_call_details = {**self.model_call_details, "litellm_params": litellm_params}
return scoped
def dispatch_attempt_failure_handlers(
self, exception: Exception, traceback_exception: str
) -> Coroutine[None, None, None]:
@ -5034,13 +5068,15 @@ class Logging(LiteLLMLoggingBaseClass):
verbose_logger.warning("LiteLLM: the anthropic_messages stream assembled no response, logging an empty one")
return litellm.ModelResponse(model=self.model)
else:
from litellm.types.llms.anthropic import AnthropicResponse
pydantic_result: Final = AnthropicResponse.model_validate({"stop_sequence": None, **(served_body or {})})
import httpx
anthropic_body: Final = (
served_body
if served_body is not None
else _SERVED_ANTHROPIC_BODY.validate_python(
result.model_dump() if isinstance(result, BaseModel) else result
)
)
result = litellm.AnthropicConfig().transform_parsed_response(
completion_response=pydantic_result.model_dump(),
completion_response={"stop_sequence": None, **anthropic_body},
raw_response=httpx.Response(
status_code=200,
headers={},
@ -7560,7 +7596,15 @@ def get_standard_logging_object_payload(
)
custom_pricing: Final = False if spilled_over else logging_obj.custom_pricing_for(init_response_obj)
raw_response_cost: Final = kwargs.get("response_cost")
llm_response_cost: Final[float] = raw_response_cost or 0.0
numeric_response_cost: Final = raw_response_cost if isinstance(raw_response_cost, int | float) else None
priced_response_cost: Final = finite_cost(numeric_response_cost)
if numeric_response_cost is not None and priced_response_cost is None:
verbose_logger.warning(
"Logging a non-finite response_cost=%s as 0 for litellm_call_id=%s",
numeric_response_cost,
single_log_line(logging_obj.litellm_call_id),
)
llm_response_cost: Final[float] = priced_response_cost or 0.0
guardrail_cost: Final = guardrail_information_cost(metadata.get("standard_logging_guardrail_information"))
finalized_cost: Final = _finalized_cost_of(logging_obj)
response_cost: Final[float] = (

21
litellm/litellm_core_utils/llm_request_utils.py
View file

@ -1,4 +1,3 @@
import datetime
from collections.abc import Iterable, Iterator, Mapping
from dataclasses import dataclass
from typing import Final, cast
@ -9,6 +8,7 @@ from typing_extensions import NotRequired, ReadOnly, TypedDict
import litellm
from litellm.constants import DEFAULT_MAX_RECURSE_DEPTH
from litellm.litellm_core_utils.get_llm_provider_logic import inferred_provider
from litellm.litellm_core_utils.model_retirement import is_past_deprecation_date
def _form_field_value(value: object) -> str:
@ -168,7 +168,9 @@ class RankedChatModel:
def rank_chat_models_from_llm_provider(custom_llm_provider: str) -> tuple[RankedChatModel, ...]:
known_models: Final[Iterable[str]] = litellm.models_by_provider.get(custom_llm_provider, ())
from litellm.utils import bundled_provider_models
known_models: Final = bundled_provider_models(custom_llm_provider)
ranked_models: Final = (
RankedChatModel(
model=model, route_rank=route_rank, cost=cost, is_past_deprecation_date=is_past_deprecation_date
@ -187,18 +189,11 @@ _COST_MAP_DEPRECATION: Final = TypeAdapter(_CostMapDeprecation)
def _is_past_deprecation_date(cost_map_key: str) -> bool:
deprecation_date: Final = _COST_MAP_DEPRECATION.validate_python(
litellm.model_cost.get(cost_map_key) or {} # pyright: ignore[reportUnknownMemberType] # untyped cost map JSON
).get("deprecation_date")
if not isinstance(deprecation_date, str):
return False
try:
return (
datetime.date.fromisoformat(deprecation_date)
<= datetime.datetime.now(datetime.timezone.utc).astimezone().date()
return is_past_deprecation_date(
_COST_MAP_DEPRECATION.validate_python(
litellm.model_cost.get(cost_map_key) or {} # pyright: ignore[reportUnknownMemberType] # untyped cost map JSON
)
except ValueError:
return False
)
def _chat_row_costs(models: Iterable[str], custom_llm_provider: str) -> Iterator[tuple[str, float, bool]]:

19
litellm/litellm_core_utils/model_retirement.py Normal file
View file

@ -0,0 +1,19 @@
import datetime
from collections.abc import Mapping
from typing import Final
PROVIDERS_HIDING_RETIRED_MODELS: Final = frozenset({"aiand", "baseten", "together_ai"})
def is_past_deprecation_date(value: Mapping[str, object]) -> bool:
deprecation_date: Final = value.get("deprecation_date")
if not isinstance(deprecation_date, str):
return False
try:
return datetime.date.fromisoformat(deprecation_date) <= datetime.datetime.now(datetime.timezone.utc).date()
except ValueError:
return False
def is_hidden_as_retired(value: Mapping[str, object]) -> bool:
return value.get("litellm_provider") in PROVIDERS_HIDING_RETIRED_MODELS and is_past_deprecation_date(value)

6
litellm/litellm_core_utils/prompt_templates/common_utils.py
View file

@ -2023,12 +2023,14 @@ def infer_content_type_from_url_and_content(
"""
from litellm.litellm_core_utils.token_counter import get_image_type
media_type: Final = current_content_type.split(";", 1)[0].strip().lower() if current_content_type else None
# If we have a valid content type that's not generic, use it
if current_content_type and current_content_type not in [
if media_type and media_type not in [
"binary/octet-stream",
"application/octet-stream",
]:
return current_content_type
return media_type
# Extension to MIME type mapping
# Supports images, documents, and other common file types

29
litellm/litellm_core_utils/prompt_templates/factory.py
View file

@ -912,9 +912,9 @@ def create_anthropic_image_param(
image_url_input: str | dict,
format: str | None = None,
is_bedrock_invoke: bool = False,
) -> AnthropicMessagesImageParam:
) -> AnthropicMessagesImageParam | AnthropicMessagesDocumentParam:
"""
Create an AnthropicMessagesImageParam from an image URL input.
Create an Anthropic content block from an image URL input.
Supports both URL references (for HTTP/HTTPS URLs) and base64 encoding.
"""
@ -933,14 +933,7 @@ def create_anthropic_image_param(
if is_bedrock_invoke or image_url.startswith("http://"):
base64_url: Final = convert_url_to_base64(url=image_url)
image_chunk = convert_to_anthropic_image_obj(openai_image_url=base64_url, format=format)
return AnthropicMessagesImageParam(
type="image",
source=AnthropicContentParamSource(
type="base64",
media_type=image_chunk["media_type"],
data=image_chunk["data"],
),
)
return _anthropic_content_element_factory(image_chunk)
else:
# HTTPS URL - pass directly for regular Anthropic
return AnthropicMessagesImageParam(
@ -953,14 +946,7 @@ def create_anthropic_image_param(
else:
# Convert to base64 for data URIs or other formats
image_chunk = convert_to_anthropic_image_obj(openai_image_url=image_url, format=format)
return AnthropicMessagesImageParam(
type="image",
source=AnthropicContentParamSource(
type="base64",
media_type=image_chunk["media_type"],
data=image_chunk["data"],
),
)
return _anthropic_content_element_factory(image_chunk)
# The following XML functions will be deprecated once JSON schema support is available on Bedrock and Vertex
@ -1910,7 +1896,12 @@ def convert_to_anthropic_tool_result(
anthropic_content_element=_anthropic_image_param,
original_content_element=content,
)
anthropic_content_list.append(cast(AnthropicMessagesImageParam, _anthropic_image_param))
anthropic_content_list.append(
cast(
AnthropicMessagesImageParam | AnthropicMessagesDocumentParam,
_anthropic_image_param,
)
)
elif content["type"] == "tool_reference":
anthropic_content_list.append(ToolReference(type="tool_reference", tool_name=content["tool_name"]))
elif content["type"] == "file":

330
litellm/litellm_core_utils/realtime_streaming.py
View file

@ -1,5 +1,6 @@
import asyncio
import json
import time
import traceback
from collections.abc import Callable, Coroutine, Mapping, MutableMapping, Sequence
from dataclasses import dataclass
@ -15,13 +16,19 @@ from litellm._logging import redact_internal_details_from_client_message, verbos
from litellm._uuid import uuid
from litellm.constants import (
BEDROCK_REALTIME_PENDING_OPENING_MESSAGE_SCOPE_KEY,
REALTIME_LAZY_SETUP_CONTENT_GRACE_SECONDS,
REALTIME_LAZY_SETUP_DEADLINE_SECONDS,
REALTIME_METERED_USAGE_KEY,
REALTIME_SESSION_FAILURE_LOGGED_KEY,
REALTIME_SESSION_SUCCESS_LOGGED_KEY,
)
from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
from litellm.llms.base_llm.realtime.transcription_protocol import RealtimeTranscriptionProtocolError
from litellm.llms.base_llm.realtime.transformation import BaseRealtimeConfig, RealtimeBackend
from litellm.llms.base_llm.realtime.transformation import (
BaseRealtimeConfig,
RealtimeBackend,
RealtimeSessionUpdateRejection,
)
from litellm.types.llms.openai import (
OpenAIRealtimeEvents,
OpenAIRealtimeOutputItemDone,
@ -125,6 +132,65 @@ class _LoggingWorker(Protocol):
_DECODED_JSON: Final = TypeAdapter(object)
_SESSION_FIELDS: Final = TypeAdapter(dict[str, object])
_NO_SESSION_FIELDS: Final[Mapping[str, object]] = MappingProxyType({})
_SESSION_MERGE_MAX_DEPTH: Final = 8
class _SessionUpdateFrame(TypedDict):
type: ReadOnly[str]
session: ReadOnly[Mapping[str, object]]
def _merge_session_fields(
base: Mapping[str, object], update: Mapping[str, object], depth: int = 0
) -> Mapping[str, object]:
def merged_value(key: str) -> object:
if key not in update:
return base[key]
value: Final = update[key]
base_value: Final = base.get(key)
if depth < _SESSION_MERGE_MAX_DEPTH and isinstance(value, Mapping) and isinstance(base_value, Mapping):
return _merge_session_fields(
_SESSION_FIELDS.validate_python(base_value), _SESSION_FIELDS.validate_python(value), depth + 1
)
return update[key]
return MappingProxyType({key: merged_value(key) for key in dict.fromkeys((*base, *update))})
def _session_update_frame(session: Mapping[str, object]) -> str:
frame: Final[_SessionUpdateFrame] = {"type": "session.update", "session": session}
return json.dumps(frame, default=_SESSION_FIELDS.validate_python)
_HELD_SESSION_UPDATE_TOO_LARGE: Final = RealtimeSessionUpdateRejection(
code="session_update_too_large",
message=(
"The session.update frames sent before the first conversation content add up to more than 10 MB, "
"so this session.update was not applied"
),
param="session",
)
def _unscreened_audio_error_event(client_event_id: object) -> Mapping[str, object]:
return {
"type": "error",
"event_id": f"event_{uuid.uuid4()}",
"error": {
"type": "invalid_request_error",
"code": "audio_input_not_screenable",
"message": (
"A realtime_input_transcription guardrail is on, and Gemini Live only transcribes a turn after it "
"starts answering it, so audio input was not sent. Send text with conversation.item.create instead"
),
"param": None,
"event_id": client_event_id if isinstance(client_event_id, str) else None,
},
}
def _decode_json_object(payload: str) -> Mapping[str, object]:
"""Decode a realtime frame into its top-level field mapping."""
return json.loads(payload)
@ -243,8 +309,17 @@ class RealTimeStreaming:
event_normalizer: RealtimeEventNormalizer | None = None,
usage_meter: RealtimeUsageMeter | None = None,
logging_worker: _LoggingWorker = GLOBAL_LOGGING_WORKER,
lazy_setup_deadline_seconds: float = REALTIME_LAZY_SETUP_DEADLINE_SECONDS,
lazy_setup_content_grace_seconds: float = REALTIME_LAZY_SETUP_CONTENT_GRACE_SECONDS,
clock: Callable[[], float] = time.monotonic,
):
self.websocket: _ClientWebSocket = websocket
self._clock: Final = clock
self._lazy_setup_deadline: Final = clock() + lazy_setup_deadline_seconds
self._lazy_setup_content_grace_seconds: Final = lazy_setup_content_grace_seconds
self._held_client_content: tuple[str, ...] = ()
self._held_client_content_bytes: int = 0
self._held_client_content_deadline: float | None = None
self._replay_scope: Final = client_websocket_scope(websocket)
self.backend_ws = backend_ws
self.logging_obj = logging_obj
@ -300,6 +375,10 @@ class RealTimeStreaming:
# Gemini Live rejects a follow-up BidiGenerateContentSetup once any
# content (realtimeInput / clientContent / toolResponse) has been sent.
self._content_sent_after_setup: bool = False
self._held_session_update: Mapping[str, object] | None = None
self._applied_session_fields: Mapping[str, object] | None = None
self._held_session_update_bytes: int = 0
self._unscreened_audio_refused: bool = False
# Whether this is a transcription-only session (session.type == "transcription",
# e.g. gpt-realtime-whisper). Such sessions must not be sent response.create and
# their input_audio_transcription.completed usage drives duration-based cost.
@ -337,6 +416,17 @@ class RealTimeStreaming:
]
)
_CLIENT_AUDIO_BUFFER_COMMIT_TYPES = frozenset(["input_audio_buffer.commit", "input_audio_buffer.end"])
_CLIENT_CONTENT_TYPES = frozenset(
[
"conversation.item.create",
"response.create",
"input_audio_buffer.append",
"input_audio_buffer.commit",
"input_audio_buffer.end",
]
)
_HELD_CONTENT_TYPES: Final = frozenset(["conversation.item.create", "input_audio_buffer.append"])
_DEFERRED_SETUP_HELD_CONTENT_TYPES: Final = frozenset(["conversation.item.create"])
_AUDIO_FORMAT_MAP: dict[str, dict[str, str | int]] = {
"pcm16": {"type": "audio/pcm", "rate": 24000},
"g711_ulaw": {"type": "audio/G711-ulaw", "rate": 8000},
@ -377,7 +467,7 @@ class RealTimeStreaming:
message_obj: dict[str, Any] = cast(dict[str, Any], message)
else:
message_obj = cast(dict[str, Any], json.loads(message))
self._collect_tool_calls_from_response_done(cast(dict, message_obj))
self._collect_tool_calls_from_response_done(message_obj)
if not self._should_store_message(message_obj):
return
try:
@ -502,7 +592,7 @@ class RealTimeStreaming:
self.store_message(flush_event)
self._capture_transcription_usage(flush_event)
def _collect_tool_calls_from_response_done(self, event_obj: dict | OpenAIRealtimeEvents) -> None:
def _collect_tool_calls_from_response_done(self, event_obj: Mapping[str, object] | OpenAIRealtimeEvents) -> None:
"""Extract function_call items from response.done events for spend logging."""
try:
if event_obj.get("type") != "response.done":
@ -607,6 +697,167 @@ class RealTimeStreaming:
self._usage_meter.observe_client_event(sent_event)
return True
def _builds_setup_lazily(self) -> bool:
if self.provider_config is None:
return False
return self.provider_config.builds_setup_from_session_update()
def _holds_session_updates_until_first_content(self) -> bool:
return self.session_configuration_request is None and self._builds_setup_lazily()
async def _rejected_setup_session_update(self, message: str) -> bool:
provider_config: Final = self.provider_config
if provider_config is None or not provider_config.builds_setup_from_session_update():
return False
client_event: Final = _decode_json_object(message)
session: Final = client_event.get("session")
update: Final = _SESSION_FIELDS.validate_python(session) if isinstance(session, Mapping) else _NO_SESSION_FIELDS
if self._leaves_applied_setup_unchanged(update):
await self._acknowledge_session_setup(self._applied_session_fields or _NO_SESSION_FIELDS)
return True
rejection: Final = provider_config.setup_session_update_rejection(
self.model, update, setup_sent=self.session_configuration_request is not None
)
if rejection is None:
return False
await self.websocket.send_text(json.dumps(rejection.error_event(client_event.get("event_id"))))
return True
def _held_session_update_message(self) -> str:
return _session_update_frame(self._held_session_update or _NO_SESSION_FIELDS)
async def _hold_session_update(self, message: str) -> None:
client_event: Final = _decode_json_object(message)
if self._held_session_update_bytes + len(message.encode("utf-8")) > RealTimeStreaming._MAX_BUFFERED_BYTES:
await self.websocket.send_text(
json.dumps(_HELD_SESSION_UPDATE_TOO_LARGE.error_event(client_event.get("event_id")))
)
return
session: Final = client_event.get("session")
update: Final = _SESSION_FIELDS.validate_python(session) if isinstance(session, Mapping) else _NO_SESSION_FIELDS
held: Final = _merge_session_fields(self._held_session_update or _NO_SESSION_FIELDS, update)
self._held_session_update = held
self._held_session_update_bytes = len(_session_update_frame(held).encode("utf-8"))
await self._acknowledge_session_setup(held)
def _setup_preview(self, session: Mapping[str, object]) -> str | None:
if self.provider_config is None:
return None
preview: Final = self.provider_config.transform_realtime_request(
_session_update_frame(session), self.model, None
)
return next(
(frame for frame in preview if isinstance(frame, str) and "setup" in _decode_json_object(frame)), None
)
def _leaves_applied_setup_unchanged(self, update: Mapping[str, object]) -> bool:
applied: Final = self._applied_session_fields
if self.session_configuration_request is None or applied is None:
return False
applied_setup: Final = self._setup_preview(applied)
merged_setup: Final = self._setup_preview(_merge_session_fields(applied, update))
return (
applied_setup is not None
and merged_setup is not None
and _decode_json_object(applied_setup) == _decode_json_object(merged_setup)
)
async def _acknowledge_session_setup(self, session: Mapping[str, object]) -> None:
if self.provider_config is None:
return
setup: Final = self._setup_preview(session)
if setup is None:
return
acknowledged: Final = self.provider_config.transform_session_created_event(
self.model, str(self.logging_obj.litellm_trace_id), setup
)
if acknowledged is None:
return
event: Final[Mapping[str, object]] = {**acknowledged, "type": "session.updated"}
event_str: Final = json.dumps(event)
self.store_message(event_str)
await self._send_event_to_client(event, event_str)
async def _send_held_session_update_as_setup(self) -> None:
held: Final = self._held_session_update
if held is None:
return
message: Final = self._held_session_update_message()
self._held_session_update = None
self._held_session_update_bytes = 0
if await self._send_to_backend(message):
self._applied_session_fields = held
await self._send_held_client_content()
def _holds_client_content(self, msg_type: str | None, message: str) -> bool:
held_types: Final = (
RealTimeStreaming._DEFERRED_SETUP_HELD_CONTENT_TYPES
if self._uses_deferred_backend_setup()
else RealTimeStreaming._HELD_CONTENT_TYPES
)
return (
msg_type in held_types
and self._held_session_update is not None
and self._holds_session_updates_until_first_content()
and not self._pending_messages_until_setup
and len(self._held_client_content) < RealTimeStreaming._MAX_BUFFERED_MESSAGES
and self._held_client_content_bytes + len(message.encode("utf-8")) <= RealTimeStreaming._MAX_BUFFERED_BYTES
)
def _hold_client_content(self, message: str) -> None:
if self._held_client_content_deadline is None:
self._held_client_content_deadline = self._clock() + self._lazy_setup_content_grace_seconds
self._held_client_content = (*self._held_client_content, message)
self._held_client_content_bytes += len(message.encode("utf-8"))
async def _send_held_client_content(self) -> None:
held: Final = self._held_client_content
self._held_client_content = ()
self._held_client_content_bytes = 0
self._held_client_content_deadline = None
for message in held:
await self._send_to_backend(message)
def _setup_deadline(self) -> float:
content_deadline: Final = self._held_client_content_deadline
return (
self._lazy_setup_deadline if content_deadline is None else min(self._lazy_setup_deadline, content_deadline)
)
async def _send_setup_before_content(self, msg_type: str | None) -> None:
if self._holds_session_updates_until_first_content():
await self._send_held_session_update_as_setup()
await self._send_default_setup_before_first_client_message(msg_type)
async def _receive_client_message_before_setup_deadline(self) -> str | None:
if not self._holds_session_updates_until_first_content():
return await self._receive_client_message()
receive: Final = asyncio.ensure_future(self._receive_client_message())
try:
done, _ = await asyncio.wait((receive,), timeout=self._setup_deadline() - self._clock())
if not done:
await self._send_setup_before_content(None)
return await receive
finally:
receive.cancel()
async def _send_default_setup_before_first_client_message(self, msg_type: str | None) -> None:
if self.session_configuration_request is not None or self.provider_config is None:
return
if msg_type == "session.update" or (
self._uses_deferred_backend_setup()
and (msg_type in RealTimeStreaming._CLIENT_AUDIO_BUFFER_TYPES or self._pending_messages_until_setup)
):
return
default_setup: Final = self.provider_config.session_configuration_request(self.model)
if not default_setup:
return
setup: Final = self.maybe_inject_guardrail_auto_response_disable(default_setup)
await self.backend_ws.send(setup)
self._cache_session_configuration_request(setup)
self._applied_session_fields = _NO_SESSION_FIELDS
await self._send_held_client_content()
def _enforce_transcription_session_model(self, message: str) -> str:
"""Force client transcription session updates to the authorized model.
@ -851,6 +1102,20 @@ class RealTimeStreaming:
self._guardrail_cancel_event_ids = self._guardrail_cancel_event_ids | {event_id}
await self._send_to_backend(json.dumps({"type": "response.cancel", "event_id": event_id}))
async def _send_blocked_turn_to_backend(self, pre_block_backend_message: str | None, item_id: str | None) -> None:
if pre_block_backend_message is None and self._is_transcription_session:
return
await self._send_setup_before_content("conversation.item.create")
# Deliver any caller-supplied backend message FIRST so that
# protocol contracts requiring a specific ordering (e.g.
# Gemini Live's mandatory ``toolResponse`` after a
# ``toolCall``) are honored before the guardrail's own
# clientContent / cancel messages are sent.
if pre_block_backend_message is not None:
await self._send_to_backend(pre_block_backend_message)
if not self._is_transcription_session:
await self._withdraw_blocked_turn(item_id)
async def _withdraw_blocked_turn(self, item_id: str | None) -> None:
await self._send_guardrail_cancel()
if item_id is not None and self._deletes_conversation_items:
@ -958,6 +1223,21 @@ class RealTimeStreaming:
"""Transcription-only sessions have no assistant turn to gate and reject realtime session updates."""
return not self._is_transcription_session and self._has_audio_transcription_guardrails()
def _refuses_unscreened_audio(self, msg_type: str | None) -> bool:
return (
msg_type in RealTimeStreaming._CLIENT_AUDIO_BUFFER_TYPES
and self._builds_setup_lazily()
and self._should_disable_vad_auto_response()
)
async def _refuse_unscreened_audio(self, msg_type: str | None, message: str) -> None:
if self._unscreened_audio_refused and msg_type not in RealTimeStreaming._CLIENT_AUDIO_BUFFER_COMMIT_TYPES:
return
self._unscreened_audio_refused = True
await self.websocket.send_text(
json.dumps(_unscreened_audio_error_event(_decode_json_object(message).get("event_id")))
)
async def run_realtime_guardrails(
self,
transcript: str,
@ -1030,15 +1310,7 @@ class RealTimeStreaming:
# Use realtime_violation_message if configured; fall back to guardrail error text.
error_msg = getattr(callback, "realtime_violation_message", None) or safe_msg
# Deliver any caller-supplied backend message FIRST so that
# protocol contracts requiring a specific ordering (e.g.
# Gemini Live's mandatory ``toolResponse`` after a
# ``toolCall``) are honored before the guardrail's own
# clientContent / cancel messages are sent.
if pre_block_backend_message is not None:
await self._send_to_backend(pre_block_backend_message)
if not self._is_transcription_session:
await self._withdraw_blocked_turn(item_id)
await self._send_blocked_turn_to_backend(pre_block_backend_message, item_id)
# Send the policy violation hint (shows as small gray status text in UI).
await self.websocket.send_text(
json.dumps(
@ -1136,6 +1408,9 @@ class RealTimeStreaming:
break
finally:
self._flushing_pending_messages_until_setup = False
if self.session_created_sent_to_client and self._builds_setup_lazily():
await self._maybe_send_guardrail_turn_detection_update()
continue
if self.session_created_sent_to_client:
# A synthetic session.created (with placeholder defaults) was
# already forwarded to the client when we connected. The
@ -1295,7 +1570,11 @@ class RealTimeStreaming:
return BackendClose(code=1011, reason="proxy failed while relaying the upstream websocket")
def _backend_refused_session(self, close: BackendClose) -> bool:
return close.code != 1000 and not self._backend_sent_frames
return (
close.code != 1000
and not self._backend_sent_frames
and not self._holds_session_updates_until_first_content()
)
async def _log_session_end(self, transport_refusal: Exception | None) -> None:
provider_refusal: Final = None if self.provider_config is None else self.provider_config.session_refusal()
@ -1488,7 +1767,7 @@ class RealTimeStreaming:
client_event: _ClientEventFrame
try:
while True:
message = await self._receive_client_message()
message = await self._receive_client_message_before_setup_deadline()
if message is None:
return ClientLoopExit.CLIENT_DISCONNECTED
@ -1600,6 +1879,7 @@ class RealTimeStreaming:
if (
msg_type == "session.update"
and self.session_configuration_request is None
and self._held_session_update is None
and not self._guardrail_turn_detection_update_sent
and self._should_disable_vad_auto_response()
and (
@ -1663,12 +1943,25 @@ class RealTimeStreaming:
except (json.JSONDecodeError, AttributeError):
pass
if self._refuses_unscreened_audio(msg_type):
await self._refuse_unscreened_audio(msg_type, message)
continue
## LOGGING
# Log after any in-place modifications (GA remap, guardrail
# turn_detection injection) so audit logs reflect what we
# actually forward to the backend.
self.store_input(message=message)
if self._holds_client_content(msg_type, message):
self._hold_client_content(message)
continue
if self._held_client_content and msg_type != "session.update":
await self._send_setup_before_content(msg_type)
if msg_type in RealTimeStreaming._CLIENT_CONTENT_TYPES and self._uses_deferred_backend_setup():
await self._send_setup_before_content(msg_type)
if self._should_buffer_client_message_until_setup(message):
self._buffer_pending_message_until_setup(message)
continue
@ -1697,6 +1990,15 @@ class RealTimeStreaming:
# ``_maybe_send_guardrail_turn_detection_update`` would retry.
if msg_type == "session.update" and self.provider_config is None:
await self._send_session_gate_once()
if msg_type == "session.update" and await self._rejected_setup_session_update(message):
continue
if msg_type == "session.update" and self._holds_session_updates_until_first_content():
await self._hold_session_update(message)
if self._pending_messages_until_setup:
await self._send_held_session_update_as_setup()
continue
if msg_type in RealTimeStreaming._CLIENT_CONTENT_TYPES:
await self._send_setup_before_content(msg_type)
try:
sent = await self._send_to_backend(message)
except RealtimeTranscriptionProtocolError as e:

77
litellm/litellm_core_utils/reasoning_effort_utils.py
View file

@ -1,6 +1,12 @@
import reprlib
from collections.abc import Collection, Mapping
from dataclasses import dataclass
from typing import Final, Literal
from pydantic import TypeAdapter
import litellm
from litellm._logging import verbose_logger
from litellm.constants import (
DEFAULT_REASONING_EFFORT_HIGH_THINKING_BUDGET,
DEFAULT_REASONING_EFFORT_LOW_THINKING_BUDGET,
@ -11,6 +17,54 @@ from litellm.litellm_core_utils.core_helpers import as_str_mapping
OpenAIStyleReasoningEffort = Literal["minimal", "low", "medium", "high"]
_DROP_WARNING: Final = "Dropping %s=%s for model=%r because %s"
_DROP_PARAMS_DROP_WARNING: Final = "Dropping %s=%s for model=%r (drop_params=True) because %s"
_NON_NUMERIC_BUDGET_REASON: Final = "it is not a number. Send budget_tokens as an integer to set a reasoning level."
_NON_STRING_EFFORT_REASON: Final = (
"it is not a string. Send reasoning_effort as a level name such as 'high' to set a reasoning level."
)
_REASONING_PARAMS: Final = ("reasoning", "reasoning_effort", "thinking")
OPAQUE_VALUE: Final = TypeAdapter(object)
def warn_dropped_reasoning(
param: str, value: object, model: str | None, reason: str, *, drop_params_gated: bool
) -> None:
verbose_logger.warning(
_DROP_PARAMS_DROP_WARNING if drop_params_gated else _DROP_WARNING, param, reprlib.repr(value), model, reason
)
def warn_non_string_reasoning_effort(value: object, model: str | None) -> None:
effort: Final = as_str_mapping(value)
requested_level: Final = effort.get("effort") if effort is not None else value
if requested_level is None or isinstance(value, str):
return
warn_dropped_reasoning("reasoning_effort", value, model, _NON_STRING_EFFORT_REASON, drop_params_gated=False)
def warn_dropped_reasoning_params(
dropped: Collection[object],
request: Mapping[str, object],
model: str | None,
reason: str,
*,
drop_params_gated: bool,
) -> None:
for param in _REASONING_PARAMS:
if param in dropped and param in request:
warn_dropped_reasoning(param, request[param], model, reason, drop_params_gated=drop_params_gated)
@dataclass(frozen=True, slots=True)
class DroppedThinkingBudget:
budget_tokens: object
def warn(self, model: str) -> None:
warn_dropped_reasoning(
"thinking.budget_tokens", self.budget_tokens, model, _NON_NUMERIC_BUDGET_REASON, drop_params_gated=True
)
def reasoning_effort_from_thinking_budget(
budget_tokens: float,
@ -29,7 +83,9 @@ def reasoning_effort_from_thinking_budget(
return "minimal"
def reasoning_effort_from_thinking(thinking: object, *, drop_params: bool) -> OpenAIStyleReasoningEffort | None:
def _effort_or_dropped_budget(
thinking: object, *, drop_params: bool
) -> OpenAIStyleReasoningEffort | DroppedThinkingBudget | None:
fields: Final = as_str_mapping(thinking)
if fields is None or fields.get("type") != "enabled":
return None
@ -37,10 +93,27 @@ def reasoning_effort_from_thinking(thinking: object, *, drop_params: bool) -> Op
if isinstance(budget, (int, float)) and not isinstance(budget, bool):
return reasoning_effort_from_thinking_budget(budget)
if drop_params or litellm.drop_params is True:
return None
return DroppedThinkingBudget(budget)
raise non_numeric_thinking_budget_error(budget)
def reasoning_effort_from_thinking(
thinking: object, *, model: str, drop_params: bool
) -> OpenAIStyleReasoningEffort | None:
effort: Final = _effort_or_dropped_budget(thinking, drop_params=drop_params)
if not isinstance(effort, DroppedThinkingBudget):
return effort
effort.warn(model)
return None
def reasoning_effort_from_thinking_silently(
thinking: object, *, drop_params: bool
) -> OpenAIStyleReasoningEffort | None:
effort: Final = _effort_or_dropped_budget(thinking, drop_params=drop_params)
return None if isinstance(effort, DroppedThinkingBudget) else effort
def non_numeric_thinking_budget_error(budget: object) -> UnsupportedParamsError:
return UnsupportedParamsError(
message=(

1
litellm/litellm_core_utils/redact_messages.py
View file

@ -249,6 +249,7 @@ def _redacted_provider_specific_fields(fields: Mapping[str, object]) -> Mapping[
"reasoningContentBlocks",
"citations",
"compaction_blocks",
"compaction_start",
"compaction_delta",
)
if key in fields

23
litellm/litellm_core_utils/safe_json_dumps.py
View file

@ -1,4 +1,5 @@
import json
import math
from collections.abc import Callable
from typing import Any, Final
@ -14,16 +15,23 @@ def strip_null_bytes(value: str) -> str:
return value.replace("\x00", "")
def _finite_or_none(value: float | None) -> float | None:
return None if isinstance(value, float) and not math.isfinite(value) else value
def safe_json_structure(
data: object,
max_depth: int = DEFAULT_MAX_RECURSE_DEPTH,
value_transform: Callable[[str | None, str], str] | None = None,
key: str | None = None,
*,
non_finite_as_null: bool = False,
) -> object:
"""
Rebuild data out of JSON-native pieces while detecting circular references.
If a circular reference is detected then a marker string is returned.
NUL bytes are stripped from strings to prevent PostgreSQL 22P05 errors.
NUL bytes are stripped from strings to prevent PostgreSQL 22P05 errors. With
non_finite_as_null, NaN and infinite floats become null, since jsonb rejects them.
value_transform, when given, is applied to every string leaf (and to the
str() fallback for non-serializable objects) with the mapping key the leaf
@ -43,7 +51,7 @@ def safe_json_structure(
cleaned = obj.replace("\x00", "") if "\x00" in obj else obj
return _transform(key, cleaned)
if isinstance(obj, (int, float, bool, type(None))):
return obj
return _finite_or_none(obj) if non_finite_as_null else obj
# Check for circular reference.
if id(obj) in seen:
return "CircularReference Detected"
@ -65,8 +73,9 @@ def safe_json_structure(
seen.remove(id(obj))
return tuple_result
elif isinstance(obj, set):
sortable_values: Final[list[Any]] = [_serialize(item, seen, depth + 1, key) for item in obj]
set_result: Final = sorted(sortable_values)
members: Final[list[Any]] = [_serialize(item, seen, depth + 1, key) for item in obj]
present: Final = sorted(member for member in members if member is not None)
set_result: Final = present + [None] * members.count(None)
seen.remove(id(obj))
return set_result
elif isinstance(obj, BaseModel):
@ -88,6 +97,10 @@ def safe_dumps(
data: object,
max_depth: int = DEFAULT_MAX_RECURSE_DEPTH,
value_transform: Callable[[str | None, str], str] | None = None,
*,
non_finite_as_null: bool = False,
) -> str:
"""Serialize data to JSON text through safe_json_structure."""
return json.dumps(safe_json_structure(data, max_depth, value_transform), default=str)
return json.dumps(
safe_json_structure(data, max_depth, value_transform, non_finite_as_null=non_finite_as_null), default=str
)

9
litellm/litellm_core_utils/streaming_chunk_builder_utils.py
View file

@ -15,6 +15,7 @@ from litellm._logging import verbose_logger
from litellm.litellm_core_utils.core_helpers import (
PROVIDERS_BILLED_FROM_THEIR_OWN_USAGE_FIELDS,
chunk_usage,
finite_cost,
provider_reported_usage_cost,
stamp_litellm_bill_on_usage,
)
@ -1014,7 +1015,9 @@ class ChunkProcessor:
prompt_tokens_details, cache_creation_token_details
)
if usage_chunk_dict["cost"] is not None:
if Usage.dropped_non_finite_cost(usage_chunk):
cost = None
elif usage_chunk_dict["cost"] is not None:
cost = usage_chunk_dict["cost"]
prompt_tokens_details = attach_cache_creation_token_details(prompt_tokens_details, cache_creation_token_details)
@ -1444,8 +1447,8 @@ def _set_stream_builder_response_cost(
def _stamp_streaming_usage_cost(usage: Usage, response: ModelResponse, logging_obj: Optional["Logging"]) -> None:
if logging_obj is None:
return
if _logged_provider(logging_obj) in PROVIDERS_BILLED_FROM_THEIR_OWN_USAGE_FIELDS and isinstance(
getattr(usage, "cost", None), (int, float)
if _logged_provider(logging_obj) in PROVIDERS_BILLED_FROM_THEIR_OWN_USAGE_FIELDS and (
finite_cost(getattr(usage, "cost", None)) is not None
):
return
computed_cost: Final = logging_obj.response_cost_calculator(result=response)

60
litellm/litellm_core_utils/streaming_handler.py
View file

@ -28,6 +28,7 @@ from litellm.litellm_core_utils.core_helpers import chunk_usage, provider_report
from litellm.litellm_core_utils.hidden_params import (
HIDDEN_PARAMS_ATTR,
get_hidden_params,
set_extra_attribute,
set_hidden_param,
stored_attribute,
)
@ -40,6 +41,7 @@ from litellm.llms.ollama.common_utils import OllamaResponseIdNormalizer
from litellm.types.llms.base import LiteLLMBaseModel
from litellm.types.router import GenericLiteLLMParams
from litellm.types.utils import (
CHOICE_LOGPROBS_OR_JSON,
CacheCreationTokenDetails,
CompletionTokensDetailsWrapper,
Delta,
@ -1034,7 +1036,7 @@ class CustomStreamWrapper:
_initial_delta: Final = model_response.choices[0].delta.model_dump()
_initial_delta.pop("role", None)
model_response.choices[0].delta = Delta(**_initial_delta)
set_extra_attribute(model_response.choices[0], "delta", Delta(**_initial_delta))
return model_response
def _prebuilt_plain_delta(self, chunk: object) -> Delta | None:
@ -1060,13 +1062,13 @@ class CustomStreamWrapper:
def _set_role_on_fresh_delta(self, model_response: ModelResponseStream) -> None:
delta: Final = model_response.choices[0].delta
if self.sent_first_chunk is False:
delta.role = "assistant"
set_extra_attribute(delta, "role", "assistant")
self.sent_first_chunk = True
return
if not _holds_only_scalars(delta):
self.strip_role_from_delta(model_response)
return
delta.role = None
set_extra_attribute(delta, "role", None)
if "provider_specific_fields" not in delta.model_fields_set:
delta.provider_specific_fields = None
@ -1185,8 +1187,8 @@ class CustomStreamWrapper:
and _holds_only_scalars(upstream_delta)
)
if can_reuse_choice and upstream_choice is not None:
model_response.choices[0].finish_reason = None # rebind-ok: keep reused choice
model_response.choices[0].index = upstream_choice.index # rebind-ok: keep reused choice
set_extra_attribute(model_response.choices[0], "finish_reason", None)
set_extra_attribute(model_response.choices[0], "index", upstream_choice.index)
_prepend_upstream_choice_extras(
model_response.choices[0],
upstream_choice,
@ -1201,18 +1203,17 @@ class CustomStreamWrapper:
choice_json.pop(
"finish_reason", None
) # for mistral etc. which return a value in their last chunk (not-openai compatible).
choices.append(StreamingChoices(**choice_json))
choice_json.pop("logprobs", None)
choices.append(
StreamingChoices(**choice_json, logprobs=getattr(choice, "logprobs", None))
)
except Exception:
choices.append(StreamingChoices())
setattr(model_response, "choices", choices)
else:
return
model_response.system_fingerprint = upstream_chunk.system_fingerprint
setattr(
model_response,
"citations",
stored_attribute(upstream_chunk, "citations"),
)
set_extra_attribute(model_response, "citations", stored_attribute(upstream_chunk, "citations"))
preserve_upstream_non_openai_attributes(
model_response=model_response,
original_chunk=original_chunk,
@ -1605,7 +1606,7 @@ class CustomStreamWrapper:
model_response.system_fingerprint = response_obj["original_chunk"].system_fingerprint
self.system_fingerprint = response_obj["original_chunk"].system_fingerprint
if response_obj["logprobs"] is not None:
model_response.choices[0].logprobs = response_obj["logprobs"]
model_response.choices[0].logprobs = CHOICE_LOGPROBS_OR_JSON.validate_python(response_obj["logprobs"])
if response_obj["usage"] is not None:
if isinstance(response_obj["usage"], dict):
@ -2482,6 +2483,25 @@ class CustomStreamWrapper:
def bill_failure_after_earlier_rounds(self, earlier: "BilledRoundsSpend") -> None:
self._earlier_rounds_spend = earlier
def partial_response(self) -> "ModelResponse | TextCompletionResponse | None":
partial: Final = (
litellm.stream_chunk_builder(
chunks=self.chunks,
messages=self.messages if isinstance(self.messages, list) else None,
logging_obj=self.logging_obj,
count_prompt_tokens=self.count_prompt_tokens,
)
if self.chunks
else None
)
usage: Final[object] = getattr(partial, "usage", None)
if partial is None or not isinstance(usage, Usage):
return None
if self.model:
partial.model = self.model
backfill_missing_cache_usage_fields(usage)
return partial
def _record_partial_usage_for_failure(self) -> None:
"""
A stream that breaks mid-flight still billed the provider for the chunks
@ -2491,23 +2511,15 @@ class CustomStreamWrapper:
moves it into terminal failure usage before opening the next attempt,
so a recovered attempt logs its own usage.
"""
if self.logging_obj is None or not self.chunks:
if self.logging_obj is None:
return
try:
partial_response: Final = litellm.stream_chunk_builder(
chunks=self.chunks,
messages=self.messages if isinstance(self.messages, list) else None,
logging_obj=self.logging_obj,
count_prompt_tokens=self.count_prompt_tokens,
)
partial_response: Final = self.partial_response()
if partial_response is None:
return
usage: Final = cast(Usage | None, getattr(partial_response, "usage", None))
if usage is None:
usage: Final[object] = getattr(partial_response, "usage", None)
if not isinstance(usage, Usage):
return
if self.model:
partial_response.model = self.model
backfill_missing_cache_usage_fields(usage)
partial_cost: Final = self.logging_obj.response_cost_calculator(result=partial_response) or 0.0
if self._earlier_rounds_spend is None:
self.logging_obj.model_call_details["combined_usage_object"] = usage

43
litellm/litellm_core_utils/token_counter.py
View file

@ -17,6 +17,7 @@ import anyio
import anyio.lowlevel
import httpx
import tiktoken
from pydantic import TypeAdapter
from tiktoken.model import MODEL_TO_ENCODING
from typing_extensions import ParamSpec, TypeVar
@ -69,6 +70,7 @@ from litellm.types.llms.anthropic import (
AnthropicMessagesToolResultParam,
AnthropicMessagesToolUseParam,
AnthropicSearchResultParam,
CompactionBlock,
)
from litellm.types.llms.openai import (
AllMessageValues,
@ -937,6 +939,8 @@ def _count_messages(
default_token_count,
params.pdf_pricing,
)
elif key == "provider_specific_fields":
num_tokens += sum(map(params.count_function, _compaction_summaries(value)))
elif key == "search_results" and isinstance(value, list):
from litellm.litellm_core_utils.prompt_templates.common_utils import (
extract_search_results_text,
@ -951,6 +955,25 @@ def _count_messages(
return num_tokens
_OBJECT_LIST: Final = TypeAdapter(list[object])
def _compaction_summaries(provider_specific_fields: object) -> tuple[str, ...]:
fields: Final = as_str_mapping(provider_specific_fields)
blocks: Final = fields.get("compaction_blocks") if fields is not None else None
block_fields: Final = (
tuple(map(as_str_mapping, _OBJECT_LIST.validate_python(blocks))) if isinstance(blocks, list) else ()
)
return tuple(
content
for block in block_fields
if block is not None
and block.get("type") == "compaction"
and isinstance(content := block.get("content"), str)
and content
)
def _count_extra(
count_function: TokenCounterFunction,
tools: list[ChatCompletionToolParam] | None,
@ -1648,6 +1671,11 @@ def _count_anthropic_content(
return tokens
_TEXT_FIELD_BY_BLOCK_TYPE: Final = MappingProxyType(
{"text": "text", "thinking": "thinking", "redacted_thinking": "thinking", "compaction": "content"}
)
def _count_content_list(
count_function: TokenCounterFunction,
content_list: str
@ -1659,6 +1687,7 @@ def _count_content_list(
| AnthropicMessagesImageParam
| AnthropicMessagesDocumentParam
| AnthropicSearchResultParam
| CompactionBlock
],
use_default_image_token_count: bool,
default_token_count: int | None,
@ -1670,8 +1699,10 @@ def _count_content_list(
for c in content_list:
if isinstance(c, str):
num_tokens += count_function(c)
elif c["type"] == "text":
num_tokens += count_function(str(c.get("text", "")))
elif c["type"] in _TEXT_FIELD_BY_BLOCK_TYPE:
block_text = str(c.get(_TEXT_FIELD_BY_BLOCK_TYPE[c["type"]]) or "")
if block_text:
num_tokens += count_function(block_text)
elif c["type"] == "image_url":
image_url = c.get("image_url")
num_tokens += _count_image_tokens(image_url, use_default_image_token_count)
@ -1704,12 +1735,6 @@ def _count_content_list(
default_token_count,
pdf_pricing,
)
elif c["type"] in ("thinking", "redacted_thinking"):
# Claude extended thinking content block
# Count the thinking text and skip the opaque blobs (signature, redacted data)
thinking_text = str(c.get("thinking", ""))
if thinking_text:
num_tokens += count_function(thinking_text)
elif c["type"] == "tool_reference":
# Anthropic tool-search reference block: a lightweight pointer to
# a deferred tool, e.g. {"type": "tool_reference", "tool_name": ...}.
@ -1727,7 +1752,7 @@ def _count_content_list(
f"Invalid content item type: {content_type}. "
f"Expected str or dict with 'type' field "
f"(text, image_url, image, document, file, tool_use, tool_result, thinking, redacted_thinking, "
f"tool_reference, search_result)."
f"compaction, tool_reference, search_result)."
)
return num_tokens
except Exception as e:

335
litellm/llms/anthropic/chat/guardrail_translation/handler.py
View file

@ -24,7 +24,7 @@ from pydantic import ConfigDict, TypeAdapter, ValidationError, with_config
from typing_extensions import ReadOnly, TypedDict, assert_never
from litellm._logging import verbose_proxy_logger
from litellm.litellm_core_utils.core_helpers import as_str_mapping
from litellm.litellm_core_utils.core_helpers import as_str_mapping, iter_items
from litellm.llms.anthropic.chat.transformation import AnthropicConfig
from litellm.llms.anthropic.messages.adapters.transformation import (
OPENAI_MAX_TOOL_NAME_LENGTH,
@ -61,6 +61,7 @@ from litellm.llms.base_llm.guardrail_translation.utils import (
unappliable_request_rewrite,
)
from litellm.types.llms.anthropic import (
ANTHROPIC_SERVER_TOOL_USE_ID_PREFIX,
AllAnthropicToolsValues,
AnthropicMessagesRequest,
)
@ -153,6 +154,7 @@ class _TextAttrBlock(Protocol):
text: str
@runtime_checkable
class _WritableMessage(Protocol):
@overload
def get(self, key: str, /) -> object | None: ...
@ -195,6 +197,140 @@ class _ToolCallShape:
arguments: str
@dataclass(frozen=True, slots=True)
class _WebSearchString:
block_idx: int
result_idx: int | None
field: str
value: str
def _streamed_web_search_result_blocks(response: ModelResponse) -> tuple[Mapping[str, object], ...]:
choice: Final = response.choices[0] if response.choices else None
fields: Final = as_str_mapping(choice.message.provider_specific_fields) if isinstance(choice, Choices) else None
blocks: Final = (
(as_str_mapping(item) for item in iter_items(fields.get("web_search_results")))
if fields is not None and isinstance(fields.get("web_search_results"), list)
else ()
)
return tuple(
deepcopy(block) for block in blocks if block is not None and block.get("type") == "web_search_tool_result"
)
def _web_search_results(block: Mapping[str, object]) -> Iterable[object]:
return iter_items(block.get("content")) if isinstance(block.get("content"), list) else ()
def _web_search_result_strings(block_idx: int, result_idx: int, result: object) -> tuple[_WebSearchString, ...]:
fields: Final = as_str_mapping(result)
if fields is None or fields.get("type") != "web_search_result":
return ()
values: Final = ((field, fields.get(field)) for field in ("title", "url"))
return tuple(
_WebSearchString(block_idx, result_idx, field, value)
for field, value in values
if isinstance(value, str) and value
)
def _is_server_tool_use(block: Mapping[str, object]) -> bool:
block_id: Final = block.get("id")
match block.get("type"):
case "server_tool_use":
return True
case "tool_use":
return isinstance(block_id, str) and block_id.startswith(ANTHROPIC_SERVER_TOOL_USE_ID_PREFIX)
case _:
return False
def _web_search_block_strings(
block_idx: int, block: Mapping[str, object], *, with_query: bool
) -> tuple[_WebSearchString, ...]:
"""The strings a client reads off a native web search block: the ``server_tool_use`` query and each
``web_search_tool_result`` result's title and url."""
match block.get("type"):
case "server_tool_use" | "tool_use" if with_query and _is_server_tool_use(block):
tool_input: Final = as_str_mapping(block.get("input"))
query: Final = None if tool_input is None else tool_input.get("query")
return (_WebSearchString(block_idx, None, "query", query),) if isinstance(query, str) and query else ()
case "web_search_tool_result":
return tuple(
chain.from_iterable(
_web_search_result_strings(block_idx, result_idx, result)
for result_idx, result in enumerate(_web_search_results(block))
)
)
case _:
return ()
def _rewritten_web_search_result(
result_idx: int, result: object, rewrites: Mapping[tuple[int | None, str], str]
) -> object:
fields: Final = as_str_mapping(result)
if fields is None:
return result
return {
**fields,
**{field: rewrites[(result_idx, field)] for field in ("title", "url") if (result_idx, field) in rewrites},
}
def _rewritten_web_search_block(
block: Mapping[str, object], rewrites: Sequence[tuple[_WebSearchString, str]]
) -> Mapping[str, object]:
by_location: Final = MappingProxyType({(target.result_idx, target.field): value for target, value in rewrites})
if _is_server_tool_use(block):
query: Final = by_location.get((None, "query"))
tool_input: Final = as_str_mapping(block.get("input")) or MappingProxyType({})
return block if query is None else {**block, "input": {**tool_input, "query": query}}
if not isinstance(block.get("content"), list):
return block
return {
**block,
"content": [
_rewritten_web_search_result(result_idx, result, by_location)
for result_idx, result in enumerate(_web_search_results(block))
],
}
def _write_back_web_search_block(content_block: object, rewrites: Sequence[tuple[_WebSearchString, str]]) -> None:
block: Final = as_str_mapping(content_block)
if block is None or not isinstance(content_block, _WritableMessage):
return
rewritten: Final = _rewritten_web_search_block(block, rewrites)
writable: Final = _as_writable(content_block)
for key in ("input", "content"):
if key in rewritten:
writable[key] = rewritten[key]
def _split_web_search_texts(
guardrailed: Sequence[str], web_search_count: int, reply_count: int
) -> tuple[Sequence[str], Sequence[str]]:
split_at: Final = web_search_count if len(guardrailed) == web_search_count + reply_count else 0
return guardrailed[:split_at], guardrailed[split_at:]
def _web_search_rewrites_by_block(
strings: Sequence[_WebSearchString], guardrailed: Sequence[object]
) -> Mapping[int, tuple[tuple[_WebSearchString, str], ...]]:
rewrites: Final = tuple(
(target, value)
for target, value in zip(strings, guardrailed)
if isinstance(value, str) and value != target.value
)
return MappingProxyType(
{
block_idx: tuple(rewrite for rewrite in rewrites if rewrite[0].block_idx == block_idx)
for block_idx in {target.block_idx for target, _ in rewrites}
}
)
def _is_client_tool_use(block: Mapping[str, object]) -> bool:
return (
block.get("type") == "tool_use"
@ -289,6 +425,7 @@ def _write_back_message_text(message: _WritableMessage, target: MessageTextTarge
_TOOL_USE_INPUT_ADAPTER: Final = TypeAdapter(dict[str, object])
_DECODED_SSE_DATA: Final = TypeAdapter(object)
_REBUILT_TOOL_CALL_BLOCK_TYPES: Final = frozenset({"tool_use", "server_tool_use"})
_RELEASED_TOOL_USE_STOP: Final = (
b"event: message_delta\n"
b'data: {"type": "message_delta", "delta": {"stop_reason": "tool_use", "stop_sequence": null}, '
@ -315,6 +452,51 @@ def _write_back_tool_use(
block["name"] = shape.name
def _streamed_query_strings(
tool_call_blocks: Sequence[tuple[int, Mapping[str, object]]], tool_calls: Sequence[_ToolCallShape]
) -> tuple[_WebSearchString, ...]:
if len(tool_call_blocks) != len(tool_calls):
return ()
return tuple(
chain.from_iterable(
_web_search_block_strings(
index, {**block, "input": _rewritten_tool_use_input(tool_call.arguments)}, with_query=True
)
for (index, block), tool_call in zip(tool_call_blocks, tool_calls)
)
)
def _query_rewrites(strings: Sequence[_WebSearchString], guardrailed: Sequence[object]) -> Mapping[int, str]:
return MappingProxyType(
{
target.block_idx: value
for target, value in zip(strings, guardrailed)
if isinstance(value, str) and value != target.value
}
)
def _with_query(tool_call: _ToolCallShape, query: str) -> _ToolCallShape:
arguments: Final = _rewritten_tool_use_input(tool_call.arguments)
if arguments is None:
return tool_call
return _ToolCallShape(name=tool_call.name, arguments=json.dumps({**arguments, "query": query}))
def _with_query_rewrites(
tool_calls: tuple[_ToolCallShape, ...],
tool_call_blocks: Sequence[tuple[int, Mapping[str, object]]],
query_rewrites: Mapping[int, str],
) -> tuple[_ToolCallShape, ...]:
if not query_rewrites or len(tool_call_blocks) != len(tool_calls):
return tool_calls
return tuple(
_with_query(tool_call, query_rewrites[index]) if index in query_rewrites else tool_call
for (index, _), tool_call in zip(tool_call_blocks, tool_calls)
)
@dataclass(frozen=True, slots=True)
class _SSEFieldRewrite:
"""One field of one nested section of a buffered SSE event, rewritten."""
@ -402,7 +584,12 @@ class AnthropicMessagesHandler(BaseTranslation):
def post_call_hook_response(self, response: object) -> object:
if not isinstance(response, ModelResponse):
return response
return self.adapter.translate_openai_response_to_anthropic(response)
translated: Final = self.adapter.translate_openai_response_to_anthropic(response)
result_blocks: Final = _streamed_web_search_result_blocks(response)
if not result_blocks:
return translated
content: Final = translated.get("content") or ()
return {**translated, "content": [*result_blocks, *content]}
@staticmethod
def _build_streaming_usage_response(
@ -1273,9 +1460,17 @@ class AnthropicMessagesHandler(BaseTranslation):
task_mappings,
tool_calls_to_check,
)
block_fields: Final = tuple(map(self._output_block_fields, response_content))
web_search_strings: Final = tuple(
chain.from_iterable(
_web_search_block_strings(content_idx, fields[1], with_query=True)
for content_idx, fields in enumerate(block_fields)
if fields is not None
)
)
# Step 2: Apply guardrail to all texts in batch
if texts_to_check or tool_calls_to_check:
if texts_to_check or tool_calls_to_check or web_search_strings:
request_data = self._prepare_request_data(
request_data,
response,
@ -1284,7 +1479,7 @@ class AnthropicMessagesHandler(BaseTranslation):
)
inputs: Final = self._build_guardrail_inputs(
texts_to_check,
[*(target.value for target in web_search_strings), *texts_to_check],
images_to_check,
tool_calls_to_check,
response,
@ -1297,14 +1492,19 @@ class AnthropicMessagesHandler(BaseTranslation):
logging_obj=litellm_logging_obj,
)
guardrailed_texts: Final = guardrailed_inputs.get("texts", [])
web_search_rewrites, reply_rewrites = _split_web_search_texts(
guardrailed_inputs.get("texts", []), len(web_search_strings), len(texts_to_check)
)
# Step 3: Map guardrail responses back to original response structure
await self._apply_guardrail_responses_to_output(
response=response,
responses=guardrailed_texts,
responses=list(reply_rewrites),
task_mappings=task_mappings,
)
self._write_back_web_search_strings(
response_content, _web_search_rewrites_by_block(web_search_strings, web_search_rewrites)
)
verbose_proxy_logger.debug("Anthropic Messages: Processed output response: %s", response)
@ -1350,8 +1550,14 @@ class AnthropicMessagesHandler(BaseTranslation):
string_so_far = first_choice.message.content
pre_guardrail_tool_calls: Final = _tool_call_shapes(tool_calls_list or ())
guardrail_inputs: Final = GenericGuardrailAPIInputs()
if string_so_far:
guardrail_inputs["texts"] = [string_so_far]
leading_texts: Final = (string_so_far,) if isinstance(string_so_far, str) and string_so_far else ()
tool_call_blocks: Final = self._streamed_tool_call_blocks(responses_so_far)
streamed_query_strings: Final = _streamed_query_strings(tool_call_blocks, pre_guardrail_tool_calls)
streamed_result_strings: Final = self._streamed_web_search_result_strings(responses_so_far)
streamed_web_search_strings: Final = (*streamed_result_strings, *streamed_query_strings)
texts_to_check: Final = [*(target.value for target in streamed_web_search_strings), *leading_texts]
if texts_to_check:
guardrail_inputs["texts"] = texts_to_check
if tool_calls_list:
guardrail_inputs["tool_calls"] = tool_calls_list
@ -1374,29 +1580,42 @@ class AnthropicMessagesHandler(BaseTranslation):
responses_so_far, request_data
)
raise
guardrailed_texts: Final = _guardrailed_inputs.get("texts")
web_search_rewrites, reply_rewrites = _split_web_search_texts(
_guardrailed_inputs.get("texts") or [], len(streamed_web_search_strings), len(leading_texts)
)
if (
deliver_ended_stream_rewrites
and isinstance(string_so_far, str)
and string_so_far
and guardrailed_texts
and guardrailed_texts[0] != string_so_far
and reply_rewrites
and reply_rewrites[0] != string_so_far
):
self._write_ended_stream_text_rewrite(
responses_so_far,
guardrailed_texts[0],
reply_rewrites[0],
guardrail_name=guardrail_to_apply.guardrail_name or "unknown",
)
if deliver_ended_stream_rewrites:
self._write_ended_stream_web_search_rewrites(
responses_so_far,
_web_search_rewrites_by_block(streamed_result_strings, web_search_rewrites),
)
if deliver_ended_stream_rewrites:
returned_tool_calls: Final = _guardrailed_inputs.get("tool_calls")
self._write_ended_stream_tool_call_rewrites(
responses_so_far,
pre_guardrail_tool_calls=pre_guardrail_tool_calls,
post_guardrail_tool_calls=_tool_call_shapes(
returned_tool_calls
if isinstance(returned_tool_calls, list)
and len(returned_tool_calls) == len(pre_guardrail_tool_calls)
else tool_calls_list or ()
post_guardrail_tool_calls=_with_query_rewrites(
_tool_call_shapes(
returned_tool_calls
if isinstance(returned_tool_calls, list)
and len(returned_tool_calls) == len(pre_guardrail_tool_calls)
else tool_calls_list or ()
),
tool_call_blocks,
_query_rewrites(
streamed_query_strings, web_search_rewrites[len(streamed_result_strings) :]
),
),
guardrail_name=guardrail_to_apply.guardrail_name or "unknown",
)
@ -1451,6 +1670,14 @@ class AnthropicMessagesHandler(BaseTranslation):
request_data["litellm_metadata"] = user_metadata
return request_data
@staticmethod
def _write_back_web_search_strings(
response_content: Sequence[object],
rewrites_by_block: Mapping[int, tuple[tuple[_WebSearchString, str], ...]],
) -> None:
for block_idx, rewrites in rewrites_by_block.items():
_write_back_web_search_block(_content_block_at(response_content, block_idx), rewrites)
@staticmethod
def _get_response_content(response: object) -> Sequence[object]:
"""Extract content list from a dict or object response."""
@ -1573,30 +1800,22 @@ class AnthropicMessagesHandler(BaseTranslation):
) -> None:
"""Deliver ended-stream guardrail tool-call rewrites by rewriting the
buffered chunks in place: the rebuilt response lists tool calls in the
order of the stream's ``tool_use`` blocks, so the nth rewritten call lands
on the nth block, its first ``input_json_delta`` carrying the full rewritten
order of the stream's ``tool_use`` and ``server_tool_use`` blocks, so the nth
rewritten call lands on the nth block, its first ``input_json_delta`` carrying the full rewritten
arguments, every later one blanked, and ``content_block_start`` carrying the
rewritten name. Blocks that do not line up with the rebuilt tool calls make
the rewrite undeliverable, so the pipeline executor discards it and releases
the original chunks."""
if post_guardrail_tool_calls == pre_guardrail_tool_calls:
return
block_indices: Final = tuple(
index
for item in responses_so_far
for event in cls._iter_sse_events(item)
if event.get("type") == "content_block_start"
and isinstance(block := event.get("content_block"), Mapping)
and block.get("type") == "tool_use"
and isinstance(index := event.get("index"), int)
)
block_indices: Final = tuple(index for index, _ in cls._streamed_tool_call_blocks(responses_so_far))
if len(block_indices) != len(post_guardrail_tool_calls):
from litellm.proxy.policy_engine.pipeline_executor import UndeliverableStreamRewrite
raise UndeliverableStreamRewrite(
guardrail_name,
f"the guardrail returned {len(post_guardrail_tool_calls)} tool calls for a stream that carried "
f"{len(block_indices)} tool_use blocks",
f"{len(block_indices)} tool_use or server_tool_use blocks",
)
rewrites_by_block: Final = MappingProxyType(
{
@ -1636,6 +1855,60 @@ class AnthropicMessagesHandler(BaseTranslation):
),
)
@classmethod
def _streamed_web_search_result_strings(cls, responses_so_far: Sequence[object]) -> tuple[_WebSearchString, ...]:
events: Final = chain.from_iterable(cls._iter_sse_events(item) for item in responses_so_far)
result_blocks: Final = (
(index, block)
for event in events
if event.get("type") == "content_block_start"
and isinstance(index := event.get("index"), int)
and (block := as_str_mapping(event.get("content_block"))) is not None
)
return tuple(
chain.from_iterable(
_web_search_block_strings(index, block, with_query=False) for index, block in result_blocks
)
)
@classmethod
def _write_ended_stream_web_search_rewrites(
cls,
responses_so_far: MutableSequence[object], # mutable-ok: rewrites the caller's buffered chunks in place
rewrites_by_block: Mapping[int, tuple[tuple[_WebSearchString, str], ...]],
) -> None:
"""Deliver ended-stream guardrail rewrites of web search result titles and urls into the
``content_block_start`` that carries each whole ``web_search_tool_result`` block."""
if not rewrites_by_block:
return
def rewrite_result_block(event: Mapping[str, object]) -> _SSEFieldRewrite | None:
index: Final = event.get("index")
block: Final = as_str_mapping(event.get("content_block"))
rewrites: Final = rewrites_by_block.get(index) if isinstance(index, int) else None
if event.get("type") != "content_block_start" or block is None or rewrites is None:
return None
return _SSEFieldRewrite(
"content_block", "content", _rewritten_web_search_block(block, rewrites).get("content")
)
cls._rewrite_ended_stream_events(responses_so_far, rewrite_result_block)
@classmethod
def _streamed_tool_call_blocks(
cls, responses_so_far: Sequence[object]
) -> tuple[tuple[int, Mapping[str, object]], ...]:
events: Final = chain.from_iterable(cls._iter_sse_events(item) for item in responses_so_far)
return tuple(
(index, block)
for event in events
if event.get("type") == "content_block_start"
and (block := as_str_mapping(event.get("content_block"))) is not None
and isinstance(block_type := block.get("type"), str)
and block_type in _REBUILT_TOOL_CALL_BLOCK_TYPES
and isinstance(index := event.get("index"), int)
)
@classmethod
def _streamed_tool_use_ids(cls, responses_so_far: Sequence[object]) -> Mapping[int, str]:
"""The id of each buffered ``tool_use`` block, by its content block index."""
@ -1738,8 +2011,10 @@ class AnthropicMessagesHandler(BaseTranslation):
def get_streaming_scan_key(self, responses_so_far: Sequence[object]) -> StreamingScanKey | None:
stream_ended: Final = self._check_streaming_has_ended(responses_so_far)
tool_use_fingerprints: Final = self._streamed_tool_use_fingerprints(responses_so_far)
streamed_text: Final = self.get_streaming_string_so_far(responses_so_far)
result_strings: Final = self._streamed_web_search_result_strings(responses_so_far) if stream_ended else ()
return StreamingScanKey(
texts=(self.get_streaming_string_so_far(responses_so_far),),
texts=(*(target.value for target in result_strings), streamed_text),
tool_calls=tool_use_fingerprints if stream_ended else (),
stream_ended=stream_ended,
tool_calls_in_flight=bool(tool_use_fingerprints) and not stream_ended,

2
litellm/llms/anthropic/chat/transformation.py
View file

@ -48,6 +48,7 @@ from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import
place_mid_conversation_system,
split_leading_system_run,
)
from litellm.litellm_core_utils.reasoning_effort_utils import OPAQUE_VALUE, warn_non_string_reasoning_effort
from litellm.llms.base_llm.base_utils import (
ResponseFormatSchema,
as_json_object,
@ -1951,6 +1952,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
if isinstance(effort_value, dict):
effort_value = effort_value.get("effort")
if not isinstance(effort_value, str):
warn_non_string_reasoning_effort(OPAQUE_VALUE.validate_python(value), model)
continue
mapped_thinking = AnthropicConfig._map_reasoning_effort(
reasoning_effort=effort_value,

148
litellm/llms/anthropic/common_utils.py
View file

@ -8,12 +8,13 @@ import re
from collections.abc import Container, Mapping, MutableMapping, Sequence
from dataclasses import dataclass
from datetime import datetime, timezone
from itertools import chain
from types import MappingProxyType
from typing import Annotated, Any, ClassVar, Final, Literal, TypeVar
from urllib.parse import quote, urlparse
import httpx
from pydantic import BeforeValidator, ConfigDict, Field, StrictBool, TypeAdapter, ValidationError
from pydantic import BeforeValidator, ConfigDict, Field, JsonValue, StrictBool, TypeAdapter, ValidationError
from typing_extensions import ReadOnly, TypedDict
import litellm
@ -24,6 +25,7 @@ from litellm.constants import (
DEFAULT_REASONING_EFFORT_XHIGH_THINKING_BUDGET,
)
from litellm.exceptions import UnsupportedParamsError
from litellm.litellm_core_utils.core_helpers import as_str_mapping
from litellm.litellm_core_utils.prompt_templates.common_utils import (
get_file_ids_from_messages,
is_encrypted_reasoning_block,
@ -35,7 +37,7 @@ from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import
message_field,
parts_of,
)
from litellm.litellm_core_utils.reasoning_effort_utils import non_numeric_thinking_budget_error
from litellm.litellm_core_utils.reasoning_effort_utils import DroppedThinkingBudget, non_numeric_thinking_budget_error
from litellm.litellm_core_utils.url_utils import join_api_base
from litellm.llms.anthropic.wif import (
aget_anthropic_wif_token,
@ -46,6 +48,7 @@ from litellm.llms.anthropic.wif import (
from litellm.llms.base_llm.base_utils import BaseLLMModelInfo, BaseTokenCounter
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.types.llms.anthropic import (
ANTHROPIC_BETA_HEADER_VALUES,
ANTHROPIC_HOSTED_TOOLS,
ANTHROPIC_MID_CONVERSATION_OUTPUT_CONFIG_BETA_HEADER,
ANTHROPIC_MID_CONVERSATION_TOOL_CHANGES_BETA_HEADER,
@ -55,6 +58,7 @@ from litellm.types.llms.anthropic import (
ANTHROPIC_THINKING_BINDING_CONTROLS_BETA_HEADER,
ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER,
ANTHROPIC_TOOL_SEARCH_TOOL_TYPES,
LEGACY_THINKING_DISPLAY_UPDATES_PROVIDERS,
AllAnthropicToolsValues,
AnthropicMcpServerTool,
AnthropicMessagesToolChoice,
@ -118,10 +122,13 @@ _CLAUDE_CODE_BILLING_HEADER_PREFIX: Final = "x-anthropic-billing-header:"
OBJECT_MAPPING_ADAPTER: Final = TypeAdapter(dict[object, object])
OBJECT_LIST_ADAPTER: Final = TypeAdapter(list[object])
_DECODED_JSON: Final = TypeAdapter(object)
_JSON_VALUE: Final = TypeAdapter(JsonValue)
_CLAUDE_CODE_USER_AGENT_PREFIXES: Final = ("claude-cli/", "claude-code/")
NATIVE_COMPACTION_PROVIDERS: Final = frozenset({"anthropic", "azure_ai", "vertex_ai"})
def is_claude_tool_target(custom_llm_provider: str | None, model: str) -> bool:
if custom_llm_provider == "anthropic":
@ -177,24 +184,89 @@ def is_anthropic_messages_url(url: str) -> bool:
return parsed_url.hostname == "api.anthropic.com" or parsed_url.path.removesuffix("/").endswith("/v1/messages")
def _message_blocks(message: object) -> Sequence[object]:
fields: Final = as_str_mapping(message)
blocks: Final = validated_object_list(fields.get("content")) if fields is not None else None
return blocks or ()
def _is_signed_compaction_block(block: object) -> bool:
fields: Final = as_str_mapping(block)
if fields is None or fields.get("type") != "compaction":
return False
signature: Final = fields.get("signature")
return isinstance(signature, str) and bool(signature)
def carries_signed_compaction_block(messages: Sequence[object]) -> bool:
return any(map(_is_signed_compaction_block, chain.from_iterable(map(_message_blocks, messages))))
def requires_native_compaction_beta(
custom_llm_provider: str,
optional_params: Mapping[str, object],
messages: Sequence[object],
) -> bool:
return custom_llm_provider == "anthropic" and (
optional_params.get("compaction") is not None
or any(
isinstance(block, Mapping)
and block.get("type") == "compaction"
and isinstance(block.get("signature"), str)
and bool(block.get("signature"))
for message in messages
if isinstance(message, Mapping)
for content in (message.get("content"),)
if isinstance(content, (list, tuple))
for block in content
return custom_llm_provider in NATIVE_COMPACTION_PROVIDERS and (
optional_params.get("compaction") is not None or carries_signed_compaction_block(messages)
)
def anthropic_context_management(context_management: JsonValue) -> JsonValue:
if not isinstance(context_management, list):
return context_management
from litellm.llms.anthropic.chat.transformation import AnthropicConfig
edits: Final = AnthropicConfig.map_openai_context_management_to_anthropic(
[entry for entry in context_management if isinstance(entry, dict)]
)
return context_management if edits is None else _JSON_VALUE.validate_python(edits)
def _context_management_edits(spec: JsonValue) -> Sequence[object]:
mapping: Final = as_str_mapping(spec)
return (validated_object_list(mapping.get("edits")) if mapping is not None else None) or ()
def context_management_with_edits(context_management: JsonValue) -> JsonValue:
spec: Final = anthropic_context_management(context_management)
return spec if _context_management_edits(spec) else None
def context_management_betas(context_management: JsonValue) -> tuple[str, ...]:
edits: Final = _context_management_edits(anthropic_context_management(context_management))
edit_types: Final = tuple(edit.get("type") for edit in map(as_str_mapping, edits) if edit is not None)
return tuple(
beta
for beta, used in (
(ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_01_12.value, "compact_20260112" in edit_types),
(
ANTHROPIC_BETA_HEADER_VALUES.CONTEXT_MANAGEMENT_2025_06_27.value,
any(edit_type != "compact_20260112" for edit_type in edit_types),
),
)
if used
)
def count_tokens_feature_betas(
custom_llm_provider: str,
optional_params: Mapping[str, object],
messages: Sequence[object],
) -> tuple[str, ...]:
return tuple(
beta
for beta, used in (
(
ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_09_04.value,
requires_native_compaction_beta(custom_llm_provider, optional_params, messages),
),
(
ANTHROPIC_MID_CONVERSATION_TOOL_CHANGES_BETA_HEADER,
AnthropicModelInfo().is_mid_conversation_tool_change_used(messages),
),
)
if used
)
@ -562,11 +634,11 @@ def _numeric_budget_tokens(budget_tokens: object) -> int | None:
return None
def _legacy_budget_tokens(budget_tokens: object, *, drop_params: bool) -> int | None:
def _legacy_budget_tokens(budget_tokens: object, *, drop_params: bool) -> int | DroppedThinkingBudget:
if (numeric := _numeric_budget_tokens(budget_tokens)) is not None:
return numeric
if drop_params or litellm.drop_params is True:
return None
return DroppedThinkingBudget(budget_tokens)
raise non_numeric_thinking_budget_error(budget_tokens)
@ -597,10 +669,12 @@ class AnthropicModelInfo(BaseLLMModelInfo):
file_ids: Final = get_file_ids_from_messages(messages)
return len(file_ids) > 0
def is_thinking_display_updates_used(self, thinking: AnthropicThinkingParam | None) -> bool:
if not isinstance(thinking, dict):
return False
return thinking.get("type") in ("adaptive", "enabled") and thinking.get("display") == "updates"
def is_thinking_display_updates_used(self, thinking: object) -> bool:
match thinking:
case {"type": "adaptive" | "enabled", "display": "updates"}:
return True
case _:
return False
def thinking_betas(self, thinking: AnthropicThinkingParam | None) -> tuple[str, ...]:
return (
@ -1049,6 +1123,22 @@ class AnthropicModelInfo(BaseLLMModelInfo):
custom_llm_provider: str,
drop_params: bool,
) -> None:
dropped: Final = AnthropicModelInfo.translate_legacy_thinking_for_adaptive_model_silently(
model=model,
optional_params=optional_params,
custom_llm_provider=custom_llm_provider,
drop_params=drop_params,
)
if dropped is not None:
dropped.warn(model)
@staticmethod
def translate_legacy_thinking_for_adaptive_model_silently(
model: str,
optional_params: MutableMapping[str, object], # mutable-ok: in-place out-param like the sibling helpers
custom_llm_provider: str,
drop_params: bool,
) -> DroppedThinkingBudget | None:
"""Translate legacy ``thinking.type=enabled`` to adaptive for the
adaptive-thinking models that reject it (4.7+ and the 5 families).
Models flagged ``supports_legacy_thinking`` (the 4.6 family) accept the
@ -1057,22 +1147,25 @@ class AnthropicModelInfo(BaseLLMModelInfo):
``output_config.effort`` is never overridden.
"""
if not AnthropicModelInfo._is_adaptive_thinking_model(model, custom_llm_provider):
return
return None
if AnthropicModelInfo._supports_legacy_thinking(model, custom_llm_provider):
return
return None
thinking: Final = validated_object_mapping(optional_params.get("thinking"))
if thinking is None or thinking.get("type") != "enabled":
return
return None
budget_tokens: Final = _legacy_budget_tokens(thinking.get("budget_tokens", 0), drop_params=drop_params)
display: Final = thinking.get("display")
supported_display: Final = {"display": display} if display in ("summarized", "omitted") else {}
keeps_display: Final = display in ("summarized", "omitted") or (
display == "updates" and custom_llm_provider in LEGACY_THINKING_DISPLAY_UPDATES_PROVIDERS
)
supported_display: Final = {"display": display} if keeps_display else {}
carried_keys: Final = carried_thinking_keys(
thinking, rebuilt_keys=frozenset(("type", "budget_tokens", "display")), drop_params=drop_params
)
optional_params["thinking"] = {**carried_keys, "type": "adaptive", **supported_display}
if budget_tokens is None:
return
if isinstance(budget_tokens, DroppedThinkingBudget):
return budget_tokens
effort: Final = AnthropicModelInfo._legacy_budget_to_effort(
model=model, budget_tokens=budget_tokens, custom_llm_provider=custom_llm_provider
)
@ -1081,6 +1174,7 @@ class AnthropicModelInfo(BaseLLMModelInfo):
"effort": effort,
**(existing_output_config if isinstance(existing_output_config, dict) else MappingProxyType({})),
}
return None
@staticmethod
def _legacy_budget_to_effort(model: str, budget_tokens: int, custom_llm_provider: str) -> str:
@ -1367,8 +1461,6 @@ class AnthropicModelInfo(BaseLLMModelInfo):
if is_vertex_request is True:
# Vertex AI requires web search beta header for web search to work
if web_search_tool_used:
from litellm.types.llms.anthropic import ANTHROPIC_BETA_HEADER_VALUES
headers["anthropic-beta"] = ANTHROPIC_BETA_HEADER_VALUES.WEB_SEARCH_2025_03_05.value
elif len(all_betas) > 0:
headers["anthropic-beta"] = ",".join(all_betas)

4
litellm/llms/anthropic/count_tokens/handler.py
View file

@ -39,6 +39,7 @@ class AnthropicCountTokensHandler(AnthropicCountTokensConfig):
tools: list[dict[str, JsonValue]] | None = None,
system: JsonValue = None,
optional_params: Mapping[str, JsonValue] | None = None,
context_management: JsonValue = None,
drop_params: bool = False,
) -> dict[str, JsonValue]:
"""
@ -70,6 +71,7 @@ class AnthropicCountTokensHandler(AnthropicCountTokensConfig):
tools=tools,
system=system,
optional_params=optional_params,
context_management=context_management,
drop_params=drop_params,
)
@ -81,7 +83,7 @@ class AnthropicCountTokensHandler(AnthropicCountTokensConfig):
verbose_logger.debug("Making request to: %s", endpoint_url)
# Get required headers
headers: Final = self.get_count_tokens_headers(auth_header)
headers: Final = self.get_count_tokens_headers(auth_header, messages, context_management)
# Use LiteLLM's async httpx client
async_client: Final = get_async_httpx_client(llm_provider=litellm.LlmProviders.ANTHROPIC)

1
litellm/llms/anthropic/count_tokens/token_counter.py
View file

@ -132,6 +132,7 @@ class AnthropicTokenCounter(BaseTokenCounter):
tools=tools,
system=system,
optional_params=count_options,
context_management=context_management.spec if context_management is not None else None,
drop_params=drops_unsupported_params(litellm_params.get("drop_params")),
)

31
litellm/llms/anthropic/count_tokens/transformation.py
View file

@ -27,11 +27,15 @@ from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import
)
from litellm.litellm_core_utils.url_utils import join_api_base
from litellm.llms.anthropic.common_utils import (
carries_signed_compaction_block,
context_management_betas,
context_management_with_edits,
is_claude_model_target,
merge_anthropic_beta_headers,
tools_without_custom_field,
)
from litellm.llms.anthropic.wif import resolve_anthropic_base
from litellm.types.llms.anthropic import ANTHROPIC_BETA_HEADER_VALUES
from litellm.types.llms.openai import AllMessageValues, ChatCompletionImageObject, ValidUserMessageContentTypes
from litellm.types.utils import TokenCountMessagesFormat, TokenCountResponse
@ -376,6 +380,7 @@ class AnthropicCountTokensConfig:
tools: list[dict[str, JsonValue]] | None = None,
system: JsonValue = None,
optional_params: Mapping[str, JsonValue] | None = None,
context_management: JsonValue = None,
drop_params: bool = False,
) -> dict[str, JsonValue]: # mutable-ok: provider transport requires JSON dictionaries
"""
@ -408,6 +413,7 @@ class AnthropicCountTokensConfig:
drop_custom=drop_params,
),
),
("context_management", context_management_with_edits(context_management)),
)
if value is not None
}
@ -419,18 +425,33 @@ class AnthropicCountTokensConfig:
)
)
def get_count_tokens_headers(self, auth_header: Mapping[str, str]) -> dict[str, str]:
def get_count_tokens_headers(
self,
auth_header: Mapping[str, str],
messages: Sequence[Mapping[str, JsonValue]],
context_management: JsonValue = None,
) -> dict[str, str]:
"""The count-tokens headers around a resolved Anthropic auth header
(``AnthropicModelInfo.get_auth_header``): x-api-key for a static key, an Authorization
bearer for ``ANTHROPIC_AUTH_TOKEN`` and for sk-ant-oat tokens, whose mandatory oauth beta
merges with the token-counting beta instead of replacing it."""
merges with the token-counting beta instead of replacing it. A history that carries a
signed compaction block also needs the compaction beta, or Anthropic rejects the block
as an unknown content type, and ``context_management`` edits need the betas inference sends."""
compaction_betas: Final = (
(ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_09_04.value,)
if carries_signed_compaction_block(messages)
else ()
)
request_betas: Final = (
ANTHROPIC_TOKEN_COUNTING_BETA_VERSION,
*compaction_betas,
*context_management_betas(context_management),
)
return {
"Content-Type": "application/json",
"anthropic-version": "2023-06-01",
**auth_header,
"anthropic-beta": merge_anthropic_beta_headers(
auth_header.get("anthropic-beta"), ANTHROPIC_TOKEN_COUNTING_BETA_VERSION
),
"anthropic-beta": merge_anthropic_beta_headers(auth_header.get("anthropic-beta"), request_betas),
}
def validate_request(

2
litellm/llms/anthropic/messages/adapters/handler.py
View file

@ -647,12 +647,14 @@ class LiteLLMMessagesToCompletionTransformationHandler:
request_data["output_config"] = extra_kwargs["output_config"]
custom_llm_provider: Final = extra_kwargs.get("custom_llm_provider")
api_base: Final = extra_kwargs.get("api_base")
(
openai_request,
tool_name_mapping,
) = ANTHROPIC_ADAPTER.translate_completion_input_params_with_tool_mapping(
request_data,
custom_llm_provider=custom_llm_provider if isinstance(custom_llm_provider, str) else None,
api_base=api_base if isinstance(api_base, str) else None,
drop_params=drops_unsupported_params(extra_kwargs.get("drop_params")),
)

48
litellm/llms/anthropic/messages/adapters/transformation.py
View file

@ -24,6 +24,7 @@ OPENAI_MAX_TOOL_NAME_LENGTH: Final = 64
TOOL_NAME_HASH_LENGTH: Final = 8
TOOL_NAME_PREFIX_LENGTH: Final = OPENAI_MAX_TOOL_NAME_LENGTH - TOOL_NAME_HASH_LENGTH - 1 # 55
PROVIDERS_PROXYING_AN_UNKNOWN_BACKEND: Final = frozenset({"litellm_proxy"})
PROVIDERS_SERVED_BY_THE_MISTRAL_CHAT_CONFIG: Final = frozenset({"mistral", "codestral"})
_COMPACTION_BLOCK: Final = TypeAdapter(dict[str, JsonValue])
@ -133,6 +134,7 @@ from litellm.litellm_core_utils.prompt_templates.factory import (
)
from litellm.litellm_core_utils.reasoning_effort_utils import (
reasoning_effort_from_thinking,
reasoning_effort_from_thinking_silently,
)
from litellm.llms.anthropic.common_utils import (
eager_input_streaming_flag,
@ -255,7 +257,12 @@ class AnthropicAdapter:
return result
def translate_completion_input_params_with_tool_mapping(
self, kwargs, *, custom_llm_provider: str | None = None, drop_params: bool = True
self,
kwargs,
*,
custom_llm_provider: str | None = None,
api_base: str | None = None,
drop_params: bool = True,
) -> tuple[ChatCompletionRequest | None, dict[str, str]]:
"""
Translate Anthropic request params to OpenAI format, returning tool name mapping.
@ -289,7 +296,9 @@ class AnthropicAdapter:
) = LiteLLMAnthropicMessagesAdapter().translate_anthropic_to_openai(
anthropic_message_request=request_body,
custom_llm_provider=custom_llm_provider,
api_base=api_base,
drop_params=drop_params,
report_reasoning_drops=True,
)
return translated_body, tool_name_mapping
@ -659,7 +668,9 @@ class LiteLLMAnthropicMessagesAdapter:
def translate_anthropic_thinking_to_reasoning_effort(
thinking: AnthropicThinkingParam,
*,
model: str,
drop_params: bool = True,
report_reasoning_drops: bool = False,
) -> str | None:
"""
Translate Anthropic's thinking parameter to OpenAI's reasoning_effort.
@ -678,7 +689,9 @@ class LiteLLMAnthropicMessagesAdapter:
if thinking_type == "disabled":
return "none"
elif thinking_type == "enabled":
return reasoning_effort_from_thinking(thinking, drop_params=drop_params)
if report_reasoning_drops:
return reasoning_effort_from_thinking(thinking, model=model, drop_params=drop_params)
return reasoning_effort_from_thinking_silently(thinking, drop_params=drop_params)
elif thinking_type == "adaptive":
# Adaptive thinking: effort is controlled by output_config.effort,
# not budget_tokens. Return a default; caller should override with
@ -745,7 +758,7 @@ class LiteLLMAnthropicMessagesAdapter:
return {"thinking": thinking}
else:
reasoning_effort: Final = LiteLLMAnthropicMessagesAdapter.translate_anthropic_thinking_to_reasoning_effort(
thinking
thinking, model=model
)
if reasoning_effort:
return {
@ -1001,7 +1014,9 @@ class LiteLLMAnthropicMessagesAdapter:
)
@staticmethod
def _supports_prompt_cache_key(model: str | None, custom_llm_provider: str | None) -> bool:
def _supports_prompt_cache_key(
prompt_cache_key: str, model: str | None, custom_llm_provider: str | None, api_base: str | None
) -> bool:
if not model or not custom_llm_provider:
return False
if custom_llm_provider in PROVIDERS_PROXYING_AN_UNKNOWN_BACKEND:
@ -1009,7 +1024,17 @@ class LiteLLMAnthropicMessagesAdapter:
supported_params: Final = litellm.get_supported_openai_params(
model=model, custom_llm_provider=custom_llm_provider
)
return "prompt_cache_key" in (supported_params or ())
if "prompt_cache_key" not in (supported_params or ()):
return False
if custom_llm_provider not in PROVIDERS_SERVED_BY_THE_MISTRAL_CHAT_CONFIG:
return True
_, _, _, resolved_api_base = litellm.get_llm_provider(
model=model, custom_llm_provider=custom_llm_provider, api_base=api_base
)
mistral_config: Final = litellm.MistralConfig()
return mistral_config.takes_prompt_cache_key_value(
prompt_cache_key
) and mistral_config.accepts_prompt_cache_key_at(resolved_api_base)
@staticmethod
def _target_declares_reasoning_effort(model: str, custom_llm_provider: str | None) -> bool:
@ -1042,6 +1067,7 @@ class LiteLLMAnthropicMessagesAdapter:
new_kwargs: ChatCompletionRequest,
*,
custom_llm_provider: str | None = None,
api_base: str | None = None,
) -> None:
"""Translate metadata fields from Anthropic request to OpenAI request."""
if "metadata" in anthropic_message_request:
@ -1050,7 +1076,7 @@ class LiteLLMAnthropicMessagesAdapter:
new_kwargs["user"] = metadata["user_id"]
prompt_cache_key: Final = prompt_cache_key_from_user_id(metadata["user_id"])
if prompt_cache_key is not None and self._supports_prompt_cache_key(
anthropic_message_request.get("model"), custom_llm_provider
prompt_cache_key, anthropic_message_request.get("model"), custom_llm_provider, api_base
):
new_kwargs["prompt_cache_key"] = prompt_cache_key
@ -1127,6 +1153,7 @@ class LiteLLMAnthropicMessagesAdapter:
*,
custom_llm_provider: str | None = None,
drop_params: bool = True,
report_reasoning_drops: bool = False,
) -> None:
"""Translate Anthropic thinking to either thinking or reasoning_effort.
@ -1187,7 +1214,10 @@ class LiteLLMAnthropicMessagesAdapter:
return
reasoning_effort: Final = declared_effort or self.translate_anthropic_thinking_to_reasoning_effort(
cast(AnthropicThinkingParam, thinking), drop_params=drop_params
cast(AnthropicThinkingParam, thinking),
model=model,
drop_params=drop_params,
report_reasoning_drops=report_reasoning_drops,
)
if not reasoning_effort:
return
@ -1244,8 +1274,10 @@ class LiteLLMAnthropicMessagesAdapter:
anthropic_message_request: AnthropicMessagesRequest,
*,
custom_llm_provider: str | None = None,
api_base: str | None = None,
preserve_midturn_system: bool = False,
drop_params: bool = True,
report_reasoning_drops: bool = False,
) -> tuple[ChatCompletionRequest, dict[str, str]]:
"""
This is used by the beta Anthropic Adapter, for translating anthropic `/v1/messages` requests to the openai format.
@ -1282,6 +1314,7 @@ class LiteLLMAnthropicMessagesAdapter:
anthropic_message_request=anthropic_message_request,
new_kwargs=new_kwargs,
custom_llm_provider=custom_llm_provider,
api_base=api_base,
)
## CONVERT TOOL CHOICE
self._translate_tool_choice_to_openai(
@ -1299,6 +1332,7 @@ class LiteLLMAnthropicMessagesAdapter:
new_kwargs=new_kwargs,
custom_llm_provider=custom_llm_provider,
drop_params=drop_params,
report_reasoning_drops=report_reasoning_drops,
)
## CONVERT STOP_SEQUENCES
self._translate_stop_sequences_to_openai(

56
litellm/llms/anthropic/messages/native/fake_stream_iterator.py
View file

@ -12,11 +12,20 @@ import json
from collections.abc import Mapping, Sequence
from typing import Final, cast
from pydantic import TypeAdapter
from litellm.llms.anthropic.messages.native.utils import anthropic_message_delta
from litellm.types.llms.anthropic_messages.anthropic_response import (
AnthropicMessagesResponse,
)
_CITATIONS: Final = TypeAdapter(tuple[object, ...])
def _citations_of(block: Mapping[str, object]) -> tuple[object, ...]:
citations: Final = block.get("citations")
return _CITATIONS.validate_python(citations) if isinstance(citations, list) else ()
class FakeAnthropicMessagesStreamIterator:
"""
@ -47,16 +56,30 @@ class FakeAnthropicMessagesStreamIterator:
def _create_content_block_chunks(self, block_dict: Mapping[str, object], index: int) -> list[bytes]:
"""Build SSE chunks for a single content block."""
chunks: Final = []
chunks: Final = list[bytes]()
block_type: Final = block_dict.get("type")
if block_type == "text":
citations: Final = _citations_of(block_dict)
content_block_start = {
"type": "content_block_start",
"index": index,
"content_block": {"type": "text", "text": ""},
"content_block": {"type": "text", "text": "", "citations": ()}
if citations
else {"type": "text", "text": ""},
}
chunks.append(f"event: content_block_start\ndata: {json.dumps(content_block_start)}\n\n".encode())
citation_deltas: Final = (
{
"type": "content_block_delta",
"index": index,
"delta": {"type": "citations_delta", "citation": citation},
}
for citation in citations
)
chunks.extend(
f"event: content_block_delta\ndata: {json.dumps(delta)}\n\n".encode() for delta in citation_deltas
)
text: Final = block_dict.get("text", "")
content_block_delta = {
"type": "content_block_delta",
@ -89,36 +112,23 @@ class FakeAnthropicMessagesStreamIterator:
}
chunks.append(f"event: content_block_delta\ndata: {json.dumps(signature_delta)}\n\n".encode())
elif block_type == "redacted_thinking":
content_block_start = {
elif block_type in ("tool_use", "server_tool_use"):
empty_input: Final[Mapping[str, object]] = {}
tool_use_start: Final = {
"type": "content_block_start",
"index": index,
"content_block": {"type": "redacted_thinking"},
"content_block": {**block_dict, "input": empty_input},
}
chunks.append(f"event: content_block_start\ndata: {json.dumps(content_block_start)}\n\n".encode())
elif block_type == "tool_use":
content_block_start = {
"type": "content_block_start",
"index": index,
"content_block": {
"type": "tool_use",
"id": block_dict.get("id"),
"name": block_dict.get("name"),
"input": {},
},
}
chunks.append(f"event: content_block_start\ndata: {json.dumps(content_block_start)}\n\n".encode())
input_data: Final = block_dict.get("input", {})
content_block_delta = {
chunks.append(f"event: content_block_start\ndata: {json.dumps(tool_use_start)}\n\n".encode())
tool_use_delta: Final = {
"type": "content_block_delta",
"index": index,
"delta": {
"type": "input_json_delta",
"partial_json": json.dumps(input_data),
"partial_json": json.dumps(block_dict.get("input", {})),
},
}
chunks.append(f"event: content_block_delta\ndata: {json.dumps(content_block_delta)}\n\n".encode())
chunks.append(f"event: content_block_delta\ndata: {json.dumps(tool_use_delta)}\n\n".encode())
else:
passthrough_start: Final = {

8
litellm/llms/anthropic/messages/native/handler.py
View file

@ -20,6 +20,7 @@ from litellm.litellm_core_utils.core_helpers import as_str_mapping
from litellm.litellm_core_utils.exception_mapping_utils import exception_type
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.llms.anthropic.common_utils import (
AnthropicModelInfo,
strip_provider_specific_fields_from_anthropic_messages,
tools_marked_for_a_tool_config_point,
)
@ -36,7 +37,6 @@ from litellm.router_utils.reasoning_effort_capability import (
keeps_effort,
model_group_from_metadata,
resolve_deployment_reasoning_effort,
with_adaptive_thinking_as_effort,
)
from litellm.types.llms.anthropic import AllAnthropicToolsValues
from litellm.types.llms.anthropic_messages.anthropic_request import AnthropicMetadata
@ -185,9 +185,11 @@ def _with_deployment_output_effort(
).value
def translate(source: Mapping[str, object]) -> Mapping[str, object]:
return with_adaptive_thinking_as_effort(
source, model=model, custom_llm_provider=custom_llm_provider, drop_params=drop_params
translated: Final = dict(source)
AnthropicModelInfo.translate_legacy_thinking_for_adaptive_model(
model=model, optional_params=translated, custom_llm_provider=custom_llm_provider, drop_params=drop_params
)
return translated
requested_output: Final = as_str_mapping(params.get("output_config"))
caller_output: Final = None if requested_output is None else resolve(requested_output)

40
litellm/llms/anthropic/messages/native/transformation.py
View file

@ -21,6 +21,7 @@ from litellm.constants import ANTHROPIC_BILLING_METADATA_PREFIX
from litellm.exceptions import AuthenticationError
from litellm.litellm_core_utils.core_helpers import drops_unsupported_params
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.litellm_core_utils.reasoning_effort_utils import OPAQUE_VALUE, warn_non_string_reasoning_effort
from litellm.litellm_core_utils.url_utils import ensure_api_route
from litellm.llms.anthropic.messages.convertible_blocks import drop_non_string_tool_result_texts
from litellm.llms.anthropic.messages.safeguards import proxy_serves_safeguards
@ -40,9 +41,11 @@ from litellm.types.llms.anthropic_tool_search import get_tool_search_beta_header
from litellm.types.router import GenericLiteLLMParams
from ...common_utils import (
NATIVE_COMPACTION_PROVIDERS,
OBJECT_LIST_ADAPTER,
AnthropicError,
AnthropicModelInfo,
context_management_betas,
is_claude_model_target,
merge_anthropic_beta_headers,
optionally_handle_anthropic_oauth,
@ -102,6 +105,7 @@ _MESSAGES_ADAPTER: Final = TypeAdapter(list[dict[str, object]])
class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
_workload_identity_eligible: ClassVar[bool] = True
_report_reasoning_drops: ClassVar[bool] = True
@property
def custom_llm_provider(self) -> str | None:
@ -125,7 +129,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
"tool_choice",
"thinking",
"context_management",
*(("compaction",) if self._resolved_provider == "anthropic" else ()),
*(("compaction",) if self._resolved_provider in NATIVE_COMPACTION_PROVIDERS else ()),
"output_format",
"inference_geo",
"speed",
@ -694,12 +698,17 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
):
anthropic_messages_optional_request_params.pop("output_format") # pyright: ignore[reportUnknownMemberType] # the request params arrive as a bare dict
requested_reasoning_effort: Final = OPAQUE_VALUE.validate_python(
anthropic_messages_optional_request_params.get("reasoning_effort") # pyright: ignore[reportUnknownMemberType] # the request params arrive as a bare dict
)
self._translate_reasoning_effort_to_anthropic(
model=model,
optional_params=anthropic_messages_optional_request_params,
max_tokens=max_tokens,
custom_llm_provider=self._resolved_provider,
)
if self._report_reasoning_drops:
warn_non_string_reasoning_effort(requested_reasoning_effort, model)
AnthropicModelInfo.maybe_drop_disabled_thinking(
model=model,
@ -707,12 +716,14 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
custom_llm_provider=self._resolved_provider,
)
AnthropicModelInfo.translate_legacy_thinking_for_adaptive_model(
dropped_budget: Final = AnthropicModelInfo.translate_legacy_thinking_for_adaptive_model_silently(
model=model,
optional_params=anthropic_messages_optional_request_params,
custom_llm_provider=self._resolved_provider,
drop_params=getattr(litellm_params, "drop_params", None) is True,
)
if dropped_budget is not None and self._report_reasoning_drops:
dropped_budget.warn(model)
self._translate_adaptive_effort_for_non_adaptive_model(
model=model,
@ -745,6 +756,8 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
)
if transformed_context_management is not None:
anthropic_messages_optional_request_params["context_management"] = transformed_context_management
elif drops_unsupported_params(litellm_params.drop_params):
anthropic_messages_optional_request_params.pop("context_management")
####### get required params for all anthropic messages requests ######
# Lazy %s: the f-string previously stringified the entire messages
@ -873,28 +886,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
if requires_native_compaction_beta(custom_llm_provider, optional_params, messages):
beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_09_04.value)
# Check for context management
context_management_param: Final = optional_params.get("context_management")
if context_management_param is not None:
# Check edits array for compact_20260112 type
edits: Final = context_management_param.get("edits", ())
has_compact = False
has_other = False
for edit in edits:
edit_type = edit.get("type", "")
if edit_type == "compact_20260112":
has_compact = True
else:
has_other = True
# Add compact header if any compact edits exist
if has_compact:
beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_01_12.value)
# Add context management header if any other edits exist
if has_other:
beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.CONTEXT_MANAGEMENT_2025_06_27.value)
beta_values.update(context_management_betas(optional_params.get("context_management")))
# Check for structured outputs. Anthropic's newer request shape nests
# the schema under output_config.format; the older top-level

4
litellm/llms/anthropic/messages/responses_adapters/transformation.py
View file

@ -534,6 +534,7 @@ class LiteLLMAnthropicToResponsesAPIAdapter:
output_config: dict[str, object] | None = None,
auto_summary_level: str = "detailed",
*,
model: str,
drop_params: bool = True,
) -> dict[str, object] | None:
"""
@ -555,7 +556,7 @@ class LiteLLMAnthropicToResponsesAPIAdapter:
if isinstance(output_config, dict) and output_config.get("effort"):
effort = output_config["effort"]
elif thinking_type == "enabled":
budget_effort: Final = reasoning_effort_from_thinking(thinking, drop_params=drop_params)
budget_effort: Final = reasoning_effort_from_thinking(thinking, model=model, drop_params=drop_params)
if budget_effort is None:
return None
effort = budget_effort
@ -667,6 +668,7 @@ class LiteLLMAnthropicToResponsesAPIAdapter:
thinking,
output_config=cast(dict[str, object] | None, output_config),
auto_summary_level=auto_summary_level,
model=model,
drop_params=drop_params,
)
if reasoning:

8
litellm/llms/anthropic/prompt_cache_prediction.py
View file

@ -9,7 +9,7 @@ from dataclasses import dataclass, field
from functools import reduce
from itertools import accumulate, groupby
from types import MappingProxyType
from typing import Annotated, Final, Literal, Protocol, TypeAlias
from typing import Annotated, ClassVar, Final, Literal, Protocol, TypeAlias
import httpx
from pydantic import ConfigDict, Field, JsonValue, StrictInt, TypeAdapter, ValidationError
@ -81,6 +81,10 @@ _DEPLOYMENT_OPTIONS: Final = frozenset(
)
class _BaselineAnthropicMessagesConfig(AnthropicMessagesConfig):
_report_reasoning_drops: ClassVar[bool] = False
class _StrictModel(LiteLLMBaseModel):
model_config = ConfigDict(extra="forbid", frozen=True, strict=True)
@ -661,7 +665,7 @@ def prepare_native_baseline_body(request: Mapping[str, object], model: str) -> M
TypeAdapter(tuple[str, ...]).validate_python(owned.get("additional_drop_params") or ()),
dict(options),
)
config: Final = AnthropicMessagesConfig()
config: Final = _BaselineAnthropicMessagesConfig()
body: Final = config.transform_anthropic_messages_request(
model=resolved_model,
messages=strip_provider_specific_fields_from_anthropic_messages(messages),

7
litellm/llms/azure/responses/transformation.py
View file

@ -14,6 +14,7 @@ from litellm.constants import AZURE_DEFAULT_RESPONSES_API_VERSION
from litellm.litellm_core_utils.url_utils import encode_url_path_segment
from litellm.llms.azure.chat.gpt_reasoning_transformation import AzureOpenAIGPT5Config
from litellm.llms.azure.common_utils import BaseAzureLLM
from litellm.llms.base_llm.responses.transformation import SUMMARY_ONLY_REASONING_FIELD_VALUES, ReasoningFieldValues
from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig
from litellm.types.llms.openai import *
from litellm.types.responses.main import *
@ -92,6 +93,12 @@ class AzureOpenAIResponsesAPIConfig(OpenAIResponsesAPIConfig):
def _effort_resolves_to_none(model: str, effort: str | None) -> bool:
return AzureOpenAIGPT5Config.effort_resolves_to_none(model, effort)
def deployment_names_wrap_openai_models(self) -> bool:
return True
def unresolved_model_reasoning_field_values(self, model: str) -> ReasoningFieldValues | None:
return SUMMARY_ONLY_REASONING_FIELD_VALUES if self._is_gpt_5_model(model) else None
def get_supported_openai_params(self, model: str) -> list:
"""
Azure Responses API does not support context_management (compaction).

1
litellm/llms/azure_ai/anthropic/count_tokens/handler.py
View file

@ -84,6 +84,7 @@ class AzureAIAnthropicCountTokensHandler(AzureAIAnthropicCountTokensConfig):
headers: Final = self.get_required_headers(
api_key=api_key,
litellm_params=litellm_params,
messages=messages,
)
if not {"x-api-key", "Authorization"} & headers.keys():
raise AnthropicError(

8
litellm/llms/azure_ai/anthropic/count_tokens/transformation.py
View file

@ -4,11 +4,13 @@ Azure AI Anthropic CountTokens API transformation logic.
Extends the base Anthropic CountTokens transformation with Azure authentication.
"""
from collections.abc import Sequence
from typing import Final
from urllib.parse import urlsplit
from litellm.constants import ANTHROPIC_TOKEN_COUNTING_BETA_VERSION
from litellm.litellm_core_utils.url_utils import join_api_base
from litellm.llms.anthropic.common_utils import count_tokens_feature_betas, merge_anthropic_beta_headers
from litellm.llms.anthropic.count_tokens.transformation import (
AnthropicCountTokensConfig,
)
@ -28,6 +30,7 @@ class AzureAIAnthropicCountTokensConfig(AnthropicCountTokensConfig):
self,
api_key: str | None,
litellm_params: dict[str, object] | None = None,
messages: Sequence[object] = (),
) -> dict[str, str]:
"""
Get the required headers for the Azure AI Anthropic CountTokens API.
@ -38,6 +41,7 @@ class AzureAIAnthropicCountTokensConfig(AnthropicCountTokensConfig):
Args:
api_key: The Azure AI API key
litellm_params: Optional LiteLLM parameters for additional auth config
messages: The request messages, read for betas the counted features need
Returns:
Dictionary of required headers with both x-api-key and Azure authentication
@ -46,7 +50,9 @@ class AzureAIAnthropicCountTokensConfig(AnthropicCountTokensConfig):
headers: Final = {
"Content-Type": "application/json",
"anthropic-version": "2023-06-01",
"anthropic-beta": ANTHROPIC_TOKEN_COUNTING_BETA_VERSION,
"anthropic-beta": merge_anthropic_beta_headers(
ANTHROPIC_TOKEN_COUNTING_BETA_VERSION, count_tokens_feature_betas("azure_ai", {}, messages)
),
**({"x-api-key": api_key} if api_key else {}),
}

136
litellm/llms/azure_ai/chat/transformation.py
View file

@ -6,16 +6,21 @@ from typing import TYPE_CHECKING, Final, cast
import httpx
from httpx import Response
from pydantic import TypeAdapter, ValidationError
import litellm
from litellm._logging import verbose_logger
from litellm.integrations.anthropic_cache_control_hook import AnthropicCacheControlHook
from litellm.litellm_core_utils.prompt_templates.common_utils import (
content_parts,
convert_content_list_to_str,
drop_tool_reference_parts_from_tool_messages,
filter_value_from_dict,
is_pdf_file_part,
prompt_cache_breakpoint_for_wire,
with_prompt_cache_breakpoint,
)
from litellm.litellm_core_utils.reasoning_effort_utils import warn_dropped_reasoning_params
from litellm.llms.azure.common_utils import BaseAzureLLM
from litellm.llms.azure_ai.common_utils import (
api_key_header_for_base,
@ -27,7 +32,7 @@ from litellm.llms.openai.common_utils import drop_params_from_unprocessable_enti
from litellm.llms.openai.openai import OpenAIChatCompletionResponseIterator, OpenAIConfig
from litellm.llms.xai.chat.transformation import XAIChatConfig
from litellm.secret_managers.main import get_secret_str
from litellm.types.llms.openai import AllMessageValues
from litellm.types.llms.openai import AllMessageValues, PromptCacheBreakpoint
from litellm.types.router import GenericLiteLLMParams
from litellm.types.utils import ModelResponse, ModelResponseStream, ProviderField
from litellm.utils import add_path_to_api_base, supports_tool_choice
@ -36,6 +41,14 @@ if TYPE_CHECKING:
from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
_REQUEST_BODY: Final = TypeAdapter(dict[str, object])
def _request_model(request: Mapping[str, object]) -> str | None:
model: Final = request.get("model")
return model if isinstance(model, str) else None
class AzureFoundryErrorStrings(str, enum.Enum):
SET_EXTRA_PARAMETERS_TO_PASS_THROUGH = "Set extra-parameters to 'pass-through'"
@ -62,6 +75,72 @@ def _has_list_only_content_part(message: Mapping[str, object]) -> bool:
return any(_is_list_only_content_part(part) for part in content_parts(message))
_CONTENT_PARTS_ADAPTER: Final = TypeAdapter(list[dict[str, object]])
def _text_parts(message: Mapping[str, object]) -> tuple[Mapping[str, object], ...]:
content: Final = message.get("content")
if not isinstance(content, list):
return ()
try:
parts: Final = _CONTENT_PARTS_ADAPTER.validate_python(content)
except ValidationError:
return ()
return tuple(part for part in parts if part.get("type") == "text")
def _with_only_text_parts(message: AllMessageValues) -> AllMessageValues:
if _has_list_only_content_part(message) or not (text_parts := _text_parts(message)):
return message
text_only: Final = {**message, "content": list(text_parts)}
return cast(AllMessageValues, text_only) # cast-ok: the same message, text parts only
_FIELDS: Final = TypeAdapter(Mapping[str, object])
def _foundry_breakpoint(marker: object) -> PromptCacheBreakpoint | None:
if prompt_cache_breakpoint_for_wire(marker, drop_params=True) is None:
return None
return PromptCacheBreakpoint(mode="explicit")
def _part_with_foundry_breakpoint(part: object) -> object:
if not isinstance(part, Mapping):
return part
fields: Final = _FIELDS.validate_python(part)
if "prompt_cache_breakpoint" not in fields:
return fields
unmarked: Final = {key: value for key, value in fields.items() if key != "prompt_cache_breakpoint"}
return with_prompt_cache_breakpoint(unmarked, _foundry_breakpoint(fields["prompt_cache_breakpoint"]))
def _with_breakpoints_for_foundry(message: AllMessageValues, drop_params: bool) -> AllMessageValues:
renamed: Final = AnthropicCacheControlHook.cache_control_as_prompt_cache_breakpoints(_with_only_text_parts(message))
parts: Final = content_parts(renamed) if drop_params else ()
if not parts:
return renamed
foundry_message: Final = {**renamed, "content": [_part_with_foundry_breakpoint(part) for part in parts]}
return cast(AllMessageValues, foundry_message) # cast-ok: the same message, its breakpoints in Foundry's shape
def _deployment_reads_breakpoints(model: str, litellm_params: Mapping[str, object]) -> bool:
base_model: Final = litellm_params.get("base_model")
named_models: Final = (base_model, model) if isinstance(base_model, str) and base_model else (model,)
model_info: Final = litellm_params.get("model_info")
if not isinstance(model_info, Mapping):
return any(
AnthropicCacheControlHook.targets_openai_prompt_cache_breakpoint(name, "azure_ai") for name in named_models
)
deployment_flag: Final = _FIELDS.validate_python(model_info).get("supports_prompt_cache_breakpoint")
if isinstance(deployment_flag, bool):
return deployment_flag
return any(
AnthropicCacheControlHook.catalog_targets_openai_prompt_cache_breakpoint(name, "azure_ai")
for name in named_models
)
class AzureAIChatCompletionResponseIterator(OpenAIChatCompletionResponseIterator):
def chunk_parser(self, chunk: dict[str, object]) -> ModelResponseStream: # mutable-ok: mirrors override contract
usage: Final = chunk.get("usage")
@ -251,6 +330,8 @@ class AzureAIStudioConfig(OpenAIConfig):
self,
messages: list[AllMessageValues],
model: str,
reads_breakpoints: bool | None = None,
drop_params: bool = False,
) -> list:
"""
- Azure AI Studio doesn't support content as a list. This handles:
@ -260,13 +341,31 @@ class AzureAIStudioConfig(OpenAIConfig):
Azure AI Foundry backends set additionalProperties=false and reject
these with "Extra inputs are not permitted", which breaks multi-turn
Anthropic-format clients that echo thinking blocks back as history.
2. Transforms list content to a string.
2. Transforms list content to a string, except on deployments that read OpenAI explicit
prompt cache breakpoints (the deployment's own model_info opt-in, else the cost-map entry
of its base_model or name, as loaded and without a sibling deployment's opt-in): only a
content part can carry the breakpoint (a message-level one is silently ignored), so those
keep their text parts, with each Anthropic `cache_control` mark renamed to a breakpoint the
way the openai provider renames it. Every other part is dropped, exactly as the string
conversion drops it, before the marks are renamed, so a message-level mark lands on the
last text part. Under drop_params each breakpoint goes out as `{"mode": "explicit"}`, the only
shape Foundry takes, and a malformed one is dropped.
3. If message contains an image, audio, or PDF file part, send as is (user-intended)
Operates on a deep copy so the caller's messages keep their thinking blocks
and provider metadata, which a fallback to another provider still needs.
"""
stripped_messages: Final = copy.deepcopy(drop_tool_reference_parts_from_tool_messages(messages))
deployment_reads_breakpoints: Final = (
AnthropicCacheControlHook.targets_openai_prompt_cache_breakpoint(model, "azure_ai")
if reads_breakpoints is None
else reads_breakpoints
)
copied_messages: Final = copy.deepcopy(drop_tool_reference_parts_from_tool_messages(messages))
stripped_messages: Final = (
[_with_breakpoints_for_foundry(message, drop_params) for message in copied_messages]
if deployment_reads_breakpoints
else copied_messages
)
for message in stripped_messages:
message_dict = cast(dict, message) # cast-ok: TypedDict is a runtime dict stripped on our copy
for field in NON_OPENAI_SPEC_MESSAGE_FIELDS:
@ -275,6 +374,10 @@ class AzureAIStudioConfig(OpenAIConfig):
if _has_list_only_content_part(message):
continue
if deployment_reads_breakpoints and (text_parts := _text_parts(message)):
message_dict["content"] = list(text_parts)
continue
texts = convert_content_list_to_str(message=message)
if texts:
message["content"] = texts
@ -329,7 +432,16 @@ class AzureAIStudioConfig(OpenAIConfig):
if extra_body and isinstance(extra_body, dict):
optional_params.update(extra_body)
optional_params.pop("max_retries", None)
return super().transform_request(model, messages, optional_params, litellm_params, headers)
return {
"model": model,
"messages": self._transform_messages(
messages,
model,
_deployment_reads_breakpoints(model, _FIELDS.validate_python(litellm_params)),
bool(litellm_params.get("drop_params") or litellm.drop_params),
),
**optional_params,
}
def transform_response(
self,
@ -407,7 +519,13 @@ class AzureAIStudioConfig(OpenAIConfig):
request_data = self._drop_extra_params_from_request_data(request_data, error_text)
if "Extra inputs are not permitted" in error_text and self._error_has_tool_level_extra_fields(error_text):
request_data = self._drop_tool_level_extra_fields(request_data, error_text)
data: Final = drop_params_from_unprocessable_entity_error(e=e, data=request_data)
data: Final = drop_params_from_unprocessable_entity_error(
e=e,
data=request_data,
model=_request_model(_REQUEST_BODY.validate_python(request_data)),
drop_params_gated="Extra inputs are not permitted" in error_text
and not self._error_has_tool_level_extra_fields(error_text),
)
return data
def _drop_tool_level_extra_fields(self, request_data: dict, error_text: str) -> dict:
@ -423,6 +541,14 @@ class AzureAIStudioConfig(OpenAIConfig):
def _drop_extra_params_from_request_data(self, request_data: dict, error_text: str) -> dict:
params_to_drop: Final = self._extract_params_to_drop_from_error_text(error_text)
if params_to_drop:
request: Final = _REQUEST_BODY.validate_python(request_data)
warn_dropped_reasoning_params(
params_to_drop,
request,
_request_model(request),
"Azure AI rejected it as an extra parameter its deployment does not pass through.",
drop_params_gated=False,
)
for param in params_to_drop:
if param in request_data:
request_data.pop(param, None)

32
litellm/llms/base_llm/realtime/transformation.py
View file

@ -1,10 +1,12 @@
import dataclasses
from abc import ABC, abstractmethod
from collections.abc import Mapping, Sequence
from types import TracebackType
from typing import TYPE_CHECKING, Any, Protocol
from typing import TYPE_CHECKING, Any, Literal, Protocol
import httpx
from litellm._uuid import uuid
from litellm.types.llms.openai import OpenAIRealtimeStreamSessionEvents
from litellm.types.realtime import (
RealtimeInputAudioTranscriptionUsage,
@ -22,6 +24,26 @@ else:
LiteLLMLoggingObj = Any
@dataclasses.dataclass(frozen=True, slots=True)
class RealtimeSessionUpdateRejection:
code: Literal["invalid_type", "invalid_value", "session_update_not_applied", "session_update_too_large"]
message: str
param: str
def error_event(self, client_event_id: object) -> dict[str, object]:
return {
"type": "error",
"event_id": f"event_{uuid.uuid4()}",
"error": {
"type": "invalid_request_error",
"code": self.code,
"message": self.message,
"param": self.param,
"event_id": client_event_id if isinstance(client_event_id, str) else None,
},
}
class RealtimeBackend(Protocol):
async def __aenter__(self) -> object: ...
@ -93,6 +115,14 @@ class BaseRealtimeConfig(ABC):
def session_configuration_request(self, model: str) -> str | None: # message sent to setup the realtime session
return None
def builds_setup_from_session_update(self) -> bool:
return False
def setup_session_update_rejection(
self, model: str, session: Mapping[str, object], *, setup_sent: bool
) -> RealtimeSessionUpdateRejection | None:
return None
def unbilled_usage_on_session_close(self, model: str) -> RealtimeInputAudioTranscriptionUsage | None:
return None

28
litellm/llms/base_llm/responses/transformation.py
View file

@ -1,7 +1,7 @@
import types
from abc import ABC, abstractmethod
from collections.abc import Mapping
from typing import TYPE_CHECKING, Any, Final, Literal, TypeAlias, cast
from typing import TYPE_CHECKING, Any, Final, Literal, TypeAlias, cast, get_args
import httpx
@ -27,6 +27,20 @@ else:
BaseLLMException = Any
ReasoningSummaryLevel: TypeAlias = Literal["auto", "concise", "detailed"]
ReasoningFieldValues: TypeAlias = Mapping[str, frozenset[str | None] | None]
REASONING_SUMMARY_VALUES: Final = frozenset((*get_args(ReasoningSummaryLevel), None))
SUMMARY_ONLY_REASONING_FIELD_VALUES: Final[ReasoningFieldValues] = types.MappingProxyType(
{"summary": REASONING_SUMMARY_VALUES}
)
STANDARD_MODE_WITHOUT_CONTEXT_REASONING_FIELD_VALUES: Final[ReasoningFieldValues] = types.MappingProxyType(
{**SUMMARY_ONLY_REASONING_FIELD_VALUES, "mode": frozenset({"standard"})}
)
STANDARD_MODE_REASONING_FIELD_VALUES: Final[ReasoningFieldValues] = types.MappingProxyType(
{**STANDARD_MODE_WITHOUT_CONTEXT_REASONING_FIELD_VALUES, "context": None}
)
ANY_REASONING_FIELD_VALUES: Final[ReasoningFieldValues] = types.MappingProxyType(
{**SUMMARY_ONLY_REASONING_FIELD_VALUES, "mode": None, "context": None}
)
class BaseResponsesAPIConfig(ABC):
@ -100,6 +114,18 @@ class BaseResponsesAPIConfig(ABC):
"""The ``reasoning.summary`` LiteLLM's opt-in auto summary asks this provider for."""
return "detailed"
def reasoning_field_values(self) -> ReasoningFieldValues | None:
return None
def deployment_names_wrap_openai_models(self) -> bool:
return False
def unresolved_model_reasoning_field_values(self, model: str) -> ReasoningFieldValues | None:
return None
def ignores_unsupported_reasoning_fields(self) -> bool:
return False
@abstractmethod
def map_openai_params(
self,

17
litellm/llms/bedrock/chat/chat_completions/transformation.py
View file

@ -33,6 +33,7 @@ from litellm.litellm_core_utils.prompt_templates.image_handling import (
inline_remote_image_urls,
inline_remote_media,
)
from litellm.litellm_core_utils.reasoning_effort_utils import warn_dropped_reasoning
from litellm.litellm_core_utils.url_utils import ensure_api_route, join_api_base
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM, bedrock_bearer_token
@ -42,6 +43,7 @@ from litellm.llms.bedrock.common_utils import (
bedrock_rejects_stop_sequences,
bedrock_runtime_chat_completions_serves_reasoning_inline,
split_bedrock_region_path,
strip_bedrock_routing_prefix,
)
from litellm.llms.openai.chat.gpt_transformation import OpenAIChatCompletionStreamingHandler
from litellm.llms.openai_like.chat.transformation import OpenAILikeChatConfig
@ -121,6 +123,13 @@ def without_refused_reasoning_effort(model: str, params: Mapping[str, object]) -
effort: Final = params.get("reasoning_effort")
if not isinstance(effort, str) or effort not in chat_completions_reasoning_efforts_refused_for(model):
return params
warn_dropped_reasoning(
"reasoning_effort",
effort,
strip_bedrock_routing_prefix(model),
"AWS's Chat Completions endpoint rejects that level for this model family, so AWS applies its default effort.",
drop_params_gated=False,
)
return _without_params(params, frozenset(("reasoning_effort",)))
@ -445,6 +454,14 @@ class AmazonBedrockRuntimeChatCompletionsConfig(OpenAILikeChatConfig):
),
status_code=400,
)
if malformed_effort:
warn_dropped_reasoning(
"reasoning_effort",
raw_params["reasoning_effort"],
strip_bedrock_routing_prefix(model),
"Bedrock's Chat Completions endpoint takes it only as a string, so AWS applies its default effort.",
drop_params_gated=True,
)
rejected_stop: Final = frozenset(("stop",)) if bedrock_rejects_stop_sequences(model) else frozenset[str]()
return dict(
without_refused_reasoning_effort(

25
litellm/llms/bedrock/chat/converse_transformation.py
View file

@ -50,6 +50,7 @@ from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import
message_field,
parts_of,
)
from litellm.litellm_core_utils.reasoning_effort_utils import OPAQUE_VALUE, warn_dropped_reasoning
from litellm.llms.anthropic.chat.transformation import (
DROP_UNSUPPORTED_ADAPTIVE_THINKING_WARNING,
DROP_UNSUPPORTED_OUTPUT_CONFIG_WARNING,
@ -63,6 +64,7 @@ from litellm.llms.bedrock.chat.tool_result_images import place_tool_result_image
from litellm.llms.bedrock.common_utils import (
bedrock_chat_uses_native_structured_output,
bedrock_cleaned_output_schema,
bedrock_model_is_unknown,
bedrock_model_supports_regex_lookaround,
bedrock_native_output_format,
bedrock_native_output_is_opt_in,
@ -186,6 +188,8 @@ def _tools_the_model_accepts(
_OUTPUT_CONFIG_BLOCK: Final = TypeAdapter(OutputConfigBlock)
_DEEPSEEK_THINKING_REASON: Final = "DeepSeek on Bedrock reasons natively and takes no thinking field."
_DEEPSEEK_R1_EFFORT_REASON: Final = "DeepSeek R1 on Bedrock always reasons and rejects reasoning_effort."
def _has_native_text_format(output_config: object) -> bool:
@ -529,13 +533,14 @@ class AmazonConverseConfig(BaseConfig):
"""
Handle the reasoning_effort parameter based on the model type.
- GPT-OSS and DeepSeek V3 models: passed through unchanged via additionalModelRequestFields.
- GPT-OSS, DeepSeek V3 and unknown models (``bedrock_model_is_unknown``): passed through unchanged
via additionalModelRequestFields.
- OpenAI GPT-5.x and GPT-6 models: mapped to ``reasoning.effort`` via additionalModelRequestFields.
- Nova 2 models: transformed to reasoningConfig.
- Anthropic models: mapped to ``thinking`` (and ``output_config.effort`` on
adaptive Claude 4.6 / 4.7).
"""
if "gpt-oss" in model or "deepseek" in model:
if "gpt-oss" in model or "deepseek" in model or bedrock_model_is_unknown(model):
optional_params["reasoning_effort"] = reasoning_effort
elif self._is_openai_gpt_reasoning_model(model):
reasoning: Final[BedrockConverseGptReasoningEffortBlock] = {"effort": reasoning_effort}
@ -638,6 +643,10 @@ class AmazonConverseConfig(BaseConfig):
or supports_reasoning(model=base_model, custom_llm_provider=self.custom_llm_provider)
)
def _warn_native_reasoning_drop(self, param: str, value: object, model: str, reason: str) -> None:
if param in self.get_supported_openai_params(model):
warn_dropped_reasoning(param, value, model, reason, drop_params_gated=False)
def _model_rejects_reasoning_effort_param(self, model: str, base_model: str) -> bool:
"""Whether the model returns a 400 for every ``reasoning_effort`` shape on Converse.
@ -681,7 +690,7 @@ class AmazonConverseConfig(BaseConfig):
"parallel_tool_calls",
]
if "arn" in model: # we can't infer the model from the arn, so just add all params
if "arn" in model or bedrock_model_is_unknown(model):
supported_params.append("tools")
supported_params.append("tool_choice")
supported_params.append("thinking")
@ -1141,9 +1150,8 @@ class AmazonConverseConfig(BaseConfig):
"tool_choice": {"type": "auto", "disable_parallel_tool_use": not value}
}
if param == "thinking" and drop_thinking_param:
verbose_logger.debug(
"Dropping unsupported `thinking` param for Bedrock model=%s; it reasons natively.",
model,
self._warn_native_reasoning_drop(
param, OPAQUE_VALUE.validate_python(value), model, _DEEPSEEK_THINKING_REASON
)
elif param == "thinking" and not self._is_openai_gpt_reasoning_model(model):
if (
@ -1177,10 +1185,7 @@ class AmazonConverseConfig(BaseConfig):
drop_params=drop_params,
)
elif param == "reasoning_effort" and isinstance(value, str) and drop_reasoning_effort_param:
verbose_logger.debug(
"Dropping unsupported `reasoning_effort` param for Bedrock model=%s; it always reasons and rejects it.",
model,
)
self._warn_native_reasoning_drop(param, value, model, _DEEPSEEK_R1_EFFORT_REASON)
elif (
param == "reasoning_effort"
and isinstance(value, str)

62
litellm/llms/bedrock/chat/invoke_transformations/base_invoke_transformation.py
View file

@ -2,7 +2,7 @@ import copy
import json
import time
from collections.abc import Mapping, Sequence
from typing import TYPE_CHECKING, Any, Final, cast, get_args
from typing import TYPE_CHECKING, Any, Final
import httpx
from pydantic import ConfigDict, TypeAdapter, ValidationError, with_config
@ -74,10 +74,6 @@ _GUARDRAIL_CONFIG_EXPECTED_FORMAT: Final = (
"{'guardrailIdentifier': str, 'guardrailVersion': str, 'trace': 'enabled'|'disabled'|'enabled_full'}"
)
_INVOKE_PROVIDERS: Final = TypeAdapter(tuple[litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL, ...]).validate_python(
get_args(litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL)
)
def _response_header(headers: Mapping[str, str], name: str) -> str | None:
return headers.get(name, None)
@ -562,62 +558,6 @@ class AmazonInvokeConfig(BaseConfig, BaseAWSLLM):
"""
return False
@staticmethod
def get_bedrock_invoke_provider(
model: str,
) -> litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL | None:
"""
Helper function to get the bedrock provider from the model
handles 4 scenarios:
1. model=invoke/anthropic.claude-3-5-sonnet-20240620-v1:0 -> Returns `anthropic`
2. model=anthropic.claude-3-5-sonnet-20240620-v1:0 -> Returns `anthropic`
3. model=llama/arn:aws:bedrock:us-east-1:086734376398:imported-model/r4c4kewx2s0n -> Returns `llama`
4. model=us.amazon.nova-pro-v1:0 -> Returns `nova`
"""
if model.startswith("invoke/"):
model = model.replace("invoke/", "", 1)
# Special case: Check for "nova" in model name first (before "amazon")
# This handles amazon.nova-* models which would otherwise match "amazon" (Titan)
if "nova" in model.lower():
if "nova" in get_args(litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL):
return cast(litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL, "nova")
_split_model: Final = model.split(".")[0]
if _split_model in get_args(litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL):
return cast(litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL, _split_model)
# If not a known provider, check for pattern with two slashes
provider = AmazonInvokeConfig._get_provider_from_model_path(model)
if provider is not None:
return provider
for provider in _INVOKE_PROVIDERS:
if provider in model:
return provider
return None
@staticmethod
def _get_provider_from_model_path(
model_path: str,
) -> litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL | None:
"""
Helper function to get the provider from a model path with format: provider/model-name
Args:
model_path (str): The model path (e.g., 'llama/arn:aws:bedrock:us-east-1:086734376398:imported-model/r4c4kewx2s0n' or 'anthropic/model-name')
Returns:
Optional[str]: The provider name, or None if no valid provider found
"""
parts: Final = model_path.split("/")
if len(parts) >= 1:
provider: Final = parts[0]
if provider in get_args(litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL):
return cast(litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL, provider)
return None
def convert_messages_to_prompt(self, model: str, messages, provider, custom_prompt_dict) -> tuple[str, list | None]:
# handle anthropic prompts and amazon titan prompts
prompt = ""

68
litellm/llms/bedrock/common_utils.py
View file

@ -1098,6 +1098,11 @@ def _model_cost_entry(key: str) -> Mapping[str, object] | None:
return None if raw is None else _MODEL_COST_ENTRY_ADAPTER.validate_python(raw)
def _cost_map_row_names_a_provider(key: str) -> bool:
entry: Final = _model_cost_entry(key)
return entry is not None and entry.get("litellm_provider") is not None
def _bedrock_price_map_entries(model: str) -> tuple[Mapping[str, object] | None, ...]:
return tuple(
_model_cost_entry(key)
@ -1567,7 +1572,8 @@ def bedrock_model_supports_regex_lookaround(model: str, litellm_params: Mapping[
The deployment's ``model_info.supports_regex_lookaround`` wins, then the
``model_prices_and_context_window.json`` entry of its ``base_model``, then the
entry of ``model`` itself. A model nobody flagged keeps its schema as sent.
entry of ``model`` itself. A model nobody flagged keeps its schema as sent, unless
no candidate names a model LiteLLM knows, whatever route prefix it carries.
"""
params: Final = litellm_params or {}
deployment_flag: Final = deployment_model_info(params).get("supports_regex_lookaround")
@ -1576,7 +1582,41 @@ def bedrock_model_supports_regex_lookaround(model: str, litellm_params: Mapping[
base_model: Final = params.get("base_model")
candidates: Final = (*((base_model,) if isinstance(base_model, str) else ()), model)
flags: Final = (_bedrock_converse_model_flag(candidate, "supports_regex_lookaround") for candidate in candidates)
return next((flag for flag in flags if flag is not None), True)
flag: Final = next((flag for flag in flags if flag is not None), None)
if flag is not None:
return flag
return not all(bedrock_model_is_unknown(split_bedrock_region_path(candidate)[1]) for candidate in candidates)
def _bedrock_model_id_without_region(model: str) -> str:
routed_id: Final = model.removeprefix("bedrock/")
region, separator, regional_id = routed_id.partition("/")
return regional_id if separator and region in get_all_bedrock_regions() else routed_id
def bedrock_model_is_unknown(model: str) -> bool:
model_id: Final = _bedrock_model_id_without_region(model)
if "/" in model_id:
return False
base: Final = get_bedrock_base_model(model_id)
regional_ids: Final = (f"{region}.{base}" for region in get_bedrock_cross_region_inference_regions())
if any(_cost_map_row_names_a_provider(key) for key in (*_bedrock_model_info_candidates(model_id), *regional_ids)):
return False
from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM
return BaseAWSLLM.get_bedrock_invoke_provider(model_id) is None
def _names_no_invoke_provider(model: str) -> bool:
from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM
model_id: Final = _bedrock_model_id_without_region(model)
names_one_model: Final = bool(model_id) and ("/" not in model_id or model_id.startswith("arn:"))
return names_one_model and BaseAWSLLM.get_bedrock_invoke_provider(model) is None
def bedrock_model_reaches_converse_by_default(model: str) -> bool:
return BedrockModelInfo.named_bedrock_route(model, None) is None and _names_no_invoke_provider(model)
_BedrockConverseModelFlag: TypeAlias = Literal[
@ -1585,10 +1625,13 @@ _BedrockConverseModelFlag: TypeAlias = Literal[
]
def _bedrock_converse_model_flag(model: str, key: _BedrockConverseModelFlag) -> bool | None:
def _bedrock_model_info_candidates(model: str) -> tuple[str, ...]:
base: Final = get_bedrock_base_model(model)
candidates: Final = dict.fromkeys((model, base, _BEDROCK_MODEL_VERSION_SUFFIX_RE.sub("", base)))
for candidate in candidates:
return tuple(dict.fromkeys((model, base, _BEDROCK_MODEL_VERSION_SUFFIX_RE.sub("", base))))
def _bedrock_converse_model_flag(model: str, key: _BedrockConverseModelFlag) -> bool | None:
for candidate in _bedrock_model_info_candidates(model):
with contextlib.suppress(Exception):
model_info = get_cached_model_info()(
model=candidate,
@ -1766,8 +1809,19 @@ class BedrockModelInfo(BaseLLMModelInfo):
default (``bedrock_runtime_chat_completions_is_default``) and ``chat_completions/`` opts any other model in;
``request_params`` (the caller's chat params) sends such a request to Converse when it needs a
feature only Converse serves, and ``converse/`` pins a model to Converse. Every other OpenAI-family
model stays on Converse without the prefix.
model stays on Converse without the prefix. A model the cost map does not list as Converse goes to
Invoke only when its id names an Invoke provider, and to Converse otherwise.
"""
named_route: Final = BedrockModelInfo.named_bedrock_route(model, request_params)
if named_route is not None:
return named_route
return "converse" if _names_no_invoke_provider(model) else "invoke"
@staticmethod
def named_bedrock_route(
model: str,
request_params: Mapping[str, object] | None,
) -> BedrockRoute | None:
route_mappings: dict[
str,
Literal[
@ -1822,7 +1876,7 @@ class BedrockModelInfo(BaseLLMModelInfo):
return "converse"
if _OPENAI_FAMILY_MODEL_RE.search(base_model):
return "converse"
return "invoke"
return None
@staticmethod
def _explicit_converse_route(model: str) -> bool:

20
litellm/llms/bedrock/realtime/handler.py
View file

@ -17,7 +17,6 @@ from pydantic import JsonValue, TypeAdapter
import litellm
from litellm._logging import redact_string, verbose_proxy_logger
from litellm._uuid import uuid
from litellm.constants import (
BEDROCK_REALTIME_CLOSE_TIMEOUT_SECONDS,
BEDROCK_REALTIME_COMMITTED_FAILURE_SCOPE_KEY,
@ -31,13 +30,14 @@ from litellm.litellm_core_utils.aws_partition import get_aws_dns_suffix
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging
from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
from litellm.litellm_core_utils.realtime_streaming import DefaultLoggedRealTimeEventTypes
from litellm.llms.base_llm.realtime.transformation import RealtimeSessionUpdateRejection
from litellm.types.llms.bedrock import AwsAuthParams
from litellm.types.llms.openai import OpenAIRealtimeEvents
from litellm.types.realtime import RealtimeResponseTransformInput
from ..base_aws_llm import BaseAWSLLM, run_aws_signing
from ..common_utils import BedrockError
from .transformation import BedrockRealtimeConfig, BedrockSessionUpdateRejection, requested_modalities
from .transformation import BedrockRealtimeConfig, requested_modalities
_CLIENT_MESSAGE_ADAPTER: Final[TypeAdapter[JsonValue]] = TypeAdapter(JsonValue)
@ -256,23 +256,11 @@ async def _ack_session_update(
async def _reject_session_update(
client_ws: RealtimeClientWebSocket,
parsed_client_message: Mapping[str, JsonValue],
rejection: BedrockSessionUpdateRejection,
rejection: RealtimeSessionUpdateRejection,
) -> bool:
"""Send an error event for a session.update Bedrock cannot apply; False means the client is gone."""
client_event_id: Final = parsed_client_message.get("event_id")
error_event: Final = {
"type": "error",
"event_id": f"event_{uuid.uuid4()}",
"error": {
"type": "invalid_request_error",
"code": rejection.code,
"message": rejection.message,
"param": rejection.param,
"event_id": client_event_id if isinstance(client_event_id, str) else None,
},
}
try:
await client_ws.send_text(json.dumps(error_event))
await client_ws.send_text(json.dumps(rejection.error_event(parsed_client_message.get("event_id"))))
except Exception as e: # noqa: BLE001 # any send failure means the client is gone
verbose_proxy_logger.debug("Client to Bedrock forwarding ended: %s", e, exc_info=True)
return False

19
litellm/llms/bedrock/realtime/transformation.py
View file

@ -11,7 +11,7 @@ import math
import uuid as uuid_lib
from collections.abc import Mapping, Sequence
from types import MappingProxyType
from typing import Final, Literal
from typing import Final
import httpx
from pydantic import ConfigDict, JsonValue, TypeAdapter
@ -19,7 +19,7 @@ from pydantic import ConfigDict, JsonValue, TypeAdapter
from litellm._logging import verbose_logger
from litellm._uuid import uuid
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.llms.base_llm.realtime.transformation import BaseRealtimeConfig
from litellm.llms.base_llm.realtime.transformation import BaseRealtimeConfig, RealtimeSessionUpdateRejection
from litellm.llms.bedrock.common_utils import BedrockError
from litellm.llms.bedrock.realtime.trigger_audio import ready_trigger_pcm
from litellm.types.llms.base import LiteLLMBaseModel
@ -196,13 +196,6 @@ def _malformed_settings(session_config: Mapping[str, JsonValue]) -> tuple[tuple[
return tuple((name, expected) for name, expected, is_malformed in malformed if is_malformed)
@dataclasses.dataclass(frozen=True, slots=True)
class BedrockSessionUpdateRejection:
code: Literal["invalid_type", "invalid_value", "session_update_not_applied"]
message: str
param: str
@dataclasses.dataclass(frozen=True, slots=True)
class BedrockLockedSessionSettings:
"""The session settings Bedrock Nova Sonic locks in once the prompt starts."""
@ -533,7 +526,7 @@ class BedrockRealtimeConfig(BaseRealtimeConfig):
max_tokens=max_tokens,
)
def session_update_rejection(self, json_message: Mapping[str, JsonValue]) -> BedrockSessionUpdateRejection | None:
def session_update_rejection(self, json_message: Mapping[str, JsonValue]) -> RealtimeSessionUpdateRejection | None:
"""The error for a session.update this session cannot apply, or None when it applies."""
session_config: Final = json_message.get("session")
if not isinstance(session_config, Mapping):
@ -541,7 +534,7 @@ class BedrockRealtimeConfig(BaseRealtimeConfig):
malformed: Final = () if self.drop_params else _malformed_settings(session_config)
if malformed:
name, expected = malformed[0]
return BedrockSessionUpdateRejection(
return RealtimeSessionUpdateRejection(
code="invalid_type",
message=(
f"Invalid type for 'session.{name}': Bedrock Nova Sonic needs {expected} there. "
@ -551,7 +544,7 @@ class BedrockRealtimeConfig(BaseRealtimeConfig):
)
unplayable_voice: Final = None if self.drop_params else _unplayable_voice(session_config)
if unplayable_voice is not None:
return BedrockSessionUpdateRejection(
return RealtimeSessionUpdateRejection(
code="invalid_value",
message=(
f"Invalid value for 'session.voice': '{unplayable_voice}' is not a Bedrock Nova Sonic voice. "
@ -562,7 +555,7 @@ class BedrockRealtimeConfig(BaseRealtimeConfig):
if not self.prompt_started:
return None
if self._apply_locked_settings(session_config, self._started_settings) != self._started_settings:
return BedrockSessionUpdateRejection(
return RealtimeSessionUpdateRejection(
code="session_update_not_applied",
message="Bedrock Nova Sonic cannot change tools, instructions, voice, output audio format, temperature or max output tokens after the session has started, so this session.update was not applied",
param="session",

8
litellm/llms/chatgpt/authenticator.py
View file

@ -122,6 +122,12 @@ class Authenticator:
return tokens["access_token"]
def get_account_id(self, access_token: str | None = None) -> str | None:
return self._account_id(access_token, persist_derived=True)
def read_account_id(self, access_token: str | None = None) -> str | None:
return self._account_id(access_token, persist_derived=False)
def _account_id(self, access_token: str | None, persist_derived: bool) -> str | None:
auth_data: Final = self._read_auth_file()
if access_token and (not auth_data or auth_data.get("access_token") != access_token):
return extract_chatgpt_account_id(access_token)
@ -133,7 +139,7 @@ class Authenticator:
id_token: Final = auth_data.get("id_token")
saved_access_token: Final = auth_data.get("access_token")
derived: Final = extract_chatgpt_account_id(_optional_str(id_token or saved_access_token))
if derived:
if derived and persist_derived:
self._write_auth_file({**auth_data, "account_id": derived})
return derived

4
litellm/llms/chatgpt/responses/transformation.py
View file

@ -11,6 +11,7 @@ from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response impo
safe_convert_created_field,
)
from litellm.litellm_core_utils.url_utils import join_api_base
from litellm.llms.base_llm.responses.transformation import STANDARD_MODE_REASONING_FIELD_VALUES, ReasoningFieldValues
from litellm.llms.custom_httpx.sse_lines import split_sse_lines
from litellm.llms.openai.common_utils import OpenAIError
from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig
@ -295,3 +296,6 @@ class ChatGPTResponsesAPIConfig(OpenAIResponsesAPIConfig):
def supports_native_websocket(self) -> bool:
"""ChatGPT does not support native WebSocket for Responses API"""
return False
def reasoning_field_values(self) -> ReasoningFieldValues:
return STANDARD_MODE_REASONING_FIELD_VALUES

15
litellm/llms/custom_httpx/llm_http_handler.py
View file

@ -249,7 +249,7 @@ if TYPE_CHECKING:
FakeAnthropicMessagesStreamIterator,
)
from litellm.llms.base_llm.passthrough.transformation import BasePassthroughConfig
from litellm.responses.streaming_iterator import ResponsesWsFrameGuard
from litellm.responses.streaming_iterator import ResponsesWsFrameGuard, ResponsesWsTurnAdmission
from litellm.types.google_genai.main import GenerateContentResponse
from litellm.types.llms.openai_evals import (
CancelEvalResponse,
@ -6973,14 +6973,19 @@ class BaseLLMHTTPHandler:
),
)
# Auto-send session setup if the provider requires it (e.g.
# Gemini/Vertex AI Live needs a `setup` before any realtime_input).
# Auto-send session setup at connect unless the provider builds it from the
# client's first session.update (Gemini/Vertex AI Live accept exactly one
# setup, so RealTimeStreaming sends theirs on the first client message).
# Build the streaming handler first so a transcription guardrail's
# auto-response disable can be folded into this one setup: Gemini
# rejects a second setup, so a follow-up disable would be dropped
# and the guardrail bypassed.
_session_config: str | None = None
if provider_config.requires_session_configuration():
sends_setup_at_connect: Final = (
provider_config.requires_session_configuration()
and not provider_config.builds_setup_from_session_update()
)
if sends_setup_at_connect:
_session_config = provider_config.session_configuration_request(model)
if _session_config:
_session_config = realtime_streaming.maybe_inject_guardrail_auto_response_disable(
@ -7264,7 +7269,7 @@ class BaseLLMHTTPHandler:
custom_llm_provider: str | None = None,
first_message: str | None = None,
request_defaults: ResponsesWebSocketRequestDefaults | None = None,
turn_admission: Callable[[], Awaitable[None]] | None = None,
turn_admission: "ResponsesWsTurnAdmission | None" = None,
frame_guard: "ResponsesWsFrameGuard | None" = None,
**kwargs: object,
) -> Exception | None:

13
litellm/llms/deepseek/chat/transformation.py
View file

@ -6,10 +6,12 @@ from collections.abc import Coroutine, Mapping, Sequence
from typing import Final, Literal, cast, overload
import litellm
from litellm.litellm_core_utils.core_helpers import as_str_mapping
from litellm.litellm_core_utils.prompt_templates.common_utils import (
convert_content_list_to_str,
extract_search_results_text,
)
from litellm.litellm_core_utils.reasoning_effort_utils import OPAQUE_VALUE, warn_dropped_reasoning
from litellm.litellm_core_utils.url_utils import ensure_api_route
from litellm.secret_managers.main import get_secret_str
from litellm.types.llms.openai import AllMessageValues
@ -18,6 +20,16 @@ from litellm.utils import supports_reasoning, supports_vision
from ...openai.chat.gpt_transformation import OpenAIGPTConfig
SUPPORTED_REASONING_EFFORTS: Final = ("minimal", "low", "medium", "high", "xhigh", "max", "ultra")
_NO_BUDGET_REASON: Final = "DeepSeek takes thinking only as enabled or disabled, with no budget."
def _warn_on_dropped_budget(thinking: object, model: str) -> None:
fields: Final = as_str_mapping(thinking)
if fields is None or fields.get("type") != "enabled" or "budget_tokens" not in fields:
return
warn_dropped_reasoning(
"thinking.budget_tokens", fields["budget_tokens"], model, _NO_BUDGET_REASON, drop_params_gated=False
)
class DeepSeekChatConfig(OpenAIGPTConfig):
@ -53,6 +65,7 @@ class DeepSeekChatConfig(OpenAIGPTConfig):
# Then re-add only if valid for DeepSeek
thinking_value: Final = optional_params.pop("thinking", None)
reasoning_effort: Final = optional_params.pop("reasoning_effort", None)
_warn_on_dropped_budget(OPAQUE_VALUE.validate_python(thinking_value), model)
# Handle thinking parameter - accept both enabled and disabled, ignore budget_tokens
if isinstance(thinking_value, dict) and thinking_value.get("type") in ("enabled", "disabled"):

13
litellm/llms/fal_ai/chat/transformation.py
View file

@ -12,7 +12,9 @@ from typing import TYPE_CHECKING, Final
import httpx
from pydantic import ConfigDict, TypeAdapter
import litellm
from litellm.litellm_core_utils.core_helpers import map_finish_reason
from litellm.litellm_core_utils.reasoning_effort_utils import warn_dropped_reasoning
from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException
from litellm.secret_managers.main import get_secret_str
from litellm.types.llms.base import LiteLLMBaseModel
@ -29,6 +31,10 @@ PROVIDER_PREFIX: Final[str] = "fal_ai/"
PASSTHROUGH_PARAMS: Final[frozenset[str]] = frozenset(("reasoning", "temperature", "top_p"))
REASONING_DISABLED_EFFORTS: Final[frozenset[str]] = frozenset(("none", "minimal"))
REASONING_ENABLED_EFFORTS: Final[frozenset[str]] = frozenset(("low", "medium", "high"))
_UNSUPPORTED_EFFORT_REASON: Final = (
f"fal only turns reasoning off for {sorted(REASONING_DISABLED_EFFORTS)} "
f"and on for {sorted(REASONING_ENABLED_EFFORTS)}."
)
class _FalUsage(LiteLLMBaseModel):
@ -120,9 +126,10 @@ class FalAIChatConfig(BaseConfig):
return False
if isinstance(value, str) and value in REASONING_ENABLED_EFFORTS:
return True
if drop_params:
return None
raise FalAIError(status_code=400, message=f"Unsupported reasoning_effort {value!r} for {model}")
if not (drop_params or litellm.drop_params is True):
raise FalAIError(status_code=400, message=f"Unsupported reasoning_effort {value!r} for {model}")
warn_dropped_reasoning("reasoning_effort", value, model, _UNSUPPORTED_EFFORT_REASON, drop_params_gated=True)
return None
def _translate_param(self, param: str, value: object, model: str, drop_params: bool) -> tuple[str, object] | None:
if param in ("temperature", "top_p"):

14
litellm/llms/fireworks_ai/chat/transformation.py
View file

@ -17,6 +17,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
strip_litellm_response_only_fields,
unpack_legacy_defs,
)
from litellm.litellm_core_utils.reasoning_effort_utils import OPAQUE_VALUE, warn_dropped_reasoning
from litellm.litellm_core_utils.url_utils import join_api_base
from litellm.secret_managers.main import get_secret_str
from litellm.types.llms.openai import (
@ -97,6 +98,7 @@ def _json_schema_response_format(schema: object, name: str) -> Mapping[str, obje
EFFORT_KWARG_KEYS: Final = frozenset({"enable_thinking", "thinking", "reasoning_budget", "low_effort"})
_NOT_A_REASONING_MODEL_REASON: Final = "litellm's model map does not list it as a reasoning model."
def _bool_from_kwargs(kwargs: Mapping[str, object], keys: tuple[str, ...]) -> bool | None:
@ -107,6 +109,13 @@ def _bool_from_kwargs(kwargs: Mapping[str, object], keys: tuple[str, ...]) -> bo
return None
def warn_dropped_effort_kwargs(chat_template_kwargs: object, effort: object, model: str) -> None:
if effort != "none":
warn_dropped_reasoning(
"chat_template_kwargs", chat_template_kwargs, model, _NOT_A_REASONING_MODEL_REASON, drop_params_gated=False
)
def effort_from_chat_template_kwargs(kwargs: Mapping[str, object]) -> object:
enable_thinking: Final = _bool_from_kwargs(kwargs, ("enable_thinking", "thinking"))
if enable_thinking is False:
@ -438,10 +447,7 @@ class FireworksAIConfig(FireworksAIMixin, OpenAIGPTConfig):
if effort is None:
return ()
if not supports_reasoning(model=model, custom_llm_provider="fireworks_ai"):
verbose_logger.debug(
"fireworks_ai model %r does not support reasoning; dropping chat_template_kwargs effort keys.",
model,
)
warn_dropped_effort_kwargs(OPAQUE_VALUE.validate_python(chat_template_kwargs), effort, model)
return ()
return (("reasoning_effort", effort),)

7
litellm/llms/fireworks_ai/completion/transformation.py
View file

@ -2,6 +2,7 @@ from collections.abc import Mapping
from typing import Final
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.reasoning_effort_utils import OPAQUE_VALUE
from litellm.types.llms.openai import AllMessageValues, OpenAITextCompletionUserMessage
from litellm.utils import supports_reasoning
@ -12,6 +13,7 @@ from ..chat.transformation import (
NIM_VLLM_STRIP_PARAMS,
FireworksAIConfig,
effort_from_chat_template_kwargs,
warn_dropped_effort_kwargs,
)
from ..common_utils import FireworksAIMixin, resolve_fireworks_resource_name
@ -129,10 +131,7 @@ class FireworksAITextCompletionConfig(FireworksAIMixin, BaseTextCompletionConfig
)
return result
if not supports_reasoning(model=model, custom_llm_provider="fireworks_ai"):
verbose_logger.debug(
"fireworks_ai model %r does not support reasoning; dropping chat_template_kwargs effort keys.",
model,
)
warn_dropped_effort_kwargs(OPAQUE_VALUE.validate_python(chat_template_kwargs), effort, model)
return result
return {**result, "reasoning_effort": effort}

10
litellm/llms/fireworks_ai/responses/transformation.py
View file

@ -6,6 +6,10 @@ from urllib.parse import unquote
import httpx
from litellm.litellm_core_utils.url_utils import join_api_base
from litellm.llms.base_llm.responses.transformation import (
STANDARD_MODE_WITHOUT_CONTEXT_REASONING_FIELD_VALUES,
ReasoningFieldValues,
)
from litellm.llms.fireworks_ai.common_utils import (
FIREWORKS_FORWARD_USER_ID_PARAM,
get_fireworks_forwarded_user_id,
@ -129,3 +133,9 @@ class FireworksAIResponsesAPIConfig(OpenAIResponsesAPIConfig):
def supports_native_file_search(self) -> bool:
return False
def reasoning_field_values(self) -> ReasoningFieldValues:
return STANDARD_MODE_WITHOUT_CONTEXT_REASONING_FIELD_VALUES
def ignores_unsupported_reasoning_fields(self) -> bool:
return True

317
litellm/llms/gemini/realtime/transformation.py
View file

@ -7,6 +7,7 @@ from collections import OrderedDict
from collections.abc import Mapping, Sequence
from typing import Final, cast
from pydantic import TypeAdapter
from typing_extensions import ReadOnly, Required, TypedDict
import litellm
@ -15,8 +16,9 @@ from litellm._uuid import uuid
from litellm.litellm_core_utils.core_helpers import as_str_mapping
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.litellm_core_utils.url_utils import join_api_base
from litellm.llms.base_llm.realtime.transformation import BaseRealtimeConfig
from litellm.llms.base_llm.realtime.transformation import BaseRealtimeConfig, RealtimeSessionUpdateRejection
from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import (
OPENAI_TO_GEMINI_TTS_VOICES,
VertexGeminiConfig,
)
from litellm.responses.litellm_completion_transformation.transformation import (
@ -76,27 +78,82 @@ MAP_GEMINI_FIELD_TO_OPENAI_EVENT: Final[dict[str, OpenAIRealtimeEventTypes | Res
# Keys the main transform loop handles; siblings like ``usageMetadata`` are skipped.
_KNOWN_GEMINI_TOP_LEVEL_KEYS: Final[set] = {map_key.split(".", 1)[0] for map_key in MAP_GEMINI_FIELD_TO_OPENAI_EVENT}
_GEMINI_LIVE_RESPONSE_MODALITIES: Final = frozenset({"TEXT", "AUDIO"})
OPENAI_STOCK_REALTIME_VOICES: Final[frozenset[str]] = frozenset(
{"alloy", "ash", "ballad", "cedar", "coral", "echo", "marin", "sage", "shimmer", "verse"}
)
# Per https://docs.cloud.google.com/vertex-ai/generative-ai/docs/live-api/configure-language-voice (2026-10-28)
GEMINI_LIVE_DOCUMENTED_VOICES: Final[frozenset[str]] = frozenset(
{
"achernar",
"achird",
"algenib",
"algieba",
"alnilam",
"aoede",
"autonoe",
"callirrhoe",
"charon",
"despina",
"enceladus",
"erinome",
"fenrir",
"gacrux",
"iapetus",
"kore",
"laomedeia",
"leda",
"orus",
"puck",
"pulcherrima",
"rasalgethi",
"sadachbia",
"sadaltager",
"schedar",
"sulafat",
"umbriel",
"vindemiatrix",
"zephyr",
"zubenelgenubi",
}
)
def _gemini_live_speech_config(voice: object) -> Mapping[str, object] | None:
def _is_undocumented_live_voice(voice: object) -> bool:
return (
isinstance(voice, str)
and voice.lower() not in OPENAI_STOCK_REALTIME_VOICES
and OPENAI_TO_GEMINI_TTS_VOICES.get(voice.lower(), voice).lower() not in GEMINI_LIVE_DOCUMENTED_VOICES
)
def _gemini_live_speech_config(voice: object, drop_undocumented: bool) -> Mapping[str, object] | None:
"""Build the Gemini Live speechConfig for a client-requested voice.
OpenAI stock voice names have no Gemini equivalent and Gemini Live closes
the session on an unknown voice, so they are dropped with a warning and
the model keeps its default voice. Every other name is forwarded verbatim.
OpenAI stock voice names have no Gemini equivalent, so they are always
dropped, and so is any name outside Google's documented Live voices when
``drop_undocumented`` is set, keeping the model's default voice. Every
other name is forwarded verbatim.
"""
if isinstance(voice, str) and voice.lower() in OPENAI_STOCK_REALTIME_VOICES:
if not isinstance(voice, str):
return VertexGeminiConfig().map_audio_params({"voice": voice})
if voice.lower() in OPENAI_STOCK_REALTIME_VOICES:
verbose_logger.warning(
"Gemini Realtime: voice %s is an OpenAI voice with no Gemini equivalent; "
"dropping it so the session keeps the model's default voice.",
voice,
)
return None
if drop_undocumented and _is_undocumented_live_voice(voice):
verbose_logger.warning(
"Gemini Realtime: voice %s is not a documented Gemini Live voice; "
"drop_params is set, so it is dropped and the session keeps the model's default voice.",
voice,
)
return None
return VertexGeminiConfig().map_audio_params({"voice": voice})
@ -142,16 +199,34 @@ def _base64_decoded_byte_count(data: str) -> int:
return max(len(data) * 3 // 4 - padding, 0)
_JSON_ARRAY: Final = TypeAdapter(tuple[object, ...])
def _chat_completion_tool(tool: object) -> dict[str, object]:
fields: Final = as_str_mapping(tool) or {}
if fields.get("type") != "function" or "function" in fields:
return {**fields}
return {"type": "function", "function": {key: value for key, value in fields.items() if key != "type"}}
def _system_instruction_text(system_instruction: HttpxContentType | None) -> str | None:
parts: Final = system_instruction.get("parts", []) if system_instruction else []
texts: Final = tuple(text for part in parts if isinstance(text := part.get("text"), str))
return "\n".join(texts) if texts else None
class GeminiRealtimeConfig(BaseRealtimeConfig):
_TOOL_CALL_ID_TO_NAME_MAX = 256 # LRU cap for call_id→name mapping
def __init__(self):
def __init__(self, drop_params: bool = False):
super().__init__()
self.drop_params = drop_params
self._tool_call_id_to_name: OrderedDict[str, str] = OrderedDict()
# Gemini Live sometimes emits usageMetadata in a standalone frame between
# turns; buffer it here so the next response.done carries the token counts.
self._pending_usage_metadata: dict | None = None
self._unbilled_input_audio_bytes: int = 0
self._manual_activity_open: bool = False
def is_setup_message(self, msg_obj: dict) -> bool:
return "setup" in msg_obj
@ -167,6 +242,10 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
"""Google AI Studio closes the socket (1007) on padding or silence tuning next to ``disabled: true``."""
return True
def _rejects_undocumented_voices(self, model: str) -> bool:
"""Google AI Studio closes the socket (1007) on an unknown voice only for its native-audio Live models."""
return bool(GeminiRealtimeConfig._model_cost_entry(model).get("gemini_native_audio"))
@staticmethod
def _usage_detail_alias(details: object, defaults: dict[str, int]) -> dict[str, object]:
detail_mapping: Final = as_str_mapping(details)
@ -272,17 +351,26 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
except (json.JSONDecodeError, TypeError, AttributeError):
return False
def _start_manual_activity(self, session_configuration_request: str | None) -> tuple[str, ...]:
if self._manual_activity_open or not self._manual_turn_detection_enabled(session_configuration_request):
return ()
self._manual_activity_open = True
activity_start: Final[BidiGenerateContentRealtimeInput] = {"activityStart": {}}
return (json.dumps({"realtimeInput": activity_start}),)
def _end_manual_activity(self) -> tuple[str, ...]:
if not self._manual_activity_open:
return ()
self._manual_activity_open = False
activity_end: Final[BidiGenerateContentRealtimeInput] = {"activityEnd": {}}
return (json.dumps({"realtimeInput": activity_end}),)
def _handle_input_audio_buffer_commit_or_end(self, session_configuration_request: str | None) -> list[str]:
"""Map OpenAI buffer commit/end to Gemini Live turn-boundary signals."""
if self._manual_turn_detection_enabled(session_configuration_request):
realtime_input_dict: BidiGenerateContentRealtimeInput = {
"activityEnd": True,
}
verbose_logger.debug("Gemini Realtime: Sending activityEnd realtimeInput to backend")
else:
realtime_input_dict = {"audioStreamEnd": True}
verbose_logger.debug("Gemini Realtime: Sending audioStreamEnd realtimeInput to backend")
return [json.dumps({"realtimeInput": realtime_input_dict})]
return list(self._end_manual_activity())
audio_stream_end: Final[BidiGenerateContentRealtimeInput] = {"audioStreamEnd": True}
return [json.dumps({"realtimeInput": audio_stream_end})]
def map_automatic_turn_detection(self, value: OpenAIRealtimeTurnDetection) -> AutomaticActivityDetection:
"""Map OpenAI ``server_vad`` to Gemini ``automaticActivityDetection``.
@ -325,7 +413,7 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
"voice",
]
def map_openai_params(self, optional_params: dict, non_default_params: dict) -> dict:
def map_openai_params(self, optional_params: dict, non_default_params: dict, *, model: str) -> dict:
if "generationConfig" not in optional_params:
optional_params["generationConfig"] = {}
for key, value in non_default_params.items():
@ -333,12 +421,12 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
optional_params["systemInstruction"] = HttpxContentType(role="user", parts=[{"text": value}])
elif key == "temperature":
optional_params["generationConfig"]["temperature"] = value
elif key == "max_response_output_tokens":
elif key == "max_response_output_tokens" and isinstance(value, int):
optional_params["generationConfig"]["maxOutputTokens"] = value
elif key == "modalities":
optional_params["generationConfig"]["responseModalities"] = [
modality.upper() for modality in cast(list[str], value)
]
optional_params["generationConfig"]["responseModalities"] = list(
GeminiRealtimeConfig._requested_response_modalities(cast(object, value))
)
elif key == "tools":
from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import (
VertexGeminiConfig,
@ -347,34 +435,43 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
vertex_gemini_config = VertexGeminiConfig()
# Tools should be at the top level of setup, not inside generationConfig
optional_params["tools"] = vertex_gemini_config.map_function(
value=value, optional_params=optional_params
value=[_chat_completion_tool(tool) for tool in _JSON_ARRAY.validate_python(value)],
optional_params=optional_params,
)
elif key == "input_audio_transcription" and value is not None:
optional_params["inputAudioTranscription"] = {}
elif key == "turn_detection" and value is not None:
value_typed = cast(OpenAIRealtimeTurnDetection, value)
if (
isinstance(value_typed, dict)
and value_typed.get("type") == "semantic_vad"
and "create_response" not in value_typed
):
# Pipecat/OpenAI GA semantic VAD — skip; Gemini uses its own VAD.
# Only skip when there is no create_response override so that
# a guardrail-injected create_response:false is not dropped.
continue
transformed_audio_activity_config = self.map_automatic_turn_detection(value_typed)
if transformed_audio_activity_config:
optional_params["realtimeInputConfig"] = BidiGenerateContentRealtimeInputConfig(
automaticActivityDetection=transformed_audio_activity_config
)
elif key == "turn_detection":
realtime_input_config = self._realtime_input_config(cast(object, value))
if realtime_input_config is not None:
optional_params["realtimeInputConfig"] = realtime_input_config
elif key == "voice":
speech_config = _gemini_live_speech_config(value)
speech_config = _gemini_live_speech_config(
value, self.drop_params and self._rejects_undocumented_voices(model)
)
if speech_config:
optional_params["generationConfig"]["speechConfig"] = speech_config
if len(optional_params["generationConfig"]) == 0:
optional_params.pop("generationConfig")
return optional_params
def _realtime_input_config(self, turn_detection: object) -> BidiGenerateContentRealtimeInputConfig | None:
if turn_detection is None:
return BidiGenerateContentRealtimeInputConfig(automaticActivityDetection={"disabled": True})
value_typed: Final = cast(OpenAIRealtimeTurnDetection, turn_detection)
if (
isinstance(value_typed, dict)
and value_typed.get("type") == "semantic_vad"
and "create_response" not in value_typed
):
# Pipecat/OpenAI GA semantic VAD — skip; Gemini uses its own VAD.
# Only skip when there is no create_response override so that
# a guardrail-injected create_response:false is not dropped.
return None
transformed_audio_activity_config: Final = self.map_automatic_turn_detection(value_typed)
if not transformed_audio_activity_config:
return None
return BidiGenerateContentRealtimeInputConfig(automaticActivityDetection=transformed_audio_activity_config)
@staticmethod
def _extract_turn_detection(session: dict) -> dict | None:
"""Extract turn_detection from a session.update payload.
@ -423,6 +520,8 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
if isinstance(input_cfg, dict):
if "input_audio_transcription" not in normalized and "transcription" in input_cfg:
normalized["input_audio_transcription"] = input_cfg["transcription"]
if "turn_detection" not in normalized and "turn_detection" in input_cfg:
normalized["turn_detection"] = input_cfg["turn_detection"]
output_cfg: Final = audio.get("output")
if isinstance(output_cfg, dict) and output_cfg.get("voice"):
normalized["voice"] = output_cfg["voice"]
@ -434,7 +533,7 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
return normalized
@staticmethod
def _model_cost_entry(model: str) -> dict:
def _model_cost_entry(model: str) -> Mapping[str, object]:
entry = litellm.model_cost.get(model)
if entry is None:
stripped: Final = model.split("/", 1)[-1]
@ -455,18 +554,122 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
return "TEXT" if GeminiRealtimeConfig._is_text_only_live_model(model) else "AUDIO"
@staticmethod
def _coerce_response_modalities(model: str, modalities: Sequence[object]) -> tuple[str, ...]:
"""Swap responseModalities a Live model cannot produce: TEXT to AUDIO for
audio-only models, AUDIO to TEXT for text-only ones (e.g. transcribe-live)."""
normalized: Final = tuple(
modality.upper() if isinstance(modality, str) else str(modality).upper() for modality in modalities
def _requested_response_modalities(value: object) -> tuple[str, ...]:
if isinstance(value, str):
return (value.upper(),)
if not isinstance(value, (list, tuple)):
return ()
return tuple(str(modality).upper() for modality in _JSON_ARRAY.validate_python(value))
@staticmethod
def _unsupported_response_modalities(model: str, requested: tuple[str, ...]) -> tuple[str, ...]:
unknown: Final = tuple(modality for modality in requested if modality not in _GEMINI_LIVE_RESPONSE_MODALITIES)
if unknown:
return unknown
if GeminiRealtimeConfig._is_audio_only_live_model(model) and frozenset(requested) == frozenset(("TEXT",)):
return ("TEXT",)
if GeminiRealtimeConfig._is_text_only_live_model(model) and frozenset(requested) == frozenset(("AUDIO",)):
return ("AUDIO",)
return ()
def setup_session_update_rejection(
self, model: str, session: Mapping[str, object], *, setup_sent: bool
) -> RealtimeSessionUpdateRejection | None:
if self.drop_params:
return None
if setup_sent:
return RealtimeSessionUpdateRejection(
code="session_update_not_applied",
message=(
"Gemini Live accepts one session configuration per connection, and this session already sent "
"its configuration, so this session.update was not applied. Put every setting in session.update "
"before the first conversation content, or set drop_params to drop it instead"
),
param="session",
)
return self._modality_rejection(model, session) or self._voice_rejection(model, session)
@staticmethod
def _modality_rejection(model: str, session: Mapping[str, object]) -> RealtimeSessionUpdateRejection | None:
param: Final = "output_modalities" if "output_modalities" in session else "modalities"
unsupported: Final = GeminiRealtimeConfig._unsupported_response_modalities(
model, GeminiRealtimeConfig._requested_response_modalities(session.get(param))
)
if not unsupported:
return None
return RealtimeSessionUpdateRejection(
code="invalid_value",
message=(
f"Invalid value for 'session.{param}': {model} cannot produce "
f"{', '.join(modality.lower() for modality in unsupported)} output. Set drop_params to drop it instead"
),
param=f"session.{param}",
)
def _voice_rejection(self, model: str, session: Mapping[str, object]) -> RealtimeSessionUpdateRejection | None:
audio_output: Final = as_str_mapping((as_str_mapping(session.get("audio")) or {}).get("output")) or {}
output_voice: Final = audio_output.get("voice")
voice: Final = output_voice or session.get("voice")
if not (self._rejects_undocumented_voices(model) and _is_undocumented_live_voice(voice)):
return None
param: Final = "session.audio.output.voice" if output_voice else "session.voice"
return RealtimeSessionUpdateRejection(
code="invalid_value",
message=(
f"Invalid value for '{param}': {voice} is not a Gemini Live voice {model} offers. "
"Set drop_params to drop it instead"
),
param=param,
)
@staticmethod
def _coerce_response_modalities(model: str, modalities: Sequence[object]) -> tuple[str, ...]:
"""Drop modalities Gemini Live cannot produce at all, swap the ones this
model cannot (TEXT to AUDIO for audio-only models, AUDIO to TEXT for
text-only ones such as transcribe-live), then keep a single modality
because Gemini Live rejects a setup naming two."""
return GeminiRealtimeConfig._single_response_modality(
GeminiRealtimeConfig._swap_unsupported_response_modalities(
model, GeminiRealtimeConfig._known_response_modalities(model, modalities)
)
)
@staticmethod
def _known_response_modalities(model: str, modalities: Sequence[object]) -> tuple[str, ...]:
normalized: Final = tuple(str(modality).upper() for modality in modalities)
known: Final = tuple(modality for modality in normalized if modality in _GEMINI_LIVE_RESPONSE_MODALITIES)
if len(known) < len(normalized):
verbose_logger.warning(
"Gemini Live: %s only produces TEXT or AUDIO; ignoring the requested %s",
model,
", ".join(modality for modality in normalized if modality not in _GEMINI_LIVE_RESPONSE_MODALITIES),
)
return known or (GeminiRealtimeConfig._default_response_modality(model),)
@staticmethod
def _swap_unsupported_response_modalities(model: str, normalized: tuple[str, ...]) -> tuple[str, ...]:
if GeminiRealtimeConfig._is_audio_only_live_model(model) and "TEXT" in normalized:
verbose_logger.warning(
"Gemini Live: %s only produces audio; downgrading the requested TEXT response modality to AUDIO",
model,
)
return tuple(modality for modality in normalized if modality != "TEXT") or ("AUDIO",)
if GeminiRealtimeConfig._is_text_only_live_model(model) and "AUDIO" in normalized:
return tuple(modality for modality in normalized if modality != "AUDIO") or ("TEXT",)
return normalized
@staticmethod
def _single_response_modality(modalities: tuple[str, ...]) -> tuple[str, ...]:
if len(modalities) <= 1:
return modalities
chosen: Final = "AUDIO" if "AUDIO" in modalities else modalities[0]
verbose_logger.warning(
"Gemini Live: at most one response modality is allowed per session; using %s for the requested %s",
chosen,
", ".join(modalities),
)
return (chosen,)
@staticmethod
def _finalize_gemini_live_setup(model: str, setup: dict[str, object]) -> dict[str, object]:
generation_config: Final = setup.get("generationConfig")
@ -504,7 +707,9 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
# would be silently dropped because ``map_openai_params`` only
# recognises the flat OpenAI-beta key names.
session_payload = self._normalize_session_payload_for_mapping(session_payload)
new_overrides: Final = self.map_openai_params(optional_params={}, non_default_params=session_payload)
new_overrides: Final = self.map_openai_params(
optional_params={}, non_default_params=session_payload, model=model
)
if session_configuration_request is None:
generation_config: Final = new_overrides.setdefault("generationConfig", {})
@ -618,14 +823,13 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
message_str = str(message)
raise ValueError(f"Invalid JSON message: {message_str}")
messages: Final[list[str]] = []
msg_type: Final = json_message.get("type")
if msg_type == "session.update":
return self._handle_session_update(json_message, model, session_configuration_request)
if msg_type == "response.create":
return [] # Gemini responds automatically; nothing to forward
return list(self._end_manual_activity())
if msg_type == "conversation.item.create":
return self._handle_conversation_item(json_message)
@ -643,8 +847,7 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
gemini_msg: Final = json.dumps({"realtimeInput": realtime_input_dict})
verbose_logger.debug("Gemini Realtime: Sending audio realtimeInput to backend")
messages.append(gemini_msg)
return messages
return [*self._start_manual_activity(session_configuration_request), gemini_msg]
if msg_type in ("input_audio_buffer.commit", "input_audio_buffer.end"):
return self._handle_input_audio_buffer_commit_or_end(session_configuration_request)
@ -660,22 +863,21 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
logging_session_id: str,
session_configuration_request: str | None = None,
) -> OpenAIRealtimeStreamSessionEvents:
if session_configuration_request:
session_configuration_request_dict: BidiGenerateContentSetup = _parse_setup(session_configuration_request)
else:
session_configuration_request_dict = {}
session_configuration_request_dict: Final = _parse_setup(
session_configuration_request or self.session_configuration_request(model)
)
_model: Final = session_configuration_request_dict.get("model") or model
generation_config: Final = session_configuration_request_dict.get("generationConfig", {}) or {}
gemini_modalities: Final = generation_config.get("responseModalities", ["AUDIO"])
_modalities: Final = [modality.lower() for modality in cast(list[str], gemini_modalities)]
_system_instruction: Final = session_configuration_request_dict.get("systemInstruction")
instructions: Final = _system_instruction_text(session_configuration_request_dict.get("systemInstruction"))
session: Final = OpenAIRealtimeStreamSession(
id=logging_session_id,
modalities=_modalities,
)
if _system_instruction is not None and isinstance(_system_instruction, str):
session["instructions"] = _system_instruction
if instructions is not None:
session["instructions"] = instructions
if _model is not None and isinstance(_model, str):
# Strip Vertex/AI Studio path prefixes to expose the bare model name.
if "/models/" in _model:
@ -1662,6 +1864,9 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
# Deferred setup opt-in: litellm.gemini_live_defer_setup = True
return not litellm.gemini_live_defer_setup
def builds_setup_from_session_update(self) -> bool:
return True
def session_configuration_request(self, model: str) -> str:
"""

7
litellm/llms/github_copilot/responses/transformation.py
View file

@ -24,6 +24,7 @@ from litellm._logging import verbose_logger
from litellm.constants import DEFAULT_MAX_RECURSE_DEPTH
from litellm.exceptions import AuthenticationError
from litellm.litellm_core_utils.url_utils import join_api_base
from litellm.llms.base_llm.responses.transformation import STANDARD_MODE_REASONING_FIELD_VALUES, ReasoningFieldValues
from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig
from litellm.types.llms.openai import (
ResponseInputParam,
@ -437,6 +438,12 @@ class GithubCopilotResponsesAPIConfig(OpenAIResponsesAPIConfig):
return False
def reasoning_field_values(self) -> ReasoningFieldValues:
return STANDARD_MODE_REASONING_FIELD_VALUES
def ignores_unsupported_reasoning_fields(self) -> bool:
return True
def supports_native_websocket(self) -> bool:
"""GitHub Copilot does not support native WebSocket for Responses API"""
return False

2
litellm/llms/hosted_vllm/chat/transformation.py
View file

@ -99,7 +99,7 @@ class HostedVLLMChatConfig(OpenAIGPTConfig):
thinking_effort: Final = (
None
if "reasoning_effort" in non_default_params
else reasoning_effort_from_thinking(thinking, drop_params=drop_params)
else reasoning_effort_from_thinking(thinking, model=model, drop_params=drop_params)
)
if thinking_effort is not None:
non_default_params["reasoning_effort"] = thinking_effort

4
litellm/llms/litellm_proxy/responses/transformation.py
View file

@ -12,6 +12,7 @@ from pydantic import TypeAdapter, ValidationError
from litellm.litellm_core_utils.url_utils import join_api_base
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.llms.base_llm.responses.transformation import ANY_REASONING_FIELD_VALUES, ReasoningFieldValues
from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig
from litellm.secret_managers.main import get_secret_str
from litellm.types.utils import LlmProviders
@ -42,6 +43,9 @@ class LiteLLMProxyResponsesAPIConfig(OpenAIResponsesAPIConfig):
def custom_llm_provider(self) -> LlmProviders:
return LlmProviders.LITELLM_PROXY
def reasoning_field_values(self) -> ReasoningFieldValues:
return ANY_REASONING_FIELD_VALUES
def get_complete_url(
self,
api_base: str | None,

176
litellm/llms/mistral/chat/transformation.py
View file

@ -9,17 +9,23 @@ Docs - https://docs.mistral.ai/api/
from collections.abc import AsyncIterator, Coroutine, Iterator, Mapping, Sequence
from itertools import accumulate, count
from typing import TYPE_CHECKING, Final, Literal, cast, get_type_hints, overload
from urllib.parse import urlparse
import httpx
from pydantic import TypeAdapter
from typing_extensions import TypeIs # noqa: TID251 # TypeIs reaches typing only on 3.13
import litellm
from litellm._logging import verbose_logger
from litellm.exceptions import UnsupportedParamsError
from litellm.litellm_core_utils.core_helpers import as_str_mapping
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.litellm_core_utils.prompt_templates.common_utils import (
convert_content_list_to_str,
strip_litellm_internal_message_fields,
strip_none_values_from_message,
)
from litellm.litellm_core_utils.reasoning_effort_utils import OPAQUE_VALUE, warn_dropped_reasoning
from litellm.litellm_core_utils.url_utils import ensure_api_route
from litellm.llms.openai.chat.gpt_transformation import (
OpenAIChatCompletionStreamingHandler,
@ -30,7 +36,15 @@ from litellm.router_utils.reasoning_effort_capability import (
nearest_declared_reasoning_effort,
)
from litellm.secret_managers.main import get_secret_str
from litellm.types.llms.mistral import MistralTextBlock, MistralThinkingBlock, MistralToolCallMessage
from litellm.types.llms.mistral import (
MistralCacheKeyParams,
MistralFilePart,
MistralReasoningEffortParams,
MistralRequestRouting,
MistralTextBlock,
MistralThinkingBlock,
MistralToolCallMessage,
)
from litellm.types.llms.openai import AllMessageValues, ChatCompletionAssistantMessage, ChatCompletionTextObject
from litellm.types.utils import ModelResponse, ModelResponseStream
from litellm.utils import convert_to_model_response_object, supports_reasoning
@ -57,6 +71,33 @@ def _accepted_reasoning_effort(model: str, requested: str, custom_llm_provider:
_NO_TOOL_CALL_TURN: Final[tuple[int, frozenset[str]]] = (-1, frozenset())
_MISTRAL_REQUEST_ROUTING: Final = TypeAdapter(MistralRequestRouting)
_MISTRAL_CACHE_KEY_PARAMS: Final = TypeAdapter(MistralCacheKeyParams)
_MISTRAL_REASONING_EFFORT_PARAMS: Final = TypeAdapter(MistralReasoningEffortParams)
_MISTRAL_FILE_PART: Final = TypeAdapter(MistralFilePart)
_REASONING_PROMPT_REASON: Final = "litellm sends Magistral a reasoning system prompt instead of the requested level."
_NON_LEVEL_EFFORT_REASON: Final = (
"the Mistral API takes it only as a level such as 'high', so Mistral applies its default effort."
)
_DOCUMENTED_MAGISTRAL_EFFORTS: Final = frozenset({"low", "medium", "high"})
_REASONING_ON_THINKING_TYPES: Final = frozenset({"enabled", "adaptive"})
def _reasoning_level_and_summary(reasoning_effort: object) -> tuple[object, object]:
requested: Final = as_str_mapping(reasoning_effort)
if requested is None:
return reasoning_effort, None
return requested.get("effort"), requested.get("summary")
def _warn_unless_reasoning_prompt_honors(param: str, value: object, model: str) -> None:
if param == "reasoning_effort" and isinstance(value, str) and value in _DOCUMENTED_MAGISTRAL_EFFORTS:
return
thinking: Final = as_str_mapping(value)
thinking_type: Final = thinking.get("type") if thinking is not None else None
if param == "thinking" and isinstance(thinking_type, str) and thinking_type in _REASONING_ON_THINKING_TYPES:
return
warn_dropped_reasoning(param, value, model, _REASONING_PROMPT_REASON, drop_params_gated=False)
def _is_json_array(value: object) -> TypeIs[Sequence[object]]: # guard-ok: a decoded JSON array is a list
@ -88,6 +129,8 @@ class MistralConfig(OpenAIGPTConfig):
- `response_format` (object or null): An object specifying the format that the model must output. Setting to { "type": "json_object" } enables JSON mode, which guarantees the message the model generates is in JSON. When using JSON mode you MUST also instruct the model to produce JSON yourself with a system or a user message.
"""
_is_base_class = False
temperature: int | None = None
top_p: int | None = None
max_tokens: int | None = None
@ -131,6 +174,7 @@ class MistralConfig(OpenAIGPTConfig):
def get_supported_openai_params(self, model: str) -> list[str]:
is_magistral: Final = "magistral" in model.lower()
accepts_reasoning_effort: Final = self._accepts_reasoning(model)
accepts_prompt_cache_key: Final = self.custom_llm_provider == "mistral"
return [
"stream",
"temperature",
@ -145,6 +189,7 @@ class MistralConfig(OpenAIGPTConfig):
"parallel_tool_calls",
*(("thinking",) if is_magistral else ()),
*(("reasoning_effort",) if accepts_reasoning_effort else ()),
*(("prompt_cache_key",) if accepts_prompt_cache_key else ()),
]
def _map_tool_choice(self, tool_choice: str) -> str:
@ -211,15 +256,90 @@ class MistralConfig(OpenAIGPTConfig):
optional_params["extra_body"] = {"random_seed": value}
if param == "response_format":
optional_params["response_format"] = value
if param == "reasoning_effort" and "magistral" not in model.lower():
optional_params["reasoning_effort"] = _accepted_reasoning_effort(model, value, self.custom_llm_provider)
if (
param == "reasoning_effort"
and "magistral" not in model.lower()
and (
reasoning_level := self._reasoning_level_to_send(
model, _MISTRAL_REASONING_EFFORT_PARAMS.validate_python(non_default_params), drop_params
)
)
is not None
):
optional_params["reasoning_effort"] = reasoning_level
if param in ("reasoning_effort", "thinking") and "magistral" in model.lower():
# Flag that we need to add reasoning system prompt
optional_params["_add_reasoning_prompt"] = True
if param == "parallel_tool_calls":
optional_params["parallel_tool_calls"] = value
_warn_unless_reasoning_prompt_honors(param, OPAQUE_VALUE.validate_python(value), model)
if param in ("parallel_tool_calls", "prompt_cache_key"):
optional_params[param] = value
return optional_params
def _reasoning_level_to_send(
self, model: str, requested: MistralReasoningEffortParams, drop_params: bool
) -> str | None:
requested_effort: Final = requested.get("reasoning_effort")
level, summary = _reasoning_level_and_summary(requested_effort)
sendable: Final = (
_accepted_reasoning_effort(model, level, self.custom_llm_provider) if isinstance(level, str) else None
)
if sendable is not None and summary is None:
return sendable
if drop_params or litellm.drop_params is True:
if sendable is None and level is not None:
warn_dropped_reasoning(
"reasoning_effort", requested_effort, model, _NON_LEVEL_EFFORT_REASON, drop_params_gated=True
)
return sendable
raise UnsupportedParamsError(
message=(
f"{self.custom_llm_provider} does not support parameters: ['reasoning_effort'], for model={model}, "
"since the Mistral API takes it only as a level such as 'high', with no reasoning summary. "
"To drop it, set `litellm.drop_params=True` or for proxy:\n\n`litellm_settings:\n drop_params: true`\n"
),
llm_provider=self.custom_llm_provider,
model=model,
)
def accepts_prompt_cache_key_at(self, api_base: str | None) -> bool:
resolved_api_base, _ = self._get_openai_compatible_provider_info(api_base, None)
hostname: Final = urlparse(resolved_api_base).hostname or ""
return hostname == "mistral.ai" or hostname.endswith(".mistral.ai")
@staticmethod
def takes_prompt_cache_key_value(prompt_cache_key: object) -> bool:
return isinstance(prompt_cache_key, str) and prompt_cache_key.isascii()
def _prompt_cache_key_rejection(self, prompt_cache_key: object, api_base: str | None) -> str | None:
if not self.accepts_prompt_cache_key_at(api_base):
return "since only the Mistral API (*.mistral.ai) takes it"
if not self.takes_prompt_cache_key_value(prompt_cache_key):
return "since the Mistral API takes it only as a string of ASCII characters"
return None
def _prompt_cache_key_withheld(self, model: str, prompt_cache_key: object, routing: MistralRequestRouting) -> bool:
requested_api_base: Final = routing.get("api_base")
api_base: Final = requested_api_base if isinstance(requested_api_base, str) else None
allowed_openai_params: Final = routing.get("allowed_openai_params")
if isinstance(allowed_openai_params, (list, tuple)) and "prompt_cache_key" in allowed_openai_params:
return False
rejection: Final = self._prompt_cache_key_rejection(prompt_cache_key, api_base)
if rejection is None:
return False
if litellm.drop_params is True or routing.get("drop_params") is True:
return True
requested_provider: Final = routing.get("custom_llm_provider")
provider: Final = requested_provider if isinstance(requested_provider, str) else self.custom_llm_provider
raise UnsupportedParamsError(
message=(
f"{provider} does not support parameters: ['prompt_cache_key'], for model={model}, {rejection}. "
"To send it anyway, pass `allowed_openai_params=['prompt_cache_key']`. To drop it, set "
"`litellm.drop_params=True` or for proxy:\n\n`litellm_settings:\n drop_params: true`\n"
),
llm_provider=provider,
model=model,
)
def _get_openai_compatible_provider_info(self, api_base: str | None, api_key: str | None) -> tuple[str, str | None]:
# mistral is openai compatible, we just need to set this to custom_openai and have the api_base be https://api.mistral.ai
api_base = (
@ -359,24 +479,31 @@ class MistralConfig(OpenAIGPTConfig):
messages = self._handle_message_with_file(messages)
return messages
@staticmethod
def _with_mistral_file_id(block: Mapping[str, object]) -> Mapping[str, object]:
file_part: Final = block.get("file") if block.get("type") == "file" else None
file_id: Final = (
_MISTRAL_FILE_PART.validate_python(file_part).get("file_id") if isinstance(file_part, Mapping) else None
)
if not file_id:
return block
return {**{key: value for key, value in block.items() if key != "file"}, "file_id": file_id}
@classmethod
def _with_mistral_file_ids(cls, message: AllMessageValues) -> AllMessageValues:
content: Final = message.get("content")
if not isinstance(content, list):
return message
blocks: Final[tuple[Mapping[str, object], ...]] = tuple(content)
return cast( # cast-ok: only file blocks change, to the file_id shape the Mistral API takes
AllMessageValues, {**message, "content": [cls._with_mistral_file_id(block) for block in blocks]}
)
def _handle_message_with_file(self, messages: list[AllMessageValues]) -> list[AllMessageValues]:
"""
Mistral API supports only 'file_id' in message content with type 'file'.
"""
for m in messages:
_content_block = m.get("content")
if _content_block and isinstance(_content_block, list):
if any(c.get("type") == "file" for c in _content_block):
# If file content is present, we get file_id from 'file' attribute of content block
# then replace 'file' with 'file_id' and assign the value of 'file_id' attribute to it.
file_contents = [c for c in _content_block if c.get("type") == "file"]
for file_content in file_contents:
file_id = file_content.get("file", {}).get("file_id")
if file_id:
# Replace 'file' with 'file_id'
file_content["file_id"] = file_id # pyright: ignore[reportGeneralTypeIssues] # legacy in-place rewrite of the block shape
file_content.pop("file", None)
return messages
return [self._with_mistral_file_ids(m) for m in messages]
def _add_reasoning_system_prompt_if_needed(
self, messages: list[AllMessageValues], optional_params: dict
@ -794,7 +921,16 @@ class MistralConfig(OpenAIGPTConfig):
if "magistral" in model.lower() and optional_params.get("_add_reasoning_prompt", False):
messages = self._add_reasoning_system_prompt_if_needed(messages, optional_params)
upstream_params: Final = {key: value for key, value in optional_params.items() if key != "client_metadata"}
cache_key_params: Final = _MISTRAL_CACHE_KEY_PARAMS.validate_python(optional_params)
withheld_params: Final = (
("client_metadata", "prompt_cache_key")
if "prompt_cache_key" in cache_key_params
and self._prompt_cache_key_withheld(
model, cache_key_params["prompt_cache_key"], _MISTRAL_REQUEST_ROUTING.validate_python(litellm_params)
)
else ("client_metadata",)
)
upstream_params: Final = {key: value for key, value in optional_params.items() if key not in withheld_params}
# Call parent transform_request which handles _transform_messages
return super().transform_request(

16
litellm/llms/oci/chat/transformation.py
View file

@ -21,6 +21,7 @@ from litellm._logging import verbose_logger
from litellm.constants import DEFAULT_OCI_CHAT_MAX_TOKENS
from litellm.litellm_core_utils.logging_utils import track_llm_api_timing
from litellm.litellm_core_utils.prompt_templates.common_utils import messages_with_file_parts_as_image_urls
from litellm.litellm_core_utils.reasoning_effort_utils import warn_dropped_reasoning
from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper
from litellm.litellm_core_utils.url_utils import join_api_base
from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException
@ -428,6 +429,13 @@ class OCIChatConfig(BaseConfig):
alias = param_map.get(key)
if key == "reasoning_effort" and alias and not _model_supports_reasoning_effort(model):
if drop_params or litellm.drop_params:
warn_dropped_reasoning(
key,
value,
model,
"the model map lists no reasoning_effort levels for it, so OCI would reject reasoningEffort.",
drop_params_gated=True,
)
continue
raise OCIError(
status_code=400,
@ -451,6 +459,14 @@ class OCIChatConfig(BaseConfig):
if key == "n" and (value is None or value == 1):
continue
if drop_params or litellm.drop_params:
if key == "reasoning_effort":
warn_dropped_reasoning(
key,
value,
model,
"OCI's Cohere chat API has no reasoning effort field.",
drop_params_gated=True,
)
continue
raise OCIError(
status_code=400,

8
litellm/llms/ollama/chat/transformation.py
View file

@ -14,6 +14,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
extract_images_from_message,
extract_reasoning_content,
)
from litellm.litellm_core_utils.reasoning_effort_utils import OPAQUE_VALUE
from litellm.litellm_core_utils.url_utils import ensure_api_route
from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator
from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException
@ -29,7 +30,7 @@ from litellm.types.llms.openai import (
)
from litellm.types.utils import ModelResponse, ModelResponseStream
from ..common_utils import OllamaError
from ..common_utils import OllamaError, ollama_think
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
@ -172,10 +173,7 @@ class OllamaChatConfig(BaseConfig):
if value.get("json_schema") and value["json_schema"].get("schema"):
optional_params["format"] = value["json_schema"]["schema"]
if param == "reasoning_effort" and value is not None:
if model.startswith("gpt-oss"):
optional_params["think"] = value
else:
optional_params["think"] = value in {"low", "medium", "high"}
optional_params["think"] = ollama_think(model, OPAQUE_VALUE.validate_python(value))
### FUNCTION CALLING LOGIC ###
# Ollama 0.4+ supports native tool calling - pass tools directly
# and let Ollama handle model capability detection

12
litellm/llms/ollama/common_utils.py
View file

@ -8,11 +8,14 @@ from pydantic import BaseModel, TypeAdapter
from litellm import verbose_logger
from litellm._uuid import uuid
from litellm.litellm_core_utils.reasoning_effort_utils import warn_dropped_reasoning
from litellm.litellm_core_utils.url_utils import join_api_base
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.types.utils import ModelResponse, ModelResponseStream
_PROVIDER_FIELDS: Final = TypeAdapter(dict[str, object])
_THINK_ON_EFFORTS: Final = frozenset({"low", "medium", "high"})
_THINK_OFF_REASON: Final = "Ollama takes think only as on or off, and litellm maps this level to off."
_DECODED_JSON: Final = TypeAdapter(object)
@ -296,3 +299,12 @@ class OllamaModelInfo(BaseLLMModelInfo):
Return the base model name for Ollama (no-op).
"""
return model
def ollama_think(model: str, reasoning_effort: object) -> object:
if model.startswith("gpt-oss"):
return reasoning_effort
think: Final = reasoning_effort in _THINK_ON_EFFORTS
if isinstance(reasoning_effort, str) and not think and reasoning_effort != "none":
warn_dropped_reasoning("reasoning_effort", reasoning_effort, model, _THINK_OFF_REASON, drop_params_gated=False)
return think

8
litellm/llms/ollama/completion/transformation.py
View file

@ -23,6 +23,7 @@ from litellm.litellm_core_utils.prompt_templates.image_handling import (
async_inline_remote_media,
inline_remote_image_urls,
)
from litellm.litellm_core_utils.reasoning_effort_utils import OPAQUE_VALUE
from litellm.litellm_core_utils.url_utils import ensure_api_route
from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator
from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException
@ -43,7 +44,7 @@ from litellm.types.utils import (
generate_id,
)
from ..common_utils import OllamaError, OllamaModelInfo, convert_image
from ..common_utils import OllamaError, OllamaModelInfo, convert_image, ollama_think
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
@ -249,10 +250,7 @@ class OllamaConfig(BaseConfig):
elif param == "stop":
optional_params["stop"] = value
elif param == "reasoning_effort" and value is not None:
if model.startswith("gpt-oss"):
optional_params["think"] = value
else:
optional_params["think"] = value in {"low", "medium", "high"}
optional_params["think"] = ollama_think(model, OPAQUE_VALUE.validate_python(value))
elif param == "response_format" and isinstance(value, dict):
if value["type"] == "json_object":
optional_params["format"] = "json"

22
litellm/llms/openai/common_utils.py
View file

@ -32,6 +32,7 @@ import litellm
from litellm.exceptions import error_response_without_credentials
from litellm.litellm_core_utils.completion_timeout import CompletionTimeout
from litellm.litellm_core_utils.core_helpers import as_str_mapping, decoded_json_object
from litellm.litellm_core_utils.reasoning_effort_utils import warn_dropped_reasoning_params
from litellm.litellm_core_utils.token_counter import token_counter, usage_tokenizer_model
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.llms.custom_httpx.http_handler import (
@ -165,17 +166,11 @@ _DECODED_JSON: Final = TypeAdapter(object)
def drop_params_from_unprocessable_entity_error(
e: openai.UnprocessableEntityError | httpx.HTTPStatusError,
data: Mapping[str, object],
*,
model: str | None,
drop_params_gated: bool,
) -> dict[str, object]:
"""
Helper function to read OpenAI UnprocessableEntityError and drop the params that raised an error from the error message.
Args:
e (UnprocessableEntityError): The UnprocessableEntityError exception
data (Dict[str, Any]): The original data dictionary containing all parameters
Returns:
Dict[str, Any]: A new dictionary with invalid parameters removed
"""
"""Return a copy of ``data`` without the params a 422 response names as invalid."""
invalid_params: Final[list[str]] = []
if isinstance(e, httpx.HTTPStatusError):
error_json: Final = decoded_json_object(_DECODED_JSON.validate_python(e.response.json()))
@ -203,6 +198,13 @@ def drop_params_from_unprocessable_entity_error(
invalid_params.append(error_dict["loc"][1])
new_data: Final = {k: v for k, v in data.items() if k not in invalid_params}
warn_dropped_reasoning_params(
invalid_params,
data,
model,
"the provider answered 422 naming it an invalid param.",
drop_params_gated=drop_params_gated,
)
return new_data

13
litellm/llms/openai/openai.py
View file

@ -27,6 +27,7 @@ import litellm
from litellm import LlmProviders
from litellm._logging import verbose_logger
from litellm.constants import DEFAULT_MAX_RETRIES
from litellm.exceptions import UnsupportedParamsError
from litellm.files.types import FileContentStreamingResult
from litellm.litellm_core_utils.core_helpers import set_provider_response_headers_in_hidden_params
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
@ -939,7 +940,9 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
except openai.UnprocessableEntityError as e:
## check if body contains unprocessable params - related issue https://github.com/BerriAI/litellm/issues/4800
if litellm.drop_params is True or drop_params is True:
inference_params = drop_params_from_unprocessable_entity_error(e, inference_params)
inference_params = drop_params_from_unprocessable_entity_error(
e, inference_params, model=model, drop_params_gated=True
)
else:
raise e
# e.message
@ -971,8 +974,8 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
litellm.remove_index_from_tool_calls(messages=messages)
else:
raise e
except OpenAIError as e:
raise e
except (OpenAIError, UnsupportedParamsError):
raise
except Exception as e:
status_code: Final = getattr(e, "status_code", 500)
error_headers = getattr(e, "headers", None)
@ -1096,7 +1099,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
except openai.UnprocessableEntityError as e:
## check if body contains unprocessable params - related issue https://github.com/BerriAI/litellm/issues/4800
if litellm.drop_params is True or drop_params is True:
data = drop_params_from_unprocessable_entity_error(e, data)
data = drop_params_from_unprocessable_entity_error(e, data, model=model, drop_params_gated=True)
else:
raise e
# e.message
@ -1251,7 +1254,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
except openai.UnprocessableEntityError as e:
## check if body contains unprocessable params - related issue https://github.com/BerriAI/litellm/issues/4800
if litellm.drop_params is True or drop_params is True:
data = drop_params_from_unprocessable_entity_error(e, data)
data = drop_params_from_unprocessable_entity_error(e, data, model=model, drop_params_gated=True)
else:
raise e
except (

8
litellm/llms/openrouter/chat/transformation.py
View file

@ -13,6 +13,7 @@ from typing import TYPE_CHECKING, Final, cast
import httpx
from pydantic import ConfigDict, TypeAdapter
from litellm.litellm_core_utils.core_helpers import finite_cost
from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR, set_hidden_params
from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator
from litellm.llms.base_llm.chat.transformation import BaseLLMException
@ -225,8 +226,11 @@ class OpenrouterConfig(OpenAIGPTConfig):
additional_headers: Final = cast( # cast-ok: preserve mapping operations on response metadata
dict[str, object], hidden_params["additional_headers"]
)
if isinstance(response_cost, str | int | float):
additional_headers["llm_provider-x-litellm-response-cost"] = float(response_cost)
reported_cost: Final = (
finite_cost(float(response_cost)) if isinstance(response_cost, str | int | float) else None
)
if reported_cost is not None:
additional_headers["llm_provider-x-litellm-response-cost"] = reported_cost
except Exception:
# If we can't extract cost, continue without it - don't fail the response
pass

6
litellm/llms/pass_through/typesafe_logging_handler.py
View file

@ -6,6 +6,7 @@ import httpx
from pydantic import TypeAdapter, ValidationError
import litellm
from litellm.litellm_core_utils.core_helpers import finite_reported_cost
from litellm.litellm_core_utils.decisions_pricing import apply_decisions_pricing
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.litellm_core_utils.litellm_logging import (
@ -100,9 +101,10 @@ class TypeSafePassthroughLoggingHandler:
f"{custom_llm_provider}/{model}" for model in candidate_models if model is not None
)
pricing: Final = _pricing_for(candidate_model_keys)
reported_cost: Final = finite_reported_cost(usage.cost) if custom_llm_provider == "openrouter" else None
response_cost: Final = (
usage.cost
if custom_llm_provider == "openrouter" and usage.cost is not None
reported_cost
if reported_cost is not None
else input_tokens * pricing.input_cost_per_token + output_tokens * pricing.output_cost_per_token
)
usage_object: Final = Usage(

8
litellm/llms/perplexity/cost_calculator.py
View file

@ -6,6 +6,7 @@ Helper util for handling perplexity-specific cost calculation
from datetime import datetime
from typing import Final
from litellm.litellm_core_utils.core_helpers import finite_reported_cost
from litellm.litellm_core_utils.llm_cost_calc.utils import TokenRates, apply_off_peak_pricing
from litellm.types.utils import Usage
from litellm.utils import get_model_info
@ -34,10 +35,9 @@ def cost_per_token(model: str, usage: Usage, current_time: datetime | None = Non
total_cost = cost_info.get("total_cost")
elif isinstance(cost_info, (int, float)) and not isinstance(cost_info, bool):
total_cost = float(cost_info)
if total_cost is not None:
# Return total cost as completion_cost (prompt_cost=0) since Perplexity
# doesn't break down by input/output in their cost object
return (0.0, float(total_cost))
reported_cost: Final = None if total_cost is None else finite_reported_cost(float(total_cost))
if reported_cost is not None:
return (0.0, reported_cost)
## FALLBACK: Calculate cost manually if Perplexity doesn't provide it
## GET MODEL INFO

11
litellm/llms/perplexity/responses/transformation.py
View file

@ -9,6 +9,7 @@ The only provider quirks:
Ref: https://docs.perplexity.ai/api-reference/responses-post
"""
from types import MappingProxyType
from typing import Any, Final
import httpx
@ -17,6 +18,7 @@ from pydantic import TypeAdapter
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.litellm_core_utils.url_utils import join_api_base
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.llms.base_llm.responses.transformation import STANDARD_MODE_REASONING_FIELD_VALUES, ReasoningFieldValues
from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig, drops_params
from litellm.secret_managers.main import get_secret_str
from litellm.types.llms.openai import ResponseInputParam, ResponsesAPIResponse
@ -24,6 +26,9 @@ from litellm.types.router import GenericLiteLLMParams
from litellm.types.utils import LlmProviders
_DECODED_JSON: Final = TypeAdapter(object)
PERPLEXITY_REASONING_FIELD_VALUES: Final[ReasoningFieldValues] = MappingProxyType(
{**STANDARD_MODE_REASONING_FIELD_VALUES, "context": frozenset({"auto", "current_turn", "all_turns"})}
)
class PerplexityResponsesConfig(OpenAIResponsesAPIConfig):
@ -127,3 +132,9 @@ class PerplexityResponsesConfig(OpenAIResponsesAPIConfig):
def supports_native_websocket(self) -> bool:
"""Perplexity does not support native WebSocket for Responses API"""
return False
def reasoning_field_values(self) -> ReasoningFieldValues:
return PERPLEXITY_REASONING_FIELD_VALUES
def ignores_unsupported_reasoning_fields(self) -> bool:
return True

14
litellm/llms/sagemaker/chat/transformation.py
View file

@ -28,7 +28,7 @@ from litellm.types.llms.openai import AllMessageValues
from litellm.types.utils import LlmProviders
from ...openai.chat.gpt_transformation import OpenAIGPTConfig
from ..common_utils import AWSEventStreamDecoder, SagemakerError
from ..common_utils import INFERENCE_COMPONENT_HEADER, OBJECT_KEYED_MAPPING, AWSEventStreamDecoder, SagemakerError
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
@ -59,7 +59,12 @@ class SagemakerChatConfig(OpenAIGPTConfig, BaseAWSLLM):
inference_component_name: Final = optional_params.get("model_id")
if not isinstance(inference_component_name, str):
return headers
return {**headers, "X-Amzn-SageMaker-Inference-Component": inference_component_name}
other_headers: Final = {
name: value
for name, value in OBJECT_KEYED_MAPPING.validate_python(headers).items()
if not isinstance(name, str) or name.lower() != INFERENCE_COMPONENT_HEADER.lower()
}
return {**other_headers, INFERENCE_COMPONENT_HEADER: inference_component_name}
def transform_request(
self,
@ -76,9 +81,8 @@ class SagemakerChatConfig(OpenAIGPTConfig, BaseAWSLLM):
litellm_params=litellm_params,
headers=headers,
)
container_request: Final = {
key: value for key, value in request.items() if key not in self.aws_authentication_params
}
header_only_params: Final = frozenset((*self.aws_authentication_params, "model_id"))
container_request: Final = {key: value for key, value in request.items() if key not in header_only_params}
served_model_name: Final = litellm_params.get("hf_model_name")
if not isinstance(served_model_name, str):
return container_request

33
litellm/llms/sagemaker/common_utils.py
View file

@ -14,9 +14,10 @@ from typing_extensions import ReadOnly, Required, TypedDict
from litellm import verbose_logger
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.types.utils import GenericStreamingChunk as GChunk
from litellm.types.utils import StreamingChatCompletionChunk
from litellm.types.utils import LlmProviders, StreamingChatCompletionChunk
STR_KEYED_MAPPING: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(hide_input_in_errors=True))
OBJECT_KEYED_MAPPING: Final = TypeAdapter(Mapping[object, object], config=ConfigDict(hide_input_in_errors=True))
@with_config(ConfigDict(extra="allow", strict=True, hide_input_in_errors=True))
@ -167,6 +168,36 @@ def get_sagemaker_response_stream_shape():
return _load_sagemaker_response_stream_shape()
INFERENCE_COMPONENT_HEADER: Final = "X-Amzn-SageMaker-Inference-Component"
_SAGEMAKER_PROVIDERS: Final = frozenset(
provider.value for provider in (LlmProviders.SAGEMAKER, LlmProviders.SAGEMAKER_CHAT, LlmProviders.SAGEMAKER_NOVA)
)
def _deployment_provider(litellm_params: Mapping[str, object]) -> str | None:
custom_llm_provider: Final = litellm_params.get("custom_llm_provider")
if isinstance(custom_llm_provider, str) and custom_llm_provider:
return custom_llm_provider
model: Final = litellm_params.get("model")
return model.split("/", 1)[0] if isinstance(model, str) else None
def deployment_inference_component(litellm_params: Mapping[str, object]) -> str | None:
headers: Final = litellm_params.get("extra_headers")
if _deployment_provider(litellm_params) not in _SAGEMAKER_PROVIDERS or not isinstance(headers, Mapping):
return None
return next(
(
value
for name, value in OBJECT_KEYED_MAPPING.validate_python(headers).items()
if isinstance(name, str) and name.lower() == INFERENCE_COMPONENT_HEADER.lower() and isinstance(value, str)
),
None,
)
def with_inference_component_hint(error_message: str) -> str:
if "Inference Component Name header is required" in error_message:
return f"{error_message}\n pass in via `litellm.embedding(..., model_id={{InferenceComponentName}})`"

6
litellm/llms/sagemaker/completion/transformation.py
View file

@ -23,7 +23,7 @@ from litellm.types.llms.openai import AllMessageValues
from litellm.types.utils import ModelResponse, Usage
from litellm.utils import token_counter
from ..common_utils import SagemakerError
from ..common_utils import STR_KEYED_MAPPING, SagemakerError
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
@ -153,7 +153,9 @@ class SagemakerConfig(BaseConfig):
litellm_params: dict,
headers: dict,
) -> dict:
inference_params: Final = optional_params.copy()
inference_params: Final = {
key: value for key, value in STR_KEYED_MAPPING.validate_python(optional_params).items() if key != "model_id"
}
stream: Final = inference_params.pop("stream", False)
data: Final[dict] = {"parameters": inference_params}
if stream is True:

37
litellm/llms/subscription_accounts.py Normal file
View file

@ -0,0 +1,37 @@
"""
Providers whose requests are covered by a flat subscription, each with the way to read
the account id of the login a deployment is signed into.
"""
from collections.abc import Callable, Mapping
from types import MappingProxyType
from typing import Final
from litellm.llms.chatgpt.authenticator import Authenticator, get_cached_authenticator, get_chatgpt_auth_file
from litellm.secret_managers.main import UnresolvableSecretReference, resolve_local_oidc_reference
from litellm.utils import with_named_credential
def _chatgpt_account_id(litellm_params: Mapping[str, object]) -> str | None:
params: Final = with_named_credential(litellm_params)
auth_file: Final = get_chatgpt_auth_file(params)
if auth_file:
return get_cached_authenticator(auth_file).read_account_id()
api_key: Final = params.get("api_key")
if not isinstance(api_key, str) or not api_key:
return Authenticator().read_account_id()
match resolve_local_oidc_reference(api_key):
case UnresolvableSecretReference():
return None
case str() as access_token:
return Authenticator().read_account_id(access_token)
SUBSCRIPTION_ACCOUNT_ID_RESOLVERS: Final[Mapping[str, Callable[[Mapping[str, object]], str | None]]] = MappingProxyType(
{"chatgpt": _chatgpt_account_id}
)
def resolve_subscription_account_id(custom_llm_provider: str, litellm_params: Mapping[str, object]) -> str | None:
resolver: Final = SUBSCRIPTION_ACCOUNT_ID_RESOLVERS.get(custom_llm_provider)
return None if resolver is None else resolver(litellm_params)

7
litellm/llms/tencent/chat/transformation.py
View file

@ -9,12 +9,15 @@ from typing import Final, TypedDict
from typing_extensions import ReadOnly
import litellm
from litellm.litellm_core_utils.reasoning_effort_utils import warn_dropped_reasoning
from litellm.litellm_core_utils.url_utils import ensure_api_route, join_api_base
from litellm.secret_managers.main import get_secret_str
from litellm.utils import supports_reasoning
from ...openai.chat.gpt_transformation import OpenAIGPTConfig
_ADAPTIVE_BUDGET_REASON: Final = "it is not an integer, so litellm sends adaptive thinking with no budget."
class ThinkingPayload(TypedDict, total=False):
"""Tencent TokenHub `thinking` object.
@ -102,6 +105,10 @@ class TencentChatConfig(OpenAIGPTConfig):
if isinstance(budget, int):
coerced_with_budget: Final[ThinkingPayload] = {"type": "adaptive", "budget_tokens": budget}
return coerced_with_budget
if budget is not None:
warn_dropped_reasoning(
"thinking.budget_tokens", budget, model, _ADAPTIVE_BUDGET_REASON, drop_params_gated=False
)
coerced: Final[ThinkingPayload] = {"type": "adaptive"}
return coerced

2
litellm/llms/vertex_ai/common_utils.py
View file

@ -1257,7 +1257,7 @@ def get_vertex_model_id_from_url(url: str) -> str | None:
`https://${LOCATION}-aiplatform.googleapis.com/v1/projects/${PROJECT_ID}/locations/${LOCATION}/publishers/google/models/${MODEL_ID}:streamGenerateContent`
"""
match: Final = re.search(r"/models/([^:]+)", url)
match: Final = re.search(r"/models/([^:?#]+)", url)
return match.group(1) if match else None

26
litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py
View file

@ -34,6 +34,7 @@ from litellm.litellm_core_utils.json_fragment_accumulator import JSONFragmentAcc
from litellm.litellm_core_utils.prompt_templates.factory import (
encode_tool_call_id_with_signature,
)
from litellm.litellm_core_utils.reasoning_effort_utils import warn_dropped_reasoning
from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException
from litellm.llms.custom_httpx.http_handler import (
AsyncHTTPHandler,
@ -201,6 +202,13 @@ def _unsupported_reasoning_effort(reasoning_effort: str) -> UnsupportedParamsErr
_SAMPLING_DEPRECATION_UNMAPPED_MODELS: Final = "models outside litellm's model map"
_NON_INTEGER_BUDGET_REASON: Final = (
"Gemini takes budget_tokens only as an integer, so it uses its default thinking budget."
)
_LEVEL_ONLY_BUDGET_REASON: Final = (
"Gemini 3 and later set reasoning by thinking level, not budget, so litellm does not send the budget. "
"Send reasoning_effort to pick a level."
)
_sampling_deprecation_lock: Final = threading.Lock()
@ -1466,6 +1474,14 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
params["includeThoughts"] = False
else:
params["includeThoughts"] = sends_thoughts
if thinking_budget is not None:
warn_dropped_reasoning(
"thinking.budget_tokens",
thinking_budget,
model,
_LEVEL_ONLY_BUDGET_REASON,
drop_params_gated=False,
)
# Follow provider defaults unless explicitly opted into legacy behavior.
if litellm.enable_gemini_default_thinking_level_low is True:
params["thinkingLevel"] = (
@ -1478,8 +1494,16 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
# For older Gemini models, use thinkingBudget
if thinking_enabled and not VertexGeminiConfig._is_thinking_budget_zero(thinking_budget):
params["includeThoughts"] = sends_thoughts
if thinking_budget is not None and isinstance(thinking_budget, int):
if isinstance(thinking_budget, int):
params["thinkingBudget"] = thinking_budget
elif thinking_budget is not None:
warn_dropped_reasoning(
"thinking.budget_tokens",
thinking_budget,
model,
_NON_INTEGER_BUDGET_REASON,
drop_params_gated=False,
)
return params

17
litellm/llms/vertex_ai/realtime/transformation.py
View file

@ -39,8 +39,8 @@ class VertexAIRealtimeConfig(GeminiRealtimeConfig):
(they require async I/O) and injected at construction time.
"""
def __init__(self, access_token: str, project: str, location: str) -> None:
super().__init__()
def __init__(self, access_token: str, project: str, location: str, drop_params: bool = False) -> None:
super().__init__(drop_params=drop_params)
self._access_token = access_token
self._project = project
self._location = location
@ -51,6 +51,9 @@ class VertexAIRealtimeConfig(GeminiRealtimeConfig):
def _rejects_vad_tuning_with_detection_off(self) -> bool:
return False
def _rejects_undocumented_voices(self, model: str) -> bool:
return True
# ------------------------------------------------------------------
# URL
# ------------------------------------------------------------------
@ -163,7 +166,7 @@ class VertexAIRealtimeConfig(GeminiRealtimeConfig):
# settings would be silently dropped because ``map_openai_params`` only
# recognises the flat OpenAI-beta key names.
session_params = self._normalize_session_payload_for_mapping(session_params)
setup_config: Final = self.map_openai_params(optional_params={}, non_default_params=session_params)
setup_config: Final = self.map_openai_params(optional_params={}, non_default_params=session_params, model=model)
# Use full Vertex AI model path
setup_config["model"] = self._vertex_model_path(model)
@ -177,15 +180,16 @@ class VertexAIRealtimeConfig(GeminiRealtimeConfig):
# ``silence_duration_ms``). ``map_automatic_turn_detection`` sets
# ``disabled=True`` whenever ``create_response`` is absent or
# ``False``. Force ``disabled=False`` only when the client did
# not explicitly request ``create_response: False`` — that path
# not explicitly request ``create_response: False`` or
# ``turn_detection: null`` — the first of those paths
# is how transcription guardrails suppress automatic responses,
# and overriding it here would silently bypass the guardrail.
# Vertex Live has no "VAD on, no auto-response" mode, so callers
# that need that behaviour must accept that VAD is off.
client_turn_detection: Final = session_params.get("turn_detection")
client_disabled_auto_response: Final = (
isinstance(client_turn_detection, dict) and client_turn_detection.get("create_response") is False
)
"turn_detection" in session_params and client_turn_detection is None
) or (isinstance(client_turn_detection, dict) and client_turn_detection.get("create_response") is False)
realtime_input_config: Final = setup_config.setdefault("realtimeInputConfig", {})
automatic_detection: Final = realtime_input_config.setdefault("automaticActivityDetection", {})
if not client_disabled_auto_response:
@ -261,4 +265,5 @@ def vertex_realtime_config(
access_token=access_token,
project=project,
location=VertexBase.get_vertex_region(vertex_region=location, model=model),
drop_params=drop_params,
)

53
litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/messages/transformation.py
View file

@ -1,9 +1,15 @@
from collections.abc import Mapping, Sequence
from typing import Final
from pydantic import TypeAdapter
from litellm.exceptions import UnsupportedParamsError
from litellm.litellm_core_utils.core_helpers import drops_unsupported_params
from litellm.llms.anthropic.common_utils import AnthropicModelInfo
from litellm.llms.anthropic.common_utils import (
AnthropicModelInfo,
context_management_betas,
requires_native_compaction_beta,
)
from litellm.llms.anthropic.messages.native.transformation import (
AnthropicMessagesConfig,
messages_carry_output_config,
@ -11,6 +17,7 @@ from litellm.llms.anthropic.messages.native.transformation import (
from litellm.types.llms.anthropic import (
ANTHROPIC_BETA_HEADER_VALUES,
ANTHROPIC_HOSTED_TOOLS,
ANTHROPIC_MID_CONVERSATION_TOOL_CHANGES_BETA_HEADER,
)
from litellm.types.llms.anthropic_tool_search import get_tool_search_beta_header
from litellm.types.llms.vertex_ai import VertexPartnerProvider
@ -19,6 +26,8 @@ from litellm.types.router import GenericLiteLLMParams
from ....vertex_llm_base import VertexBase
from ..output_params_utils import sanitize_vertex_anthropic_output_params
_OPTIONAL_PARAMS: Final = TypeAdapter(Mapping[str, object])
_VERSION_HEADER_VERTEX_READS_FROM_BODY: Final = "anthropic-version"
_VERTEX_DEFAULT_VERSION: Final = "vertex-2023-10-16"
_VERSIONS_SERVED_AS_VERTEX_DEFAULT: Final = frozenset({"", "2023-01-01", "2023-06-01"})
@ -171,28 +180,7 @@ class VertexAIPartnerModelsAnthropicMessagesConfig(AnthropicMessagesConfig, Vert
if existing_beta:
beta_values.update(b.strip() for b in existing_beta.split(","))
# Check for context management
context_management_param: Final = optional_params.get("context_management")
if context_management_param is not None:
# Check edits array for compact_20260112 type
edits: Final = context_management_param.get("edits", [])
has_compact = False
has_other = False
for edit in edits:
edit_type = edit.get("type", "")
if edit_type == "compact_20260112":
has_compact = True
else:
has_other = True
# Add compact header if any compact edits exist
if has_compact:
beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_01_12.value)
# Add context management header if any other edits exist
if has_other:
beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.CONTEXT_MANAGEMENT_2025_06_27.value)
beta_values.update(context_management_betas(optional_params.get("context_management")))
# Check for web search tool
for tool in tools:
@ -211,14 +199,23 @@ class VertexAIPartnerModelsAnthropicMessagesConfig(AnthropicMessagesConfig, Vert
if messages_carry_output_config(messages):
beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.PER_TURN_CONTROL_2026_07_01.value)
beta_values.update(
typed_params: Final = _OPTIONAL_PARAMS.validate_python(optional_params)
if requires_native_compaction_beta("vertex_ai", typed_params, messages):
beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_09_04.value)
tool_change_betas: Final = (
(ANTHROPIC_MID_CONVERSATION_TOOL_CHANGES_BETA_HEADER,)
if anthropic_model_info.is_mid_conversation_tool_change_used(messages)
else ()
)
all_beta_values: Final = beta_values.union(
anthropic_model_info.thinking_betas(
optional_params.get("thinking"), # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # optional_params is the base config's bare dict
)
),
tool_change_betas,
)
if beta_values:
request_headers["anthropic-beta"] = ",".join(beta_values)
if all_beta_values:
request_headers["anthropic-beta"] = ",".join(sorted(all_beta_values))
return request_headers, api_base

82
litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/transformation.py
View file

@ -2,19 +2,23 @@
## Handler file for calling claude-3 on vertex ai
import base64
import json
from collections.abc import Mapping
from collections.abc import Mapping, Sequence
from typing import TYPE_CHECKING, Final
import httpx
from pydantic import TypeAdapter
from pydantic import TypeAdapter, ValidationError
import litellm
from litellm.anthropic_beta_headers_manager import get_header_only_betas
from litellm.litellm_core_utils.core_helpers import drops_unsupported_params
from litellm.litellm_core_utils.prompt_templates.image_handling import RemoteMedia, inline_remote_image_urls
from litellm.llms.base_llm.chat.transformation import LiteLLMLoggingObj
from litellm.types.llms.anthropic import ANTHROPIC_BETA_HEADER_VALUES
from litellm.types.llms.openai import AllMessageValues
from litellm.types.utils import ModelResponse
from ....anthropic.chat.transformation import AnthropicConfig
from ....anthropic.common_utils import AnthropicError, requires_native_compaction_beta
from .output_params_utils import sanitize_vertex_anthropic_output_params
if TYPE_CHECKING:
@ -36,6 +40,32 @@ def _vertex_labels_header(labels: object, header_names: Mapping[str, object]) ->
return base64.b64encode(json.dumps(string_labels).encode()).decode() if string_labels else None
_BETA_VALUE: Final[TypeAdapter[str | tuple[str, ...] | None]] = TypeAdapter(str | tuple[str, ...] | None)
_REQUEST_MESSAGES: Final = TypeAdapter(Sequence[object])
def _beta_names(value: object) -> frozenset[str] | None:
try:
parsed: Final = _BETA_VALUE.validate_python(value)
except ValidationError:
return None
names: Final = parsed.split(",") if isinstance(parsed, str) else parsed or ()
return frozenset(name.strip() for name in names if name.strip())
def _client_betas(value: object, field: str, drop_params: bool) -> frozenset[str]:
names: Final = _beta_names(value)
if names is not None:
return names
if drop_params:
return frozenset()
raise AnthropicError(
status_code=400,
message=f"{field} must be a string or a list of strings, got {type(value).__name__}",
)
class VertexAIError(Exception):
def __init__(self, status_code, message):
self.status_code = status_code
@ -157,21 +187,43 @@ class VertexAIAnthropicConfig(AnthropicConfig):
if context_management:
self._add_context_management_beta_headers(beta_set, context_management)
extra_headers: Final = optional_params.get("extra_headers") or {}
anthropic_beta_value: Final = extra_headers.get("anthropic-beta", "")
if isinstance(anthropic_beta_value, str) and anthropic_beta_value:
for beta in anthropic_beta_value.split(","):
beta = beta.strip()
if beta:
beta_set.add(beta)
elif isinstance(anthropic_beta_value, list):
beta_set.update(anthropic_beta_value)
request_params: Final = _NON_DEFAULT_PARAMS.validate_python(optional_params)
compaction_betas: Final = (
frozenset({ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_09_04.value})
if requires_native_compaction_beta(
"vertex_ai", request_params, _REQUEST_MESSAGES.validate_python(data["messages"])
)
else frozenset[str]()
)
drop_params: Final = drops_unsupported_params(litellm_params.get("drop_params"))
extra_headers: Final = _NON_DEFAULT_PARAMS.validate_python(request_params.get("extra_headers") or {})
data.pop("extra_headers", None)
injected_betas: Final = frozenset(beta_set) | _client_betas(
extra_headers.get("anthropic-beta"), "The anthropic-beta header", drop_params
)
client_body_betas: Final = _client_betas(
_NON_DEFAULT_PARAMS.validate_python(data).get("anthropic_beta"), "anthropic_beta", drop_params
)
if beta_set:
data["anthropic_beta"] = list(beta_set)
headers["anthropic-beta"] = ",".join(beta_set)
requested_betas: Final = injected_betas | client_body_betas
header_only_betas: Final = get_header_only_betas("vertex_ai") if requested_betas else frozenset[str]()
body_betas: Final = requested_betas - header_only_betas
if body_betas:
data["anthropic_beta"] = sorted(body_betas)
elif "anthropic_beta" in data:
del data["anthropic_beta"]
header_betas: Final = (
injected_betas
| compaction_betas
| (client_body_betas & header_only_betas)
| _client_betas(
_NON_DEFAULT_PARAMS.validate_python(headers).get("anthropic-beta"),
"The anthropic-beta header",
drop_params,
)
)
if header_betas:
headers["anthropic-beta"] = ",".join(sorted(header_betas))
return data

25
litellm/llms/vertex_ai/vertex_ai_partner_models/count_tokens/handler.py
View file

@ -6,12 +6,19 @@ Unlike Gemini models which use Google's token counting API, partner models use
their respective publisher-specific count-tokens endpoints.
"""
from collections.abc import Mapping
from collections.abc import Mapping, Sequence
from typing import Any, Final
from pydantic import TypeAdapter
from litellm.litellm_core_utils.core_helpers import drops_unsupported_params
from litellm.litellm_core_utils.url_utils import join_api_base
from litellm.llms.anthropic.common_utils import OBJECT_LIST_ADAPTER, is_claude_model_target, tools_without_custom_field
from litellm.llms.anthropic.common_utils import (
OBJECT_LIST_ADAPTER,
count_tokens_feature_betas,
is_claude_model_target,
tools_without_custom_field,
)
from litellm.llms.custom_httpx.http_handler import get_async_httpx_client
from litellm.llms.vertex_ai.common_utils import get_vertex_base_url
from litellm.llms.vertex_ai.vertex_llm_base import VertexBase
@ -23,6 +30,9 @@ _COUNT_TOKENS_PUBLISHERS: Final = (
)
_REQUEST_MESSAGES: Final = TypeAdapter(Sequence[object])
def count_tokens_request_body(
request_data: Mapping[str, object], model: str, drop_params: object
) -> Mapping[str, object]:
@ -43,6 +53,14 @@ def count_tokens_publisher(model: str) -> str | None:
return next((publisher for marker, publisher in _COUNT_TOKENS_PUBLISHERS if marker in model), None)
def count_tokens_headers(access_token: str, request_data: Mapping[str, object]) -> Mapping[str, str]:
authorization: Final = {"Authorization": f"Bearer {access_token}"}
betas: Final = count_tokens_feature_betas(
"vertex_ai", request_data, _REQUEST_MESSAGES.validate_python(request_data["messages"])
)
return {**authorization, "anthropic-beta": ",".join(betas)} if betas else authorization
class VertexAIPartnerModelsTokenCounter(VertexBase):
"""
Token counter for Vertex AI Partner Models.
@ -175,8 +193,7 @@ class VertexAIPartnerModelsTokenCounter(VertexBase):
api_base=litellm_params.get("api_base"),
)
# Prepare headers
headers: Final = {"Authorization": f"Bearer {access_token}"}
headers: Final = dict(count_tokens_headers(access_token, request_data))
# Get async HTTP client
from litellm import LlmProviders

20
litellm/llms/vllm/common_utils.py
View file

@ -1,3 +1,4 @@
from collections.abc import Mapping
from typing import Final
import httpx
@ -10,6 +11,14 @@ from litellm.types.llms.openai import AllMessageValues
from litellm.utils import add_path_to_api_base
def endpoint_under_api_base(endpoint: str, api_base: str | None) -> str:
return (
endpoint.lstrip("/").removeprefix("v1/")
if api_base is not None and api_base.rstrip("/").endswith("/v1")
else endpoint
)
class VLLMError(BaseLLMException):
def __init__(
self,
@ -39,9 +48,14 @@ class VLLMModelInfo(BaseLLMModelInfo):
api_key: str | None = None,
api_base: str | None = None,
) -> dict:
if api_key is not None:
headers["x-api-key"] = api_key
return headers
upstream_key: Final = self.upstream_api_key(api_key, api_base)
return {**headers, **self.upstream_auth_headers(upstream_key)} if upstream_key else headers
def upstream_api_key(self, api_key: str | None, api_base: str | None) -> str | None:
return api_key
def upstream_auth_headers(self, api_key: str) -> Mapping[str, str]:
return {"Authorization": f"Bearer {api_key}", "x-api-key": api_key}
@staticmethod
def get_api_base(api_base: str | None = None) -> str | None:

35
litellm/llms/vllm/passthrough/transformation.py
View file

@ -3,6 +3,7 @@ from typing import TYPE_CHECKING, Final
from httpx import Response
from litellm.constants import PLACEHOLDER_API_KEY
from litellm.litellm_core_utils.litellm_logging import Logging
from litellm.llms.base_llm.passthrough.transformation import BasePassthroughConfig
from litellm.llms.openai_like.passthrough.transformation import (
@ -12,8 +13,14 @@ from litellm.llms.openai_like.passthrough.transformation import (
logged_text_completion_stream,
relays_text_completion,
)
from litellm.secret_managers.main import (
GOOGLE_OIDC_SECRET_PREFIX,
UnresolvableSecretReference,
get_secret_str,
resolve_local_oidc_reference,
)
from ..common_utils import VLLMModelInfo
from ..common_utils import VLLMModelInfo, endpoint_under_api_base
if TYPE_CHECKING:
from httpx import URL
@ -22,6 +29,30 @@ if TYPE_CHECKING:
class VLLMPassthroughConfig(VLLMModelInfo, BasePassthroughConfig):
@staticmethod
def get_api_key(api_key: str | None = None) -> str | None:
return None if api_key == PLACEHOLDER_API_KEY else api_key
def upstream_api_key(self, api_key: str | None, api_base: str | None) -> str | None:
if api_key is not None or not self._is_vllm_api_base(api_base):
return api_key
vllm_api_key: Final = get_secret_str("VLLM_API_KEY")
if vllm_api_key is None or vllm_api_key.startswith(GOOGLE_OIDC_SECRET_PREFIX):
return None
match resolve_local_oidc_reference(vllm_api_key):
case UnresolvableSecretReference():
return None
case str() as resolved:
return resolved
@staticmethod
def _is_vllm_api_base(api_base: str | None) -> bool:
vllm_api_base: Final = get_secret_str("VLLM_API_BASE")
return api_base is not None and vllm_api_base is not None and api_base.rstrip("/") == vllm_api_base.rstrip("/")
def upstream_auth_headers(self, api_key: str) -> Mapping[str, str]:
return {"Authorization": f"Bearer {api_key}"}
def is_streaming_request(self, endpoint: str, request_data: dict) -> bool:
return "stream" in request_data
@ -40,7 +71,7 @@ class VLLMPassthroughConfig(VLLMModelInfo, BasePassthroughConfig):
raise Exception("VLLM api base not found")
return (
self.format_url(endpoint, base_target_url, request_query_params),
self.format_url(endpoint_under_api_base(endpoint, base_target_url), base_target_url, request_query_params),
base_target_url,
)

10
litellm/llms/xai/responses/transformation.py
View file

@ -10,6 +10,10 @@ from litellm._logging import verbose_logger
from litellm.constants import XAI_API_BASE
from litellm.exceptions import AuthenticationError
from litellm.litellm_core_utils.url_utils import join_api_base
from litellm.llms.base_llm.responses.transformation import (
STANDARD_MODE_WITHOUT_CONTEXT_REASONING_FIELD_VALUES,
ReasoningFieldValues,
)
from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig
from litellm.llms.xai.common_utils import XAIModelInfo, xai_reported_cost_in_usd
from litellm.secret_managers.main import get_secret_str
@ -290,3 +294,9 @@ class XAIResponsesAPIConfig(OpenAIResponsesAPIConfig):
def supports_native_websocket(self) -> bool:
"""XAI does not support native WebSocket for Responses API"""
return False
def reasoning_field_values(self) -> ReasoningFieldValues:
return STANDARD_MODE_WITHOUT_CONTEXT_REASONING_FIELD_VALUES
def ignores_unsupported_reasoning_fields(self) -> bool:
return True

67
litellm/main.py
View file

@ -681,7 +681,7 @@ async def acompletion(
- The `completion` function is called using `run_in_executor` to execute synchronously in the event loop.
- If `stream` is True, the function returns an async generator that yields completion lines.
"""
fallbacks = kwargs.get("fallbacks", None)
fallbacks: Final = kwargs.get("fallbacks", None) or litellm.model_fallbacks
mock_timeout = kwargs.get("mock_timeout", None)
if mock_timeout is True:
@ -697,6 +697,7 @@ async def acompletion(
loop: Final = asyncio.get_event_loop()
custom_llm_provider = kwargs.get("custom_llm_provider", None)
_ = _resolve_control_options(kwargs, model)
caller_kwargs: Final = cast("Mapping[str, object]", {**kwargs}) # cast-ok: acompletion takes untyped kwargs
## PROMPT MANAGEMENT HOOKS ##
#########################################################
@ -715,15 +716,34 @@ async def acompletion(
custom_llm_provider=cast(str | None, custom_llm_provider), # cast-ok: read from untyped kwargs
tools=tools,
enable_prompt_caching=cast(bool | None, kwargs.get("enable_prompt_caching")), # cast-ok: untyped kwargs
api_base=kwargs.get("api_base") or base_url,
api_base=base_url or kwargs.get("api_base"),
)
if isinstance(litellm_logging_obj, LiteLLMLoggingObj) and (
litellm_logging_obj.should_run_prompt_management_hooks(
prompt_id=kwargs.get("prompt_id", None),
prompt_id: Final = cast(str | None, kwargs.get("prompt_id", None)) # cast-ok: read from untyped kwargs
prompt_management_logger: Final = (
litellm_logging_obj.get_custom_logger_for_prompt_management(
model=model,
non_default_params=kwargs,
tools=tools,
prompt_id=prompt_id,
dynamic_callback_params=litellm_logging_obj.standard_callback_dynamic_params,
)
if isinstance(litellm_logging_obj, LiteLLMLoggingObj)
and litellm_logging_obj.should_run_prompt_management_hooks(
prompt_id=prompt_id,
non_default_params=kwargs,
tools=tools,
)
else None
)
each_fallback_writes_its_own_cache_dialect: Final = fallbacks is not None and isinstance(
prompt_management_logger, AnthropicCacheControlHook
)
if (
isinstance(litellm_logging_obj, LiteLLMLoggingObj)
and prompt_management_logger is not None
and not each_fallback_writes_its_own_cache_dialect
):
(
model,
@ -733,8 +753,9 @@ async def acompletion(
model=model,
messages=messages,
non_default_params=kwargs,
prompt_id=kwargs.get("prompt_id", None),
prompt_id=prompt_id,
prompt_variables=kwargs.get("prompt_variables", None),
prompt_management_logger=prompt_management_logger,
tools=tools,
prompt_label=kwargs.get("prompt_label", None),
prompt_version=kwargs.get("prompt_version", None),
@ -826,9 +847,10 @@ async def acompletion(
litellm_params=(GenericLiteLLMParams.model_validate(_supplemental) if _supplemental else None),
)
fallbacks = fallbacks or litellm.model_fallbacks
if fallbacks is not None:
response = await async_completion_with_fallbacks(**completion_kwargs, kwargs={"fallbacks": fallbacks, **kwargs})
response = await async_completion_with_fallbacks(
**completion_kwargs, kwargs={"fallbacks": fallbacks, **caller_kwargs}
)
if response is None:
raise Exception(
"No response from fallbacks. Got none. Turn on `litellm.set_verbose=True` to see more details."
@ -5765,7 +5787,7 @@ def completion(
custom_llm_provider=cast(str | None, kwargs.get("custom_llm_provider")), # cast-ok: untyped kwargs
tools=tools,
enable_prompt_caching=cast(bool | None, kwargs.get("enable_prompt_caching")), # cast-ok: untyped kwargs
api_base=kwargs.get("api_base") or base_url,
api_base=base_url or kwargs.get("api_base"),
)
if isinstance(litellm_logging_obj, LiteLLMLoggingObj) and (
@ -6199,6 +6221,7 @@ def completion(
PROVIDER_AFFINITY_HEADER_KWARG_KEY,
"fireworks_forward_user_id",
"responses_api_version",
"allowed_openai_params",
)
if key in kwargs
},
@ -6266,6 +6289,9 @@ def completion(
# reasoningSummary/reasoning_summary without tools (AI SDK) that the first
# (early) check doesn't cover.
_reasoning_summary_for_bridge: Final = peek_reasoning_summary_aliases(optional_params)
typed_deployment_model_info: Final = str_keyed_mapping(
cast(object, model_info) # cast-ok: model_info is the untyped dict kwarg
)
if responses_api_model_info.get("mode") != "responses":
responses_api_model_info, model = responses_api_bridge_check(
model=model,
@ -6275,9 +6301,7 @@ def completion(
reasoning_effort=bridge_reasoning_effort(resolved_reasoning_effort, optional_params.get("extra_body")),
reasoning_summary=_reasoning_summary_for_bridge,
api_base=api_base,
deployment_model_info=str_keyed_mapping(
cast(object, model_info) # cast-ok: model_info is the untyped dict kwarg
),
deployment_model_info=typed_deployment_model_info,
)
# Use base_model (the true underlying model) for Azure model-type
@ -6289,19 +6313,28 @@ def completion(
optional_params, rs_val = strip_reasoning_summary_aliases_from_optional_params(optional_params)
if custom_llm_provider == "openai" and isinstance(resolved_reasoning_effort, dict):
from litellm.responses.utils import openai_bridge_reasoning
if isinstance(resolved_reasoning_effort, dict):
from litellm.responses.utils import (
RefusedReasoningFields,
bridge_reasoning,
raise_refused_reasoning_fields,
)
optional_params["reasoning_effort"] = openai_bridge_reasoning(
bridged_reasoning: Final = bridge_reasoning(
requested=resolved_reasoning_effort,
effort_kept="reasoning_effort" in optional_params,
summary_alias=rs_val,
model=model,
base_model=base_model if isinstance(base_model, str) else None,
custom_llm_provider=custom_llm_provider,
api_base=api_base,
deployment_model_info=typed_deployment_model_info,
drop_params=drop_unsupported_reasoning,
additional_drop_params=routing_kwargs.get("additional_drop_params"),
)
elif isinstance(resolved_reasoning_effort, dict) and "summary" in resolved_reasoning_effort:
optional_params["reasoning_effort"] = resolved_reasoning_effort
if isinstance(bridged_reasoning, RefusedReasoningFields):
raise_refused_reasoning_fields(bridged_reasoning, model, custom_llm_provider)
optional_params["reasoning_effort"] = bridged_reasoning
elif rs_val is not None:
eff: Final = optional_params.get("reasoning_effort", resolved_reasoning_effort)
if isinstance(eff, dict):

80
litellm/model_prices_and_context_window_backup.json
View file

@ -8715,7 +8715,7 @@
"search_context_size_low": 0.014,
"search_context_size_medium": 0.014
},
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'",
"source": "https://azure.microsoft.com/en-us/pricing/details/cognitive-services/openai-service/",
"supported_endpoints": [
"/v1/chat/completions",
"/v1/responses"
@ -8742,7 +8742,11 @@
"supports_tool_choice": true,
"supports_vision": true,
"supports_web_search": true,
"supports_xhigh_reasoning_effort": true
"supports_xhigh_reasoning_effort": true,
"cache_creation_input_token_cost_above_272k_tokens_flex": 1.25e-05,
"cache_read_input_token_cost_above_272k_tokens_flex": 1e-06,
"input_cost_per_token_above_272k_tokens_flex": 1e-05,
"output_cost_per_token_above_272k_tokens_flex": 3.75e-05
},
"azure/gpt-6-astra-2026-09-03": {
"cache_creation_input_token_cost": 1.25e-05,
@ -8763,7 +8767,7 @@
"search_context_size_low": 0.014,
"search_context_size_medium": 0.014
},
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'",
"source": "https://azure.microsoft.com/en-us/pricing/details/cognitive-services/openai-service/",
"supported_endpoints": [
"/v1/chat/completions",
"/v1/responses"
@ -8790,7 +8794,11 @@
"supports_tool_choice": true,
"supports_vision": true,
"supports_web_search": true,
"supports_xhigh_reasoning_effort": true
"supports_xhigh_reasoning_effort": true,
"cache_creation_input_token_cost_above_272k_tokens_flex": 1.25e-05,
"cache_read_input_token_cost_above_272k_tokens_flex": 1e-06,
"input_cost_per_token_above_272k_tokens_flex": 1e-05,
"output_cost_per_token_above_272k_tokens_flex": 3.75e-05
},
"azure/gpt-6-luna": {
"cache_creation_input_token_cost": 1.25e-07,
@ -30856,8 +30864,9 @@
"output_cost_per_token": 0,
"output_vector_size": 3072,
"rpm": 10000,
"source": "https://ai.google.dev/gemini-api/docs/embeddings#model-versions",
"tpm": 10000000
"source": "https://ai.google.dev/gemini-api/docs/pricing",
"tpm": 10000000,
"input_cost_per_token_batches": 7.5e-08
},
"gemini/gemini-embedding-2": {
"input_cost_per_audio_token": 6.5e-06,
@ -31713,7 +31722,10 @@
"input_cost_per_token_flex": 6.25e-07,
"output_cost_per_token_batches": 5e-06,
"output_cost_per_token_flex": 5e-06,
"supports_url_context": true
"supports_url_context": true,
"cache_read_input_token_cost_above_200k_tokens_batches": 2.5e-07,
"input_cost_per_token_above_200k_tokens_batches": 1.25e-06,
"output_cost_per_token_above_200k_tokens_batches": 7.5e-06
},
"gemini/gemini-2.5-computer-use-preview-10-2025": {
"deprecation_date": "2026-07-28",
@ -31871,7 +31883,8 @@
},
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"web_search_billing_unit": "per_query",
"google_maps_grounding_cost_per_query": 0.014
"google_maps_grounding_cost_per_query": 0.014,
"supports_computer_use": true
},
"gemini/gemini-3-flash-preview": {
"cache_read_input_audio_token_cost": 1e-07,
@ -31937,7 +31950,8 @@
"input_cost_per_token_flex": 2.5e-07,
"output_cost_per_token_batches": 1.5e-06,
"output_cost_per_token_flex": 1.5e-06,
"supports_audio_input": true
"supports_audio_input": true,
"supports_computer_use": true
},
"gemini/gemini-3.5-flash": {
"prompt_cache_min_tokens": 4096,
@ -32184,7 +32198,8 @@
},
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"web_search_billing_unit": "per_query",
"google_maps_grounding_cost_per_query": 0.014
"google_maps_grounding_cost_per_query": 0.014,
"supports_computer_use": true
},
"gemini/gemini-omni-flash-preview": {
"input_cost_per_audio_token": 1.5e-06,
@ -32280,7 +32295,10 @@
"web_search_billing_unit": "per_query",
"google_maps_grounding_cost_per_query": 0.014,
"input_cost_per_token_flex": 1e-06,
"output_cost_per_token_flex": 6e-06
"output_cost_per_token_flex": 6e-06,
"cache_read_input_token_cost_above_200k_tokens_batches": 4e-07,
"input_cost_per_token_above_200k_tokens_batches": 2e-06,
"output_cost_per_token_above_200k_tokens_batches": 9e-06
},
"gemini/gemini-3.1-pro-preview-customtools": {
"prompt_cache_min_tokens": 4096,
@ -32343,7 +32361,10 @@
"web_search_billing_unit": "per_query",
"google_maps_grounding_cost_per_query": 0.014,
"input_cost_per_token_flex": 1e-06,
"output_cost_per_token_flex": 6e-06
"output_cost_per_token_flex": 6e-06,
"cache_read_input_token_cost_above_200k_tokens_batches": 4e-07,
"input_cost_per_token_above_200k_tokens_batches": 2e-06,
"output_cost_per_token_above_200k_tokens_batches": 9e-06
},
"gemini-3-flash-preview": {
"cache_read_input_audio_token_cost": 1e-07,
@ -72228,6 +72249,7 @@
},
"aiand/deepseek-ai/deepseek-v4-flash": {
"cache_read_input_token_cost": 8e-08,
"deprecation_date": "2026-10-27",
"input_cost_per_token": 1.5e-07,
"litellm_provider": "aiand",
"max_input_tokens": 1048576,
@ -72257,6 +72279,22 @@
"supports_vision": false
},
"aiand/deepseek-ai/deepseek-v4.1-flash": {
"cache_read_input_token_cost": 2e-08,
"deprecation_date": "2026-11-05",
"input_cost_per_token": 3e-07,
"litellm_provider": "aiand",
"max_input_tokens": 1048576,
"max_output_tokens": 384000,
"mode": "chat",
"output_cost_per_token": 6e-07,
"source": "https://api.aiand.com/v1/models",
"supports_function_calling": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_tool_choice": true,
"supports_vision": true
},
"aiand/deepseek-ai/deepseek-v4.1-flash-1017": {
"cache_read_input_token_cost": 2e-08,
"input_cost_per_token": 3e-07,
"litellm_provider": "aiand",
@ -72286,8 +72324,24 @@
"supports_tool_choice": true,
"supports_vision": true
},
"aiand/meta/muse-spark-open": {
"cache_read_input_token_cost": 1.5e-07,
"input_cost_per_token": 1.25e-06,
"litellm_provider": "aiand",
"max_input_tokens": 1048576,
"max_output_tokens": 943718,
"mode": "chat",
"output_cost_per_token": 4.25e-06,
"source": "https://api.aiand.com/v1/models",
"supports_function_calling": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_tool_choice": true,
"supports_vision": true
},
"aiand/moonshotai/kimi-k2.7-code": {
"cache_read_input_token_cost": 2e-07,
"deprecation_date": "2026-10-27",
"input_cost_per_token": 7.5e-07,
"litellm_provider": "aiand",
"max_input_tokens": 262144,
@ -72378,6 +72432,7 @@
},
"aiand/zai-org/glm-5.2": {
"cache_read_input_token_cost": 3e-07,
"deprecation_date": "2026-10-27",
"input_cost_per_token": 1e-06,
"litellm_provider": "aiand",
"max_input_tokens": 1048576,
@ -76540,6 +76595,7 @@
],
"supports_audio_input": true,
"supports_audio_output": false,
"supports_computer_use": true,
"supports_function_calling": true,
"supports_minimal_reasoning_effort": false,
"supports_native_streaming": true,

2
litellm/models/__init__.py
View file

@ -30,6 +30,7 @@ from litellm.models.organization_membership import LiteLLM_OrganizationMembershi
from litellm.models.project import LiteLLM_ProjectTable
from litellm.models.skills import LiteLLM_SkillsTable
from litellm.models.spend_logs import LiteLLM_ErrorLogs, LiteLLM_SpendLogs
from litellm.models.subscription_account import LiteLLM_SubscriptionAccountTable
from litellm.models.tag import LiteLLM_TagTable
from litellm.models.team import LiteLLM_TeamTable
from litellm.models.team_membership import LiteLLM_TeamMembership
@ -59,6 +60,7 @@ __all__ = [
"LiteLLM_ProxyModelTable",
"LiteLLM_SkillsTable",
"LiteLLM_SpendLogs",
"LiteLLM_SubscriptionAccountTable",
"LiteLLM_TagTable",
"LiteLLM_TeamMemberTable",
"LiteLLM_TeamMembership",

Some files were not shown because too many files have changed in this diff Show more