Merge remote-tracking branch 'origin/main' into litellm_mistral_magistral_reasoning_effort
Some checks are pending
Cost map pins recheck / rerun-cost-map-guard (push) Waiting to run
LiteLLM Rust DB / db-queries (push) Waiting to run
LiteLLM Rust / rust-lint (push) Waiting to run
LiteLLM Rust / rust-test (push) Waiting to run
LiteLLM Rust / rust-wheel (push) Waiting to run
Terraform Provider / gofmt, vet, build, test (push) Waiting to run
Terraform Provider / Provider endpoints vs proxy OpenAPI schema (push) Waiting to run

# Conflicts:
#	litellm/llms/mistral/chat/transformation.py
This commit is contained in:
mateo-berri 2026-10-30 12:38:48 -07:00
commit 08e01d3dbe
293 changed files with 20579 additions and 1677 deletions

View file

@ -227,6 +227,7 @@ jobs:
tests/unit/llms/test_oss_decision.py
tests/unit/llms/test_polling_url_origin_match.py
tests/unit/llms/test_predibase_transformation.py
tests/unit/llms/test_subscription_accounts.py
tests/unit/llms/a2a
tests/unit/llms/aiml
tests/unit/llms/aiohttp_openai

View file

@ -55,6 +55,7 @@ BACKEND_PATH_PREFIXES: tuple[str, ...] = (
"/cache_settings",
"/coordination_redis/",
"/cost_tracking",
"/subscription_accounts/",
"/cost_optimization/",
"/cost/",
"/credentials",

View file

@ -0,0 +1,21 @@
SET LOCAL lock_timeout = '10s';
-- CreateTable
CREATE TABLE IF NOT EXISTS "LiteLLM_SubscriptionAccountTable" (
"subscription_account_id" TEXT NOT NULL,
"custom_llm_provider" TEXT NOT NULL,
"account_id" TEXT NOT NULL,
"label" TEXT,
"monthly_fee" DOUBLE PRECISION NOT NULL,
"currency" TEXT NOT NULL,
"billing_period_start" TEXT NOT NULL,
"created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
"created_by" TEXT NOT NULL,
"updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
"updated_by" TEXT NOT NULL,
CONSTRAINT "LiteLLM_SubscriptionAccountTable_pkey" PRIMARY KEY ("subscription_account_id")
);
-- CreateIndex
CREATE UNIQUE INDEX IF NOT EXISTS "LiteLLM_SubscriptionAccountTable_provider_account_key" ON "LiteLLM_SubscriptionAccountTable"("custom_llm_provider", "account_id");

View file

@ -2131,3 +2131,19 @@ model LiteLLM_AccountingDelivery {
@@id([event_id, target, version])
@@index([target, state, next_attempt_at])
}
model LiteLLM_SubscriptionAccountTable {
subscription_account_id String @id @default(uuid())
custom_llm_provider String
account_id String
label String?
monthly_fee Float
currency String
billing_period_start String
created_at DateTime @default(now()) @map("created_at")
created_by String
updated_at DateTime @default(now()) @updatedAt @map("updated_at")
updated_by String
@@unique([custom_llm_provider, account_id], map: "LiteLLM_SubscriptionAccountTable_provider_account_key")
}

View file

@ -199,7 +199,7 @@ impl AnthropicBeta {
&[BetaProvider::BedrockConverse, BetaProvider::Databricks]
}
Self::ContextManagement20250627 => &[BetaProvider::BedrockConverse],
Self::AdvisorTool20260301 | Self::Compact20260904 => &[
Self::AdvisorTool20260301 => &[
BetaProvider::AzureAi,
BetaProvider::BedrockConverse,
BetaProvider::Bedrock,
@ -207,6 +207,12 @@ impl AnthropicBeta {
BetaProvider::VertexAi,
BetaProvider::Databricks,
],
Self::Compact20260904 => &[
BetaProvider::BedrockConverse,
BetaProvider::Bedrock,
BetaProvider::BedrockMantle,
BetaProvider::Databricks,
],
Self::Bash20241022
| Self::Bash20250124
| Self::McpServers20251204
@ -256,14 +262,18 @@ impl AnthropicBeta {
Self::InterleavedThinking20250514 | Self::WebSearch20250305 => {
&[BetaProvider::BedrockConverse, BetaProvider::Bedrock]
}
Self::MidConversationOutputConfig20260701
| Self::MidConversationToolChanges20260701 => &[
Self::MidConversationOutputConfig20260701 => &[
BetaProvider::AzureAi,
BetaProvider::BedrockConverse,
BetaProvider::BedrockMantle,
BetaProvider::VertexAi,
BetaProvider::Databricks,
],
Self::MidConversationToolChanges20260701 => &[
BetaProvider::BedrockConverse,
BetaProvider::BedrockMantle,
BetaProvider::Databricks,
],
Self::Output128k20250219 | Self::TokenEfficientTools20250219 => &[
BetaProvider::AzureAi,
BetaProvider::BedrockConverse,
@ -483,7 +493,7 @@ mod tests {
let config = beta_headers_config();
let config_columns: Vec<String> = config
.keys()
.filter(|column| column.as_str() != "description")
.filter(|column| column.as_str() != "description" && !column.ends_with("_header_only"))
.cloned()
.collect();
let beta_provider_columns: Vec<String> = BetaProvider::VARIANTS
@ -502,12 +512,16 @@ mod tests {
let config = beta_headers_config();
for provider in BetaProvider::VARIANTS {
for beta in &AnthropicBeta::KNOWN {
let expected = config
.get(provider.as_ref())
.and_then(serde_json::Value::as_object)
.and_then(|column| column.get(beta.as_str()))
.and_then(serde_json::Value::as_str)
.map(str::to_string);
let cell = |column: &str| {
config
.get(column)
.and_then(serde_json::Value::as_object)
.and_then(|values| values.get(beta.as_str()))
.and_then(serde_json::Value::as_str)
.map(str::to_string)
};
let expected =
cell(&format!("{provider}_header_only")).or_else(|| cell(provider.as_ref()));
let actual = beta.on(*provider).map(|name| name.to_string());
assert_eq!(actual, expected, "provider {provider}, beta {beta}");
if matches!(beta, AnthropicBeta::AdvancedToolUse20251120)

View file

@ -1,7 +1,8 @@
use litellm_auth::CredentialPlacement;
use litellm_llms_types::{
formats::messages::{
ContextEdit, ContextManagement, Message, MessagesOptionalParams, MessagesRequest, Speed,
ContentBlock, ContentBlockType, ContextEdit, ContextManagement, Message,
MessagesOptionalParams, MessagesRequest, Speed, ThinkingConfig, ThinkingDisplay,
},
providers::anthropic::{AnthropicBeta, BetaProvider, BetaSet, DEFAULT_HEADERS},
recognized::Recognized,
@ -203,6 +204,10 @@ fn feature_betas(request: &MessagesRequest) -> BetaSet {
.then_some(AnthropicBeta::FastMode20260201),
messages_carry_output_config(&request.messages)
.then_some(AnthropicBeta::PerTurnControl20260701),
uses_thinking_display_updates(params)
.then_some(AnthropicBeta::ThinkingDisplayUpdates20260818),
messages_carry_tool_changes(&request.messages)
.then_some(AnthropicBeta::MidConversationToolChanges20260701),
has_advisor_tool(tools).then_some(AnthropicBeta::AdvisorTool20260301),
is_tool_search_used(tools).then_some(AnthropicBeta::AdvancedToolUse20251120),
]
@ -253,6 +258,36 @@ fn messages_carry_output_config(messages: &[Message]) -> bool {
.any(|message| message.extra.contains_key("output_config"))
}
fn uses_thinking_display_updates(params: &MessagesOptionalParams) -> bool {
let display = match params.thinking.as_ref().and_then(Recognized::known) {
Some(ThinkingConfig::Enabled(thinking)) => thinking.display.as_ref(),
Some(ThinkingConfig::Adaptive(thinking)) => thinking.display.as_ref(),
_ => None,
};
display.and_then(Recognized::known) == Some(&ThinkingDisplay::Updates)
}
fn is_tool_reference_change(block: &ContentBlock) -> bool {
matches!(
&block.block_type,
Some(ContentBlockType::Other(block_type))
if block_type == "tool_addition" || block_type == "tool_removal"
) && block
.extra
.get("tool")
.and_then(|tool| tool.get("type"))
.and_then(Value::as_str)
== Some("tool_reference")
}
fn messages_carry_tool_changes(messages: &[Message]) -> bool {
messages
.iter()
.filter(|message| message.role == "system")
.flat_map(Message::blocks)
.any(is_tool_reference_change)
}
fn unsupported_param(model: &str, param: &str, value: &str, hint: &str) -> Error {
Error::InvalidRequest(crate::ErrorDetail::UnsupportedParameter {
model: model.into(),
@ -1071,6 +1106,41 @@ mod tests {
json!({"safeguards": [{"type": "dangerous_tool_use", "classifier_context": {"v": 1, "permission_mode": "auto"}}]}),
&[]
)]
#[case::adaptive_display_updates(
json!({"thinking": {"type": "adaptive", "display": "updates"}}),
&["thinking-display-updates-2026-08-18"]
)]
#[case::enabled_display_updates(
json!({"thinking": {"type": "enabled", "budget_tokens": 2048, "display": "updates"}}),
&["thinking-display-updates-2026-08-18"]
)]
#[case::summarized_display(json!({"thinking": {"type": "adaptive", "display": "summarized"}}), &[])]
#[case::disabled_thinking(json!({"thinking": {"type": "disabled"}}), &[])]
#[case::system_tool_addition(
json!({"messages": [
{"role": "user", "content": "Use the ping tool"},
{"role": "system", "content": [{"type": "tool_addition", "tool": {"type": "tool_reference", "name": "ping"}}]},
]}),
&["mid-conversation-tool-changes-2026-07-01"]
)]
#[case::system_tool_removal(
json!({"messages": [
{"role": "system", "content": [{"type": "tool_removal", "tool": {"type": "tool_reference", "name": "ping"}}]},
]}),
&["mid-conversation-tool-changes-2026-07-01"]
)]
#[case::user_tool_addition(
json!({"messages": [
{"role": "user", "content": [{"type": "tool_addition", "tool": {"type": "tool_reference", "name": "ping"}}]},
]}),
&[]
)]
#[case::tool_addition_without_a_tool_reference(
json!({"messages": [
{"role": "system", "content": [{"type": "tool_addition", "tool": {"type": "tool_definition"}}]},
]}),
&[]
)]
fn feature_betas_follow_the_request(#[case] fields: Value, #[case] expected: &[&str]) {
assert_eq!(feature_betas(&request(fields)), betas(expected));
}
@ -1093,6 +1163,33 @@ mod tests {
);
}
#[rstest]
#[case::vertex_ai(
BetaProvider::VertexAi,
&["mid-conversation-tool-changes-2026-07-01", "thinking-display-updates-2026-08-18"]
)]
#[case::azure_ai(
BetaProvider::AzureAi,
&["mid-conversation-tool-changes-2026-07-01", "thinking-display-updates-2026-08-18"]
)]
#[case::bedrock_converse(BetaProvider::BedrockConverse, &["thinking-display-updates-2026-08-18"])]
#[case::databricks(BetaProvider::Databricks, &["thinking-display-updates-2026-08-18"])]
fn tool_change_and_display_betas_reach_the_providers_that_accept_them(
#[case] provider: BetaProvider,
#[case] expected: &[&str],
) {
let fields = json!({
"thinking": {"type": "adaptive", "display": "updates"},
"messages": [
{"role": "system", "content": [{"type": "tool_addition", "tool": {"type": "tool_reference", "name": "ping"}}]},
],
});
assert_eq!(
provider_feature_betas(&request(fields), provider),
betas(expected)
);
}
#[rstest]
fn hosts_without_a_beta_map_column_get_safeguards_without_a_derived_beta() {
let client = headers(&[("anthropic-beta", "claude-code-20250219")]);

View file

@ -545,6 +545,11 @@ mod tests {
&[("anthropic-beta", "web-search-2025-03-05")],
&[("x-api-key", "k"), ("anthropic-beta", "structured-outputs-2025-11-13,web-search-2025-03-05")]
)]
#[case::compaction_needs_the_compact_2026_09_04_beta(
json!({"compaction": {"type": "summarize"}}),
&[],
&[("x-api-key", "k"), ("anthropic-beta", "compact-2026-09-04")]
)]
#[case::no_feature_needs_a_beta(json!({}), &[], &[("x-api-key", "k")])]
#[case::safeguards_need_the_dangerous_tool_use_beta(
json!({"safeguards": [{"type": "dangerous_tool_use"}]}),

View file

@ -798,6 +798,18 @@ mod tests {
]}}),
&["compact-2026-01-12", "context-management-2025-06-27"]
)]
#[case::compaction(json!({"compaction": {"type": "summarize"}}), &["compact-2026-09-04"])]
#[case::thinking_display_updates(
json!({"thinking": {"type": "adaptive", "display": "updates"}}),
&["thinking-display-updates-2026-08-18"]
)]
#[case::tool_addition(
json!({"messages": [
{"role": "user", "content": "Use the ping tool"},
{"role": "system", "content": [{"type": "tool_addition", "tool": {"type": "tool_reference", "name": "ping"}}]}
]}),
&["mid-conversation-tool-changes-2026-07-01"]
)]
#[case::advisor_tool_is_not_a_vertex_beta(
json!({"tools": [{"type": "advisor_20260301", "name": "advisor"}]}),
&[]

View file

@ -7,7 +7,6 @@ _apply_process_defaults()
### INIT VARIABLES #########################
import threading
import os
import datetime
from collections.abc import Mapping, Sequence
from typing import (
@ -550,6 +549,7 @@ _key_management_system: Optional["KeyManagementSystem"] = None
output_parse_pii: bool = False
#############################################
from litellm.litellm_core_utils.get_model_cost_map import get_model_cost_map, mark_litellm_import_complete
from litellm.litellm_core_utils.model_retirement import is_hidden_as_retired
model_cost = get_model_cost_map(url=model_cost_map_url)
cost_discount_config: Dict[str, float] = {} # Provider-specific cost discounts {"vertex_ai": 0.05} = 5% discount
@ -756,16 +756,6 @@ def is_openai_finetune_model(key: str) -> bool:
return key.startswith("ft:") and not key.count(":") > 1
def _is_past_deprecation_date(value: Mapping[str, object]) -> bool:
deprecation_date: Final = value.get("deprecation_date")
if not isinstance(deprecation_date, str):
return False
try:
return datetime.date.fromisoformat(deprecation_date) <= datetime.date.today()
except ValueError:
return False
def _populate_provider_model_sets(model_cost_map: Mapping[str, Mapping[str, object]]) -> None:
for key, value in tuple(model_cost_map.items()):
if value.get("litellm_provider") == "openai" and not is_openai_finetune_model(key):
@ -873,12 +863,12 @@ def _populate_provider_model_sets(model_cost_map: Mapping[str, Mapping[str, obje
if "-to-" not in key and "fireworks-ai-default" not in key:
fireworks_ai_models.add(key)
elif value.get("litellm_provider") == "together_ai":
if _is_past_deprecation_date(value):
if is_hidden_as_retired(value):
together_ai_models.discard(key)
elif not key.startswith("together-ai-"):
together_ai_models.add(key)
elif value.get("litellm_provider") == "baseten":
if _is_past_deprecation_date(value):
if is_hidden_as_retired(value):
baseten_models.discard(key)
else:
baseten_models.add(key)
@ -1051,7 +1041,10 @@ def _populate_provider_model_sets(model_cost_map: Mapping[str, Mapping[str, obje
elif value.get("litellm_provider") == "coralbricks":
coralbricks_models.add(key)
elif value.get("litellm_provider") == "aiand":
aiand_models.add(key)
if is_hidden_as_retired(value):
aiand_models.discard(key)
else:
aiand_models.add(key)
elif value.get("litellm_provider") == "reducto":
reducto_models.add(key)
elif value.get("litellm_provider") == "bedrock_mantle":

View file

@ -1,5 +1,5 @@
{
"description": "Mapping of Anthropic beta headers for each provider. Keys are input header names, values are provider-specific header names (or null if unsupported). Only headers present in mapping keys with non-null values can be forwarded.",
"description": "Mapping of Anthropic beta headers for each provider. Keys are input header names, values are provider-specific header names (or null if unsupported). Only headers present in mapping keys with non-null values can be forwarded. A <provider>_header_only column lists betas that provider accepts only as the anthropic-beta HTTP header: they are forwarded there and dropped from the anthropic_beta body field. Proxies that predate these columns ignore them and read the provider column alone.",
"anthropic": {
"advisor-tool-2026-03-01": "advisor-tool-2026-03-01",
"advanced-tool-use-2025-11-20": "advanced-tool-use-2025-11-20",
@ -46,6 +46,7 @@
"bash_20250124": null,
"code-execution-2025-08-25": "code-execution-2025-08-25",
"compact-2026-01-12": null,
"compact-2026-09-04": "compact-2026-09-04",
"computer-use-2025-01-24": "computer-use-2025-01-24",
"computer-use-2025-11-24": "computer-use-2025-11-24",
"context-1m-2025-08-07": "context-1m-2025-08-07",
@ -59,6 +60,7 @@
"mcp-client-2025-11-20": "mcp-client-2025-11-20",
"mcp-client-2025-04-04": "mcp-client-2025-04-04",
"mcp-servers-2025-12-04": null,
"mid-conversation-tool-changes-2026-07-01": "mid-conversation-tool-changes-2026-07-01",
"output-128k-2025-02-19": null,
"structured-output-2024-03-01": null,
"per-turn-control-2026-07-01": "per-turn-control-2026-07-01",
@ -202,6 +204,7 @@
"mcp-client-2025-11-20": null,
"mcp-client-2025-04-04": null,
"mcp-servers-2025-12-04": null,
"mid-conversation-tool-changes-2026-07-01": "mid-conversation-tool-changes-2026-07-01",
"output-128k-2025-02-19": null,
"structured-output-2024-03-01": null,
"per-turn-control-2026-07-01": "per-turn-control-2026-07-01",
@ -218,6 +221,10 @@
"web-search-2025-03-05": "web-search-2025-03-05",
"inline-tools-2026-09-15": "inline-tools-2026-09-15"
},
"vertex_ai_header_only": {
"compact-2026-09-04": "compact-2026-09-04",
"inline-tools-2026-09-15": "inline-tools-2026-09-15"
},
"databricks": {
"advisor-tool-2026-03-01": null,
"advanced-tool-use-2025-11-20": null,

View file

@ -30,7 +30,7 @@ from collections.abc import Mapping
from concurrent.futures import ThreadPoolExecutor
from functools import cache
from importlib.resources import files
from typing import Final, TypeAlias
from typing import Final, Literal, TypeAlias
import httpx
from pydantic import TypeAdapter
@ -47,6 +47,9 @@ BetaHeadersConfig: TypeAlias = Mapping[str, str | Mapping[str, str | None]]
_BETA_HEADERS_CONFIG_ADAPTER: Final = TypeAdapter(BetaHeadersConfig)
_DECODED_JSON: Final = TypeAdapter(object)
BetaPlacement: TypeAlias = Literal["header", "body"]
_HEADER_ONLY_COLUMN_SUFFIX: Final = "_header_only"
class GetAnthropicBetaHeadersConfig:
"""
@ -241,6 +244,31 @@ def get_provider_name(provider: str) -> str:
return aliases.get(provider, provider)
def _beta_column(name: str) -> Mapping[str, str | None]:
column: Final[Mapping[str, str | None]] = _load_beta_headers_config().get(name, {})
return column
def _header_only_column(canonical_provider: str) -> Mapping[str, str | None]:
return _beta_column(f"{canonical_provider}{_HEADER_ONLY_COLUMN_SUFFIX}")
def _provider_beta_mapping(provider: str, placement: BetaPlacement) -> Mapping[str, str | None]:
canonical_provider: Final = get_provider_name(provider)
column: Final = _beta_column(canonical_provider)
header_only: Final = _header_only_column(canonical_provider)
match placement:
case "header":
return {**column, **header_only}
case "body":
return {beta: name for beta, name in column.items() if beta not in header_only}
def get_header_only_betas(provider: str) -> frozenset[str]:
header_only: Final = _header_only_column(get_provider_name(provider))
return frozenset(beta for beta, name in header_only.items() if name is not None)
def has_beta_headers_config(provider: str) -> bool:
config: Final = _bundled_beta_headers_config()
aliases: Final = config.get("provider_aliases", {})
@ -251,6 +279,7 @@ def has_beta_headers_config(provider: str) -> bool:
def filter_and_transform_beta_headers(
beta_headers: list[str],
provider: str,
placement: BetaPlacement = "header",
) -> list[str]:
"""
Filter and transform beta headers based on provider's mapping configuration.
@ -263,6 +292,7 @@ def filter_and_transform_beta_headers(
Args:
beta_headers: List of Anthropic beta header values
provider: Provider name (e.g., "anthropic", "bedrock", "vertex_ai")
placement: Whether the betas travel as the anthropic-beta header or the anthropic_beta body field
Returns:
List of filtered and transformed beta headers for the provider
@ -270,11 +300,7 @@ def filter_and_transform_beta_headers(
if not beta_headers:
return []
config: Final = _load_beta_headers_config()
provider = get_provider_name(provider)
# Get the header mapping for this provider
provider_mapping: Final = config.get(provider, {})
provider_mapping: Final = _provider_beta_mapping(provider, placement)
filtered_headers: Final[set[str]] = set()
@ -426,6 +452,7 @@ def update_request_with_filtered_beta(
filtered_body_betas: Final = filter_and_transform_beta_headers(
beta_headers=existing_body_betas,
provider=provider,
placement="body",
)
if filtered_body_betas:

View file

@ -68,6 +68,7 @@ RUNTIME_UPDATABLE_ROUTER_SETTINGS: Final[frozenset[str]] = frozenset(
"retry_after",
"fallbacks",
"context_window_fallbacks",
"content_policy_fallbacks",
"retry_policy",
"model_group_retry_policy",
"model_group_alias",
@ -899,6 +900,7 @@ PROVIDERS_THAT_AUTHENTICATE_ON_PROVIDER_INFO: Final = frozenset(
}
)
SUBSCRIPTION_BACKED_PROVIDERS: Final = frozenset({"chatgpt"})
A2A_CHAT_COMPLETION_BRIDGE_PROVIDERS: Final = frozenset({"databricks_agent"})
LITELLM_EMBEDDING_PROVIDERS_SUPPORTING_INPUT_ARRAY_OF_TOKENS: Final = [

View file

@ -116,12 +116,12 @@ def configured_injection_points(value: object) -> Sequence[CacheControlInjection
return tuple(cast(CacheControlInjectionPoint, entry) for entry in value if isinstance(entry, dict))
def _served_provider_for_model(model: str) -> str | None:
def _served_provider_for_model(model: str, api_base: object = None) -> str | None:
from litellm.exceptions import BadRequestError
from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
try:
_, provider, _, _ = get_llm_provider(model=model)
_, provider, _, _ = get_llm_provider(model=model, api_base=api_base if isinstance(api_base, str) else None)
except BadRequestError:
return None
return provider
@ -536,7 +536,7 @@ class AnthropicCacheControlHook(CustomPromptManagement):
if model is None:
return False
hosted_flag: Final = _hosted_openai_dialect_flag(
model, custom_llm_provider, AnthropicCacheControlHook._resolve_provider
model, custom_llm_provider, lambda name: AnthropicCacheControlHook._resolve_provider(name, api_base)
)
if hosted_flag is not None:
return hosted_flag
@ -559,8 +559,8 @@ class AnthropicCacheControlHook(CustomPromptManagement):
)
@staticmethod
def _resolve_provider(model: str) -> str | None:
return _served_provider_for_model(model)
def _resolve_provider(model: str, api_base: object = None) -> str | None:
return _served_provider_for_model(model, api_base)
@staticmethod
def count_request_cache_breakpoints(

View file

@ -361,6 +361,7 @@ _PRISMA_MODELS: Final[frozenset[str]] = frozenset(
"LiteLLM_LensReview",
"LiteLLM_LensIngestionKey",
"LiteLLM_LensWorker",
"LiteLLM_SubscriptionAccountTable",
"LiteLLM_LensSignalConfig",
"LiteLLM_LensTraceSignal",
)

View file

@ -82,6 +82,7 @@ from litellm.types.proxy.carried_budget_state import (
UserBudgetSnapshot,
)
from litellm.types.utils import (
CallTypes,
StandardLoggingGuardrailInformation,
StandardLoggingPayload,
StandardLoggingZeroCostDiagnostic,
@ -279,6 +280,11 @@ def _logged_fields(value: object) -> Mapping[str, object]:
return _NO_LOGGED_FIELDS
_GOOGLE_PASS_THROUGH_PROVIDERS: Final = frozenset(
(litellm.LlmProviders.GEMINI.value, litellm.LlmProviders.VERTEX_AI.value)
)
def _is_llm_provider(name: str) -> bool:
routable: Final = (
name in litellm.provider_list or JSONProviderRegistry.exists(name) or is_registered_custom_provider(name)
@ -2942,6 +2948,22 @@ class PrometheusLogger(CustomLogger):
return False
@staticmethod
def _google_pass_through_provider(request_data: Mapping[str, object]) -> str | None:
declared_provider: Final = request_data.get("custom_llm_provider")
if request_data.get("call_type") != CallTypes.pass_through.value or not isinstance(declared_provider, str):
return None
return declared_provider if declared_provider in _GOOGLE_PASS_THROUGH_PROVIDERS else None
@staticmethod
def _failed_request_requested_model(request_data: Mapping[str, object]) -> object:
requested_model: Final = request_data.get("model", "")
if PrometheusLogger._google_pass_through_provider(request_data) is None:
return requested_model
request_metadata: Final = _logged_fields(_logged_fields(request_data.get("litellm_params")).get("metadata"))
model_group: Final = request_metadata.get("model_group")
return model_group if isinstance(model_group, str) and model_group else requested_model
@staticmethod
def _extract_api_provider_from_request_data(request_data: dict) -> str | None:
"""
@ -2950,22 +2972,27 @@ class PrometheusLogger(CustomLogger):
A request can fail before a deployment is resolved, so the provider is
not always known. Prefer the resolved ``custom_llm_provider`` on
``litellm_params``, then any provider recovered onto a partial
``standard_logging_object`` (e.g. a stream that broke mid-flight), and
finally infer it from the requested model name (e.g. ``gpt-4o-mini`` ->
``openai``) since the proxy's failure ``request_data`` usually carries
only the client-supplied model. Return ``None`` when it cannot be
determined so the label emits empty rather than a guess.
``standard_logging_object`` (e.g. a stream that broke mid-flight). On a
Gemini or Vertex pass-through failure only, next take the provider the
failure payload names (``gemini`` or ``vertex_ai``), so every other
pass-through route keeps its earlier label. Finally infer it from the
requested model name (e.g. ``gpt-4o-mini`` -> ``openai``) since the
proxy's failure ``request_data`` usually carries only the client-supplied
model. Return ``None`` when it cannot be determined so the label emits
empty rather than a guess.
"""
litellm_params: Final = request_data.get("litellm_params") or {}
provider = litellm_params.get("custom_llm_provider")
if provider:
return provider
standard_logging_object: Final = request_data.get("standard_logging_object") or {}
provider = standard_logging_object.get("custom_llm_provider")
if provider:
return provider
resolved_provider: Final = litellm_params.get("custom_llm_provider") or standard_logging_object.get(
"custom_llm_provider"
)
if resolved_provider:
return resolved_provider
google_provider: Final = PrometheusLogger._google_pass_through_provider(request_data)
if google_provider is not None:
return google_provider
model: Final = litellm_params.get("model") or request_data.get("model")
if not model:
if not isinstance(model, str) or not model:
return None
try:
return litellm.get_llm_provider(model=model)[1] or None
@ -3052,7 +3079,7 @@ class PrometheusLogger(CustomLogger):
team_alias=user_api_key_dict.team_alias,
org_id=user_api_key_dict.org_id,
org_alias=user_api_key_dict.organization_alias,
requested_model=_bounded_requested_model_label(request_data.get("model", "")),
requested_model=_bounded_requested_model_label(self._failed_request_requested_model(request_data)),
status_code=str(status_code),
exception_status=str(status_code),
exception_class=self._get_exception_class_name(original_exception),

View file

@ -9,12 +9,13 @@ from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Final, Literal, Protocol, TypeVar, cast, runtime_checkable
import httpx
from pydantic import TypeAdapter, ValidationError
from pydantic import BaseModel, TypeAdapter, ValidationError
from typing_extensions import TypeIs # noqa: TID251 # narrows a duck-typed value to the Protocol its caller reads
import litellm
from litellm._logging import verbose_logger
from litellm.constants import KUBERNETES_POD_ROUTING_KEY
from litellm.litellm_core_utils.hidden_params import set_extra_attribute
from litellm.litellm_core_utils.llm_response_utils.get_headers import get_provider_request_id
from litellm.types.llms.openai import AllMessageValues, OpenAIChatCompletionFinishReason
@ -710,9 +711,9 @@ def preserve_upstream_non_openai_attributes(
_assign_attributes(model_response, original_chunk.model_dump(include=frozenset(candidate_keys)).items())
def _assign_attributes(target: object, values: Iterable[tuple[str, object]]) -> None:
def _assign_attributes(target: BaseModel, values: Iterable[tuple[str, object]]) -> None:
for key, value in values:
setattr(target, key, value)
set_extra_attribute(target, key, value)
def safe_deep_copy(data):
@ -982,25 +983,36 @@ def _usage_field(usage: object, name: str) -> object:
return getattr(usage, name, None)
def _cost_number(value: object) -> float | None:
if isinstance(value, bool) or not isinstance(value, int | float):
def finite_cost(value: object) -> float | None:
if isinstance(value, bool) or not isinstance(value, int | float) or not math.isfinite(value):
return None
return float(value)
def single_log_line(value: object) -> str:
return str(value).replace("\r", "").replace("\n", "")
def finite_reported_cost(value: object) -> float | None:
cost: Final = finite_cost(value)
if cost is None and isinstance(value, float):
verbose_logger.warning("Ignoring a non-finite provider-reported cost %s", value)
return cost
def _reported_cost_value(value: object) -> float | None:
"""A reported cost is a number, or a breakdown whose total lives under ``total_cost``."""
raw: Final[object] = value
if isinstance(value, Mapping):
return _cost_number(_usage_field(raw, "total_cost"))
return _cost_number(value)
return finite_reported_cost(_usage_field(raw, "total_cost"))
return finite_reported_cost(value)
def response_cost_in_hidden_params(response: _CarriesHiddenParams) -> float | None:
additional_headers: Final[object] = response.hidden_params.get("additional_headers")
if not isinstance(additional_headers, Mapping):
return None
return _cost_number(_USAGE_BLOCK_ADAPTER.validate_python(additional_headers).get(RESPONSE_COST_HEADER))
return finite_reported_cost(_USAGE_BLOCK_ADAPTER.validate_python(additional_headers).get(RESPONSE_COST_HEADER))
def override_stamped_response_cost(response: _CarriesHiddenParams, cost: float) -> None:

View file

@ -18,6 +18,7 @@ from litellm.litellm_core_utils.bug_report import (
)
from litellm.litellm_core_utils.core_helpers import as_str_mapping, attribute_or_none
from litellm.litellm_core_utils.secret_redaction import redact_string as redact_secret_string
from litellm.llms.bedrock.common_utils import bedrock_model_reaches_converse_by_default
from litellm.llms.bigmodel.common_utils import raise_bigmodel_business_error
from litellm.types.utils import LlmProviders
@ -938,13 +939,22 @@ _BEDROCK_MANTLE_CONTEXT_WINDOW_GENERIC_MESSAGE: Final = (
)
def _bedrock_mantle_context_window_response(original_exception: _ProviderHTTPException) -> httpx.Response:
def _bedrock_response_with_status(original_exception: _ProviderHTTPException, status_code: int) -> httpx.Response:
original_response: Final = getattr(original_exception, "response", None)
if not isinstance(original_response, httpx.Response) or getattr(original_response, "_request", None) is None:
return httpx.Response(status_code=400, request=httpx.Request(method="POST", url="https://api.openai.com/v1/"))
if original_response.status_code == 400:
return httpx.Response(
status_code=status_code, request=httpx.Request(method="POST", url="https://api.openai.com/v1/")
)
if original_response.status_code == status_code:
return original_response
return httpx.Response(status_code=400, headers=original_response.headers, request=original_response.request)
return httpx.Response(status_code=status_code, headers=original_response.headers, request=original_response.request)
def _came_from_bedrock_converse(original_exception: _ProviderHTTPException) -> bool:
original_response: Final = getattr(original_exception, "response", None)
if not isinstance(original_response, httpx.Response) or getattr(original_response, "_request", None) is None:
return False
return original_response.request.url.path.endswith(("/converse", "/converse-stream"))
def _get_bedrock_mantle_context_window_message(error_str: str) -> str | None:
@ -980,7 +990,7 @@ def _map_bedrock_exception(
message=mantle_context_window_message,
model=model,
llm_provider=custom_llm_provider,
response=_bedrock_mantle_context_window_response(original_exception),
response=_bedrock_response_with_status(original_exception, 400),
)
if (
"too many tokens" in error_str
@ -1003,6 +1013,18 @@ def _map_bedrock_exception(
llm_provider="bedrock",
response=getattr(original_exception, "response", None),
)
elif (
"The provided model identifier is invalid" in error_str
and bool(model)
and _came_from_bedrock_converse(original_exception)
and bedrock_model_reaches_converse_by_default(model)
):
raise NotFoundError(
message=f"BedrockException - {error_str}",
model=model,
llm_provider="bedrock",
response=_bedrock_response_with_status(original_exception, 404),
)
elif "Malformed input request" in error_str:
raise BadRequestError(
message=f"BedrockException - {error_str}",

View file

@ -1,8 +1,9 @@
from collections.abc import Iterator, Mapping, MutableMapping
from functools import cache, lru_cache
from types import MappingProxyType
from typing import Final, Protocol, cast
from pydantic import BaseModel
from pydantic import BaseModel, ConfigDict, create_model
from litellm.types.llms.base import HiddenParams
@ -60,6 +61,38 @@ def stored_attribute(obj: object, name: str) -> object:
return extras.get(name)
@cache
def _assigned_extras_join_fields_set() -> bool:
probe: Final = create_model("_ExtraAssignmentProbe", __config__=ConfigDict(extra="allow"))()
setattr(probe, "assigned", None)
return "assigned" in probe.model_fields_set
@lru_cache(maxsize=1024)
def _writes_plain_extra(model_class: type[BaseModel], name: str) -> bool:
config: Final = model_class.model_config
return (
model_class.__setattr__ is BaseModel.__setattr__
and config.get("extra") == "allow"
and not config.get("frozen")
and not config.get("validate_assignment")
and not name.startswith("_")
and name not in model_class.__pydantic_fields__
and name not in model_class.__class_vars__
and not hasattr(model_class, name)
and _assigned_extras_join_fields_set()
)
def set_extra_attribute(model: BaseModel, name: str, value: object) -> None:
extras: Final = model.__pydantic_extra__
if extras is None or not _writes_plain_extra(type(model), name):
setattr(model, name, value)
return
extras[name] = value
model.__pydantic_fields_set__.add(name)
def _get_hidden_params_storage(obj: object) -> object | None:
if isinstance(obj, dict):
return obj.get(_HIDDEN_PARAMS_ATTR)

View file

@ -35,6 +35,12 @@ from litellm.types.utils import (
BUDGET_RESERVATION_METADATA_KEYS: Final = frozenset({"user_api_key_budget_reservation"})
ITPM_RESERVED_KEY: Final = "_litellm_itpm_reserved"
OTPM_RESERVED_KEY: Final = "_litellm_otpm_reserved"
ITPM_CACHE_KEY: Final = "_litellm_itpm_cache_key"
OTPM_CACHE_KEY: Final = "_litellm_otpm_cache_key"
IO_TOKEN_RESERVATION_KEYS: Final = frozenset({ITPM_RESERVED_KEY, OTPM_RESERVED_KEY, ITPM_CACHE_KEY, OTPM_CACHE_KEY})
MODEL_ACCESS_GROUP_METADATA_KEY: Final = "user_api_key_matched_model_access_groups"
"""Where auth records the model access groups that authorized the request, for the spend writer.

View file

@ -76,6 +76,7 @@ from litellm.litellm_core_utils.classifier_logging import (
is_classifier_call,
)
from litellm.litellm_core_utils.core_helpers import (
finite_cost,
get_provider_response_headers_from_hidden_params,
is_expected_client_error,
native_messages_reported_cost,
@ -84,6 +85,7 @@ from litellm.litellm_core_utils.core_helpers import (
proxy_stamped_used_client_oauth_token,
reconstruct_model_name,
set_response_cost_in_hidden_params,
single_log_line,
)
from litellm.litellm_core_utils.error_normalization import normalize_error
from litellm.litellm_core_utils.get_litellm_params import get_litellm_params
@ -567,7 +569,13 @@ def _resolve_mantle_region_for_cost(
_PER_CALL_LITELLM_PARAM_DICTS: Final = frozenset({"metadata", "litellm_metadata"})
_LITELLM_PARAMS: Final = TypeAdapter(dict[str, object])
_SERVED_ANTHROPIC_BODY: Final = TypeAdapter(dict[str, object])
_REDACTED_RESULT: Final = TypeAdapter(object)
_GUARDRAIL_INFORMATION_KEY: Final = "standard_logging_guardrail_information"
def _without_guardrail_information(metadata: Mapping[str, object]) -> Mapping[str, object]:
return {key: value for key, value in metadata.items() if key != _GUARDRAIL_INFORMATION_KEY}
def _provider_response_id(source: object) -> str | None:
@ -2088,7 +2096,7 @@ class Logging(LiteLLMLoggingBaseClass):
litellm_model_name=litellm_model_name,
router_model_id=router_model_id or hidden_params.get("model_id"),
)
return hidden_params["response_cost"]
return self._recorded_response_cost(hidden_params["response_cost"])
elif router_model_id is None and "model_id" in hidden_params: # use model_id if not already set
router_model_id = hidden_params["model_id"]
@ -2188,7 +2196,7 @@ class Logging(LiteLLMLoggingBaseClass):
litellm_model_name=litellm_model_name,
router_model_id=router_model_id,
)
return total_response_cost
return self._recorded_response_cost(total_response_cost)
except Exception as e: # error calculating cost
debug_info = StandardLoggingModelCostFailureDebugInformation(
error_str=str(e),
@ -2214,6 +2222,15 @@ class Logging(LiteLLMLoggingBaseClass):
_response_cost_calculator = response_cost_calculator
def _recorded_response_cost(self, cost: float | None) -> float | None:
if cost is None or finite_cost(cost) is not None:
return cost
verbose_logger.warning(
"Recording a non-finite response_cost as 0 for litellm_call_id=%s", single_log_line(self.litellm_call_id)
)
self.cost_breakdown = None
return 0.0
def uncached_price(self, result: object) -> UncachedPrice:
previous_cost_breakdown: Final = self.cost_breakdown
previous_billed_token_rates: Final = self.billed_token_rates
@ -2566,6 +2583,23 @@ class Logging(LiteLLMLoggingBaseClass):
attempt.model_call_details = dict(self.model_call_details)
return attempt
def guardrail_scoped_copy(self) -> "Logging":
request_params: Final = _LITELLM_PARAMS.validate_python(
self.litellm_params or {} # pyright: ignore[reportUnknownMemberType] # Logging.litellm_params is untyped
)
litellm_params: Final = {
**request_params,
**{
key: _without_guardrail_information(_METADATA_MAPPING.validate_python(value))
for key, value in request_params.items()
if key in _PER_CALL_LITELLM_PARAM_DICTS and isinstance(value, dict)
},
}
scoped: Final = copy.copy(self)
scoped.litellm_params = litellm_params
scoped.model_call_details = {**self.model_call_details, "litellm_params": litellm_params}
return scoped
def dispatch_attempt_failure_handlers(
self, exception: Exception, traceback_exception: str
) -> Coroutine[None, None, None]:
@ -5034,13 +5068,15 @@ class Logging(LiteLLMLoggingBaseClass):
verbose_logger.warning("LiteLLM: the anthropic_messages stream assembled no response, logging an empty one")
return litellm.ModelResponse(model=self.model)
else:
from litellm.types.llms.anthropic import AnthropicResponse
pydantic_result: Final = AnthropicResponse.model_validate({"stop_sequence": None, **(served_body or {})})
import httpx
anthropic_body: Final = (
served_body
if served_body is not None
else _SERVED_ANTHROPIC_BODY.validate_python(
result.model_dump() if isinstance(result, BaseModel) else result
)
)
result = litellm.AnthropicConfig().transform_parsed_response(
completion_response=pydantic_result.model_dump(),
completion_response={"stop_sequence": None, **anthropic_body},
raw_response=httpx.Response(
status_code=200,
headers={},
@ -7560,7 +7596,15 @@ def get_standard_logging_object_payload(
)
custom_pricing: Final = False if spilled_over else logging_obj.custom_pricing_for(init_response_obj)
raw_response_cost: Final = kwargs.get("response_cost")
llm_response_cost: Final[float] = raw_response_cost or 0.0
numeric_response_cost: Final = raw_response_cost if isinstance(raw_response_cost, int | float) else None
priced_response_cost: Final = finite_cost(numeric_response_cost)
if numeric_response_cost is not None and priced_response_cost is None:
verbose_logger.warning(
"Logging a non-finite response_cost=%s as 0 for litellm_call_id=%s",
numeric_response_cost,
single_log_line(logging_obj.litellm_call_id),
)
llm_response_cost: Final[float] = priced_response_cost or 0.0
guardrail_cost: Final = guardrail_information_cost(metadata.get("standard_logging_guardrail_information"))
finalized_cost: Final = _finalized_cost_of(logging_obj)
response_cost: Final[float] = (

View file

@ -1,4 +1,3 @@
import datetime
from collections.abc import Iterable, Iterator, Mapping
from dataclasses import dataclass
from typing import Final, cast
@ -9,6 +8,7 @@ from typing_extensions import NotRequired, ReadOnly, TypedDict
import litellm
from litellm.constants import DEFAULT_MAX_RECURSE_DEPTH
from litellm.litellm_core_utils.get_llm_provider_logic import inferred_provider
from litellm.litellm_core_utils.model_retirement import is_past_deprecation_date
def _form_field_value(value: object) -> str:
@ -168,7 +168,9 @@ class RankedChatModel:
def rank_chat_models_from_llm_provider(custom_llm_provider: str) -> tuple[RankedChatModel, ...]:
known_models: Final[Iterable[str]] = litellm.models_by_provider.get(custom_llm_provider, ())
from litellm.utils import bundled_provider_models
known_models: Final = bundled_provider_models(custom_llm_provider)
ranked_models: Final = (
RankedChatModel(
model=model, route_rank=route_rank, cost=cost, is_past_deprecation_date=is_past_deprecation_date
@ -187,18 +189,11 @@ _COST_MAP_DEPRECATION: Final = TypeAdapter(_CostMapDeprecation)
def _is_past_deprecation_date(cost_map_key: str) -> bool:
deprecation_date: Final = _COST_MAP_DEPRECATION.validate_python(
litellm.model_cost.get(cost_map_key) or {} # pyright: ignore[reportUnknownMemberType] # untyped cost map JSON
).get("deprecation_date")
if not isinstance(deprecation_date, str):
return False
try:
return (
datetime.date.fromisoformat(deprecation_date)
<= datetime.datetime.now(datetime.timezone.utc).astimezone().date()
return is_past_deprecation_date(
_COST_MAP_DEPRECATION.validate_python(
litellm.model_cost.get(cost_map_key) or {} # pyright: ignore[reportUnknownMemberType] # untyped cost map JSON
)
except ValueError:
return False
)
def _chat_row_costs(models: Iterable[str], custom_llm_provider: str) -> Iterator[tuple[str, float, bool]]:

View file

@ -0,0 +1,19 @@
import datetime
from collections.abc import Mapping
from typing import Final
PROVIDERS_HIDING_RETIRED_MODELS: Final = frozenset({"aiand", "baseten", "together_ai"})
def is_past_deprecation_date(value: Mapping[str, object]) -> bool:
deprecation_date: Final = value.get("deprecation_date")
if not isinstance(deprecation_date, str):
return False
try:
return datetime.date.fromisoformat(deprecation_date) <= datetime.datetime.now(datetime.timezone.utc).date()
except ValueError:
return False
def is_hidden_as_retired(value: Mapping[str, object]) -> bool:
return value.get("litellm_provider") in PROVIDERS_HIDING_RETIRED_MODELS and is_past_deprecation_date(value)

View file

@ -2023,12 +2023,14 @@ def infer_content_type_from_url_and_content(
"""
from litellm.litellm_core_utils.token_counter import get_image_type
media_type: Final = current_content_type.split(";", 1)[0].strip().lower() if current_content_type else None
# If we have a valid content type that's not generic, use it
if current_content_type and current_content_type not in [
if media_type and media_type not in [
"binary/octet-stream",
"application/octet-stream",
]:
return current_content_type
return media_type
# Extension to MIME type mapping
# Supports images, documents, and other common file types

View file

@ -912,9 +912,9 @@ def create_anthropic_image_param(
image_url_input: str | dict,
format: str | None = None,
is_bedrock_invoke: bool = False,
) -> AnthropicMessagesImageParam:
) -> AnthropicMessagesImageParam | AnthropicMessagesDocumentParam:
"""
Create an AnthropicMessagesImageParam from an image URL input.
Create an Anthropic content block from an image URL input.
Supports both URL references (for HTTP/HTTPS URLs) and base64 encoding.
"""
@ -933,14 +933,7 @@ def create_anthropic_image_param(
if is_bedrock_invoke or image_url.startswith("http://"):
base64_url: Final = convert_url_to_base64(url=image_url)
image_chunk = convert_to_anthropic_image_obj(openai_image_url=base64_url, format=format)
return AnthropicMessagesImageParam(
type="image",
source=AnthropicContentParamSource(
type="base64",
media_type=image_chunk["media_type"],
data=image_chunk["data"],
),
)
return _anthropic_content_element_factory(image_chunk)
else:
# HTTPS URL - pass directly for regular Anthropic
return AnthropicMessagesImageParam(
@ -953,14 +946,7 @@ def create_anthropic_image_param(
else:
# Convert to base64 for data URIs or other formats
image_chunk = convert_to_anthropic_image_obj(openai_image_url=image_url, format=format)
return AnthropicMessagesImageParam(
type="image",
source=AnthropicContentParamSource(
type="base64",
media_type=image_chunk["media_type"],
data=image_chunk["data"],
),
)
return _anthropic_content_element_factory(image_chunk)
# The following XML functions will be deprecated once JSON schema support is available on Bedrock and Vertex
@ -1910,7 +1896,12 @@ def convert_to_anthropic_tool_result(
anthropic_content_element=_anthropic_image_param,
original_content_element=content,
)
anthropic_content_list.append(cast(AnthropicMessagesImageParam, _anthropic_image_param))
anthropic_content_list.append(
cast(
AnthropicMessagesImageParam | AnthropicMessagesDocumentParam,
_anthropic_image_param,
)
)
elif content["type"] == "tool_reference":
anthropic_content_list.append(ToolReference(type="tool_reference", tool_name=content["tool_name"]))
elif content["type"] == "file":

View file

@ -1,6 +1,12 @@
import reprlib
from collections.abc import Collection, Mapping
from dataclasses import dataclass
from typing import Final, Literal
from pydantic import TypeAdapter
import litellm
from litellm._logging import verbose_logger
from litellm.constants import (
DEFAULT_REASONING_EFFORT_HIGH_THINKING_BUDGET,
DEFAULT_REASONING_EFFORT_LOW_THINKING_BUDGET,
@ -11,6 +17,54 @@ from litellm.litellm_core_utils.core_helpers import as_str_mapping
OpenAIStyleReasoningEffort = Literal["minimal", "low", "medium", "high"]
_DROP_WARNING: Final = "Dropping %s=%s for model=%r because %s"
_DROP_PARAMS_DROP_WARNING: Final = "Dropping %s=%s for model=%r (drop_params=True) because %s"
_NON_NUMERIC_BUDGET_REASON: Final = "it is not a number. Send budget_tokens as an integer to set a reasoning level."
_NON_STRING_EFFORT_REASON: Final = (
"it is not a string. Send reasoning_effort as a level name such as 'high' to set a reasoning level."
)
_REASONING_PARAMS: Final = ("reasoning", "reasoning_effort", "thinking")
OPAQUE_VALUE: Final = TypeAdapter(object)
def warn_dropped_reasoning(
param: str, value: object, model: str | None, reason: str, *, drop_params_gated: bool
) -> None:
verbose_logger.warning(
_DROP_PARAMS_DROP_WARNING if drop_params_gated else _DROP_WARNING, param, reprlib.repr(value), model, reason
)
def warn_non_string_reasoning_effort(value: object, model: str | None) -> None:
effort: Final = as_str_mapping(value)
requested_level: Final = effort.get("effort") if effort is not None else value
if requested_level is None or isinstance(value, str):
return
warn_dropped_reasoning("reasoning_effort", value, model, _NON_STRING_EFFORT_REASON, drop_params_gated=False)
def warn_dropped_reasoning_params(
dropped: Collection[object],
request: Mapping[str, object],
model: str | None,
reason: str,
*,
drop_params_gated: bool,
) -> None:
for param in _REASONING_PARAMS:
if param in dropped and param in request:
warn_dropped_reasoning(param, request[param], model, reason, drop_params_gated=drop_params_gated)
@dataclass(frozen=True, slots=True)
class DroppedThinkingBudget:
budget_tokens: object
def warn(self, model: str) -> None:
warn_dropped_reasoning(
"thinking.budget_tokens", self.budget_tokens, model, _NON_NUMERIC_BUDGET_REASON, drop_params_gated=True
)
def reasoning_effort_from_thinking_budget(
budget_tokens: float,
@ -29,7 +83,9 @@ def reasoning_effort_from_thinking_budget(
return "minimal"
def reasoning_effort_from_thinking(thinking: object, *, drop_params: bool) -> OpenAIStyleReasoningEffort | None:
def _effort_or_dropped_budget(
thinking: object, *, drop_params: bool
) -> OpenAIStyleReasoningEffort | DroppedThinkingBudget | None:
fields: Final = as_str_mapping(thinking)
if fields is None or fields.get("type") != "enabled":
return None
@ -37,10 +93,27 @@ def reasoning_effort_from_thinking(thinking: object, *, drop_params: bool) -> Op
if isinstance(budget, (int, float)) and not isinstance(budget, bool):
return reasoning_effort_from_thinking_budget(budget)
if drop_params or litellm.drop_params is True:
return None
return DroppedThinkingBudget(budget)
raise non_numeric_thinking_budget_error(budget)
def reasoning_effort_from_thinking(
thinking: object, *, model: str, drop_params: bool
) -> OpenAIStyleReasoningEffort | None:
effort: Final = _effort_or_dropped_budget(thinking, drop_params=drop_params)
if not isinstance(effort, DroppedThinkingBudget):
return effort
effort.warn(model)
return None
def reasoning_effort_from_thinking_silently(
thinking: object, *, drop_params: bool
) -> OpenAIStyleReasoningEffort | None:
effort: Final = _effort_or_dropped_budget(thinking, drop_params=drop_params)
return None if isinstance(effort, DroppedThinkingBudget) else effort
def non_numeric_thinking_budget_error(budget: object) -> UnsupportedParamsError:
return UnsupportedParamsError(
message=(

View file

@ -249,6 +249,7 @@ def _redacted_provider_specific_fields(fields: Mapping[str, object]) -> Mapping[
"reasoningContentBlocks",
"citations",
"compaction_blocks",
"compaction_start",
"compaction_delta",
)
if key in fields

View file

@ -1,4 +1,5 @@
import json
import math
from collections.abc import Callable
from typing import Any, Final
@ -14,16 +15,23 @@ def strip_null_bytes(value: str) -> str:
return value.replace("\x00", "")
def _finite_or_none(value: float | None) -> float | None:
return None if isinstance(value, float) and not math.isfinite(value) else value
def safe_json_structure(
data: object,
max_depth: int = DEFAULT_MAX_RECURSE_DEPTH,
value_transform: Callable[[str | None, str], str] | None = None,
key: str | None = None,
*,
non_finite_as_null: bool = False,
) -> object:
"""
Rebuild data out of JSON-native pieces while detecting circular references.
If a circular reference is detected then a marker string is returned.
NUL bytes are stripped from strings to prevent PostgreSQL 22P05 errors.
NUL bytes are stripped from strings to prevent PostgreSQL 22P05 errors. With
non_finite_as_null, NaN and infinite floats become null, since jsonb rejects them.
value_transform, when given, is applied to every string leaf (and to the
str() fallback for non-serializable objects) with the mapping key the leaf
@ -43,7 +51,7 @@ def safe_json_structure(
cleaned = obj.replace("\x00", "") if "\x00" in obj else obj
return _transform(key, cleaned)
if isinstance(obj, (int, float, bool, type(None))):
return obj
return _finite_or_none(obj) if non_finite_as_null else obj
# Check for circular reference.
if id(obj) in seen:
return "CircularReference Detected"
@ -65,8 +73,9 @@ def safe_json_structure(
seen.remove(id(obj))
return tuple_result
elif isinstance(obj, set):
sortable_values: Final[list[Any]] = [_serialize(item, seen, depth + 1, key) for item in obj]
set_result: Final = sorted(sortable_values)
members: Final[list[Any]] = [_serialize(item, seen, depth + 1, key) for item in obj]
present: Final = sorted(member for member in members if member is not None)
set_result: Final = present + [None] * members.count(None)
seen.remove(id(obj))
return set_result
elif isinstance(obj, BaseModel):
@ -88,6 +97,10 @@ def safe_dumps(
data: object,
max_depth: int = DEFAULT_MAX_RECURSE_DEPTH,
value_transform: Callable[[str | None, str], str] | None = None,
*,
non_finite_as_null: bool = False,
) -> str:
"""Serialize data to JSON text through safe_json_structure."""
return json.dumps(safe_json_structure(data, max_depth, value_transform), default=str)
return json.dumps(
safe_json_structure(data, max_depth, value_transform, non_finite_as_null=non_finite_as_null), default=str
)

View file

@ -14,11 +14,18 @@ import litellm
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.core_helpers import (
PROVIDERS_BILLED_FROM_THEIR_OWN_USAGE_FIELDS,
UPSTREAM_USAGE_COST,
chunk_usage,
finite_cost,
provider_reported_usage_cost,
stamp_litellm_bill_on_usage,
)
from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR, set_hidden_params
from litellm.litellm_core_utils.hidden_params import (
HIDDEN_PARAMS_ATTR,
get_hidden_params,
set_hidden_param,
set_hidden_params,
)
from litellm.litellm_core_utils.llm_cost_calc.guardrail_cost import BEDROCK_TRACE_FIELDS, object_items, object_mapping
from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import partition_annotations
from litellm.litellm_core_utils.prompt_templates.common_utils import (
@ -1014,7 +1021,9 @@ class ChunkProcessor:
prompt_tokens_details, cache_creation_token_details
)
if usage_chunk_dict["cost"] is not None:
if Usage.dropped_non_finite_cost(usage_chunk):
cost = None
elif usage_chunk_dict["cost"] is not None:
cost = usage_chunk_dict["cost"]
prompt_tokens_details = attach_cache_creation_token_details(prompt_tokens_details, cache_creation_token_details)
@ -1441,11 +1450,19 @@ def _set_stream_builder_response_cost(
_set_hidden_response_cost(response, response_cost)
def _carry_upstream_usage_cost(response: ModelResponse, chunks: Sequence[object]) -> None:
chunk_hidden_params: Final = (get_hidden_params(chunk) for chunk in reversed(chunks))
stamped: Final = (hidden for hidden in chunk_hidden_params if hidden is not None and UPSTREAM_USAGE_COST in hidden)
for hidden in stamped:
set_hidden_param(response, UPSTREAM_USAGE_COST, hidden[UPSTREAM_USAGE_COST])
return
def _stamp_streaming_usage_cost(usage: Usage, response: ModelResponse, logging_obj: Optional["Logging"]) -> None:
if logging_obj is None:
return
if _logged_provider(logging_obj) in PROVIDERS_BILLED_FROM_THEIR_OWN_USAGE_FIELDS and isinstance(
getattr(usage, "cost", None), (int, float)
if _logged_provider(logging_obj) in PROVIDERS_BILLED_FROM_THEIR_OWN_USAGE_FIELDS and (
finite_cost(getattr(usage, "cost", None)) is not None
):
return
computed_cost: Final = logging_obj.response_cost_calculator(result=response)
@ -1791,6 +1808,7 @@ def stream_chunk_builder(
response,
logging_obj,
)
_carry_upstream_usage_cost(response, chunks) # pyright: ignore[reportUnknownArgumentType] # chunks is an untyped list param
_stamp_streaming_usage_cost(usage, response, logging_obj)
_set_stream_builder_response_cost(
response,
@ -1833,6 +1851,7 @@ def stream_chunk_builder(
)
break
_carry_upstream_usage_cost(response, chunks) # pyright: ignore[reportUnknownArgumentType] # chunks is an untyped list param
_stamp_streaming_usage_cost(usage, response, logging_obj)
_set_stream_builder_response_cost(
response,

View file

@ -24,10 +24,16 @@ import litellm
from litellm import verbose_logger
from litellm._uuid import uuid
from litellm.litellm_core_utils.asyncify import asyncify
from litellm.litellm_core_utils.core_helpers import chunk_usage, provider_reported_usage_cost
from litellm.litellm_core_utils.core_helpers import (
UPSTREAM_USAGE_COST,
chunk_usage,
provider_reported_usage_cost,
stamp_litellm_bill_on_usage,
)
from litellm.litellm_core_utils.hidden_params import (
HIDDEN_PARAMS_ATTR,
get_hidden_params,
set_extra_attribute,
set_hidden_param,
stored_attribute,
)
@ -40,6 +46,7 @@ from litellm.llms.ollama.common_utils import OllamaResponseIdNormalizer
from litellm.types.llms.base import LiteLLMBaseModel
from litellm.types.router import GenericLiteLLMParams
from litellm.types.utils import (
CHOICE_LOGPROBS_OR_JSON,
CacheCreationTokenDetails,
CompletionTokensDetailsWrapper,
Delta,
@ -96,12 +103,17 @@ def _stamp_usage_cost(
usage_chunk: ModelResponse | TextCompletionResponse | ModelResponseStream,
complete_response: ModelResponse | TextCompletionResponse | ModelResponseStream,
) -> None:
bill: Final = (get_hidden_params(complete_response) or {}).get("response_cost")
complete_hidden: Final = get_hidden_params(complete_response) or {}
bill: Final = complete_hidden.get("response_cost")
usage: Final[object] = getattr(usage_chunk, "usage", None)
if usage is None or isinstance(bill, bool) or not isinstance(bill, (int, float)):
return
if bill > 0 or isinstance(getattr(usage, "cost", None), (int, float)):
setattr(usage, "cost", float(bill)) # noqa: B010 # cost is an optional extra on the usage model
if bill <= 0 and not isinstance(getattr(usage, "cost", None), (int, float)):
return
chunk_hidden: Final = get_hidden_params(usage_chunk)
if chunk_hidden is not None and UPSTREAM_USAGE_COST in complete_hidden:
chunk_hidden[UPSTREAM_USAGE_COST] = complete_hidden[UPSTREAM_USAGE_COST]
stamp_litellm_bill_on_usage(usage_chunk, usage, float(bill))
def _stamp_served_service_tier(response: ModelResponseStream, complete_streaming_response: ModelResponse) -> None:
@ -1034,7 +1046,7 @@ class CustomStreamWrapper:
_initial_delta: Final = model_response.choices[0].delta.model_dump()
_initial_delta.pop("role", None)
model_response.choices[0].delta = Delta(**_initial_delta)
set_extra_attribute(model_response.choices[0], "delta", Delta(**_initial_delta))
return model_response
def _prebuilt_plain_delta(self, chunk: object) -> Delta | None:
@ -1060,13 +1072,13 @@ class CustomStreamWrapper:
def _set_role_on_fresh_delta(self, model_response: ModelResponseStream) -> None:
delta: Final = model_response.choices[0].delta
if self.sent_first_chunk is False:
delta.role = "assistant"
set_extra_attribute(delta, "role", "assistant")
self.sent_first_chunk = True
return
if not _holds_only_scalars(delta):
self.strip_role_from_delta(model_response)
return
delta.role = None
set_extra_attribute(delta, "role", None)
if "provider_specific_fields" not in delta.model_fields_set:
delta.provider_specific_fields = None
@ -1185,8 +1197,8 @@ class CustomStreamWrapper:
and _holds_only_scalars(upstream_delta)
)
if can_reuse_choice and upstream_choice is not None:
model_response.choices[0].finish_reason = None # rebind-ok: keep reused choice
model_response.choices[0].index = upstream_choice.index # rebind-ok: keep reused choice
set_extra_attribute(model_response.choices[0], "finish_reason", None)
set_extra_attribute(model_response.choices[0], "index", upstream_choice.index)
_prepend_upstream_choice_extras(
model_response.choices[0],
upstream_choice,
@ -1201,18 +1213,17 @@ class CustomStreamWrapper:
choice_json.pop(
"finish_reason", None
) # for mistral etc. which return a value in their last chunk (not-openai compatible).
choices.append(StreamingChoices(**choice_json))
choice_json.pop("logprobs", None)
choices.append(
StreamingChoices(**choice_json, logprobs=getattr(choice, "logprobs", None))
)
except Exception:
choices.append(StreamingChoices())
setattr(model_response, "choices", choices)
else:
return
model_response.system_fingerprint = upstream_chunk.system_fingerprint
setattr(
model_response,
"citations",
stored_attribute(upstream_chunk, "citations"),
)
set_extra_attribute(model_response, "citations", stored_attribute(upstream_chunk, "citations"))
preserve_upstream_non_openai_attributes(
model_response=model_response,
original_chunk=original_chunk,
@ -1605,7 +1616,7 @@ class CustomStreamWrapper:
model_response.system_fingerprint = response_obj["original_chunk"].system_fingerprint
self.system_fingerprint = response_obj["original_chunk"].system_fingerprint
if response_obj["logprobs"] is not None:
model_response.choices[0].logprobs = response_obj["logprobs"]
model_response.choices[0].logprobs = CHOICE_LOGPROBS_OR_JSON.validate_python(response_obj["logprobs"])
if response_obj["usage"] is not None:
if isinstance(response_obj["usage"], dict):
@ -2482,6 +2493,25 @@ class CustomStreamWrapper:
def bill_failure_after_earlier_rounds(self, earlier: "BilledRoundsSpend") -> None:
self._earlier_rounds_spend = earlier
def partial_response(self) -> "ModelResponse | TextCompletionResponse | None":
partial: Final = (
litellm.stream_chunk_builder(
chunks=self.chunks,
messages=self.messages if isinstance(self.messages, list) else None,
logging_obj=self.logging_obj,
count_prompt_tokens=self.count_prompt_tokens,
)
if self.chunks
else None
)
usage: Final[object] = getattr(partial, "usage", None)
if partial is None or not isinstance(usage, Usage):
return None
if self.model:
partial.model = self.model
backfill_missing_cache_usage_fields(usage)
return partial
def _record_partial_usage_for_failure(self) -> None:
"""
A stream that breaks mid-flight still billed the provider for the chunks
@ -2491,23 +2521,15 @@ class CustomStreamWrapper:
moves it into terminal failure usage before opening the next attempt,
so a recovered attempt logs its own usage.
"""
if self.logging_obj is None or not self.chunks:
if self.logging_obj is None:
return
try:
partial_response: Final = litellm.stream_chunk_builder(
chunks=self.chunks,
messages=self.messages if isinstance(self.messages, list) else None,
logging_obj=self.logging_obj,
count_prompt_tokens=self.count_prompt_tokens,
)
partial_response: Final = self.partial_response()
if partial_response is None:
return
usage: Final = cast(Usage | None, getattr(partial_response, "usage", None))
if usage is None:
usage: Final[object] = getattr(partial_response, "usage", None)
if not isinstance(usage, Usage):
return
if self.model:
partial_response.model = self.model
backfill_missing_cache_usage_fields(usage)
partial_cost: Final = self.logging_obj.response_cost_calculator(result=partial_response) or 0.0
if self._earlier_rounds_spend is None:
self.logging_obj.model_call_details["combined_usage_object"] = usage

View file

@ -48,6 +48,7 @@ from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import
place_mid_conversation_system,
split_leading_system_run,
)
from litellm.litellm_core_utils.reasoning_effort_utils import OPAQUE_VALUE, warn_non_string_reasoning_effort
from litellm.llms.base_llm.base_utils import (
ResponseFormatSchema,
as_json_object,
@ -1951,6 +1952,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
if isinstance(effort_value, dict):
effort_value = effort_value.get("effort")
if not isinstance(effort_value, str):
warn_non_string_reasoning_effort(OPAQUE_VALUE.validate_python(value), model)
continue
mapped_thinking = AnthropicConfig._map_reasoning_effort(
reasoning_effort=effort_value,

View file

@ -37,7 +37,7 @@ from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import
message_field,
parts_of,
)
from litellm.litellm_core_utils.reasoning_effort_utils import non_numeric_thinking_budget_error
from litellm.litellm_core_utils.reasoning_effort_utils import DroppedThinkingBudget, non_numeric_thinking_budget_error
from litellm.litellm_core_utils.url_utils import join_api_base
from litellm.llms.anthropic.wif import (
aget_anthropic_wif_token,
@ -58,6 +58,7 @@ from litellm.types.llms.anthropic import (
ANTHROPIC_THINKING_BINDING_CONTROLS_BETA_HEADER,
ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER,
ANTHROPIC_TOOL_SEARCH_TOOL_TYPES,
LEGACY_THINKING_DISPLAY_UPDATES_PROVIDERS,
AllAnthropicToolsValues,
AnthropicMcpServerTool,
AnthropicMessagesToolChoice,
@ -126,6 +127,8 @@ _JSON_VALUE: Final = TypeAdapter(JsonValue)
_CLAUDE_CODE_USER_AGENT_PREFIXES: Final = ("claude-cli/", "claude-code/")
NATIVE_COMPACTION_PROVIDERS: Final = frozenset({"anthropic", "azure_ai", "vertex_ai"})
def is_claude_tool_target(custom_llm_provider: str | None, model: str) -> bool:
if custom_llm_provider == "anthropic":
@ -204,7 +207,7 @@ def requires_native_compaction_beta(
optional_params: Mapping[str, object],
messages: Sequence[object],
) -> bool:
return custom_llm_provider == "anthropic" and (
return custom_llm_provider in NATIVE_COMPACTION_PROVIDERS and (
optional_params.get("compaction") is not None or carries_signed_compaction_block(messages)
)
@ -246,6 +249,27 @@ def context_management_betas(context_management: JsonValue) -> tuple[str, ...]:
)
def count_tokens_feature_betas(
custom_llm_provider: str,
optional_params: Mapping[str, object],
messages: Sequence[object],
) -> tuple[str, ...]:
return tuple(
beta
for beta, used in (
(
ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_09_04.value,
requires_native_compaction_beta(custom_llm_provider, optional_params, messages),
),
(
ANTHROPIC_MID_CONVERSATION_TOOL_CHANGES_BETA_HEADER,
AnthropicModelInfo().is_mid_conversation_tool_change_used(messages),
),
)
if used
)
def supports_anthropic_cache_control(model: str, custom_llm_provider: str | None) -> bool:
from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
from litellm.utils import supports_prompt_caching
@ -610,11 +634,11 @@ def _numeric_budget_tokens(budget_tokens: object) -> int | None:
return None
def _legacy_budget_tokens(budget_tokens: object, *, drop_params: bool) -> int | None:
def _legacy_budget_tokens(budget_tokens: object, *, drop_params: bool) -> int | DroppedThinkingBudget:
if (numeric := _numeric_budget_tokens(budget_tokens)) is not None:
return numeric
if drop_params or litellm.drop_params is True:
return None
return DroppedThinkingBudget(budget_tokens)
raise non_numeric_thinking_budget_error(budget_tokens)
@ -645,10 +669,12 @@ class AnthropicModelInfo(BaseLLMModelInfo):
file_ids: Final = get_file_ids_from_messages(messages)
return len(file_ids) > 0
def is_thinking_display_updates_used(self, thinking: AnthropicThinkingParam | None) -> bool:
if not isinstance(thinking, dict):
return False
return thinking.get("type") in ("adaptive", "enabled") and thinking.get("display") == "updates"
def is_thinking_display_updates_used(self, thinking: object) -> bool:
match thinking:
case {"type": "adaptive" | "enabled", "display": "updates"}:
return True
case _:
return False
def thinking_betas(self, thinking: AnthropicThinkingParam | None) -> tuple[str, ...]:
return (
@ -1097,6 +1123,22 @@ class AnthropicModelInfo(BaseLLMModelInfo):
custom_llm_provider: str,
drop_params: bool,
) -> None:
dropped: Final = AnthropicModelInfo.translate_legacy_thinking_for_adaptive_model_silently(
model=model,
optional_params=optional_params,
custom_llm_provider=custom_llm_provider,
drop_params=drop_params,
)
if dropped is not None:
dropped.warn(model)
@staticmethod
def translate_legacy_thinking_for_adaptive_model_silently(
model: str,
optional_params: MutableMapping[str, object], # mutable-ok: in-place out-param like the sibling helpers
custom_llm_provider: str,
drop_params: bool,
) -> DroppedThinkingBudget | None:
"""Translate legacy ``thinking.type=enabled`` to adaptive for the
adaptive-thinking models that reject it (4.7+ and the 5 families).
Models flagged ``supports_legacy_thinking`` (the 4.6 family) accept the
@ -1105,22 +1147,25 @@ class AnthropicModelInfo(BaseLLMModelInfo):
``output_config.effort`` is never overridden.
"""
if not AnthropicModelInfo._is_adaptive_thinking_model(model, custom_llm_provider):
return
return None
if AnthropicModelInfo._supports_legacy_thinking(model, custom_llm_provider):
return
return None
thinking: Final = validated_object_mapping(optional_params.get("thinking"))
if thinking is None or thinking.get("type") != "enabled":
return
return None
budget_tokens: Final = _legacy_budget_tokens(thinking.get("budget_tokens", 0), drop_params=drop_params)
display: Final = thinking.get("display")
supported_display: Final = {"display": display} if display in ("summarized", "omitted") else {}
keeps_display: Final = display in ("summarized", "omitted") or (
display == "updates" and custom_llm_provider in LEGACY_THINKING_DISPLAY_UPDATES_PROVIDERS
)
supported_display: Final = {"display": display} if keeps_display else {}
carried_keys: Final = carried_thinking_keys(
thinking, rebuilt_keys=frozenset(("type", "budget_tokens", "display")), drop_params=drop_params
)
optional_params["thinking"] = {**carried_keys, "type": "adaptive", **supported_display}
if budget_tokens is None:
return
if isinstance(budget_tokens, DroppedThinkingBudget):
return budget_tokens
effort: Final = AnthropicModelInfo._legacy_budget_to_effort(
model=model, budget_tokens=budget_tokens, custom_llm_provider=custom_llm_provider
)
@ -1129,6 +1174,7 @@ class AnthropicModelInfo(BaseLLMModelInfo):
"effort": effort,
**(existing_output_config if isinstance(existing_output_config, dict) else MappingProxyType({})),
}
return None
@staticmethod
def _legacy_budget_to_effort(model: str, budget_tokens: int, custom_llm_provider: str) -> str:

View file

@ -134,6 +134,7 @@ from litellm.litellm_core_utils.prompt_templates.factory import (
)
from litellm.litellm_core_utils.reasoning_effort_utils import (
reasoning_effort_from_thinking,
reasoning_effort_from_thinking_silently,
)
from litellm.llms.anthropic.common_utils import (
eager_input_streaming_flag,
@ -297,6 +298,7 @@ class AnthropicAdapter:
custom_llm_provider=custom_llm_provider,
api_base=api_base,
drop_params=drop_params,
report_reasoning_drops=True,
)
return translated_body, tool_name_mapping
@ -666,7 +668,9 @@ class LiteLLMAnthropicMessagesAdapter:
def translate_anthropic_thinking_to_reasoning_effort(
thinking: AnthropicThinkingParam,
*,
model: str,
drop_params: bool = True,
report_reasoning_drops: bool = False,
) -> str | None:
"""
Translate Anthropic's thinking parameter to OpenAI's reasoning_effort.
@ -685,7 +689,9 @@ class LiteLLMAnthropicMessagesAdapter:
if thinking_type == "disabled":
return "none"
elif thinking_type == "enabled":
return reasoning_effort_from_thinking(thinking, drop_params=drop_params)
if report_reasoning_drops:
return reasoning_effort_from_thinking(thinking, model=model, drop_params=drop_params)
return reasoning_effort_from_thinking_silently(thinking, drop_params=drop_params)
elif thinking_type == "adaptive":
# Adaptive thinking: effort is controlled by output_config.effort,
# not budget_tokens. Return a default; caller should override with
@ -752,7 +758,7 @@ class LiteLLMAnthropicMessagesAdapter:
return {"thinking": thinking}
else:
reasoning_effort: Final = LiteLLMAnthropicMessagesAdapter.translate_anthropic_thinking_to_reasoning_effort(
thinking
thinking, model=model
)
if reasoning_effort:
return {
@ -1147,6 +1153,7 @@ class LiteLLMAnthropicMessagesAdapter:
*,
custom_llm_provider: str | None = None,
drop_params: bool = True,
report_reasoning_drops: bool = False,
) -> None:
"""Translate Anthropic thinking to either thinking or reasoning_effort.
@ -1207,7 +1214,10 @@ class LiteLLMAnthropicMessagesAdapter:
return
reasoning_effort: Final = declared_effort or self.translate_anthropic_thinking_to_reasoning_effort(
cast(AnthropicThinkingParam, thinking), drop_params=drop_params
cast(AnthropicThinkingParam, thinking),
model=model,
drop_params=drop_params,
report_reasoning_drops=report_reasoning_drops,
)
if not reasoning_effort:
return
@ -1267,6 +1277,7 @@ class LiteLLMAnthropicMessagesAdapter:
api_base: str | None = None,
preserve_midturn_system: bool = False,
drop_params: bool = True,
report_reasoning_drops: bool = False,
) -> tuple[ChatCompletionRequest, dict[str, str]]:
"""
This is used by the beta Anthropic Adapter, for translating anthropic `/v1/messages` requests to the openai format.
@ -1321,6 +1332,7 @@ class LiteLLMAnthropicMessagesAdapter:
new_kwargs=new_kwargs,
custom_llm_provider=custom_llm_provider,
drop_params=drop_params,
report_reasoning_drops=report_reasoning_drops,
)
## CONVERT STOP_SEQUENCES
self._translate_stop_sequences_to_openai(

View file

@ -20,6 +20,7 @@ from litellm.litellm_core_utils.core_helpers import as_str_mapping
from litellm.litellm_core_utils.exception_mapping_utils import exception_type
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.llms.anthropic.common_utils import (
AnthropicModelInfo,
strip_provider_specific_fields_from_anthropic_messages,
tools_marked_for_a_tool_config_point,
)
@ -36,7 +37,6 @@ from litellm.router_utils.reasoning_effort_capability import (
keeps_effort,
model_group_from_metadata,
resolve_deployment_reasoning_effort,
with_adaptive_thinking_as_effort,
)
from litellm.types.llms.anthropic import AllAnthropicToolsValues
from litellm.types.llms.anthropic_messages.anthropic_request import AnthropicMetadata
@ -185,9 +185,11 @@ def _with_deployment_output_effort(
).value
def translate(source: Mapping[str, object]) -> Mapping[str, object]:
return with_adaptive_thinking_as_effort(
source, model=model, custom_llm_provider=custom_llm_provider, drop_params=drop_params
translated: Final = dict(source)
AnthropicModelInfo.translate_legacy_thinking_for_adaptive_model(
model=model, optional_params=translated, custom_llm_provider=custom_llm_provider, drop_params=drop_params
)
return translated
requested_output: Final = as_str_mapping(params.get("output_config"))
caller_output: Final = None if requested_output is None else resolve(requested_output)

View file

@ -21,6 +21,7 @@ from litellm.constants import ANTHROPIC_BILLING_METADATA_PREFIX
from litellm.exceptions import AuthenticationError
from litellm.litellm_core_utils.core_helpers import drops_unsupported_params
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.litellm_core_utils.reasoning_effort_utils import OPAQUE_VALUE, warn_non_string_reasoning_effort
from litellm.litellm_core_utils.url_utils import ensure_api_route
from litellm.llms.anthropic.messages.convertible_blocks import drop_non_string_tool_result_texts
from litellm.llms.anthropic.messages.safeguards import proxy_serves_safeguards
@ -40,6 +41,7 @@ from litellm.types.llms.anthropic_tool_search import get_tool_search_beta_header
from litellm.types.router import GenericLiteLLMParams
from ...common_utils import (
NATIVE_COMPACTION_PROVIDERS,
OBJECT_LIST_ADAPTER,
AnthropicError,
AnthropicModelInfo,
@ -103,6 +105,7 @@ _MESSAGES_ADAPTER: Final = TypeAdapter(list[dict[str, object]])
class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
_workload_identity_eligible: ClassVar[bool] = True
_report_reasoning_drops: ClassVar[bool] = True
@property
def custom_llm_provider(self) -> str | None:
@ -126,7 +129,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
"tool_choice",
"thinking",
"context_management",
*(("compaction",) if self._resolved_provider == "anthropic" else ()),
*(("compaction",) if self._resolved_provider in NATIVE_COMPACTION_PROVIDERS else ()),
"output_format",
"inference_geo",
"speed",
@ -695,12 +698,17 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
):
anthropic_messages_optional_request_params.pop("output_format") # pyright: ignore[reportUnknownMemberType] # the request params arrive as a bare dict
requested_reasoning_effort: Final = OPAQUE_VALUE.validate_python(
anthropic_messages_optional_request_params.get("reasoning_effort") # pyright: ignore[reportUnknownMemberType] # the request params arrive as a bare dict
)
self._translate_reasoning_effort_to_anthropic(
model=model,
optional_params=anthropic_messages_optional_request_params,
max_tokens=max_tokens,
custom_llm_provider=self._resolved_provider,
)
if self._report_reasoning_drops:
warn_non_string_reasoning_effort(requested_reasoning_effort, model)
AnthropicModelInfo.maybe_drop_disabled_thinking(
model=model,
@ -708,12 +716,14 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
custom_llm_provider=self._resolved_provider,
)
AnthropicModelInfo.translate_legacy_thinking_for_adaptive_model(
dropped_budget: Final = AnthropicModelInfo.translate_legacy_thinking_for_adaptive_model_silently(
model=model,
optional_params=anthropic_messages_optional_request_params,
custom_llm_provider=self._resolved_provider,
drop_params=getattr(litellm_params, "drop_params", None) is True,
)
if dropped_budget is not None and self._report_reasoning_drops:
dropped_budget.warn(model)
self._translate_adaptive_effort_for_non_adaptive_model(
model=model,

View file

@ -534,6 +534,7 @@ class LiteLLMAnthropicToResponsesAPIAdapter:
output_config: dict[str, object] | None = None,
auto_summary_level: str = "detailed",
*,
model: str,
drop_params: bool = True,
) -> dict[str, object] | None:
"""
@ -555,7 +556,7 @@ class LiteLLMAnthropicToResponsesAPIAdapter:
if isinstance(output_config, dict) and output_config.get("effort"):
effort = output_config["effort"]
elif thinking_type == "enabled":
budget_effort: Final = reasoning_effort_from_thinking(thinking, drop_params=drop_params)
budget_effort: Final = reasoning_effort_from_thinking(thinking, model=model, drop_params=drop_params)
if budget_effort is None:
return None
effort = budget_effort
@ -667,6 +668,7 @@ class LiteLLMAnthropicToResponsesAPIAdapter:
thinking,
output_config=cast(dict[str, object] | None, output_config),
auto_summary_level=auto_summary_level,
model=model,
drop_params=drop_params,
)
if reasoning:

View file

@ -9,7 +9,7 @@ from dataclasses import dataclass, field
from functools import reduce
from itertools import accumulate, groupby
from types import MappingProxyType
from typing import Annotated, Final, Literal, Protocol, TypeAlias
from typing import Annotated, ClassVar, Final, Literal, Protocol, TypeAlias
import httpx
from pydantic import ConfigDict, Field, JsonValue, StrictInt, TypeAdapter, ValidationError
@ -81,6 +81,10 @@ _DEPLOYMENT_OPTIONS: Final = frozenset(
)
class _BaselineAnthropicMessagesConfig(AnthropicMessagesConfig):
_report_reasoning_drops: ClassVar[bool] = False
class _StrictModel(LiteLLMBaseModel):
model_config = ConfigDict(extra="forbid", frozen=True, strict=True)
@ -661,7 +665,7 @@ def prepare_native_baseline_body(request: Mapping[str, object], model: str) -> M
TypeAdapter(tuple[str, ...]).validate_python(owned.get("additional_drop_params") or ()),
dict(options),
)
config: Final = AnthropicMessagesConfig()
config: Final = _BaselineAnthropicMessagesConfig()
body: Final = config.transform_anthropic_messages_request(
model=resolved_model,
messages=strip_provider_specific_fields_from_anthropic_messages(messages),

View file

@ -14,6 +14,7 @@ from litellm.constants import AZURE_DEFAULT_RESPONSES_API_VERSION
from litellm.litellm_core_utils.url_utils import encode_url_path_segment
from litellm.llms.azure.chat.gpt_reasoning_transformation import AzureOpenAIGPT5Config
from litellm.llms.azure.common_utils import BaseAzureLLM
from litellm.llms.base_llm.responses.transformation import SUMMARY_ONLY_REASONING_FIELD_VALUES, ReasoningFieldValues
from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig
from litellm.types.llms.openai import *
from litellm.types.responses.main import *
@ -92,6 +93,12 @@ class AzureOpenAIResponsesAPIConfig(OpenAIResponsesAPIConfig):
def _effort_resolves_to_none(model: str, effort: str | None) -> bool:
return AzureOpenAIGPT5Config.effort_resolves_to_none(model, effort)
def deployment_names_wrap_openai_models(self) -> bool:
return True
def unresolved_model_reasoning_field_values(self, model: str) -> ReasoningFieldValues | None:
return SUMMARY_ONLY_REASONING_FIELD_VALUES if self._is_gpt_5_model(model) else None
def get_supported_openai_params(self, model: str) -> list:
"""
Azure Responses API does not support context_management (compaction).

View file

@ -84,6 +84,7 @@ class AzureAIAnthropicCountTokensHandler(AzureAIAnthropicCountTokensConfig):
headers: Final = self.get_required_headers(
api_key=api_key,
litellm_params=litellm_params,
messages=messages,
)
if not {"x-api-key", "Authorization"} & headers.keys():
raise AnthropicError(

View file

@ -4,11 +4,13 @@ Azure AI Anthropic CountTokens API transformation logic.
Extends the base Anthropic CountTokens transformation with Azure authentication.
"""
from collections.abc import Sequence
from typing import Final
from urllib.parse import urlsplit
from litellm.constants import ANTHROPIC_TOKEN_COUNTING_BETA_VERSION
from litellm.litellm_core_utils.url_utils import join_api_base
from litellm.llms.anthropic.common_utils import count_tokens_feature_betas, merge_anthropic_beta_headers
from litellm.llms.anthropic.count_tokens.transformation import (
AnthropicCountTokensConfig,
)
@ -28,6 +30,7 @@ class AzureAIAnthropicCountTokensConfig(AnthropicCountTokensConfig):
self,
api_key: str | None,
litellm_params: dict[str, object] | None = None,
messages: Sequence[object] = (),
) -> dict[str, str]:
"""
Get the required headers for the Azure AI Anthropic CountTokens API.
@ -38,6 +41,7 @@ class AzureAIAnthropicCountTokensConfig(AnthropicCountTokensConfig):
Args:
api_key: The Azure AI API key
litellm_params: Optional LiteLLM parameters for additional auth config
messages: The request messages, read for betas the counted features need
Returns:
Dictionary of required headers with both x-api-key and Azure authentication
@ -46,7 +50,9 @@ class AzureAIAnthropicCountTokensConfig(AnthropicCountTokensConfig):
headers: Final = {
"Content-Type": "application/json",
"anthropic-version": "2023-06-01",
"anthropic-beta": ANTHROPIC_TOKEN_COUNTING_BETA_VERSION,
"anthropic-beta": merge_anthropic_beta_headers(
ANTHROPIC_TOKEN_COUNTING_BETA_VERSION, count_tokens_feature_betas("azure_ai", {}, messages)
),
**({"x-api-key": api_key} if api_key else {}),
}

View file

@ -20,6 +20,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
prompt_cache_breakpoint_for_wire,
with_prompt_cache_breakpoint,
)
from litellm.litellm_core_utils.reasoning_effort_utils import warn_dropped_reasoning_params
from litellm.llms.azure.common_utils import BaseAzureLLM
from litellm.llms.azure_ai.common_utils import (
api_key_header_for_base,
@ -40,6 +41,14 @@ if TYPE_CHECKING:
from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
_REQUEST_BODY: Final = TypeAdapter(dict[str, object])
def _request_model(request: Mapping[str, object]) -> str | None:
model: Final = request.get("model")
return model if isinstance(model, str) else None
class AzureFoundryErrorStrings(str, enum.Enum):
SET_EXTRA_PARAMETERS_TO_PASS_THROUGH = "Set extra-parameters to 'pass-through'"
@ -510,7 +519,13 @@ class AzureAIStudioConfig(OpenAIConfig):
request_data = self._drop_extra_params_from_request_data(request_data, error_text)
if "Extra inputs are not permitted" in error_text and self._error_has_tool_level_extra_fields(error_text):
request_data = self._drop_tool_level_extra_fields(request_data, error_text)
data: Final = drop_params_from_unprocessable_entity_error(e=e, data=request_data)
data: Final = drop_params_from_unprocessable_entity_error(
e=e,
data=request_data,
model=_request_model(_REQUEST_BODY.validate_python(request_data)),
drop_params_gated="Extra inputs are not permitted" in error_text
and not self._error_has_tool_level_extra_fields(error_text),
)
return data
def _drop_tool_level_extra_fields(self, request_data: dict, error_text: str) -> dict:
@ -526,6 +541,14 @@ class AzureAIStudioConfig(OpenAIConfig):
def _drop_extra_params_from_request_data(self, request_data: dict, error_text: str) -> dict:
params_to_drop: Final = self._extract_params_to_drop_from_error_text(error_text)
if params_to_drop:
request: Final = _REQUEST_BODY.validate_python(request_data)
warn_dropped_reasoning_params(
params_to_drop,
request,
_request_model(request),
"Azure AI rejected it as an extra parameter its deployment does not pass through.",
drop_params_gated=False,
)
for param in params_to_drop:
if param in request_data:
request_data.pop(param, None)

View file

@ -1,7 +1,7 @@
import types
from abc import ABC, abstractmethod
from collections.abc import Mapping
from typing import TYPE_CHECKING, Any, Final, Literal, TypeAlias, cast
from typing import TYPE_CHECKING, Any, Final, Literal, TypeAlias, cast, get_args
import httpx
@ -27,6 +27,20 @@ else:
BaseLLMException = Any
ReasoningSummaryLevel: TypeAlias = Literal["auto", "concise", "detailed"]
ReasoningFieldValues: TypeAlias = Mapping[str, frozenset[str | None] | None]
REASONING_SUMMARY_VALUES: Final = frozenset((*get_args(ReasoningSummaryLevel), None))
SUMMARY_ONLY_REASONING_FIELD_VALUES: Final[ReasoningFieldValues] = types.MappingProxyType(
{"summary": REASONING_SUMMARY_VALUES}
)
STANDARD_MODE_WITHOUT_CONTEXT_REASONING_FIELD_VALUES: Final[ReasoningFieldValues] = types.MappingProxyType(
{**SUMMARY_ONLY_REASONING_FIELD_VALUES, "mode": frozenset({"standard"})}
)
STANDARD_MODE_REASONING_FIELD_VALUES: Final[ReasoningFieldValues] = types.MappingProxyType(
{**STANDARD_MODE_WITHOUT_CONTEXT_REASONING_FIELD_VALUES, "context": None}
)
ANY_REASONING_FIELD_VALUES: Final[ReasoningFieldValues] = types.MappingProxyType(
{**SUMMARY_ONLY_REASONING_FIELD_VALUES, "mode": None, "context": None}
)
class BaseResponsesAPIConfig(ABC):
@ -100,6 +114,18 @@ class BaseResponsesAPIConfig(ABC):
"""The ``reasoning.summary`` LiteLLM's opt-in auto summary asks this provider for."""
return "detailed"
def reasoning_field_values(self) -> ReasoningFieldValues | None:
return None
def deployment_names_wrap_openai_models(self) -> bool:
return False
def unresolved_model_reasoning_field_values(self, model: str) -> ReasoningFieldValues | None:
return None
def ignores_unsupported_reasoning_fields(self) -> bool:
return False
@abstractmethod
def map_openai_params(
self,

View file

@ -33,6 +33,7 @@ from litellm.litellm_core_utils.prompt_templates.image_handling import (
inline_remote_image_urls,
inline_remote_media,
)
from litellm.litellm_core_utils.reasoning_effort_utils import warn_dropped_reasoning
from litellm.litellm_core_utils.url_utils import ensure_api_route, join_api_base
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM, bedrock_bearer_token
@ -42,6 +43,7 @@ from litellm.llms.bedrock.common_utils import (
bedrock_rejects_stop_sequences,
bedrock_runtime_chat_completions_serves_reasoning_inline,
split_bedrock_region_path,
strip_bedrock_routing_prefix,
)
from litellm.llms.openai.chat.gpt_transformation import OpenAIChatCompletionStreamingHandler
from litellm.llms.openai_like.chat.transformation import OpenAILikeChatConfig
@ -121,6 +123,13 @@ def without_refused_reasoning_effort(model: str, params: Mapping[str, object]) -
effort: Final = params.get("reasoning_effort")
if not isinstance(effort, str) or effort not in chat_completions_reasoning_efforts_refused_for(model):
return params
warn_dropped_reasoning(
"reasoning_effort",
effort,
strip_bedrock_routing_prefix(model),
"AWS's Chat Completions endpoint rejects that level for this model family, so AWS applies its default effort.",
drop_params_gated=False,
)
return _without_params(params, frozenset(("reasoning_effort",)))
@ -445,6 +454,14 @@ class AmazonBedrockRuntimeChatCompletionsConfig(OpenAILikeChatConfig):
),
status_code=400,
)
if malformed_effort:
warn_dropped_reasoning(
"reasoning_effort",
raw_params["reasoning_effort"],
strip_bedrock_routing_prefix(model),
"Bedrock's Chat Completions endpoint takes it only as a string, so AWS applies its default effort.",
drop_params_gated=True,
)
rejected_stop: Final = frozenset(("stop",)) if bedrock_rejects_stop_sequences(model) else frozenset[str]()
return dict(
without_refused_reasoning_effort(

View file

@ -50,6 +50,7 @@ from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import
message_field,
parts_of,
)
from litellm.litellm_core_utils.reasoning_effort_utils import OPAQUE_VALUE, warn_dropped_reasoning
from litellm.llms.anthropic.chat.transformation import (
DROP_UNSUPPORTED_ADAPTIVE_THINKING_WARNING,
DROP_UNSUPPORTED_OUTPUT_CONFIG_WARNING,
@ -63,6 +64,7 @@ from litellm.llms.bedrock.chat.tool_result_images import place_tool_result_image
from litellm.llms.bedrock.common_utils import (
bedrock_chat_uses_native_structured_output,
bedrock_cleaned_output_schema,
bedrock_model_is_unknown,
bedrock_model_supports_regex_lookaround,
bedrock_native_output_format,
bedrock_native_output_is_opt_in,
@ -186,6 +188,8 @@ def _tools_the_model_accepts(
_OUTPUT_CONFIG_BLOCK: Final = TypeAdapter(OutputConfigBlock)
_DEEPSEEK_THINKING_REASON: Final = "DeepSeek on Bedrock reasons natively and takes no thinking field."
_DEEPSEEK_R1_EFFORT_REASON: Final = "DeepSeek R1 on Bedrock always reasons and rejects reasoning_effort."
def _has_native_text_format(output_config: object) -> bool:
@ -529,13 +533,14 @@ class AmazonConverseConfig(BaseConfig):
"""
Handle the reasoning_effort parameter based on the model type.
- GPT-OSS and DeepSeek V3 models: passed through unchanged via additionalModelRequestFields.
- GPT-OSS, DeepSeek V3 and unknown models (``bedrock_model_is_unknown``): passed through unchanged
via additionalModelRequestFields.
- OpenAI GPT-5.x and GPT-6 models: mapped to ``reasoning.effort`` via additionalModelRequestFields.
- Nova 2 models: transformed to reasoningConfig.
- Anthropic models: mapped to ``thinking`` (and ``output_config.effort`` on
adaptive Claude 4.6 / 4.7).
"""
if "gpt-oss" in model or "deepseek" in model:
if "gpt-oss" in model or "deepseek" in model or bedrock_model_is_unknown(model):
optional_params["reasoning_effort"] = reasoning_effort
elif self._is_openai_gpt_reasoning_model(model):
reasoning: Final[BedrockConverseGptReasoningEffortBlock] = {"effort": reasoning_effort}
@ -638,6 +643,10 @@ class AmazonConverseConfig(BaseConfig):
or supports_reasoning(model=base_model, custom_llm_provider=self.custom_llm_provider)
)
def _warn_native_reasoning_drop(self, param: str, value: object, model: str, reason: str) -> None:
if param in self.get_supported_openai_params(model):
warn_dropped_reasoning(param, value, model, reason, drop_params_gated=False)
def _model_rejects_reasoning_effort_param(self, model: str, base_model: str) -> bool:
"""Whether the model returns a 400 for every ``reasoning_effort`` shape on Converse.
@ -681,7 +690,7 @@ class AmazonConverseConfig(BaseConfig):
"parallel_tool_calls",
]
if "arn" in model: # we can't infer the model from the arn, so just add all params
if "arn" in model or bedrock_model_is_unknown(model):
supported_params.append("tools")
supported_params.append("tool_choice")
supported_params.append("thinking")
@ -1141,9 +1150,8 @@ class AmazonConverseConfig(BaseConfig):
"tool_choice": {"type": "auto", "disable_parallel_tool_use": not value}
}
if param == "thinking" and drop_thinking_param:
verbose_logger.debug(
"Dropping unsupported `thinking` param for Bedrock model=%s; it reasons natively.",
model,
self._warn_native_reasoning_drop(
param, OPAQUE_VALUE.validate_python(value), model, _DEEPSEEK_THINKING_REASON
)
elif param == "thinking" and not self._is_openai_gpt_reasoning_model(model):
if (
@ -1177,10 +1185,7 @@ class AmazonConverseConfig(BaseConfig):
drop_params=drop_params,
)
elif param == "reasoning_effort" and isinstance(value, str) and drop_reasoning_effort_param:
verbose_logger.debug(
"Dropping unsupported `reasoning_effort` param for Bedrock model=%s; it always reasons and rejects it.",
model,
)
self._warn_native_reasoning_drop(param, value, model, _DEEPSEEK_R1_EFFORT_REASON)
elif (
param == "reasoning_effort"
and isinstance(value, str)

View file

@ -2,7 +2,7 @@ import copy
import json
import time
from collections.abc import Mapping, Sequence
from typing import TYPE_CHECKING, Any, Final, cast, get_args
from typing import TYPE_CHECKING, Any, Final
import httpx
from pydantic import ConfigDict, TypeAdapter, ValidationError, with_config
@ -74,10 +74,6 @@ _GUARDRAIL_CONFIG_EXPECTED_FORMAT: Final = (
"{'guardrailIdentifier': str, 'guardrailVersion': str, 'trace': 'enabled'|'disabled'|'enabled_full'}"
)
_INVOKE_PROVIDERS: Final = TypeAdapter(tuple[litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL, ...]).validate_python(
get_args(litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL)
)
def _response_header(headers: Mapping[str, str], name: str) -> str | None:
return headers.get(name, None)
@ -562,62 +558,6 @@ class AmazonInvokeConfig(BaseConfig, BaseAWSLLM):
"""
return False
@staticmethod
def get_bedrock_invoke_provider(
model: str,
) -> litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL | None:
"""
Helper function to get the bedrock provider from the model
handles 4 scenarios:
1. model=invoke/anthropic.claude-3-5-sonnet-20240620-v1:0 -> Returns `anthropic`
2. model=anthropic.claude-3-5-sonnet-20240620-v1:0 -> Returns `anthropic`
3. model=llama/arn:aws:bedrock:us-east-1:086734376398:imported-model/r4c4kewx2s0n -> Returns `llama`
4. model=us.amazon.nova-pro-v1:0 -> Returns `nova`
"""
if model.startswith("invoke/"):
model = model.replace("invoke/", "", 1)
# Special case: Check for "nova" in model name first (before "amazon")
# This handles amazon.nova-* models which would otherwise match "amazon" (Titan)
if "nova" in model.lower():
if "nova" in get_args(litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL):
return cast(litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL, "nova")
_split_model: Final = model.split(".")[0]
if _split_model in get_args(litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL):
return cast(litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL, _split_model)
# If not a known provider, check for pattern with two slashes
provider = AmazonInvokeConfig._get_provider_from_model_path(model)
if provider is not None:
return provider
for provider in _INVOKE_PROVIDERS:
if provider in model:
return provider
return None
@staticmethod
def _get_provider_from_model_path(
model_path: str,
) -> litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL | None:
"""
Helper function to get the provider from a model path with format: provider/model-name
Args:
model_path (str): The model path (e.g., 'llama/arn:aws:bedrock:us-east-1:086734376398:imported-model/r4c4kewx2s0n' or 'anthropic/model-name')
Returns:
Optional[str]: The provider name, or None if no valid provider found
"""
parts: Final = model_path.split("/")
if len(parts) >= 1:
provider: Final = parts[0]
if provider in get_args(litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL):
return cast(litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL, provider)
return None
def convert_messages_to_prompt(self, model: str, messages, provider, custom_prompt_dict) -> tuple[str, list | None]:
# handle anthropic prompts and amazon titan prompts
prompt = ""

View file

@ -1098,6 +1098,11 @@ def _model_cost_entry(key: str) -> Mapping[str, object] | None:
return None if raw is None else _MODEL_COST_ENTRY_ADAPTER.validate_python(raw)
def _cost_map_row_names_a_provider(key: str) -> bool:
entry: Final = _model_cost_entry(key)
return entry is not None and entry.get("litellm_provider") is not None
def _bedrock_price_map_entries(model: str) -> tuple[Mapping[str, object] | None, ...]:
return tuple(
_model_cost_entry(key)
@ -1567,7 +1572,8 @@ def bedrock_model_supports_regex_lookaround(model: str, litellm_params: Mapping[
The deployment's ``model_info.supports_regex_lookaround`` wins, then the
``model_prices_and_context_window.json`` entry of its ``base_model``, then the
entry of ``model`` itself. A model nobody flagged keeps its schema as sent.
entry of ``model`` itself. A model nobody flagged keeps its schema as sent, unless
no candidate names a model LiteLLM knows, whatever route prefix it carries.
"""
params: Final = litellm_params or {}
deployment_flag: Final = deployment_model_info(params).get("supports_regex_lookaround")
@ -1576,7 +1582,41 @@ def bedrock_model_supports_regex_lookaround(model: str, litellm_params: Mapping[
base_model: Final = params.get("base_model")
candidates: Final = (*((base_model,) if isinstance(base_model, str) else ()), model)
flags: Final = (_bedrock_converse_model_flag(candidate, "supports_regex_lookaround") for candidate in candidates)
return next((flag for flag in flags if flag is not None), True)
flag: Final = next((flag for flag in flags if flag is not None), None)
if flag is not None:
return flag
return not all(bedrock_model_is_unknown(split_bedrock_region_path(candidate)[1]) for candidate in candidates)
def _bedrock_model_id_without_region(model: str) -> str:
routed_id: Final = model.removeprefix("bedrock/")
region, separator, regional_id = routed_id.partition("/")
return regional_id if separator and region in get_all_bedrock_regions() else routed_id
def bedrock_model_is_unknown(model: str) -> bool:
model_id: Final = _bedrock_model_id_without_region(model)
if "/" in model_id:
return False
base: Final = get_bedrock_base_model(model_id)
regional_ids: Final = (f"{region}.{base}" for region in get_bedrock_cross_region_inference_regions())
if any(_cost_map_row_names_a_provider(key) for key in (*_bedrock_model_info_candidates(model_id), *regional_ids)):
return False
from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM
return BaseAWSLLM.get_bedrock_invoke_provider(model_id) is None
def _names_no_invoke_provider(model: str) -> bool:
from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM
model_id: Final = _bedrock_model_id_without_region(model)
names_one_model: Final = bool(model_id) and ("/" not in model_id or model_id.startswith("arn:"))
return names_one_model and BaseAWSLLM.get_bedrock_invoke_provider(model) is None
def bedrock_model_reaches_converse_by_default(model: str) -> bool:
return BedrockModelInfo.named_bedrock_route(model, None) is None and _names_no_invoke_provider(model)
_BedrockConverseModelFlag: TypeAlias = Literal[
@ -1585,10 +1625,13 @@ _BedrockConverseModelFlag: TypeAlias = Literal[
]
def _bedrock_converse_model_flag(model: str, key: _BedrockConverseModelFlag) -> bool | None:
def _bedrock_model_info_candidates(model: str) -> tuple[str, ...]:
base: Final = get_bedrock_base_model(model)
candidates: Final = dict.fromkeys((model, base, _BEDROCK_MODEL_VERSION_SUFFIX_RE.sub("", base)))
for candidate in candidates:
return tuple(dict.fromkeys((model, base, _BEDROCK_MODEL_VERSION_SUFFIX_RE.sub("", base))))
def _bedrock_converse_model_flag(model: str, key: _BedrockConverseModelFlag) -> bool | None:
for candidate in _bedrock_model_info_candidates(model):
with contextlib.suppress(Exception):
model_info = get_cached_model_info()(
model=candidate,
@ -1766,8 +1809,19 @@ class BedrockModelInfo(BaseLLMModelInfo):
default (``bedrock_runtime_chat_completions_is_default``) and ``chat_completions/`` opts any other model in;
``request_params`` (the caller's chat params) sends such a request to Converse when it needs a
feature only Converse serves, and ``converse/`` pins a model to Converse. Every other OpenAI-family
model stays on Converse without the prefix.
model stays on Converse without the prefix. A model the cost map does not list as Converse goes to
Invoke only when its id names an Invoke provider, and to Converse otherwise.
"""
named_route: Final = BedrockModelInfo.named_bedrock_route(model, request_params)
if named_route is not None:
return named_route
return "converse" if _names_no_invoke_provider(model) else "invoke"
@staticmethod
def named_bedrock_route(
model: str,
request_params: Mapping[str, object] | None,
) -> BedrockRoute | None:
route_mappings: dict[
str,
Literal[
@ -1822,7 +1876,7 @@ class BedrockModelInfo(BaseLLMModelInfo):
return "converse"
if _OPENAI_FAMILY_MODEL_RE.search(base_model):
return "converse"
return "invoke"
return None
@staticmethod
def _explicit_converse_route(model: str) -> bool:

View file

@ -122,6 +122,12 @@ class Authenticator:
return tokens["access_token"]
def get_account_id(self, access_token: str | None = None) -> str | None:
return self._account_id(access_token, persist_derived=True)
def read_account_id(self, access_token: str | None = None) -> str | None:
return self._account_id(access_token, persist_derived=False)
def _account_id(self, access_token: str | None, persist_derived: bool) -> str | None:
auth_data: Final = self._read_auth_file()
if access_token and (not auth_data or auth_data.get("access_token") != access_token):
return extract_chatgpt_account_id(access_token)
@ -133,7 +139,7 @@ class Authenticator:
id_token: Final = auth_data.get("id_token")
saved_access_token: Final = auth_data.get("access_token")
derived: Final = extract_chatgpt_account_id(_optional_str(id_token or saved_access_token))
if derived:
if derived and persist_derived:
self._write_auth_file({**auth_data, "account_id": derived})
return derived

View file

@ -11,6 +11,7 @@ from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response impo
safe_convert_created_field,
)
from litellm.litellm_core_utils.url_utils import join_api_base
from litellm.llms.base_llm.responses.transformation import STANDARD_MODE_REASONING_FIELD_VALUES, ReasoningFieldValues
from litellm.llms.custom_httpx.sse_lines import split_sse_lines
from litellm.llms.openai.common_utils import OpenAIError
from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig
@ -295,3 +296,6 @@ class ChatGPTResponsesAPIConfig(OpenAIResponsesAPIConfig):
def supports_native_websocket(self) -> bool:
"""ChatGPT does not support native WebSocket for Responses API"""
return False
def reasoning_field_values(self) -> ReasoningFieldValues:
return STANDARD_MODE_REASONING_FIELD_VALUES

View file

@ -249,7 +249,7 @@ if TYPE_CHECKING:
FakeAnthropicMessagesStreamIterator,
)
from litellm.llms.base_llm.passthrough.transformation import BasePassthroughConfig
from litellm.responses.streaming_iterator import ResponsesWsFrameGuard
from litellm.responses.streaming_iterator import ResponsesWsFrameGuard, ResponsesWsTurnAdmission
from litellm.types.google_genai.main import GenerateContentResponse
from litellm.types.llms.openai_evals import (
CancelEvalResponse,
@ -7269,7 +7269,7 @@ class BaseLLMHTTPHandler:
custom_llm_provider: str | None = None,
first_message: str | None = None,
request_defaults: ResponsesWebSocketRequestDefaults | None = None,
turn_admission: Callable[[], Awaitable[None]] | None = None,
turn_admission: "ResponsesWsTurnAdmission | None" = None,
frame_guard: "ResponsesWsFrameGuard | None" = None,
**kwargs: object,
) -> Exception | None:

View file

@ -6,10 +6,12 @@ from collections.abc import Coroutine, Mapping, Sequence
from typing import Final, Literal, cast, overload
import litellm
from litellm.litellm_core_utils.core_helpers import as_str_mapping
from litellm.litellm_core_utils.prompt_templates.common_utils import (
convert_content_list_to_str,
extract_search_results_text,
)
from litellm.litellm_core_utils.reasoning_effort_utils import OPAQUE_VALUE, warn_dropped_reasoning
from litellm.litellm_core_utils.url_utils import ensure_api_route
from litellm.secret_managers.main import get_secret_str
from litellm.types.llms.openai import AllMessageValues
@ -18,6 +20,16 @@ from litellm.utils import supports_reasoning, supports_vision
from ...openai.chat.gpt_transformation import OpenAIGPTConfig
SUPPORTED_REASONING_EFFORTS: Final = ("minimal", "low", "medium", "high", "xhigh", "max", "ultra")
_NO_BUDGET_REASON: Final = "DeepSeek takes thinking only as enabled or disabled, with no budget."
def _warn_on_dropped_budget(thinking: object, model: str) -> None:
fields: Final = as_str_mapping(thinking)
if fields is None or fields.get("type") != "enabled" or "budget_tokens" not in fields:
return
warn_dropped_reasoning(
"thinking.budget_tokens", fields["budget_tokens"], model, _NO_BUDGET_REASON, drop_params_gated=False
)
class DeepSeekChatConfig(OpenAIGPTConfig):
@ -53,6 +65,7 @@ class DeepSeekChatConfig(OpenAIGPTConfig):
# Then re-add only if valid for DeepSeek
thinking_value: Final = optional_params.pop("thinking", None)
reasoning_effort: Final = optional_params.pop("reasoning_effort", None)
_warn_on_dropped_budget(OPAQUE_VALUE.validate_python(thinking_value), model)
# Handle thinking parameter - accept both enabled and disabled, ignore budget_tokens
if isinstance(thinking_value, dict) and thinking_value.get("type") in ("enabled", "disabled"):

View file

@ -12,7 +12,9 @@ from typing import TYPE_CHECKING, Final
import httpx
from pydantic import ConfigDict, TypeAdapter
import litellm
from litellm.litellm_core_utils.core_helpers import map_finish_reason
from litellm.litellm_core_utils.reasoning_effort_utils import warn_dropped_reasoning
from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException
from litellm.secret_managers.main import get_secret_str
from litellm.types.llms.base import LiteLLMBaseModel
@ -29,6 +31,10 @@ PROVIDER_PREFIX: Final[str] = "fal_ai/"
PASSTHROUGH_PARAMS: Final[frozenset[str]] = frozenset(("reasoning", "temperature", "top_p"))
REASONING_DISABLED_EFFORTS: Final[frozenset[str]] = frozenset(("none", "minimal"))
REASONING_ENABLED_EFFORTS: Final[frozenset[str]] = frozenset(("low", "medium", "high"))
_UNSUPPORTED_EFFORT_REASON: Final = (
f"fal only turns reasoning off for {sorted(REASONING_DISABLED_EFFORTS)} "
f"and on for {sorted(REASONING_ENABLED_EFFORTS)}."
)
class _FalUsage(LiteLLMBaseModel):
@ -120,9 +126,10 @@ class FalAIChatConfig(BaseConfig):
return False
if isinstance(value, str) and value in REASONING_ENABLED_EFFORTS:
return True
if drop_params:
return None
raise FalAIError(status_code=400, message=f"Unsupported reasoning_effort {value!r} for {model}")
if not (drop_params or litellm.drop_params is True):
raise FalAIError(status_code=400, message=f"Unsupported reasoning_effort {value!r} for {model}")
warn_dropped_reasoning("reasoning_effort", value, model, _UNSUPPORTED_EFFORT_REASON, drop_params_gated=True)
return None
def _translate_param(self, param: str, value: object, model: str, drop_params: bool) -> tuple[str, object] | None:
if param in ("temperature", "top_p"):

View file

@ -17,6 +17,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
strip_litellm_response_only_fields,
unpack_legacy_defs,
)
from litellm.litellm_core_utils.reasoning_effort_utils import OPAQUE_VALUE, warn_dropped_reasoning
from litellm.litellm_core_utils.url_utils import join_api_base
from litellm.secret_managers.main import get_secret_str
from litellm.types.llms.openai import (
@ -97,6 +98,7 @@ def _json_schema_response_format(schema: object, name: str) -> Mapping[str, obje
EFFORT_KWARG_KEYS: Final = frozenset({"enable_thinking", "thinking", "reasoning_budget", "low_effort"})
_NOT_A_REASONING_MODEL_REASON: Final = "litellm's model map does not list it as a reasoning model."
def _bool_from_kwargs(kwargs: Mapping[str, object], keys: tuple[str, ...]) -> bool | None:
@ -107,6 +109,13 @@ def _bool_from_kwargs(kwargs: Mapping[str, object], keys: tuple[str, ...]) -> bo
return None
def warn_dropped_effort_kwargs(chat_template_kwargs: object, effort: object, model: str) -> None:
if effort != "none":
warn_dropped_reasoning(
"chat_template_kwargs", chat_template_kwargs, model, _NOT_A_REASONING_MODEL_REASON, drop_params_gated=False
)
def effort_from_chat_template_kwargs(kwargs: Mapping[str, object]) -> object:
enable_thinking: Final = _bool_from_kwargs(kwargs, ("enable_thinking", "thinking"))
if enable_thinking is False:
@ -438,10 +447,7 @@ class FireworksAIConfig(FireworksAIMixin, OpenAIGPTConfig):
if effort is None:
return ()
if not supports_reasoning(model=model, custom_llm_provider="fireworks_ai"):
verbose_logger.debug(
"fireworks_ai model %r does not support reasoning; dropping chat_template_kwargs effort keys.",
model,
)
warn_dropped_effort_kwargs(OPAQUE_VALUE.validate_python(chat_template_kwargs), effort, model)
return ()
return (("reasoning_effort", effort),)

View file

@ -2,6 +2,7 @@ from collections.abc import Mapping
from typing import Final
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.reasoning_effort_utils import OPAQUE_VALUE
from litellm.types.llms.openai import AllMessageValues, OpenAITextCompletionUserMessage
from litellm.utils import supports_reasoning
@ -12,6 +13,7 @@ from ..chat.transformation import (
NIM_VLLM_STRIP_PARAMS,
FireworksAIConfig,
effort_from_chat_template_kwargs,
warn_dropped_effort_kwargs,
)
from ..common_utils import FireworksAIMixin, resolve_fireworks_resource_name
@ -129,10 +131,7 @@ class FireworksAITextCompletionConfig(FireworksAIMixin, BaseTextCompletionConfig
)
return result
if not supports_reasoning(model=model, custom_llm_provider="fireworks_ai"):
verbose_logger.debug(
"fireworks_ai model %r does not support reasoning; dropping chat_template_kwargs effort keys.",
model,
)
warn_dropped_effort_kwargs(OPAQUE_VALUE.validate_python(chat_template_kwargs), effort, model)
return result
return {**result, "reasoning_effort": effort}

View file

@ -6,6 +6,10 @@ from urllib.parse import unquote
import httpx
from litellm.litellm_core_utils.url_utils import join_api_base
from litellm.llms.base_llm.responses.transformation import (
STANDARD_MODE_WITHOUT_CONTEXT_REASONING_FIELD_VALUES,
ReasoningFieldValues,
)
from litellm.llms.fireworks_ai.common_utils import (
FIREWORKS_FORWARD_USER_ID_PARAM,
get_fireworks_forwarded_user_id,
@ -129,3 +133,9 @@ class FireworksAIResponsesAPIConfig(OpenAIResponsesAPIConfig):
def supports_native_file_search(self) -> bool:
return False
def reasoning_field_values(self) -> ReasoningFieldValues:
return STANDARD_MODE_WITHOUT_CONTEXT_REASONING_FIELD_VALUES
def ignores_unsupported_reasoning_fields(self) -> bool:
return True

View file

@ -24,6 +24,7 @@ from litellm._logging import verbose_logger
from litellm.constants import DEFAULT_MAX_RECURSE_DEPTH
from litellm.exceptions import AuthenticationError
from litellm.litellm_core_utils.url_utils import join_api_base
from litellm.llms.base_llm.responses.transformation import STANDARD_MODE_REASONING_FIELD_VALUES, ReasoningFieldValues
from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig
from litellm.types.llms.openai import (
ResponseInputParam,
@ -437,6 +438,12 @@ class GithubCopilotResponsesAPIConfig(OpenAIResponsesAPIConfig):
return False
def reasoning_field_values(self) -> ReasoningFieldValues:
return STANDARD_MODE_REASONING_FIELD_VALUES
def ignores_unsupported_reasoning_fields(self) -> bool:
return True
def supports_native_websocket(self) -> bool:
"""GitHub Copilot does not support native WebSocket for Responses API"""
return False

View file

@ -99,7 +99,7 @@ class HostedVLLMChatConfig(OpenAIGPTConfig):
thinking_effort: Final = (
None
if "reasoning_effort" in non_default_params
else reasoning_effort_from_thinking(thinking, drop_params=drop_params)
else reasoning_effort_from_thinking(thinking, model=model, drop_params=drop_params)
)
if thinking_effort is not None:
non_default_params["reasoning_effort"] = thinking_effort

View file

@ -12,6 +12,7 @@ from pydantic import TypeAdapter, ValidationError
from litellm.litellm_core_utils.url_utils import join_api_base
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.llms.base_llm.responses.transformation import ANY_REASONING_FIELD_VALUES, ReasoningFieldValues
from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig
from litellm.secret_managers.main import get_secret_str
from litellm.types.utils import LlmProviders
@ -42,6 +43,9 @@ class LiteLLMProxyResponsesAPIConfig(OpenAIResponsesAPIConfig):
def custom_llm_provider(self) -> LlmProviders:
return LlmProviders.LITELLM_PROXY
def reasoning_field_values(self) -> ReasoningFieldValues:
return ANY_REASONING_FIELD_VALUES
def get_complete_url(
self,
api_base: str | None,

View file

@ -25,7 +25,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
strip_litellm_internal_message_fields,
strip_none_values_from_message,
)
from litellm.litellm_core_utils.reasoning_effort_utils import reasoning_effort_from_thinking
from litellm.litellm_core_utils.reasoning_effort_utils import reasoning_effort_from_thinking, warn_dropped_reasoning
from litellm.litellm_core_utils.url_utils import ensure_api_route
from litellm.llms.openai.chat.gpt_transformation import (
OpenAIChatCompletionStreamingHandler,
@ -70,12 +70,12 @@ def _accepted_reasoning_effort(model: str, requested: str, custom_llm_provider:
return accepted
def _thinking_as_reasoning_effort(thinking: object, *, drop_params: bool) -> str | None:
def _thinking_as_reasoning_effort(thinking: object, *, model: str, drop_params: bool) -> str | None:
match thinking:
case {"type": "disabled"}:
return "none"
case _:
return reasoning_effort_from_thinking(thinking, drop_params=drop_params)
return reasoning_effort_from_thinking(thinking, model=model, drop_params=drop_params)
_NO_TOOL_CALL_TURN: Final[tuple[int, frozenset[str]]] = (-1, frozenset())
@ -83,6 +83,9 @@ _MISTRAL_REQUEST_ROUTING: Final = TypeAdapter(MistralRequestRouting)
_MISTRAL_CACHE_KEY_PARAMS: Final = TypeAdapter(MistralCacheKeyParams)
_MISTRAL_REASONING_EFFORT_PARAMS: Final = TypeAdapter(MistralReasoningEffortParams)
_MISTRAL_FILE_PART: Final = TypeAdapter(MistralFilePart)
_NON_LEVEL_EFFORT_REASON: Final = (
"the Mistral API takes it only as a level such as 'high', so Mistral applies its default effort."
)
def _reasoning_level_and_summary(reasoning_effort: object) -> tuple[object, object]:
@ -234,7 +237,8 @@ class MistralConfig(OpenAIGPTConfig):
if (
param == "thinking"
and "reasoning_effort" not in non_default_params
and (thinking_effort := _thinking_as_reasoning_effort(value, drop_params=drop_params)) is not None
and (thinking_effort := _thinking_as_reasoning_effort(value, model=model, drop_params=drop_params))
is not None
):
optional_params["reasoning_effort"] = _accepted_reasoning_effort(
model, thinking_effort, self.custom_llm_provider
@ -246,13 +250,18 @@ class MistralConfig(OpenAIGPTConfig):
def _reasoning_level_to_send(
self, model: str, requested: MistralReasoningEffortParams, drop_params: bool
) -> str | None:
level, summary = _reasoning_level_and_summary(requested.get("reasoning_effort"))
requested_effort: Final = requested.get("reasoning_effort")
level, summary = _reasoning_level_and_summary(requested_effort)
sendable: Final = (
_accepted_reasoning_effort(model, level, self.custom_llm_provider) if isinstance(level, str) else None
)
if sendable is not None and summary is None:
return sendable
if drop_params or litellm.drop_params is True:
if sendable is None and level is not None:
warn_dropped_reasoning(
"reasoning_effort", requested_effort, model, _NON_LEVEL_EFFORT_REASON, drop_params_gated=True
)
return sendable
raise UnsupportedParamsError(
message=(

View file

@ -21,6 +21,7 @@ from litellm._logging import verbose_logger
from litellm.constants import DEFAULT_OCI_CHAT_MAX_TOKENS
from litellm.litellm_core_utils.logging_utils import track_llm_api_timing
from litellm.litellm_core_utils.prompt_templates.common_utils import messages_with_file_parts_as_image_urls
from litellm.litellm_core_utils.reasoning_effort_utils import warn_dropped_reasoning
from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper
from litellm.litellm_core_utils.url_utils import join_api_base
from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException
@ -428,6 +429,13 @@ class OCIChatConfig(BaseConfig):
alias = param_map.get(key)
if key == "reasoning_effort" and alias and not _model_supports_reasoning_effort(model):
if drop_params or litellm.drop_params:
warn_dropped_reasoning(
key,
value,
model,
"the model map lists no reasoning_effort levels for it, so OCI would reject reasoningEffort.",
drop_params_gated=True,
)
continue
raise OCIError(
status_code=400,
@ -451,6 +459,14 @@ class OCIChatConfig(BaseConfig):
if key == "n" and (value is None or value == 1):
continue
if drop_params or litellm.drop_params:
if key == "reasoning_effort":
warn_dropped_reasoning(
key,
value,
model,
"OCI's Cohere chat API has no reasoning effort field.",
drop_params_gated=True,
)
continue
raise OCIError(
status_code=400,

View file

@ -14,6 +14,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
extract_images_from_message,
extract_reasoning_content,
)
from litellm.litellm_core_utils.reasoning_effort_utils import OPAQUE_VALUE
from litellm.litellm_core_utils.url_utils import ensure_api_route
from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator
from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException
@ -29,7 +30,7 @@ from litellm.types.llms.openai import (
)
from litellm.types.utils import ModelResponse, ModelResponseStream
from ..common_utils import OllamaError
from ..common_utils import OllamaError, ollama_think
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
@ -172,10 +173,7 @@ class OllamaChatConfig(BaseConfig):
if value.get("json_schema") and value["json_schema"].get("schema"):
optional_params["format"] = value["json_schema"]["schema"]
if param == "reasoning_effort" and value is not None:
if model.startswith("gpt-oss"):
optional_params["think"] = value
else:
optional_params["think"] = value in {"low", "medium", "high"}
optional_params["think"] = ollama_think(model, OPAQUE_VALUE.validate_python(value))
### FUNCTION CALLING LOGIC ###
# Ollama 0.4+ supports native tool calling - pass tools directly
# and let Ollama handle model capability detection

View file

@ -8,11 +8,14 @@ from pydantic import BaseModel, TypeAdapter
from litellm import verbose_logger
from litellm._uuid import uuid
from litellm.litellm_core_utils.reasoning_effort_utils import warn_dropped_reasoning
from litellm.litellm_core_utils.url_utils import join_api_base
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.types.utils import ModelResponse, ModelResponseStream
_PROVIDER_FIELDS: Final = TypeAdapter(dict[str, object])
_THINK_ON_EFFORTS: Final = frozenset({"low", "medium", "high"})
_THINK_OFF_REASON: Final = "Ollama takes think only as on or off, and litellm maps this level to off."
_DECODED_JSON: Final = TypeAdapter(object)
@ -296,3 +299,12 @@ class OllamaModelInfo(BaseLLMModelInfo):
Return the base model name for Ollama (no-op).
"""
return model
def ollama_think(model: str, reasoning_effort: object) -> object:
if model.startswith("gpt-oss"):
return reasoning_effort
think: Final = reasoning_effort in _THINK_ON_EFFORTS
if isinstance(reasoning_effort, str) and not think and reasoning_effort != "none":
warn_dropped_reasoning("reasoning_effort", reasoning_effort, model, _THINK_OFF_REASON, drop_params_gated=False)
return think

View file

@ -23,6 +23,7 @@ from litellm.litellm_core_utils.prompt_templates.image_handling import (
async_inline_remote_media,
inline_remote_image_urls,
)
from litellm.litellm_core_utils.reasoning_effort_utils import OPAQUE_VALUE
from litellm.litellm_core_utils.url_utils import ensure_api_route
from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator
from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException
@ -43,7 +44,7 @@ from litellm.types.utils import (
generate_id,
)
from ..common_utils import OllamaError, OllamaModelInfo, convert_image
from ..common_utils import OllamaError, OllamaModelInfo, convert_image, ollama_think
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
@ -249,10 +250,7 @@ class OllamaConfig(BaseConfig):
elif param == "stop":
optional_params["stop"] = value
elif param == "reasoning_effort" and value is not None:
if model.startswith("gpt-oss"):
optional_params["think"] = value
else:
optional_params["think"] = value in {"low", "medium", "high"}
optional_params["think"] = ollama_think(model, OPAQUE_VALUE.validate_python(value))
elif param == "response_format" and isinstance(value, dict):
if value["type"] == "json_object":
optional_params["format"] = "json"

View file

@ -32,6 +32,7 @@ import litellm
from litellm.exceptions import error_response_without_credentials
from litellm.litellm_core_utils.completion_timeout import CompletionTimeout
from litellm.litellm_core_utils.core_helpers import as_str_mapping, decoded_json_object
from litellm.litellm_core_utils.reasoning_effort_utils import warn_dropped_reasoning_params
from litellm.litellm_core_utils.token_counter import token_counter, usage_tokenizer_model
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.llms.custom_httpx.http_handler import (
@ -165,17 +166,11 @@ _DECODED_JSON: Final = TypeAdapter(object)
def drop_params_from_unprocessable_entity_error(
e: openai.UnprocessableEntityError | httpx.HTTPStatusError,
data: Mapping[str, object],
*,
model: str | None,
drop_params_gated: bool,
) -> dict[str, object]:
"""
Helper function to read OpenAI UnprocessableEntityError and drop the params that raised an error from the error message.
Args:
e (UnprocessableEntityError): The UnprocessableEntityError exception
data (Dict[str, Any]): The original data dictionary containing all parameters
Returns:
Dict[str, Any]: A new dictionary with invalid parameters removed
"""
"""Return a copy of ``data`` without the params a 422 response names as invalid."""
invalid_params: Final[list[str]] = []
if isinstance(e, httpx.HTTPStatusError):
error_json: Final = decoded_json_object(_DECODED_JSON.validate_python(e.response.json()))
@ -203,6 +198,13 @@ def drop_params_from_unprocessable_entity_error(
invalid_params.append(error_dict["loc"][1])
new_data: Final = {k: v for k, v in data.items() if k not in invalid_params}
warn_dropped_reasoning_params(
invalid_params,
data,
model,
"the provider answered 422 naming it an invalid param.",
drop_params_gated=drop_params_gated,
)
return new_data

View file

@ -940,7 +940,9 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
except openai.UnprocessableEntityError as e:
## check if body contains unprocessable params - related issue https://github.com/BerriAI/litellm/issues/4800
if litellm.drop_params is True or drop_params is True:
inference_params = drop_params_from_unprocessable_entity_error(e, inference_params)
inference_params = drop_params_from_unprocessable_entity_error(
e, inference_params, model=model, drop_params_gated=True
)
else:
raise e
# e.message
@ -1097,7 +1099,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
except openai.UnprocessableEntityError as e:
## check if body contains unprocessable params - related issue https://github.com/BerriAI/litellm/issues/4800
if litellm.drop_params is True or drop_params is True:
data = drop_params_from_unprocessable_entity_error(e, data)
data = drop_params_from_unprocessable_entity_error(e, data, model=model, drop_params_gated=True)
else:
raise e
# e.message
@ -1252,7 +1254,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
except openai.UnprocessableEntityError as e:
## check if body contains unprocessable params - related issue https://github.com/BerriAI/litellm/issues/4800
if litellm.drop_params is True or drop_params is True:
data = drop_params_from_unprocessable_entity_error(e, data)
data = drop_params_from_unprocessable_entity_error(e, data, model=model, drop_params_gated=True)
else:
raise e
except (

View file

@ -13,6 +13,7 @@ from typing import TYPE_CHECKING, Final, cast
import httpx
from pydantic import ConfigDict, TypeAdapter
from litellm.litellm_core_utils.core_helpers import finite_cost
from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR, set_hidden_params
from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator
from litellm.llms.base_llm.chat.transformation import BaseLLMException
@ -225,8 +226,11 @@ class OpenrouterConfig(OpenAIGPTConfig):
additional_headers: Final = cast( # cast-ok: preserve mapping operations on response metadata
dict[str, object], hidden_params["additional_headers"]
)
if isinstance(response_cost, str | int | float):
additional_headers["llm_provider-x-litellm-response-cost"] = float(response_cost)
reported_cost: Final = (
finite_cost(float(response_cost)) if isinstance(response_cost, str | int | float) else None
)
if reported_cost is not None:
additional_headers["llm_provider-x-litellm-response-cost"] = reported_cost
except Exception:
# If we can't extract cost, continue without it - don't fail the response
pass

View file

@ -6,6 +6,7 @@ import httpx
from pydantic import TypeAdapter, ValidationError
import litellm
from litellm.litellm_core_utils.core_helpers import finite_reported_cost
from litellm.litellm_core_utils.decisions_pricing import apply_decisions_pricing
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.litellm_core_utils.litellm_logging import (
@ -100,9 +101,10 @@ class TypeSafePassthroughLoggingHandler:
f"{custom_llm_provider}/{model}" for model in candidate_models if model is not None
)
pricing: Final = _pricing_for(candidate_model_keys)
reported_cost: Final = finite_reported_cost(usage.cost) if custom_llm_provider == "openrouter" else None
response_cost: Final = (
usage.cost
if custom_llm_provider == "openrouter" and usage.cost is not None
reported_cost
if reported_cost is not None
else input_tokens * pricing.input_cost_per_token + output_tokens * pricing.output_cost_per_token
)
usage_object: Final = Usage(

View file

@ -6,6 +6,7 @@ Helper util for handling perplexity-specific cost calculation
from datetime import datetime
from typing import Final
from litellm.litellm_core_utils.core_helpers import finite_reported_cost
from litellm.litellm_core_utils.llm_cost_calc.utils import TokenRates, apply_off_peak_pricing
from litellm.types.utils import Usage
from litellm.utils import get_model_info
@ -34,10 +35,9 @@ def cost_per_token(model: str, usage: Usage, current_time: datetime | None = Non
total_cost = cost_info.get("total_cost")
elif isinstance(cost_info, (int, float)) and not isinstance(cost_info, bool):
total_cost = float(cost_info)
if total_cost is not None:
# Return total cost as completion_cost (prompt_cost=0) since Perplexity
# doesn't break down by input/output in their cost object
return (0.0, float(total_cost))
reported_cost: Final = None if total_cost is None else finite_reported_cost(float(total_cost))
if reported_cost is not None:
return (0.0, reported_cost)
## FALLBACK: Calculate cost manually if Perplexity doesn't provide it
## GET MODEL INFO

View file

@ -9,6 +9,7 @@ The only provider quirks:
Ref: https://docs.perplexity.ai/api-reference/responses-post
"""
from types import MappingProxyType
from typing import Any, Final
import httpx
@ -17,6 +18,7 @@ from pydantic import TypeAdapter
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.litellm_core_utils.url_utils import join_api_base
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.llms.base_llm.responses.transformation import STANDARD_MODE_REASONING_FIELD_VALUES, ReasoningFieldValues
from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig, drops_params
from litellm.secret_managers.main import get_secret_str
from litellm.types.llms.openai import ResponseInputParam, ResponsesAPIResponse
@ -24,6 +26,9 @@ from litellm.types.router import GenericLiteLLMParams
from litellm.types.utils import LlmProviders
_DECODED_JSON: Final = TypeAdapter(object)
PERPLEXITY_REASONING_FIELD_VALUES: Final[ReasoningFieldValues] = MappingProxyType(
{**STANDARD_MODE_REASONING_FIELD_VALUES, "context": frozenset({"auto", "current_turn", "all_turns"})}
)
class PerplexityResponsesConfig(OpenAIResponsesAPIConfig):
@ -127,3 +132,9 @@ class PerplexityResponsesConfig(OpenAIResponsesAPIConfig):
def supports_native_websocket(self) -> bool:
"""Perplexity does not support native WebSocket for Responses API"""
return False
def reasoning_field_values(self) -> ReasoningFieldValues:
return PERPLEXITY_REASONING_FIELD_VALUES
def ignores_unsupported_reasoning_fields(self) -> bool:
return True

View file

@ -28,7 +28,7 @@ from litellm.types.llms.openai import AllMessageValues
from litellm.types.utils import LlmProviders
from ...openai.chat.gpt_transformation import OpenAIGPTConfig
from ..common_utils import AWSEventStreamDecoder, SagemakerError
from ..common_utils import INFERENCE_COMPONENT_HEADER, OBJECT_KEYED_MAPPING, AWSEventStreamDecoder, SagemakerError
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
@ -59,7 +59,12 @@ class SagemakerChatConfig(OpenAIGPTConfig, BaseAWSLLM):
inference_component_name: Final = optional_params.get("model_id")
if not isinstance(inference_component_name, str):
return headers
return {**headers, "X-Amzn-SageMaker-Inference-Component": inference_component_name}
other_headers: Final = {
name: value
for name, value in OBJECT_KEYED_MAPPING.validate_python(headers).items()
if not isinstance(name, str) or name.lower() != INFERENCE_COMPONENT_HEADER.lower()
}
return {**other_headers, INFERENCE_COMPONENT_HEADER: inference_component_name}
def transform_request(
self,
@ -76,9 +81,8 @@ class SagemakerChatConfig(OpenAIGPTConfig, BaseAWSLLM):
litellm_params=litellm_params,
headers=headers,
)
container_request: Final = {
key: value for key, value in request.items() if key not in self.aws_authentication_params
}
header_only_params: Final = frozenset((*self.aws_authentication_params, "model_id"))
container_request: Final = {key: value for key, value in request.items() if key not in header_only_params}
served_model_name: Final = litellm_params.get("hf_model_name")
if not isinstance(served_model_name, str):
return container_request

View file

@ -14,9 +14,10 @@ from typing_extensions import ReadOnly, Required, TypedDict
from litellm import verbose_logger
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.types.utils import GenericStreamingChunk as GChunk
from litellm.types.utils import StreamingChatCompletionChunk
from litellm.types.utils import LlmProviders, StreamingChatCompletionChunk
STR_KEYED_MAPPING: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(hide_input_in_errors=True))
OBJECT_KEYED_MAPPING: Final = TypeAdapter(Mapping[object, object], config=ConfigDict(hide_input_in_errors=True))
@with_config(ConfigDict(extra="allow", strict=True, hide_input_in_errors=True))
@ -167,6 +168,36 @@ def get_sagemaker_response_stream_shape():
return _load_sagemaker_response_stream_shape()
INFERENCE_COMPONENT_HEADER: Final = "X-Amzn-SageMaker-Inference-Component"
_SAGEMAKER_PROVIDERS: Final = frozenset(
provider.value for provider in (LlmProviders.SAGEMAKER, LlmProviders.SAGEMAKER_CHAT, LlmProviders.SAGEMAKER_NOVA)
)
def _deployment_provider(litellm_params: Mapping[str, object]) -> str | None:
custom_llm_provider: Final = litellm_params.get("custom_llm_provider")
if isinstance(custom_llm_provider, str) and custom_llm_provider:
return custom_llm_provider
model: Final = litellm_params.get("model")
return model.split("/", 1)[0] if isinstance(model, str) else None
def deployment_inference_component(litellm_params: Mapping[str, object]) -> str | None:
headers: Final = litellm_params.get("extra_headers")
if _deployment_provider(litellm_params) not in _SAGEMAKER_PROVIDERS or not isinstance(headers, Mapping):
return None
return next(
(
value
for name, value in OBJECT_KEYED_MAPPING.validate_python(headers).items()
if isinstance(name, str) and name.lower() == INFERENCE_COMPONENT_HEADER.lower() and isinstance(value, str)
),
None,
)
def with_inference_component_hint(error_message: str) -> str:
if "Inference Component Name header is required" in error_message:
return f"{error_message}\n pass in via `litellm.embedding(..., model_id={{InferenceComponentName}})`"

View file

@ -23,7 +23,7 @@ from litellm.types.llms.openai import AllMessageValues
from litellm.types.utils import ModelResponse, Usage
from litellm.utils import token_counter
from ..common_utils import SagemakerError
from ..common_utils import STR_KEYED_MAPPING, SagemakerError
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
@ -153,7 +153,9 @@ class SagemakerConfig(BaseConfig):
litellm_params: dict,
headers: dict,
) -> dict:
inference_params: Final = optional_params.copy()
inference_params: Final = {
key: value for key, value in STR_KEYED_MAPPING.validate_python(optional_params).items() if key != "model_id"
}
stream: Final = inference_params.pop("stream", False)
data: Final[dict] = {"parameters": inference_params}
if stream is True:

View file

@ -0,0 +1,37 @@
"""
Providers whose requests are covered by a flat subscription, each with the way to read
the account id of the login a deployment is signed into.
"""
from collections.abc import Callable, Mapping
from types import MappingProxyType
from typing import Final
from litellm.llms.chatgpt.authenticator import Authenticator, get_cached_authenticator, get_chatgpt_auth_file
from litellm.secret_managers.main import UnresolvableSecretReference, resolve_local_oidc_reference
from litellm.utils import with_named_credential
def _chatgpt_account_id(litellm_params: Mapping[str, object]) -> str | None:
params: Final = with_named_credential(litellm_params)
auth_file: Final = get_chatgpt_auth_file(params)
if auth_file:
return get_cached_authenticator(auth_file).read_account_id()
api_key: Final = params.get("api_key")
if not isinstance(api_key, str) or not api_key:
return Authenticator().read_account_id()
match resolve_local_oidc_reference(api_key):
case UnresolvableSecretReference():
return None
case str() as access_token:
return Authenticator().read_account_id(access_token)
SUBSCRIPTION_ACCOUNT_ID_RESOLVERS: Final[Mapping[str, Callable[[Mapping[str, object]], str | None]]] = MappingProxyType(
{"chatgpt": _chatgpt_account_id}
)
def resolve_subscription_account_id(custom_llm_provider: str, litellm_params: Mapping[str, object]) -> str | None:
resolver: Final = SUBSCRIPTION_ACCOUNT_ID_RESOLVERS.get(custom_llm_provider)
return None if resolver is None else resolver(litellm_params)

View file

@ -9,12 +9,15 @@ from typing import Final, TypedDict
from typing_extensions import ReadOnly
import litellm
from litellm.litellm_core_utils.reasoning_effort_utils import warn_dropped_reasoning
from litellm.litellm_core_utils.url_utils import ensure_api_route, join_api_base
from litellm.secret_managers.main import get_secret_str
from litellm.utils import supports_reasoning
from ...openai.chat.gpt_transformation import OpenAIGPTConfig
_ADAPTIVE_BUDGET_REASON: Final = "it is not an integer, so litellm sends adaptive thinking with no budget."
class ThinkingPayload(TypedDict, total=False):
"""Tencent TokenHub `thinking` object.
@ -102,6 +105,10 @@ class TencentChatConfig(OpenAIGPTConfig):
if isinstance(budget, int):
coerced_with_budget: Final[ThinkingPayload] = {"type": "adaptive", "budget_tokens": budget}
return coerced_with_budget
if budget is not None:
warn_dropped_reasoning(
"thinking.budget_tokens", budget, model, _ADAPTIVE_BUDGET_REASON, drop_params_gated=False
)
coerced: Final[ThinkingPayload] = {"type": "adaptive"}
return coerced

View file

@ -1257,7 +1257,7 @@ def get_vertex_model_id_from_url(url: str) -> str | None:
`https://${LOCATION}-aiplatform.googleapis.com/v1/projects/${PROJECT_ID}/locations/${LOCATION}/publishers/google/models/${MODEL_ID}:streamGenerateContent`
"""
match: Final = re.search(r"/models/([^:]+)", url)
match: Final = re.search(r"/models/([^:?#]+)", url)
return match.group(1) if match else None

View file

@ -34,6 +34,7 @@ from litellm.litellm_core_utils.json_fragment_accumulator import JSONFragmentAcc
from litellm.litellm_core_utils.prompt_templates.factory import (
encode_tool_call_id_with_signature,
)
from litellm.litellm_core_utils.reasoning_effort_utils import warn_dropped_reasoning
from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException
from litellm.llms.custom_httpx.http_handler import (
AsyncHTTPHandler,
@ -201,6 +202,13 @@ def _unsupported_reasoning_effort(reasoning_effort: str) -> UnsupportedParamsErr
_SAMPLING_DEPRECATION_UNMAPPED_MODELS: Final = "models outside litellm's model map"
_NON_INTEGER_BUDGET_REASON: Final = (
"Gemini takes budget_tokens only as an integer, so it uses its default thinking budget."
)
_LEVEL_ONLY_BUDGET_REASON: Final = (
"Gemini 3 and later set reasoning by thinking level, not budget, so litellm does not send the budget. "
"Send reasoning_effort to pick a level."
)
_sampling_deprecation_lock: Final = threading.Lock()
@ -1466,6 +1474,14 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
params["includeThoughts"] = False
else:
params["includeThoughts"] = sends_thoughts
if thinking_budget is not None:
warn_dropped_reasoning(
"thinking.budget_tokens",
thinking_budget,
model,
_LEVEL_ONLY_BUDGET_REASON,
drop_params_gated=False,
)
# Follow provider defaults unless explicitly opted into legacy behavior.
if litellm.enable_gemini_default_thinking_level_low is True:
params["thinkingLevel"] = (
@ -1478,8 +1494,16 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
# For older Gemini models, use thinkingBudget
if thinking_enabled and not VertexGeminiConfig._is_thinking_budget_zero(thinking_budget):
params["includeThoughts"] = sends_thoughts
if thinking_budget is not None and isinstance(thinking_budget, int):
if isinstance(thinking_budget, int):
params["thinkingBudget"] = thinking_budget
elif thinking_budget is not None:
warn_dropped_reasoning(
"thinking.budget_tokens",
thinking_budget,
model,
_NON_INTEGER_BUDGET_REASON,
drop_params_gated=False,
)
return params

View file

@ -1,9 +1,15 @@
from collections.abc import Mapping, Sequence
from typing import Final
from pydantic import TypeAdapter
from litellm.exceptions import UnsupportedParamsError
from litellm.litellm_core_utils.core_helpers import drops_unsupported_params
from litellm.llms.anthropic.common_utils import AnthropicModelInfo, context_management_betas
from litellm.llms.anthropic.common_utils import (
AnthropicModelInfo,
context_management_betas,
requires_native_compaction_beta,
)
from litellm.llms.anthropic.messages.native.transformation import (
AnthropicMessagesConfig,
messages_carry_output_config,
@ -11,6 +17,7 @@ from litellm.llms.anthropic.messages.native.transformation import (
from litellm.types.llms.anthropic import (
ANTHROPIC_BETA_HEADER_VALUES,
ANTHROPIC_HOSTED_TOOLS,
ANTHROPIC_MID_CONVERSATION_TOOL_CHANGES_BETA_HEADER,
)
from litellm.types.llms.anthropic_tool_search import get_tool_search_beta_header
from litellm.types.llms.vertex_ai import VertexPartnerProvider
@ -19,6 +26,8 @@ from litellm.types.router import GenericLiteLLMParams
from ....vertex_llm_base import VertexBase
from ..output_params_utils import sanitize_vertex_anthropic_output_params
_OPTIONAL_PARAMS: Final = TypeAdapter(Mapping[str, object])
_VERSION_HEADER_VERTEX_READS_FROM_BODY: Final = "anthropic-version"
_VERTEX_DEFAULT_VERSION: Final = "vertex-2023-10-16"
_VERSIONS_SERVED_AS_VERTEX_DEFAULT: Final = frozenset({"", "2023-01-01", "2023-06-01"})
@ -190,14 +199,23 @@ class VertexAIPartnerModelsAnthropicMessagesConfig(AnthropicMessagesConfig, Vert
if messages_carry_output_config(messages):
beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.PER_TURN_CONTROL_2026_07_01.value)
beta_values.update(
typed_params: Final = _OPTIONAL_PARAMS.validate_python(optional_params)
if requires_native_compaction_beta("vertex_ai", typed_params, messages):
beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_09_04.value)
tool_change_betas: Final = (
(ANTHROPIC_MID_CONVERSATION_TOOL_CHANGES_BETA_HEADER,)
if anthropic_model_info.is_mid_conversation_tool_change_used(messages)
else ()
)
all_beta_values: Final = beta_values.union(
anthropic_model_info.thinking_betas(
optional_params.get("thinking"), # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # optional_params is the base config's bare dict
)
),
tool_change_betas,
)
if beta_values:
request_headers["anthropic-beta"] = ",".join(beta_values)
if all_beta_values:
request_headers["anthropic-beta"] = ",".join(sorted(all_beta_values))
return request_headers, api_base

View file

@ -2,19 +2,23 @@
## Handler file for calling claude-3 on vertex ai
import base64
import json
from collections.abc import Mapping
from collections.abc import Mapping, Sequence
from typing import TYPE_CHECKING, Final
import httpx
from pydantic import TypeAdapter
from pydantic import TypeAdapter, ValidationError
import litellm
from litellm.anthropic_beta_headers_manager import get_header_only_betas
from litellm.litellm_core_utils.core_helpers import drops_unsupported_params
from litellm.litellm_core_utils.prompt_templates.image_handling import RemoteMedia, inline_remote_image_urls
from litellm.llms.base_llm.chat.transformation import LiteLLMLoggingObj
from litellm.types.llms.anthropic import ANTHROPIC_BETA_HEADER_VALUES
from litellm.types.llms.openai import AllMessageValues
from litellm.types.utils import ModelResponse
from ....anthropic.chat.transformation import AnthropicConfig
from ....anthropic.common_utils import AnthropicError, requires_native_compaction_beta
from .output_params_utils import sanitize_vertex_anthropic_output_params
if TYPE_CHECKING:
@ -36,6 +40,32 @@ def _vertex_labels_header(labels: object, header_names: Mapping[str, object]) ->
return base64.b64encode(json.dumps(string_labels).encode()).decode() if string_labels else None
_BETA_VALUE: Final[TypeAdapter[str | tuple[str, ...] | None]] = TypeAdapter(str | tuple[str, ...] | None)
_REQUEST_MESSAGES: Final = TypeAdapter(Sequence[object])
def _beta_names(value: object) -> frozenset[str] | None:
try:
parsed: Final = _BETA_VALUE.validate_python(value)
except ValidationError:
return None
names: Final = parsed.split(",") if isinstance(parsed, str) else parsed or ()
return frozenset(name.strip() for name in names if name.strip())
def _client_betas(value: object, field: str, drop_params: bool) -> frozenset[str]:
names: Final = _beta_names(value)
if names is not None:
return names
if drop_params:
return frozenset()
raise AnthropicError(
status_code=400,
message=f"{field} must be a string or a list of strings, got {type(value).__name__}",
)
class VertexAIError(Exception):
def __init__(self, status_code, message):
self.status_code = status_code
@ -157,21 +187,43 @@ class VertexAIAnthropicConfig(AnthropicConfig):
if context_management:
self._add_context_management_beta_headers(beta_set, context_management)
extra_headers: Final = optional_params.get("extra_headers") or {}
anthropic_beta_value: Final = extra_headers.get("anthropic-beta", "")
if isinstance(anthropic_beta_value, str) and anthropic_beta_value:
for beta in anthropic_beta_value.split(","):
beta = beta.strip()
if beta:
beta_set.add(beta)
elif isinstance(anthropic_beta_value, list):
beta_set.update(anthropic_beta_value)
request_params: Final = _NON_DEFAULT_PARAMS.validate_python(optional_params)
compaction_betas: Final = (
frozenset({ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_09_04.value})
if requires_native_compaction_beta(
"vertex_ai", request_params, _REQUEST_MESSAGES.validate_python(data["messages"])
)
else frozenset[str]()
)
drop_params: Final = drops_unsupported_params(litellm_params.get("drop_params"))
extra_headers: Final = _NON_DEFAULT_PARAMS.validate_python(request_params.get("extra_headers") or {})
data.pop("extra_headers", None)
injected_betas: Final = frozenset(beta_set) | _client_betas(
extra_headers.get("anthropic-beta"), "The anthropic-beta header", drop_params
)
client_body_betas: Final = _client_betas(
_NON_DEFAULT_PARAMS.validate_python(data).get("anthropic_beta"), "anthropic_beta", drop_params
)
if beta_set:
data["anthropic_beta"] = list(beta_set)
headers["anthropic-beta"] = ",".join(beta_set)
requested_betas: Final = injected_betas | client_body_betas
header_only_betas: Final = get_header_only_betas("vertex_ai") if requested_betas else frozenset[str]()
body_betas: Final = requested_betas - header_only_betas
if body_betas:
data["anthropic_beta"] = sorted(body_betas)
elif "anthropic_beta" in data:
del data["anthropic_beta"]
header_betas: Final = (
injected_betas
| compaction_betas
| (client_body_betas & header_only_betas)
| _client_betas(
_NON_DEFAULT_PARAMS.validate_python(headers).get("anthropic-beta"),
"The anthropic-beta header",
drop_params,
)
)
if header_betas:
headers["anthropic-beta"] = ",".join(sorted(header_betas))
return data

View file

@ -6,12 +6,19 @@ Unlike Gemini models which use Google's token counting API, partner models use
their respective publisher-specific count-tokens endpoints.
"""
from collections.abc import Mapping
from collections.abc import Mapping, Sequence
from typing import Any, Final
from pydantic import TypeAdapter
from litellm.litellm_core_utils.core_helpers import drops_unsupported_params
from litellm.litellm_core_utils.url_utils import join_api_base
from litellm.llms.anthropic.common_utils import OBJECT_LIST_ADAPTER, is_claude_model_target, tools_without_custom_field
from litellm.llms.anthropic.common_utils import (
OBJECT_LIST_ADAPTER,
count_tokens_feature_betas,
is_claude_model_target,
tools_without_custom_field,
)
from litellm.llms.custom_httpx.http_handler import get_async_httpx_client
from litellm.llms.vertex_ai.common_utils import get_vertex_base_url
from litellm.llms.vertex_ai.vertex_llm_base import VertexBase
@ -23,6 +30,9 @@ _COUNT_TOKENS_PUBLISHERS: Final = (
)
_REQUEST_MESSAGES: Final = TypeAdapter(Sequence[object])
def count_tokens_request_body(
request_data: Mapping[str, object], model: str, drop_params: object
) -> Mapping[str, object]:
@ -43,6 +53,14 @@ def count_tokens_publisher(model: str) -> str | None:
return next((publisher for marker, publisher in _COUNT_TOKENS_PUBLISHERS if marker in model), None)
def count_tokens_headers(access_token: str, request_data: Mapping[str, object]) -> Mapping[str, str]:
authorization: Final = {"Authorization": f"Bearer {access_token}"}
betas: Final = count_tokens_feature_betas(
"vertex_ai", request_data, _REQUEST_MESSAGES.validate_python(request_data["messages"])
)
return {**authorization, "anthropic-beta": ",".join(betas)} if betas else authorization
class VertexAIPartnerModelsTokenCounter(VertexBase):
"""
Token counter for Vertex AI Partner Models.
@ -175,8 +193,7 @@ class VertexAIPartnerModelsTokenCounter(VertexBase):
api_base=litellm_params.get("api_base"),
)
# Prepare headers
headers: Final = {"Authorization": f"Bearer {access_token}"}
headers: Final = dict(count_tokens_headers(access_token, request_data))
# Get async HTTP client
from litellm import LlmProviders

View file

@ -1,3 +1,4 @@
from collections.abc import Mapping
from typing import Final
import httpx
@ -10,6 +11,14 @@ from litellm.types.llms.openai import AllMessageValues
from litellm.utils import add_path_to_api_base
def endpoint_under_api_base(endpoint: str, api_base: str | None) -> str:
return (
endpoint.lstrip("/").removeprefix("v1/")
if api_base is not None and api_base.rstrip("/").endswith("/v1")
else endpoint
)
class VLLMError(BaseLLMException):
def __init__(
self,
@ -39,9 +48,14 @@ class VLLMModelInfo(BaseLLMModelInfo):
api_key: str | None = None,
api_base: str | None = None,
) -> dict:
if api_key is not None:
headers["x-api-key"] = api_key
return headers
upstream_key: Final = self.upstream_api_key(api_key, api_base)
return {**headers, **self.upstream_auth_headers(upstream_key)} if upstream_key else headers
def upstream_api_key(self, api_key: str | None, api_base: str | None) -> str | None:
return api_key
def upstream_auth_headers(self, api_key: str) -> Mapping[str, str]:
return {"Authorization": f"Bearer {api_key}", "x-api-key": api_key}
@staticmethod
def get_api_base(api_base: str | None = None) -> str | None:

View file

@ -3,6 +3,7 @@ from typing import TYPE_CHECKING, Final
from httpx import Response
from litellm.constants import PLACEHOLDER_API_KEY
from litellm.litellm_core_utils.litellm_logging import Logging
from litellm.llms.base_llm.passthrough.transformation import BasePassthroughConfig
from litellm.llms.openai_like.passthrough.transformation import (
@ -12,8 +13,14 @@ from litellm.llms.openai_like.passthrough.transformation import (
logged_text_completion_stream,
relays_text_completion,
)
from litellm.secret_managers.main import (
GOOGLE_OIDC_SECRET_PREFIX,
UnresolvableSecretReference,
get_secret_str,
resolve_local_oidc_reference,
)
from ..common_utils import VLLMModelInfo
from ..common_utils import VLLMModelInfo, endpoint_under_api_base
if TYPE_CHECKING:
from httpx import URL
@ -22,6 +29,30 @@ if TYPE_CHECKING:
class VLLMPassthroughConfig(VLLMModelInfo, BasePassthroughConfig):
@staticmethod
def get_api_key(api_key: str | None = None) -> str | None:
return None if api_key == PLACEHOLDER_API_KEY else api_key
def upstream_api_key(self, api_key: str | None, api_base: str | None) -> str | None:
if api_key is not None or not self._is_vllm_api_base(api_base):
return api_key
vllm_api_key: Final = get_secret_str("VLLM_API_KEY")
if vllm_api_key is None or vllm_api_key.startswith(GOOGLE_OIDC_SECRET_PREFIX):
return None
match resolve_local_oidc_reference(vllm_api_key):
case UnresolvableSecretReference():
return None
case str() as resolved:
return resolved
@staticmethod
def _is_vllm_api_base(api_base: str | None) -> bool:
vllm_api_base: Final = get_secret_str("VLLM_API_BASE")
return api_base is not None and vllm_api_base is not None and api_base.rstrip("/") == vllm_api_base.rstrip("/")
def upstream_auth_headers(self, api_key: str) -> Mapping[str, str]:
return {"Authorization": f"Bearer {api_key}"}
def is_streaming_request(self, endpoint: str, request_data: dict) -> bool:
return "stream" in request_data
@ -40,7 +71,7 @@ class VLLMPassthroughConfig(VLLMModelInfo, BasePassthroughConfig):
raise Exception("VLLM api base not found")
return (
self.format_url(endpoint, base_target_url, request_query_params),
self.format_url(endpoint_under_api_base(endpoint, base_target_url), base_target_url, request_query_params),
base_target_url,
)

View file

@ -10,6 +10,10 @@ from litellm._logging import verbose_logger
from litellm.constants import XAI_API_BASE
from litellm.exceptions import AuthenticationError
from litellm.litellm_core_utils.url_utils import join_api_base
from litellm.llms.base_llm.responses.transformation import (
STANDARD_MODE_WITHOUT_CONTEXT_REASONING_FIELD_VALUES,
ReasoningFieldValues,
)
from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig
from litellm.llms.xai.common_utils import XAIModelInfo, xai_reported_cost_in_usd
from litellm.secret_managers.main import get_secret_str
@ -290,3 +294,9 @@ class XAIResponsesAPIConfig(OpenAIResponsesAPIConfig):
def supports_native_websocket(self) -> bool:
"""XAI does not support native WebSocket for Responses API"""
return False
def reasoning_field_values(self) -> ReasoningFieldValues:
return STANDARD_MODE_WITHOUT_CONTEXT_REASONING_FIELD_VALUES
def ignores_unsupported_reasoning_fields(self) -> bool:
return True

View file

@ -681,7 +681,7 @@ async def acompletion(
- The `completion` function is called using `run_in_executor` to execute synchronously in the event loop.
- If `stream` is True, the function returns an async generator that yields completion lines.
"""
fallbacks = kwargs.get("fallbacks", None)
fallbacks: Final = kwargs.get("fallbacks", None) or litellm.model_fallbacks
mock_timeout = kwargs.get("mock_timeout", None)
if mock_timeout is True:
@ -697,6 +697,7 @@ async def acompletion(
loop: Final = asyncio.get_event_loop()
custom_llm_provider = kwargs.get("custom_llm_provider", None)
_ = _resolve_control_options(kwargs, model)
caller_kwargs: Final = cast("Mapping[str, object]", {**kwargs}) # cast-ok: acompletion takes untyped kwargs
## PROMPT MANAGEMENT HOOKS ##
#########################################################
@ -715,15 +716,34 @@ async def acompletion(
custom_llm_provider=cast(str | None, custom_llm_provider), # cast-ok: read from untyped kwargs
tools=tools,
enable_prompt_caching=cast(bool | None, kwargs.get("enable_prompt_caching")), # cast-ok: untyped kwargs
api_base=kwargs.get("api_base") or base_url,
api_base=base_url or kwargs.get("api_base"),
)
if isinstance(litellm_logging_obj, LiteLLMLoggingObj) and (
litellm_logging_obj.should_run_prompt_management_hooks(
prompt_id=kwargs.get("prompt_id", None),
prompt_id: Final = cast(str | None, kwargs.get("prompt_id", None)) # cast-ok: read from untyped kwargs
prompt_management_logger: Final = (
litellm_logging_obj.get_custom_logger_for_prompt_management(
model=model,
non_default_params=kwargs,
tools=tools,
prompt_id=prompt_id,
dynamic_callback_params=litellm_logging_obj.standard_callback_dynamic_params,
)
if isinstance(litellm_logging_obj, LiteLLMLoggingObj)
and litellm_logging_obj.should_run_prompt_management_hooks(
prompt_id=prompt_id,
non_default_params=kwargs,
tools=tools,
)
else None
)
each_fallback_writes_its_own_cache_dialect: Final = fallbacks is not None and isinstance(
prompt_management_logger, AnthropicCacheControlHook
)
if (
isinstance(litellm_logging_obj, LiteLLMLoggingObj)
and prompt_management_logger is not None
and not each_fallback_writes_its_own_cache_dialect
):
(
model,
@ -733,8 +753,9 @@ async def acompletion(
model=model,
messages=messages,
non_default_params=kwargs,
prompt_id=kwargs.get("prompt_id", None),
prompt_id=prompt_id,
prompt_variables=kwargs.get("prompt_variables", None),
prompt_management_logger=prompt_management_logger,
tools=tools,
prompt_label=kwargs.get("prompt_label", None),
prompt_version=kwargs.get("prompt_version", None),
@ -826,9 +847,10 @@ async def acompletion(
litellm_params=(GenericLiteLLMParams.model_validate(_supplemental) if _supplemental else None),
)
fallbacks = fallbacks or litellm.model_fallbacks
if fallbacks is not None:
response = await async_completion_with_fallbacks(**completion_kwargs, kwargs={"fallbacks": fallbacks, **kwargs})
response = await async_completion_with_fallbacks(
**completion_kwargs, kwargs={"fallbacks": fallbacks, **caller_kwargs}
)
if response is None:
raise Exception(
"No response from fallbacks. Got none. Turn on `litellm.set_verbose=True` to see more details."
@ -5765,7 +5787,7 @@ def completion(
custom_llm_provider=cast(str | None, kwargs.get("custom_llm_provider")), # cast-ok: untyped kwargs
tools=tools,
enable_prompt_caching=cast(bool | None, kwargs.get("enable_prompt_caching")), # cast-ok: untyped kwargs
api_base=kwargs.get("api_base") or base_url,
api_base=base_url or kwargs.get("api_base"),
)
if isinstance(litellm_logging_obj, LiteLLMLoggingObj) and (
@ -6267,6 +6289,9 @@ def completion(
# reasoningSummary/reasoning_summary without tools (AI SDK) that the first
# (early) check doesn't cover.
_reasoning_summary_for_bridge: Final = peek_reasoning_summary_aliases(optional_params)
typed_deployment_model_info: Final = str_keyed_mapping(
cast(object, model_info) # cast-ok: model_info is the untyped dict kwarg
)
if responses_api_model_info.get("mode") != "responses":
responses_api_model_info, model = responses_api_bridge_check(
model=model,
@ -6276,9 +6301,7 @@ def completion(
reasoning_effort=bridge_reasoning_effort(resolved_reasoning_effort, optional_params.get("extra_body")),
reasoning_summary=_reasoning_summary_for_bridge,
api_base=api_base,
deployment_model_info=str_keyed_mapping(
cast(object, model_info) # cast-ok: model_info is the untyped dict kwarg
),
deployment_model_info=typed_deployment_model_info,
)
# Use base_model (the true underlying model) for Azure model-type
@ -6290,19 +6313,28 @@ def completion(
optional_params, rs_val = strip_reasoning_summary_aliases_from_optional_params(optional_params)
if custom_llm_provider == "openai" and isinstance(resolved_reasoning_effort, dict):
from litellm.responses.utils import openai_bridge_reasoning
if isinstance(resolved_reasoning_effort, dict):
from litellm.responses.utils import (
RefusedReasoningFields,
bridge_reasoning,
raise_refused_reasoning_fields,
)
optional_params["reasoning_effort"] = openai_bridge_reasoning(
bridged_reasoning: Final = bridge_reasoning(
requested=resolved_reasoning_effort,
effort_kept="reasoning_effort" in optional_params,
summary_alias=rs_val,
model=model,
base_model=base_model if isinstance(base_model, str) else None,
custom_llm_provider=custom_llm_provider,
api_base=api_base,
deployment_model_info=typed_deployment_model_info,
drop_params=drop_unsupported_reasoning,
additional_drop_params=routing_kwargs.get("additional_drop_params"),
)
elif isinstance(resolved_reasoning_effort, dict) and "summary" in resolved_reasoning_effort:
optional_params["reasoning_effort"] = resolved_reasoning_effort
if isinstance(bridged_reasoning, RefusedReasoningFields):
raise_refused_reasoning_fields(bridged_reasoning, model, custom_llm_provider)
optional_params["reasoning_effort"] = bridged_reasoning
elif rs_val is not None:
eff: Final = optional_params.get("reasoning_effort", resolved_reasoning_effort)
if isinstance(eff, dict):

View file

@ -8715,7 +8715,7 @@
"search_context_size_low": 0.014,
"search_context_size_medium": 0.014
},
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'",
"source": "https://azure.microsoft.com/en-us/pricing/details/cognitive-services/openai-service/",
"supported_endpoints": [
"/v1/chat/completions",
"/v1/responses"
@ -8742,7 +8742,11 @@
"supports_tool_choice": true,
"supports_vision": true,
"supports_web_search": true,
"supports_xhigh_reasoning_effort": true
"supports_xhigh_reasoning_effort": true,
"cache_creation_input_token_cost_above_272k_tokens_flex": 1.25e-05,
"cache_read_input_token_cost_above_272k_tokens_flex": 1e-06,
"input_cost_per_token_above_272k_tokens_flex": 1e-05,
"output_cost_per_token_above_272k_tokens_flex": 3.75e-05
},
"azure/gpt-6-astra-2026-09-03": {
"cache_creation_input_token_cost": 1.25e-05,
@ -8763,7 +8767,7 @@
"search_context_size_low": 0.014,
"search_context_size_medium": 0.014
},
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'",
"source": "https://azure.microsoft.com/en-us/pricing/details/cognitive-services/openai-service/",
"supported_endpoints": [
"/v1/chat/completions",
"/v1/responses"
@ -8790,7 +8794,11 @@
"supports_tool_choice": true,
"supports_vision": true,
"supports_web_search": true,
"supports_xhigh_reasoning_effort": true
"supports_xhigh_reasoning_effort": true,
"cache_creation_input_token_cost_above_272k_tokens_flex": 1.25e-05,
"cache_read_input_token_cost_above_272k_tokens_flex": 1e-06,
"input_cost_per_token_above_272k_tokens_flex": 1e-05,
"output_cost_per_token_above_272k_tokens_flex": 3.75e-05
},
"azure/gpt-6-luna": {
"cache_creation_input_token_cost": 1.25e-07,
@ -30856,8 +30864,9 @@
"output_cost_per_token": 0,
"output_vector_size": 3072,
"rpm": 10000,
"source": "https://ai.google.dev/gemini-api/docs/embeddings#model-versions",
"tpm": 10000000
"source": "https://ai.google.dev/gemini-api/docs/pricing",
"tpm": 10000000,
"input_cost_per_token_batches": 7.5e-08
},
"gemini/gemini-embedding-2": {
"input_cost_per_audio_token": 6.5e-06,
@ -31713,7 +31722,10 @@
"input_cost_per_token_flex": 6.25e-07,
"output_cost_per_token_batches": 5e-06,
"output_cost_per_token_flex": 5e-06,
"supports_url_context": true
"supports_url_context": true,
"cache_read_input_token_cost_above_200k_tokens_batches": 2.5e-07,
"input_cost_per_token_above_200k_tokens_batches": 1.25e-06,
"output_cost_per_token_above_200k_tokens_batches": 7.5e-06
},
"gemini/gemini-2.5-computer-use-preview-10-2025": {
"deprecation_date": "2026-07-28",
@ -31871,7 +31883,8 @@
},
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"web_search_billing_unit": "per_query",
"google_maps_grounding_cost_per_query": 0.014
"google_maps_grounding_cost_per_query": 0.014,
"supports_computer_use": true
},
"gemini/gemini-3-flash-preview": {
"cache_read_input_audio_token_cost": 1e-07,
@ -31937,7 +31950,8 @@
"input_cost_per_token_flex": 2.5e-07,
"output_cost_per_token_batches": 1.5e-06,
"output_cost_per_token_flex": 1.5e-06,
"supports_audio_input": true
"supports_audio_input": true,
"supports_computer_use": true
},
"gemini/gemini-3.5-flash": {
"prompt_cache_min_tokens": 4096,
@ -32184,7 +32198,8 @@
},
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"web_search_billing_unit": "per_query",
"google_maps_grounding_cost_per_query": 0.014
"google_maps_grounding_cost_per_query": 0.014,
"supports_computer_use": true
},
"gemini/gemini-omni-flash-preview": {
"input_cost_per_audio_token": 1.5e-06,
@ -32280,7 +32295,10 @@
"web_search_billing_unit": "per_query",
"google_maps_grounding_cost_per_query": 0.014,
"input_cost_per_token_flex": 1e-06,
"output_cost_per_token_flex": 6e-06
"output_cost_per_token_flex": 6e-06,
"cache_read_input_token_cost_above_200k_tokens_batches": 4e-07,
"input_cost_per_token_above_200k_tokens_batches": 2e-06,
"output_cost_per_token_above_200k_tokens_batches": 9e-06
},
"gemini/gemini-3.1-pro-preview-customtools": {
"prompt_cache_min_tokens": 4096,
@ -32343,7 +32361,10 @@
"web_search_billing_unit": "per_query",
"google_maps_grounding_cost_per_query": 0.014,
"input_cost_per_token_flex": 1e-06,
"output_cost_per_token_flex": 6e-06
"output_cost_per_token_flex": 6e-06,
"cache_read_input_token_cost_above_200k_tokens_batches": 4e-07,
"input_cost_per_token_above_200k_tokens_batches": 2e-06,
"output_cost_per_token_above_200k_tokens_batches": 9e-06
},
"gemini-3-flash-preview": {
"cache_read_input_audio_token_cost": 1e-07,
@ -72236,6 +72257,7 @@
},
"aiand/deepseek-ai/deepseek-v4-flash": {
"cache_read_input_token_cost": 8e-08,
"deprecation_date": "2026-10-27",
"input_cost_per_token": 1.5e-07,
"litellm_provider": "aiand",
"max_input_tokens": 1048576,
@ -72265,6 +72287,22 @@
"supports_vision": false
},
"aiand/deepseek-ai/deepseek-v4.1-flash": {
"cache_read_input_token_cost": 2e-08,
"deprecation_date": "2026-11-05",
"input_cost_per_token": 3e-07,
"litellm_provider": "aiand",
"max_input_tokens": 1048576,
"max_output_tokens": 384000,
"mode": "chat",
"output_cost_per_token": 6e-07,
"source": "https://api.aiand.com/v1/models",
"supports_function_calling": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_tool_choice": true,
"supports_vision": true
},
"aiand/deepseek-ai/deepseek-v4.1-flash-1017": {
"cache_read_input_token_cost": 2e-08,
"input_cost_per_token": 3e-07,
"litellm_provider": "aiand",
@ -72294,8 +72332,24 @@
"supports_tool_choice": true,
"supports_vision": true
},
"aiand/meta/muse-spark-open": {
"cache_read_input_token_cost": 1.5e-07,
"input_cost_per_token": 1.25e-06,
"litellm_provider": "aiand",
"max_input_tokens": 1048576,
"max_output_tokens": 943718,
"mode": "chat",
"output_cost_per_token": 4.25e-06,
"source": "https://api.aiand.com/v1/models",
"supports_function_calling": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_tool_choice": true,
"supports_vision": true
},
"aiand/moonshotai/kimi-k2.7-code": {
"cache_read_input_token_cost": 2e-07,
"deprecation_date": "2026-10-27",
"input_cost_per_token": 7.5e-07,
"litellm_provider": "aiand",
"max_input_tokens": 262144,
@ -72386,6 +72440,7 @@
},
"aiand/zai-org/glm-5.2": {
"cache_read_input_token_cost": 3e-07,
"deprecation_date": "2026-10-27",
"input_cost_per_token": 1e-06,
"litellm_provider": "aiand",
"max_input_tokens": 1048576,
@ -76548,6 +76603,7 @@
],
"supports_audio_input": true,
"supports_audio_output": false,
"supports_computer_use": true,
"supports_function_calling": true,
"supports_minimal_reasoning_effort": false,
"supports_native_streaming": true,

View file

@ -30,6 +30,7 @@ from litellm.models.organization_membership import LiteLLM_OrganizationMembershi
from litellm.models.project import LiteLLM_ProjectTable
from litellm.models.skills import LiteLLM_SkillsTable
from litellm.models.spend_logs import LiteLLM_ErrorLogs, LiteLLM_SpendLogs
from litellm.models.subscription_account import LiteLLM_SubscriptionAccountTable
from litellm.models.tag import LiteLLM_TagTable
from litellm.models.team import LiteLLM_TeamTable
from litellm.models.team_membership import LiteLLM_TeamMembership
@ -59,6 +60,7 @@ __all__ = [
"LiteLLM_ProxyModelTable",
"LiteLLM_SkillsTable",
"LiteLLM_SpendLogs",
"LiteLLM_SubscriptionAccountTable",
"LiteLLM_TagTable",
"LiteLLM_TeamMemberTable",
"LiteLLM_TeamMembership",

View file

@ -0,0 +1,28 @@
"""
Subscription account table model.
Canonical definition for ``litellm_subscriptionaccounttable``: the operator-entered
recurring fee of a provider account whose requests are covered by a flat subscription.
"""
from datetime import datetime
from pydantic import ConfigDict
from litellm.types.llms.base import LiteLLMPydanticObjectBase
class LiteLLM_SubscriptionAccountTable(LiteLLMPydanticObjectBase):
model_config = ConfigDict(protected_namespaces=(), frozen=True)
subscription_account_id: str
custom_llm_provider: str
account_id: str
label: str | None = None
monthly_fee: float
currency: str
billing_period_start: str
created_at: datetime
created_by: str
updated_at: datetime
updated_by: str

View file

@ -1648,7 +1648,7 @@
},
"/access_group/{access_group}/update": {
"put": {
"description": "Update an access group's model names.\n\nThis will:\n1. Remove the access group from all current deployments\n2. Add the access group to all deployments for the new model_names list\n\nExample:\n```bash\ncurl -X PUT 'http://localhost:4000/access_group/production-models/update' \\\n -H \"Authorization: Bearer $LITELLM_MASTER_KEY\" \\\n -H 'Content-Type: application/json' \\\n -d '{\n \"model_names\": [\"gpt-4\", \"claude-3-sonnet\"]\n }'\n```\n\nParameters:\n- access_group: str - The access group name (URL path parameter)\n- model_names: List[str] - New list of model groups to include\n\nReturns:\n- NewModelGroupResponse with the updated access group details\n\nRaises:\n- HTTPException 400: If any model names don't exist\n- HTTPException 404: If access group not found",
"description": "Update an access group's model names.\n\nThis will:\n1. Add the access group to all deployments for the new model_names list\n2. Remove the access group from every other deployment\n\nExample:\n```bash\ncurl -X PUT 'http://localhost:4000/access_group/production-models/update' \\\n -H \"Authorization: Bearer $LITELLM_MASTER_KEY\" \\\n -H 'Content-Type: application/json' \\\n -d '{\n \"model_names\": [\"gpt-4\", \"claude-3-sonnet\"]\n }'\n```\n\nParameters:\n- access_group: str - The access group name (URL path parameter)\n- model_names: List[str] - New list of model groups to include\n\nReturns:\n- NewModelGroupResponse with the updated access group details\n\nRaises:\n- HTTPException 400: If any model names don't exist\n- HTTPException 404: If access group not found\n- HTTPException 409: If another edit holds the group or one of its models for 30 s",
"operationId": "update_access_group_access_group__access_group__update_put",
"parameters": [
{
@ -3491,6 +3491,7 @@
"additionalProperties": {
"$ref": "#/components/schemas/GuardrailMetric"
},
"description": "Spend by guardrail ID, already included in key and model spend. Populated on user, team and organization aggregated activity only, without a model filter",
"title": "Guardrails",
"type": "object"
},
@ -3812,10 +3813,12 @@
"GuardrailKeyMetric": {
"properties": {
"metadata": {
"$ref": "#/components/schemas/KeyMetadata"
"$ref": "#/components/schemas/KeyMetadata",
"description": "API key alias and existence information"
},
"metrics": {
"$ref": "#/components/schemas/GuardrailSpendMetrics"
"$ref": "#/components/schemas/GuardrailSpendMetrics",
"description": "Guardrail spend and unpriced units for this API key"
}
},
"required": [
@ -3828,6 +3831,7 @@
"GuardrailMetadata": {
"properties": {
"guardrail_name": {
"description": "Guardrail display name, falling back to the guardrail ID",
"title": "Guardrail Name",
"type": "string"
}
@ -3844,14 +3848,17 @@
"additionalProperties": {
"$ref": "#/components/schemas/GuardrailKeyMetric"
},
"description": "Per-key guardrail spend and unpriced units, limited to keys in the day's API key breakdown",
"title": "Api Key Breakdown",
"type": "object"
},
"metadata": {
"$ref": "#/components/schemas/GuardrailMetadata"
"$ref": "#/components/schemas/GuardrailMetadata",
"description": "Guardrail name information"
},
"metrics": {
"$ref": "#/components/schemas/GuardrailSpendMetrics"
"$ref": "#/components/schemas/GuardrailSpendMetrics",
"description": "Guardrail spend and unpriced units across all matching API keys"
}
},
"required": [
@ -3865,11 +3872,13 @@
"properties": {
"spend": {
"default": 0.0,
"description": "Guardrail spend in USD, already included in key and model spend",
"title": "Spend",
"type": "number"
},
"untracked_units": {
"default": 0,
"description": "Guardrail units with no known price, excluded from spend",
"title": "Untracked Units",
"type": "integer"
}

View file

@ -667,6 +667,10 @@ class LiteLLMRoutes(enum.Enum):
"/team/spend/by_user",
# gateway request counts (SGR); deployment-wide, admin-only
"/gateway/daily/activity",
"/subscription_accounts/usage",
"/subscription_accounts/new",
"/subscription_accounts/update",
"/subscription_accounts/delete",
# model
"/model/new",
"/model/update",

View file

@ -84,6 +84,7 @@ from litellm.proxy._types import (
UserAPIKeyAuth,
)
from litellm.proxy.agent_endpoints.auth.agent_access_groups import (
AgentAccessGroupCeiling,
CeilingResolver,
resolve_agent_access_group_ceiling,
)
@ -1684,6 +1685,7 @@ def get_actual_routes(allowed_routes: list) -> list:
KEY_END_USER_BUDGET_ID_METADATA_FIELD: Final = "end_user_budget_id"
_KEY_METADATA_ADAPTER: Final[TypeAdapter[Mapping[str, object] | None]] = TypeAdapter(Mapping[str, object] | None)
_OBJECTS_ADAPTER: Final[TypeAdapter[tuple[object, ...]]] = TypeAdapter(tuple[object, ...])
def get_key_end_user_budget_id(key_metadata: Mapping[str, object] | None) -> str | None:
@ -3455,28 +3457,35 @@ async def delete_cache_key_objects(
proxy_logging_obj: ProxyLogging | None,
) -> None:
"""
Evict a batch of key objects, for callers that delete keys in bulk rather than through
`/key/delete`. Auth resolves a cached key object without re-reading its team, so a key left
cached after its row is gone keeps buying access until its TTL expires.
Evicting locally only reaches this worker; `_delete_cache_key_object` itself broadcasts each
token, so a deleted key left in a peer worker's in-memory cache still authenticates there until
its TTL expires.
Evict a batch of deleted key objects here and on every other worker. Auth resolves a cached key
object without re-reading its team, so a key left cached after its row is gone keeps buying
access until its TTL expires.
Best-effort per key: the rows are already deleted by the time this runs, so an unreachable
cache backend must not abort the caller partway through its own cascade.
Every token is dropped from this worker's memory first, then from the auth cache and the
internal usage cache in one batch each, so a deleted key stops authenticating here the instant
its rows are gone instead of refilling from Redis on the next miss. Every other worker then
gets the eviction in one pipelined broadcast: a publish per key would pass the publisher's
pending cap on a bulk delete, and the dropped keys would keep authenticating there until their
TTL.
Every token is dropped from this worker's memory first, then from the auth cache's Redis layer,
so a deleted key stops authenticating here the instant its rows are gone instead of refilling
from Redis on the next miss. Every other worker then gets the eviction in one pipelined
broadcast: a publish per key would pass the publisher's pending cap on a bulk delete, and the
dropped keys would keep authenticating there until their TTL. The internal usage cache is not
an auth refill source, so its batch delete runs beside that chain and a hung Redis costs one
bounded wait there instead of another in line.
"""
evict_local(cache_keys=hashed_tokens, user_api_key_cache=user_api_key_cache)
usage_eviction: Final = (
()
if proxy_logging_obj is None
else (evict_shared(cache_keys=hashed_tokens, cache=proxy_logging_obj.internal_usage_cache.dual_cache),)
)
_ = await asyncio.gather(
_evict_auth_cache_then_broadcast(hashed_tokens=hashed_tokens, user_api_key_cache=user_api_key_cache),
*usage_eviction,
)
async def _evict_auth_cache_then_broadcast(hashed_tokens: Sequence[str], user_api_key_cache: UserApiKeyCache) -> None:
await evict_shared(cache_keys=hashed_tokens, cache=user_api_key_cache)
if proxy_logging_obj is not None:
await evict_shared(cache_keys=hashed_tokens, cache=proxy_logging_obj.internal_usage_cache.dual_cache)
await broadcast_evictions(cache_keys=hashed_tokens)
@ -5085,6 +5094,36 @@ async def can_agent_call_model(
)
async def _agent_access_group_ceilings(
valid_token: UserAPIKeyAuth, resolve_ceiling: CeilingResolver
) -> tuple[AgentAccessGroupCeiling, ...]:
if not valid_token.agent_id:
return ()
from litellm.proxy.agent_endpoints.auth.agent_access_groups import resolve_managed_agent_ceilings
managed: Final = managed_agent_policy(valid_token)
if managed is not None:
return await resolve_managed_agent_ceilings(managed)
unmanaged: Final = await resolve_ceiling(valid_token.agent_id)
return (unmanaged,) if unmanaged is not None else ()
def _managed_agent_model_grants(valid_token: UserAPIKeyAuth) -> frozenset[str]:
managed: Final = managed_agent_policy(valid_token)
granted: Final = (managed.object_permission or {}).get("models") if managed is not None else None
if not isinstance(granted, (list, tuple)):
return frozenset()
return frozenset(name for name in _OBJECTS_ADAPTER.validate_python(granted) if isinstance(name, str))
async def agent_model_grants(
valid_token: UserAPIKeyAuth, resolve_ceiling: CeilingResolver = resolve_agent_access_group_ceiling
) -> frozenset[str]:
ceilings: Final = await _agent_access_group_ceilings(valid_token, resolve_ceiling)
return _managed_agent_model_grants(valid_token).union(*(ceiling.models for ceiling in ceilings))
async def _check_agent_access_group_model_access(
model: str | list[str] | None, # mutable-ok: _can_object_call_model and the client message helper take list[str]
valid_token: UserAPIKeyAuth | None,
@ -5092,20 +5131,9 @@ async def _check_agent_access_group_model_access(
resolve_ceiling: CeilingResolver = resolve_agent_access_group_ceiling,
) -> Literal[True]:
"""Attached groups naming no model deny every model; the empty allowlist in ``_can_object_call_model`` allows."""
if not model or valid_token is None or not valid_token.agent_id:
if not model or valid_token is None:
return True
from litellm.proxy.agent_endpoints.auth.agent_access_groups import resolve_managed_agent_ceilings
managed: Final = managed_agent_policy(valid_token)
unmanaged: Final = await resolve_ceiling(valid_token.agent_id) if managed is None else None
ceilings: Final = (
await resolve_managed_agent_ceilings(managed)
if managed is not None
else (unmanaged,)
if unmanaged is not None
else ()
)
ceilings: Final = await _agent_access_group_ceilings(valid_token, resolve_ceiling)
dispatched: Final = _resolve_team_alias(
model, team_model_aliases_for_auth_check(valid_token), valid_token.team_id, llm_router
)

View file

@ -2199,15 +2199,19 @@ class JWTAuthManager:
# not mistaken for claimless and left with stale DB memberships the fallback
# could later attribute. With the flag off, keep the upstream plural-only
# reconciliation so existing deployments are unchanged.
jwt_team_ids: Final = set(
jwt_handler.get_all_jwt_team_ids(jwt_valid_token)
if jwt_handler.litellm_jwtauth.fallback_to_db_teams
else jwt_handler.get_team_ids_from_jwt(jwt_valid_token)
claim_team_ids: Final = tuple(
dict.fromkeys(
jwt_handler.get_all_jwt_team_ids(jwt_valid_token)
if jwt_handler.litellm_jwtauth.fallback_to_db_teams
else jwt_handler.get_team_ids_from_jwt(jwt_valid_token)
)
)
existing_teams: Final = set(user_object.teams or [])
teams_to_add: Final = jwt_team_ids - existing_teams
jwt_team_ids: Final = frozenset(claim_team_ids)
existing_team_order: Final = tuple(dict.fromkeys(user_object.teams or []))
existing_teams: Final = frozenset(existing_team_order)
teams_to_add: Final = tuple(team_id for team_id in claim_team_ids if team_id not in existing_teams)
preserve_db_teams_without_claims: Final = jwt_handler.litellm_jwtauth.fallback_to_db_teams and not jwt_team_ids
teams_to_remove: Final = set() if preserve_db_teams_without_claims else existing_teams - jwt_team_ids
teams_to_remove: Final = frozenset() if preserve_db_teams_without_claims else existing_teams - jwt_team_ids
if teams_to_add or teams_to_remove:
from litellm.proxy.management_endpoints.scim.scim_v2 import (
patch_team_membership,
@ -2218,7 +2222,10 @@ class JWTAuthManager:
teams_ids_to_add_user_to=list(teams_to_add),
teams_ids_to_remove_user_from=list(teams_to_remove),
)
user_object.teams = list(jwt_team_ids)
user_object.teams = [
*(team_id for team_id in existing_team_order if team_id not in teams_to_remove),
*teams_to_add,
]
if user_api_key_cache is not None:
await user_api_key_cache.async_set_cache(
key=user_object.user_id,

View file

@ -60,6 +60,7 @@ from litellm.litellm_core_utils.bug_report import (
strip_bug_report_notice,
)
from litellm.litellm_core_utils.core_helpers import (
finite_cost,
get_or_create_metadata_bucket,
independent_snapshot,
is_expected_client_error,
@ -140,6 +141,7 @@ from litellm.proxy.utils import ( # noqa: F401 # legacy module exports
_check_and_merge_model_level_guardrails, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export
check_and_merge_model_level_guardrails,
)
from litellm.responses.mcp.parked_stream_dispatch import in_flight_round_biller, parked_success
from litellm.responses.streaming_iterator import partial_responses_incomplete_event_on_disconnect, spawn_logging_task
from litellm.router import Router
from litellm.router_api import RoutingContext
@ -535,6 +537,16 @@ async def _bill_partial_streamed_spend_on_disconnect(request_data: dict, respons
return await _dispatch_disconnect_billing_event(logging_obj, partial_response)
async def _bill_in_flight_round_on_disconnect(request_data: Mapping[str, object]) -> None:
bill_in_flight_round: Final = in_flight_round_biller(request_data.get("litellm_logging_obj"))
if bill_in_flight_round is None:
return
try:
await bill_in_flight_round()
except Exception as e: # noqa: BLE001 # disconnect billing is best-effort; never break stream teardown
verbose_proxy_logger.debug("Failed to bill the in-flight round after client disconnect: %s", e)
async def _dispatch_disconnect_billing_event(logging_obj: LiteLLMLoggingObj, partial_response: object) -> bool:
try:
await logging_obj.dispatch_success_handlers(
@ -1801,14 +1813,16 @@ def _get_cost_breakdown_from_logging_obj(
def _header_value(value: float | None) -> str | None:
return str(value) if value is not None else None
return str(value) if finite_cost(value) is not None else None
def _response_cost_header_value(hidden_params: Mapping[str, object]) -> float | str:
if hidden_params.get("cache_hit") is True:
return 0.0
response_cost: Final = hidden_params.get("response_cost")
return response_cost if isinstance(response_cost, (int, float, str)) and response_cost else ""
if isinstance(response_cost, str):
return response_cost
return response_cost if isinstance(response_cost, int | float) and finite_cost(response_cost) else ""
def cost_headers(
@ -2083,6 +2097,13 @@ class _ClientStreamFailureReport:
)
def _metadata_logged_on(logged_on: LiteLLMLoggingObj) -> Mapping[str, object]:
logged_params: Final = object_mapping(logged_on.model_call_details.get("litellm_params")) or MappingProxyType({})
return MappingProxyType(
{key: logged_params[key] for key in ("metadata", "litellm_metadata") if key in logged_params}
)
class ProxyBaseLLMRequestProcessing:
def __init__(self, data: dict, served_model: str | None = None):
self.data = data
@ -4151,11 +4172,21 @@ class ProxyBaseLLMRequestProcessing:
_captured_user_api_key_dict: Final = user_api_key_dict
_captured_logging_obj: Final = logging_obj
async def _on_deferred_stream_complete(assembled_response: object, cache_hit: object) -> None:
async def _on_deferred_stream_complete(
assembled_response: object, cache_hit: object, logged_on: LiteLLMLoggingObj | None = None
) -> None:
await ProxyBaseLLMRequestProcessing._run_deferred_stream_guardrails(
captured_data=_captured_data,
captured_data=_captured_data
if logged_on is None
else {
**_captured_data,
**{
key: value for key, value in _metadata_logged_on(logged_on).items() if key in _captured_data
},
"litellm_logging_obj": logged_on,
},
captured_user_api_key_dict=_captured_user_api_key_dict,
captured_logging_obj=_captured_logging_obj,
captured_logging_obj=_captured_logging_obj if logged_on is None else logged_on,
assembled_response=assembled_response,
cache_hit=cache_hit,
)
@ -4180,14 +4211,8 @@ class ProxyBaseLLMRequestProcessing:
GLOBAL_LOGGING_WORKER.ensure_initialized_and_enqueue(async_coroutine=logging_coroutine)
case (logging_coroutine,) if asyncio.iscoroutine(logging_coroutine):
GLOBAL_LOGGING_WORKER.ensure_initialized_and_enqueue(async_coroutine=logging_coroutine)
case (assembled_response, cache_hit):
await _as_success_dispatcher(_captured_native_logging_obj).dispatch_success_handlers(
assembled_response,
cache_hit=cache_hit,
start_time=None,
end_time=None,
prefer_async_handlers=True,
)
case _ if (success := parked_success(args)) is not None:
await success.dispatch(_captured_native_logging_obj)
case _:
verbose_proxy_logger.error(
"Deferred stream logging dropped: unexpected stored args shape %s",
@ -4487,7 +4512,7 @@ class ProxyBaseLLMRequestProcessing:
"""
if isinstance(chunk, dict):
# Use safe_dumps for proper JSON serialization with circular reference detection
chunk_str: Final = safe_dumps(chunk)
chunk_str: Final = safe_dumps(chunk, non_finite_as_null=True)
return f"{STREAM_SSE_DATA_PREFIX}{chunk_str}\n\n"
else:
return chunk
@ -4524,6 +4549,7 @@ class ProxyBaseLLMRequestProcessing:
client_disconnected,
)
if recorded_client_disconnect:
await _bill_in_flight_round_on_disconnect(_REQUEST_FIELDS.validate_python(request_data))
deferred_stream_logging_armed: Final = _deferred_stream_logging_is_armed(request_data)
ProxyLogging.fire_deferred_stream_logging(request_data)
# A disconnect-time success event (the deferred-guardrail flush
@ -4862,7 +4888,11 @@ class ProxyBaseLLMRequestProcessing:
obj, model_name, litellm_logging_obj
)
if maybe_modified is not None:
lines[idx] = "data: " + safe_dumps(maybe_modified) + ("\r" if ln.endswith("\r") else "")
lines[idx] = (
"data: "
+ safe_dumps(maybe_modified, non_finite_as_null=True)
+ ("\r" if ln.endswith("\r") else "")
)
return "\n".join(lines)
return None
except Exception:

View file

@ -2,9 +2,12 @@ import asyncio
import json
import random
import time
from collections.abc import Awaitable, Callable
from collections.abc import AsyncIterator, Awaitable, Callable
from dataclasses import asdict, dataclass
from typing import TYPE_CHECKING, Final, Protocol, cast # noqa: TID251 # untyped prisma/redis boundary needs cast
from itertools import chain
from typing import TYPE_CHECKING, Final, Protocol, TypeAlias, cast # noqa: TID251 # untyped prisma/redis boundary
from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError
from litellm._logging import verbose_proxy_logger
from litellm.repositories.prisma_protocols import RowT_co, TableActions
@ -106,24 +109,69 @@ _pubsub_capable_client: Final = pubsub_capable_client
@dataclass(frozen=True, slots=True)
class _ConfigChangeMessage:
object_type: str
model_ids: tuple[str, ...] = ()
def _config_change_message_json(object_type: str) -> str:
return json.dumps(asdict(_ConfigChangeMessage(object_type=object_type)))
@dataclass(frozen=True, slots=True)
class _PubSubEnvelope:
data: str | bytes
async def publish_config_change(redis_cache: "RedisCache | None", object_type: str) -> None:
class _WrittenModelRow(BaseModel):
model_config = ConfigDict(from_attributes=True, frozen=True)
model_id: str
_CONFIG_CHANGE_MESSAGE: Final = TypeAdapter(_ConfigChangeMessage)
_PUBSUB_ENVELOPE: Final = TypeAdapter(_PubSubEnvelope)
_MODEL_TABLE_NAME: Final = "litellm_proxymodeltable"
_TARGETED_WRITE_ACTION_NAMES: Final[frozenset[str]] = frozenset({"create", "update", "upsert"})
PublishConfigChange: TypeAlias = Callable[[str, tuple[str, ...]], Awaitable[None]]
def _config_change_message_json(object_type: str, model_ids: tuple[str, ...]) -> str:
return json.dumps(asdict(_ConfigChangeMessage(object_type=object_type, model_ids=model_ids)))
def parse_config_change(message: object) -> _ConfigChangeMessage | None:
try:
payload: Final = (
message if isinstance(message, (str, bytes)) else _PUBSUB_ENVELOPE.validate_python(message).data
)
return _CONFIG_CHANGE_MESSAGE.validate_json(payload)
except ValidationError:
return None
def written_model_ids(object_type: str, action_name: str, result: object) -> tuple[str, ...]:
if object_type != _MODEL_TABLE_NAME or action_name not in _TARGETED_WRITE_ACTION_NAMES:
return ()
try:
return (_WrittenModelRow.model_validate(result).model_id,)
except ValidationError:
return ()
async def publish_config_change(
redis_cache: "RedisCache | None", object_type: str, model_ids: tuple[str, ...] = ()
) -> None:
if redis_cache is None:
return
try:
client: Final = pubsub_capable_client(redis_cache)
await client.publish(config_sync_channel(redis_cache), _config_change_message_json(object_type))
await client.publish(config_sync_channel(redis_cache), _config_change_message_json(object_type, model_ids))
except Exception as e: # noqa: BLE001 # best-effort publish; writes must never fail on redis errors
verbose_proxy_logger.warning("config sync publish for %s failed: %s", object_type, e)
async def publish_config_change_for_object_type(object_type: str) -> None:
await publish_config_change(redis_cache=coordination_redis_cache(), object_type=object_type)
async def publish_config_change_for_object_type(object_type: str, model_ids: tuple[str, ...] = ()) -> None:
await publish_config_change(redis_cache=coordination_redis_cache(), object_type=object_type, model_ids=model_ids)
async def publish_model_rows_written(model_ids: tuple[str, ...]) -> None:
await publish_config_change_for_object_type(_MODEL_TABLE_NAME, model_ids)
async def publish_config_param_change(param_name: str) -> None:
@ -139,7 +187,7 @@ async def publish_config_param_change(param_name: str) -> None:
class _PublishOnWriteActions:
__slots__ = ("_actions", "_object_type", "_publish")
def __init__(self, actions: object, object_type: str, publish: Callable[[str], Awaitable[None]]) -> None:
def __init__(self, actions: object, object_type: str, publish: PublishConfigChange) -> None:
self._actions = actions
self._object_type = object_type
self._publish = publish
@ -157,7 +205,7 @@ class _PublishOnWriteActions:
**kwargs: object, # kwargs-ok: transparent passthrough to untyped prisma action
) -> object:
result: Final = await write_action(*args, **kwargs)
await publish(object_type)
await publish(object_type, written_model_ids(object_type, name, result))
return result
return _write_then_publish
@ -166,7 +214,7 @@ class _PublishOnWriteActions:
def wrap_table_actions_for_config_sync(
actions: "TableActions[RowT_co]",
table_name: str,
publish: Callable[[str], Awaitable[None]] = publish_config_change_for_object_type,
publish: PublishConfigChange = publish_config_change_for_object_type,
) -> "TableActions[RowT_co]":
if table_name not in _CONFIG_SYNCED_TABLE_NAMES:
return actions
@ -174,8 +222,14 @@ def wrap_table_actions_for_config_sync(
return cast("TableActions[RowT_co]", wrapped) # cast-ok: dynamic write-through proxy keeps the wrapped row type
def _published_model_ids(messages: tuple[object, ...]) -> tuple[str, ...]:
changes: Final = (parse_config_change(message) for message in messages)
return tuple(dict.fromkeys(chain.from_iterable(change.model_ids for change in changes if change is not None)))
class ConfigSyncSubscriber:
__slots__ = (
"_apply_model_ids",
"_backoff_initial_seconds",
"_backoff_max_seconds",
"_debounce_seconds",
@ -195,6 +249,7 @@ class ConfigSyncSubscriber:
self,
redis_cache: "RedisCache",
resync_callbacks: tuple[Callable[[], Awaitable[None]], ...],
apply_model_ids: Callable[[tuple[str, ...]], Awaitable[None]] | None = None,
debounce_seconds: float = CONFIG_SYNC_DEBOUNCE_SECONDS,
jitter_max_seconds: float = CONFIG_SYNC_JITTER_MAX_SECONDS,
min_resync_interval_seconds: float = CONFIG_SYNC_MIN_RESYNC_INTERVAL_SECONDS,
@ -206,6 +261,7 @@ class ConfigSyncSubscriber:
) -> None:
self._redis_cache = redis_cache
self._resync_callbacks = resync_callbacks
self._apply_model_ids = apply_model_ids
self._debounce_seconds = debounce_seconds
self._jitter_max_seconds = jitter_max_seconds
self._min_resync_interval_seconds = min_resync_interval_seconds
@ -266,32 +322,64 @@ class ConfigSyncSubscriber:
message = await pubsub.get_message(ignore_subscribe_messages=True, timeout=_POLL_TIMEOUT_SECONDS)
if message is None:
continue
await self._apply(pubsub, message)
await self._debounce_and_resync(pubsub)
async def _debounce_and_resync(self, pubsub: _ConfigSyncPubSub) -> None:
await self._sleep(self._debounce_seconds + self._rng.uniform(0.0, self._jitter_max_seconds))
await self._wait_for_min_resync_interval()
async def _debounce_and_resync(self, pubsub: ConfigSyncPubSub) -> None:
await self._apply_until(
pubsub, self._monotonic() + self._debounce_seconds + self._rng.uniform(0.0, self._jitter_max_seconds)
)
await self._apply_until(pubsub, self._next_resync_at())
await self._drain_pending(pubsub)
await self._run_resync_callbacks()
self._last_resync_at = self._monotonic()
async def _wait_for_min_resync_interval(self) -> None:
def _next_resync_at(self) -> float:
if self._last_resync_at is None:
return
seconds_until_next_resync = self._min_resync_interval_seconds - (self._monotonic() - self._last_resync_at)
if seconds_until_next_resync <= 0:
return
verbose_proxy_logger.debug(
"config sync resync throttled for %.1fs to cap fleet-wide reload rate",
seconds_until_next_resync,
)
await self._sleep(seconds_until_next_resync)
return self._monotonic()
next_resync_at: Final = self._last_resync_at + self._min_resync_interval_seconds
throttled_for: Final = next_resync_at - self._monotonic()
if throttled_for > 0:
verbose_proxy_logger.debug(
"config sync resync throttled for %.1fs to cap fleet-wide reload rate",
throttled_for,
)
return next_resync_at
async def _apply_until(self, pubsub: ConfigSyncPubSub, deadline: float) -> None:
while True:
remaining = deadline - self._monotonic()
if remaining <= 0:
return
message = await pubsub.get_message(
ignore_subscribe_messages=True, timeout=min(remaining, _POLL_TIMEOUT_SECONDS)
)
if message is not None:
await self._apply(pubsub, message)
@staticmethod
async def _drain_pending(pubsub: ConfigSyncPubSub) -> None:
while await pubsub.get_message(ignore_subscribe_messages=True, timeout=0) is not None:
pass
@staticmethod
async def _ready_messages(pubsub: ConfigSyncPubSub) -> AsyncIterator[object]:
while (message := await pubsub.get_message(ignore_subscribe_messages=True, timeout=0)) is not None:
yield message
async def _apply(self, pubsub: ConfigSyncPubSub, message: object) -> None:
apply: Final = self._apply_model_ids
if apply is None:
return
ready: Final = (message, *[ready_message async for ready_message in self._ready_messages(pubsub)])
model_ids: Final = _published_model_ids(ready)
if not model_ids:
return
try:
await apply(model_ids)
except Exception as e: # noqa: BLE001 # the debounced full resync retries what a targeted apply could not
verbose_proxy_logger.warning("config sync targeted apply of %s failed: %s", model_ids, e)
async def _run_resync_callbacks(self) -> None:
for callback in self._resync_callbacks:
try:

View file

@ -219,7 +219,7 @@ def caller_alias_maps(
def alias_map(aliases: object) -> Mapping[str, str]:
try:
entries: Final = _ALIAS_ENTRIES.validate_python(aliases, strict=True)
entries: Final = _ALIAS_ENTRIES.validate_python(aliases)
except ValidationError:
return _NO_ALIASES
return MappingProxyType(
@ -287,6 +287,15 @@ def _dispatch(model_id: str, layers: Sequence[_AliasLayer]) -> tuple[str, bool]:
return reduce(_through_layer, layers, (model_id, False))
def dispatched_model(model_id: str, aliases: CallerAliases) -> str:
target, _ = _dispatch(model_id, _alias_layers(aliases))
return target
def requestable_alias_names(aliases: CallerAliases) -> frozenset[str]:
return frozenset(chain.from_iterable(layer.aliases for layer in _alias_layers(aliases)))
def _lookup_id(name: str, lookup_by_response: Mapping[str, str], lookup_ids: Container[str]) -> str | None:
if name in lookup_by_response:
return lookup_by_response[name]

View file

@ -28,6 +28,8 @@ _OPENAI_ERROR_TYPE_BY_STATUS: Final[Mapping[int, str]] = MappingProxyType(
_AUTH_FAILURE_STATUS_CODES: Final = frozenset({status.HTTP_401_UNAUTHORIZED, status.HTTP_403_FORBIDDEN})
_HTTP_ERROR_STATUSES: Final = range(status.HTTP_400_BAD_REQUEST, 600)
_ERROR_BODY: Final = TypeAdapter(dict[str, object])
_FAILOVER_SQLSTATES: Final = frozenset({"25006", "57P01", "57P02", "57P03"})
@ -47,13 +49,26 @@ def attribute_of(value: object, name: str, default: object = None) -> object:
def error_status_code(exc: object, default: int) -> int:
"""The HTTP status an exception carries as ``status_code`` or, the way ``ProxyException``
stores it, as a stringified ``code``; ``default`` when it carries neither."""
carried: Final = attribute_of(exc, "status_code")
if isinstance(carried, int) and not isinstance(carried, bool):
"""The 4xx or 5xx HTTP status an exception carries as ``status_code`` (an int or a decimal string) or,
the way ``ProxyException`` stores it, as a stringified ``code``; ``default`` when it carries neither."""
carried: Final = _http_error_status(attribute_of(exc, "status_code"))
if carried is not None:
return carried
stringified: Final = attribute_of(exc, "code")
return int(stringified) if isinstance(stringified, str) and stringified.isdecimal() else default
from_code: Final = _http_error_status(stringified) if isinstance(stringified, str) else None
return default if from_code is None else from_code
def _http_error_status(value: object) -> int | None:
match value:
case bool():
return None
case int() if value in _HTTP_ERROR_STATUSES:
return value
case str() if len(value) == 3 and value.isascii() and value.isdigit() and int(value) in _HTTP_ERROR_STATUSES:
return int(value)
case _:
return None
def openai_error_type(exc: object, status_code: int) -> str:
@ -105,9 +120,9 @@ def proxy_exception_for(exc: Exception, default_status_code: int) -> ProxyExcept
code=status.HTTP_503_SERVICE_UNAVAILABLE,
)
case HTTPException(status_code=status_code, detail=detail):
return _openai_shaped_proxy_exception(exc, str(detail), status_code)
return openai_shaped_proxy_exception(exc, str(detail), status_code)
case _:
return _openai_shaped_proxy_exception(
return openai_shaped_proxy_exception(
exc, str(exc) or type(exc).__name__, error_status_code(exc, default_status_code)
)
@ -176,14 +191,19 @@ def _route_failure_error_type(exc: Exception, raised_status: int, answered_statu
return openai_error_type(exc, answered_status)
def _openai_shaped_proxy_exception(exc: Exception, message: str, status_code: int) -> ProxyException:
def openai_shaped_proxy_exception(exc: Exception, message: str, status_code: int) -> ProxyException:
answered_status: Final = answered_status_code(message, status_code)
return ProxyException(
message=message,
type=_catch_all_error_type(exc, answered_status),
param=openai_error_param(exc),
code=answered_status,
)
return _proxy_exception(exc, message, answered_status, _catch_all_error_type(exc, answered_status))
def status_typed_proxy_exception(exc: HTTPException) -> ProxyException:
message: Final = str(exc.detail)
answered_status: Final = answered_status_code(message, exc.status_code)
return _proxy_exception(exc, message, answered_status, openai_error_type(exc, answered_status))
def _proxy_exception(exc: Exception, message: str, answered_status: int, error_type: str) -> ProxyException:
return ProxyException(message=message, type=error_type, param=openai_error_param(exc), code=answered_status)
def _catch_all_error_type(exc: Exception, status_code: int) -> str:

View file

@ -15,12 +15,13 @@ or the first routed request on this replica would pick a tier it has not loaded.
import asyncio
import time
from collections.abc import Awaitable, Callable, Mapping, Sequence
from typing import TYPE_CHECKING, Final
from typing import TYPE_CHECKING, Final, Literal, TypeAlias
from pydantic import TypeAdapter, ValidationError
from litellm._logging import verbose_proxy_logger
from litellm.caching.in_memory_cache import InMemoryCache
from litellm.litellm_core_utils.credential_accessor import CredentialAccessor
from litellm.proxy.common_utils.encrypt_decrypt_utils import decrypt_value_helper
if TYPE_CHECKING:
@ -33,8 +34,14 @@ if TYPE_CHECKING:
)
from litellm.integrations.custom_guardrail import CustomGuardrail
from litellm.repositories.prisma_protocols import TableActions
from litellm.types.agents import AgentResponse
_ModelTable: TypeAlias = TableActions[prisma_models.LiteLLM_ProxyModelTable]
_ModelRows: TypeAlias = Sequence[prisma_models.LiteLLM_ProxyModelTable]
ModelLoadOutcome: TypeAlias = Literal["loaded", "missing", "no_router", "auto_router"]
READ_THROUGH_MISS_TTL_SECONDS: Final = 2.0
READ_THROUGH_RESYNC_WINDOW_SECONDS: Final = 5.0
READ_THROUGH_MAX_RESYNCS_PER_WINDOW: Final = 20
@ -123,47 +130,114 @@ async def _resync_model_deployments(model_name: str) -> bool:
return False
prisma_client: Final = proxy_server.prisma_client
assert prisma_client is not None
table: Final = ModelRepository(prisma_client).table
name_filter: Final[LiteLLM_ProxyModelTableWhereInput] = {"model_name": model_name}
id_filter: Final[LiteLLM_ProxyModelTableWhereInput] = {"model_id": model_name}
rows: Final = await table.find_many(where=name_filter) or await table.find_many(where=id_filter)
rows: Final = await _find_rows_by_name_or_id(model_name, ModelRepository(prisma_client).table)
if not rows:
return False
router: Final = proxy_server.llm_router
if router is None:
await proxy_server.proxy_config.add_deployment(
prisma_client=prisma_client, proxy_logging_obj=proxy_server.proxy_logging_obj
)
return proxy_server.llm_router is not None
if _holds_an_auto_router(rows):
await proxy_server.proxy_config.add_deployment(
prisma_client=prisma_client, proxy_logging_obj=proxy_server.proxy_logging_obj
)
return _model_is_loaded(model_name)
async with proxy_server.MODEL_RECONCILE_LOCK:
outcome: Final = await _install_model_rows(rows, _always)
if outcome in ("no_router", "auto_router"):
await _reload_every_model()
return outcome == "loaded" or _model_is_loaded(model_name)
async def resync_model_deployments_by_ids(model_ids: tuple[str, ...]) -> None:
from litellm.proxy import proxy_server
from litellm.proxy.db.routing_prisma_wrapper import WriterPinnedClient
from litellm.repositories.model_repository import ModelRepository
if not _db_backed_registries_enabled("models"):
return
prisma_client: Final = proxy_server.prisma_client
assert prisma_client is not None
writer_table: Final = ModelRepository(WriterPinnedClient(prisma_client.db)).table
async with proxy_server.MODEL_RECONCILE_LOCK:
rows: Final = await _find_rows_by_ids(model_ids, writer_table)
_ = await _install_model_rows(_without_auto_routers(rows), _names_an_unloaded_credential)
async def _find_rows_by_name_or_id(model_name: str, table: "_ModelTable") -> "_ModelRows":
name_filter: Final[LiteLLM_ProxyModelTableWhereInput] = {"model_name": model_name}
id_filter: Final[LiteLLM_ProxyModelTableWhereInput] = {"model_id": model_name}
return await table.find_many(where=name_filter) or await table.find_many(where=id_filter)
async def _find_rows_by_ids(model_ids: tuple[str, ...], table: "_ModelTable") -> "_ModelRows":
from litellm.repositories.chunked_in import find_many_in
return list(await find_many_in(table, "model_id", model_ids))
async def _install_model_rows(
rows: "_ModelRows",
needs_credentials: Callable[["_ModelRows"], bool],
) -> ModelLoadOutcome:
from litellm.proxy import proxy_server
router: Final = proxy_server.llm_router
if not rows:
return "missing"
if router is None:
return "no_router"
if _holds_an_auto_router(rows):
return "auto_router"
if needs_credentials(rows):
prisma_client: Final = proxy_server.prisma_client
assert prisma_client is not None
await proxy_server.proxy_config.get_credentials(prisma_client=prisma_client)
proxy_server.proxy_config._add_deployment(db_models=rows)
proxy_server.llm_model_list = router.registry().deployments()
proxy_server.proxy_config._add_deployment(db_models=rows)
proxy_server.llm_model_list = router.registry().deployments()
return "loaded"
async def _reload_every_model() -> None:
from litellm.proxy import proxy_server
prisma_client: Final = proxy_server.prisma_client
assert prisma_client is not None
await proxy_server.proxy_config.add_deployment(
prisma_client=prisma_client, proxy_logging_obj=proxy_server.proxy_logging_obj
)
def _always(rows: "_ModelRows") -> bool:
return True
def _holds_an_auto_router(rows: Sequence["prisma_models.LiteLLM_ProxyModelTable"]) -> bool:
def _holds_an_auto_router(rows: "_ModelRows") -> bool:
return any(_names_an_auto_router(row.litellm_params) for row in rows)
def _without_auto_routers(rows: "_ModelRows") -> "_ModelRows":
return list(row for row in rows if not _names_an_auto_router(row.litellm_params))
def _names_an_unloaded_credential(rows: "_ModelRows") -> bool:
return any(_names_an_unloaded_credential_param(row.litellm_params) for row in rows)
_STORED_LITELLM_PARAMS: Final = TypeAdapter(Mapping[str, object])
def _names_an_auto_router(stored_litellm_params: object) -> bool:
def _decrypted_param(stored_litellm_params: object, key: str) -> str | None:
try:
params: Final = _STORED_LITELLM_PARAMS.validate_python(stored_litellm_params)
except ValidationError:
return False
stored_model: Final = params.get("model")
if not isinstance(stored_model, str):
return False
model: Final = decrypt_value_helper(value=stored_model, key="model", return_original_value=True)
return isinstance(model, str) and model.startswith("auto_router/")
return None
stored_value: Final = params.get(key)
if not isinstance(stored_value, str):
return None
value: Final = decrypt_value_helper(value=stored_value, key=key, return_original_value=True)
return value if isinstance(value, str) else None
def _names_an_auto_router(stored_litellm_params: object) -> bool:
model: Final = _decrypted_param(stored_litellm_params, "model")
return model is not None and model.startswith("auto_router/")
def _names_an_unloaded_credential_param(stored_litellm_params: object) -> bool:
credential_name: Final = _decrypted_param(stored_litellm_params, "litellm_credential_name")
return credential_name is not None and CredentialAccessor.find_credential(credential_name) is None
async def _resync_guardrails(guardrail_name: str) -> bool:

View file

@ -34,6 +34,7 @@ from litellm.constants import (
DB_SPEND_UPDATE_JOB_NAME,
INTERNAL_CALL_ORIGIN_METADATA_KEY,
)
from litellm.litellm_core_utils.core_helpers import finite_cost, single_log_line
from litellm.litellm_core_utils.litellm_logging import coerce_model_access_groups
from litellm.litellm_core_utils.safe_json_loads import safe_json_loads
from litellm.proxy._types import (
@ -652,7 +653,14 @@ class DBSpendUpdateWriter:
end_time=end_time,
llm_router=llm_router,
)
payload["spend"] = response_cost or 0.0
billable_cost: Final = finite_cost(response_cost)
if response_cost is not None and billable_cost is None:
verbose_proxy_logger.warning(
"Spend tracking - recording a non-finite response_cost=%s as 0 for request_id=%s",
response_cost,
single_log_line(payload["request_id"]),
)
payload["spend"] = billable_cost or 0.0
if isinstance(payload["startTime"], datetime):
payload["startTime"] = payload["startTime"].isoformat()
if isinstance(payload["endTime"], datetime):
@ -702,7 +710,7 @@ class DBSpendUpdateWriter:
spawn_logging_task(
asyncio.get_running_loop(),
self._batch_database_updates(
response_cost=response_cost,
response_cost=billable_cost,
user_id=user_id,
hashed_token=hashed_token,
team_id=team_id,

View file

@ -8,7 +8,7 @@ import asyncio
import json
from collections.abc import Mapping, Sequence
from datetime import datetime
from functools import reduce
from functools import partial, reduce
from itertools import chain
from typing import TYPE_CHECKING, Any, Final, Literal, TypeAlias, TypeVar, cast
@ -109,6 +109,16 @@ _ValueT = TypeVar("_ValueT")
_DECODED_JSON: Final = TypeAdapter(object)
def _zero_for_non_finite_constant(constant: str) -> float:
verbose_proxy_logger.warning(
"Redis spend buffer - reading a parked %s spend as 0, the spend it held is lost", constant
)
return 0.0
_load_buffered_transaction: Final = partial(json.loads, parse_constant=_zero_for_non_finite_constant)
def _spend_log_json_default(value: object) -> str:
return value.isoformat() if isinstance(value, datetime) else str(value)
@ -118,7 +128,7 @@ def _encode_spend_log_row(row: SpendLogRow) -> str:
def _decode_spend_log_row(encoded: str) -> dict[str, object] | None:
decoded: Final = _DECODED_JSON.validate_python(json.loads(encoded))
decoded: Final = _DECODED_JSON.validate_python(_load_buffered_transaction(encoded))
return decoded if isinstance(decoded, dict) else None
@ -825,13 +835,13 @@ class RedisUpdateBuffer:
if slot_result is None:
daily_results.append(None)
else:
list_of_daily = [json.loads(t) for t in slot_result]
list_of_daily = [_load_buffered_transaction(t) for t in slot_result]
aggregated = DailySpendUpdateQueue.get_aggregated_daily_spend_update_transactions(list_of_daily)
daily_results.append(aggregated)
window_spend: Final = (
WindowSpendUpdateQueue.get_aggregated_window_spend_transactions(
tuple(json.loads(transaction) for transaction in raw_results[6])
tuple(_load_buffered_transaction(transaction) for transaction in raw_results[6])
)
if raw_results[6] is not None
else None
@ -880,7 +890,7 @@ class RedisUpdateBuffer:
)
if list_of_transactions is None:
return None
return [json.loads(transaction) for transaction in list_of_transactions]
return [_load_buffered_transaction(transaction) for transaction in list_of_transactions]
async def get_all_daily_spend_update_transactions_from_redis_buffer(
self,
@ -1010,9 +1020,9 @@ class RedisUpdateBuffer:
Parses the list of transactions from Redis
"""
if isinstance(list_of_transactions, list):
return [json.loads(transaction) for transaction in list_of_transactions]
return [_load_buffered_transaction(transaction) for transaction in list_of_transactions]
else:
return [json.loads(list_of_transactions)]
return [_load_buffered_transaction(list_of_transactions)]
@staticmethod
def _combine_list_of_transactions(

View file

@ -22,6 +22,7 @@ from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
from litellm.proxy.common_utils.callback_utils import CALLBACK_VAR_ENCRYPTED_PREFIX
from litellm.proxy.common_utils.openai_error_payload import status_typed_proxy_exception
from litellm.proxy.common_utils.path_utils import is_within, safe_join
from litellm.proxy.guardrails.content_filter_data import CATEGORIES_DIR, DATA_ROOTS, category_dirs, find_category_file
from litellm.proxy.guardrails.guardrail_hooks.custom_code.bounded_execution import (
@ -2533,7 +2534,7 @@ async def apply_guardrail(
e = transformed_exception
except Exception:
verbose_proxy_logger.exception("apply_guardrail: post_call_failure_hook failed")
raise handle_exception_on_proxy(e)
raise handle_exception_on_proxy(status_typed_proxy_exception(e) if isinstance(e, HTTPException) else e)
# Success logging outside except so a hook error never triggers failure handlers.
response = await _emit_guardrail_success_logs(

View file

@ -32,7 +32,7 @@ from litellm.router_strategy.auto_router.auto_router_model_naming import (
classify_strategy_router_model,
strategy_router_dependencies,
)
from litellm.router_utils.common_utils import is_model_capability_field
from litellm.router_utils.common_utils import deployment_model_id_pin, is_model_capability_field
from litellm.types.utils import secret_bearing_wif_litellm_params, server_owned_wif_litellm_params
# Provider routing and server-owned federation or OAuth token-exchange fields. Allowed for proxy
@ -855,6 +855,8 @@ def update_litellm_params_for_health_check(
`deployment_model` (default: the incoming `litellm_params` model), so a `health_check_model` or a `model`
in `health_check_params` is judged on its own. Keys already set through `health_check_params` are kept,
and `modalities` set there without "audio" turns the audio fields off
- sends a SageMaker deployment's `X-Amzn-SageMaker-Inference-Component` extra header as `model_id` when the
deployment sets none, so legacy `sagemaker/` probes reach that component the way routed requests do
"""
incoming_model: Final = _PROBE_PARAMS_ADAPTER.validate_python(litellm_params).get("model")
base_model_owner: Final = deployment_model or (incoming_model if isinstance(incoming_model, str) else None)
@ -935,9 +937,11 @@ def update_litellm_params_for_health_check(
audio_output_params: Final = _health_check_audio_output_params(
_MODEL_INFO_ADAPTER.validate_python(model_info), probe_params, mode, base_model_owner
)
model_id_pin: Final = deployment_model_id_pin(probe_params)
model_id_params: Final = {} if model_id_pin is None else {"model_id": model_id_pin}
return {
key: value
for key, value in {**audio_output_params, **probe_params}.items()
for key, value in {**audio_output_params, **model_id_params, **probe_params}.items()
if not is_model_capability_field(key)
}

View file

@ -7,13 +7,15 @@ import secrets
import time
import traceback
from collections.abc import Iterable, Mapping, Sequence
from dataclasses import dataclass
from datetime import datetime, timedelta, timezone
from itertools import chain, groupby
from types import MappingProxyType
from typing import Final, Literal, TypedDict
import fastapi
from fastapi import APIRouter, Depends, HTTPException, Request, Response, status
from pydantic import TypeAdapter
from pydantic import BaseModel, ConfigDict, TypeAdapter
from typing_extensions import ReadOnly
import litellm
@ -31,7 +33,9 @@ from litellm.proxy._types import (
CallInfo,
EnterpriseLicenseData,
Litellm_EntityType,
LiteLLM_UserTable,
LitellmUserRoles,
ModelAccessDeniedProxyException,
ProxyException,
SpecialModelNames,
UserAPIKeyAuth,
@ -39,13 +43,28 @@ from litellm.proxy._types import (
)
from litellm.proxy.auth.auth_checks import (
_resolve_key_models_for_auth_check, # pyright: ignore[reportPrivateUsage] # the auth layer's sentinel resolution, reused so /health scopes exactly like a request
agent_model_grants,
can_agent_call_model,
can_key_call_resolved_model,
can_user_call_model,
get_user_object,
key_model_aliases_for_auth_check,
model_matches_any_wildcard_pattern_in_list,
team_model_aliases_for_auth_check,
)
from litellm.proxy.auth.auth_exception_handler import (
_as_proxy_exception, # pyright: ignore[reportPrivateUsage] # one auth-failure mapping, shared with user_api_key_auth
)
from litellm.proxy.auth.auth_utils import (
_BANNED_REQUEST_BODY_PARAMS, # pyright: ignore[reportPrivateUsage] # one canonical list, shared with the request-body check
)
from litellm.proxy.auth.model_checks import get_key_models
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
from litellm.proxy.common_utils import http_parsing_utils
from litellm.proxy.common_utils.model_listing_utils import (
CallerAliases,
dispatched_model,
requestable_alias_names,
)
from litellm.proxy.common_utils.openai_error_payload import proxy_exception_for
from litellm.proxy.db.db_lookup_gate import db_lookup_stall_tracker
from litellm.proxy.db.exception_handler import PrismaDBExceptionHandler
@ -79,6 +98,7 @@ from litellm.router_utils.clientside_credential_handler import (
_ADMIN_CONFIG_FIELDS_TO_CLEAR_ON_BASE_OVERRIDE, # pyright: ignore[reportPrivateUsage] # one canonical list, shared with the router path
clientside_credential_keys,
)
from litellm.router_utils.common_utils import resolve_model_group_alias
from litellm.secret_managers.main import get_secret_bool
from litellm.types.proxy.litellm_pre_call_utils import RedactedDict, SecretFields
from litellm.types.router import reject_server_owned_wif_fields
@ -253,6 +273,28 @@ def get_callback_identifier(callback):
router: Final = APIRouter()
_OBJECT_MAPPING: Final = TypeAdapter(Mapping[str, object])
_OBJECTS: Final = TypeAdapter(tuple[object, ...])
_DEPLOYMENTS: Final = TypeAdapter(tuple[Mapping[str, object], ...])
class _ModelGrants(BaseModel):
model_config = ConfigDict(frozen=True, from_attributes=True)
models: tuple[object, ...] = ()
class _ModelInfoAccessGroups(BaseModel):
model_config = ConfigDict(frozen=True)
access_groups: tuple[object, ...]
class _DeploymentAccessGroups(BaseModel):
model_config = ConfigDict(frozen=True)
model_info: _ModelInfoAccessGroups
services = (
Literal[
"slack_budget_alerts",
@ -1045,42 +1087,339 @@ def _strip_admin_only_fields_from_health_result(result: dict) -> dict:
return out
def _health_accessible_model_names(
user_api_key_dict: UserAPIKeyAuth, llm_router: Router | None
) -> frozenset[str] | None:
"""Model names the caller may health-check, or None when the key is unrestricted."""
def _key_models_unrestricted(user_api_key_dict: UserAPIKeyAuth) -> bool:
granted_models: Final = _resolve_key_models_for_auth_check(user_api_key_dict)
if not granted_models or SpecialModelNames.all_proxy_models.value in granted_models:
return not granted_models or SpecialModelNames.all_proxy_models.value in granted_models
def _own_team_names(deployment: object, team_id: str | None) -> tuple[str, str] | None:
match deployment:
case {
"model_name": str(internal_name),
"model_info": {"team_id": str(owner_team_id), "team_public_model_name": str(public_name)},
} if owner_team_id == team_id:
return (internal_name, public_name)
case _:
return None
def _names_caller_may_request(deployment: object, team_id: str | None) -> frozenset[str]:
team_names: Final = _own_team_names(deployment, team_id)
if team_names is not None:
return frozenset(team_names)
match deployment:
case {"model_name": str(name)}:
return frozenset((name,))
case _:
return frozenset()
def _public_names_by_internal_name(llm_model_list: Sequence[object], team_id: str | None) -> Mapping[str, str]:
team_names: Final = (_own_team_names(deployment, team_id) for deployment in llm_model_list)
return MappingProxyType(dict(names for names in team_names if names is not None))
async def _key_owner(user_api_key_dict: UserAPIKeyAuth) -> LiteLLM_UserTable | None:
if user_api_key_dict.team_id is not None or user_api_key_dict.user_id is None:
return None
from litellm.proxy.proxy_server import prisma_client, proxy_logging_obj, user_api_key_cache
try:
return await get_user_object(
user_id=user_api_key_dict.user_id,
prisma_client=prisma_client,
user_api_key_cache=user_api_key_cache,
user_id_upsert=False,
parent_otel_span=user_api_key_dict.parent_otel_span,
proxy_logging_obj=proxy_logging_obj,
)
except Exception: # noqa: BLE001 # auth reads an unloadable user as no user limit, so /health does too
return None
async def _key_may_call_model(
model_name: str,
user_api_key_dict: UserAPIKeyAuth,
owner: LiteLLM_UserTable | None,
llm_router: Router | None,
) -> bool:
from litellm.proxy.proxy_server import prisma_client, proxy_logging_obj, user_api_key_cache
await asyncio.sleep(0)
try:
await can_key_call_resolved_model(
model=model_name, llm_model_list=None, valid_token=user_api_key_dict, llm_router=llm_router
)
await can_agent_call_model(
model=model_name,
valid_token=user_api_key_dict,
llm_router=llm_router,
prisma_client=prisma_client,
user_api_key_cache=user_api_key_cache,
proxy_logging_obj=proxy_logging_obj,
)
await can_user_call_model(
model=model_name,
llm_router=llm_router,
user_object=owner,
key_model_aliases=key_model_aliases_for_auth_check(user_api_key_dict),
)
except ModelAccessDeniedProxyException:
return False
except ProxyException:
raise
except Exception as failure:
if isinstance(failure, HTTPException) and failure.status_code == status.HTTP_403_FORBIDDEN:
return False
raise _as_proxy_exception(failure) from failure
return True
async def _caller_aliases(user_api_key_dict: UserAPIKeyAuth) -> CallerAliases:
from litellm.proxy.proxy_server import prisma_client, proxy_config, proxy_logging_obj
router_settings: Final = await proxy_config.get_hierarchical_router_settings(
user_api_key_dict, prisma_client, proxy_logging_obj
)
return CallerAliases(
team_model_aliases_for_auth_check(user_api_key_dict),
key_model_aliases_for_auth_check(user_api_key_dict),
router_settings.get("model_group_alias") if router_settings is not None else None,
)
def _model_auth_checks(model_name: str, aliases: CallerAliases) -> str:
return resolve_model_group_alias(aliases.router, model_name) or model_name
def _deployment_id(deployment: object) -> str | None:
match deployment:
case {"model_info": {"id": str(model_id)}}:
return model_id
case _:
return None
def _granted_names(user_api_key_dict: UserAPIKeyAuth) -> frozenset[str]:
grants: Final = (
*_ModelGrants.model_validate(user_api_key_dict).models,
*_OBJECTS.validate_python(user_api_key_dict.team_models),
)
return frozenset(name for name in grants if isinstance(name, str))
async def _names_any_limit_grants(user_api_key_dict: UserAPIKeyAuth, owner: LiteLLM_UserTable | None) -> frozenset[str]:
owner_grants: Final = _ModelGrants.model_validate(owner).models if owner is not None else ()
return (
_granted_names(user_api_key_dict)
| frozenset(name for name in owner_grants if isinstance(name, str))
| await agent_model_grants(user_api_key_dict)
)
def _alias_names_caller_may_request(aliases: CallerAliases, llm_router: Router | None) -> frozenset[str]:
router_aliases: Final = frozenset(llm_router.registry().aliases()) if llm_router is not None else frozenset[str]()
return requestable_alias_names(aliases) | router_aliases
def _served_group(model_name: str, aliases: CallerAliases, llm_router: Router | None) -> str:
dispatched: Final = dispatched_model(model_name, aliases)
return llm_router.registry().resolve(dispatched) if llm_router is not None else dispatched
def _name_id_pairs(deployment: object, team_id: str | None) -> tuple[tuple[str, str], ...]:
model_id: Final = _deployment_id(deployment)
if model_id is None:
return ()
return tuple((name, model_id) for name in _names_caller_may_request(deployment, team_id) | {model_id})
def _ids_by_requested_name(llm_model_list: Sequence[object], team_id: str | None) -> Mapping[str, frozenset[str]]:
pairs: Final = sorted(chain.from_iterable(_name_id_pairs(deployment, team_id) for deployment in llm_model_list))
return MappingProxyType(
{name: frozenset(model_id for _, model_id in group) for name, group in groupby(pairs, key=lambda pair: pair[0])}
)
def _ids_of(deployments: Iterable[object]) -> frozenset[str]:
return frozenset(model_id for model_id in map(_deployment_id, deployments) if model_id is not None)
def _access_groups_of(deployment: object) -> frozenset[str]:
match deployment:
case {"model_info": {"access_groups": list()}}:
access_groups: Final = _DeploymentAccessGroups.model_validate(deployment).model_info.access_groups
return frozenset(access_group for access_group in access_groups if isinstance(access_group, str))
case _:
return frozenset()
@dataclass(frozen=True, slots=True)
class _AccessGroupGrants:
granted_names: frozenset[str]
ids_in_granted_groups: frozenset[str]
def requested_name(self, name: str, public_names: Mapping[str, str], admitted_names: frozenset[str]) -> str:
public_name: Final = public_names.get(name)
if public_name is None or public_name not in admitted_names or self._grants(name):
return name
return public_name
def _grants(self, name: str) -> bool:
return name in self.granted_names or model_matches_any_wildcard_pattern_in_list(name, tuple(self.granted_names))
def narrow(self, serving_ids: frozenset[str], served_names: frozenset[str]) -> frozenset[str]:
in_granted_groups: Final = serving_ids & self.ids_in_granted_groups
granted_directly: Final = not self.granted_names.isdisjoint(served_names)
return serving_ids if granted_directly or not in_granted_groups else in_granted_groups
def _access_group_grants(llm_model_list: Sequence[object], user_api_key_dict: UserAPIKeyAuth) -> _AccessGroupGrants:
granted: Final = _granted_names(user_api_key_dict)
in_granted_groups: Final = (
deployment for deployment in llm_model_list if not granted.isdisjoint(_access_groups_of(deployment))
)
return _AccessGroupGrants(granted, _ids_of(in_granted_groups))
def _ids_serving_group(
group: str,
ids_by_name: Mapping[str, frozenset[str]],
user_api_key_dict: UserAPIKeyAuth,
llm_router: Router | None,
access_group_grants: _AccessGroupGrants,
) -> frozenset[str]:
named: Final = ids_by_name.get(group)
if named is not None:
return access_group_grants.narrow(named, frozenset((group,)))
if llm_router is None:
return frozenset(granted_models)
registry: Final = llm_router.registry()
return frozenset()
team_id: Final = user_api_key_dict.team_id
wildcard_served: Final = llm_router.registry().deployments(group, source="wildcard", team_id=team_id)
served_names: Final = frozenset((group,)).union(
*(_names_caller_may_request(deployment, team_id) for deployment in wildcard_served)
)
return access_group_grants.narrow(_ids_of(wildcard_served), served_names)
async def _names_key_may_request(
names: Iterable[str],
aliases: CallerAliases,
user_api_key_dict: UserAPIKeyAuth,
owner: LiteLLM_UserTable | None,
llm_router: Router | None,
) -> frozenset[str]:
return frozenset(
get_key_models(
user_api_key_dict=user_api_key_dict,
proxy_model_list=registry.names(team_id=user_api_key_dict.team_id),
model_access_groups=registry.access_groups(),
[
name
for name in names
if await _key_may_call_model(_model_auth_checks(name, aliases), user_api_key_dict, owner, llm_router)
]
)
def _ids_serving(
groups: Iterable[str],
ids_by_name: Mapping[str, frozenset[str]],
user_api_key_dict: UserAPIKeyAuth,
llm_router: Router | None,
access_group_grants: _AccessGroupGrants,
) -> frozenset[str]:
return frozenset[str]().union(
*(
_ids_serving_group(group, ids_by_name, user_api_key_dict, llm_router, access_group_grants)
for group in groups
)
)
def _caller_may_probe_deployment(
deployment: Mapping[str, object],
allowed_models: frozenset[str] | None,
def _shadowed_ids(
llm_model_list: Sequence[object], admitted_names: frozenset[str], reached_ids: frozenset[str], team_id: str | None
) -> frozenset[str]:
ids_of_admitted_names: Final = (
_deployment_id(deployment)
for deployment in llm_model_list
if not admitted_names.isdisjoint(_names_caller_may_request(deployment, team_id))
)
return frozenset(model_id for model_id in ids_of_admitted_names if model_id is not None) - reached_ids
@dataclass(frozen=True, slots=True)
class _ReachableDeployments:
model_ids: frozenset[str]
groups: frozenset[str]
def include(self, deployment: object, team_id: str | None) -> bool:
model_id: Final = _deployment_id(deployment)
if model_id is None:
return not self.groups.isdisjoint(_names_caller_may_request(deployment, team_id))
return model_id in self.model_ids
_NOTHING_REACHABLE: Final = _ReachableDeployments(frozenset(), frozenset())
async def _deployments_key_may_reach(
llm_model_list: Sequence[object],
aliases: CallerAliases,
user_api_key_dict: UserAPIKeyAuth,
owner: LiteLLM_UserTable | None,
llm_router: Router | None,
) -> _ReachableDeployments:
candidate_names: Final = (await _names_any_limit_grants(user_api_key_dict, owner)).union(
_alias_names_caller_may_request(aliases, llm_router),
*(_names_caller_may_request(deployment, user_api_key_dict.team_id) for deployment in llm_model_list),
)
admitted_names: Final = await _names_key_may_request(candidate_names, aliases, user_api_key_dict, owner, llm_router)
access_group_grants: Final = _access_group_grants(llm_model_list, user_api_key_dict)
public_names: Final = _public_names_by_internal_name(llm_model_list, user_api_key_dict.team_id)
groups: Final = frozenset(
_served_group(access_group_grants.requested_name(name, public_names, admitted_names), aliases, llm_router)
for name in admitted_names
)
ids_by_name: Final = _ids_by_requested_name(llm_model_list, user_api_key_dict.team_id)
reached_ids: Final = _ids_serving(groups, ids_by_name, user_api_key_dict, llm_router, access_group_grants)
admitted_ids: Final = await _names_key_may_request(
_shadowed_ids(llm_model_list, admitted_names, reached_ids, user_api_key_dict.team_id),
aliases,
user_api_key_dict,
owner,
llm_router,
)
id_groups: Final = frozenset(_served_group(model_id, aliases, llm_router) for model_id in admitted_ids)
return _ReachableDeployments(
reached_ids | _ids_serving(id_groups, ids_by_name, user_api_key_dict, llm_router, access_group_grants), groups
)
def _model_param_target(
model: str,
aliases: CallerAliases,
llm_model_list: Sequence[Mapping[str, object]],
team_id: str | None,
caller_is_admin: bool,
llm_router: Router | None,
) -> str:
group: Final = _served_group(model, aliases, llm_router)
if llm_router is None or deployments_targeted_by_name(llm_model_list, group, team_id):
return group
patterns: Final = frozenset(
deployment.get("model_name")
for deployment in llm_router.registry().deployments(group, source="wildcard", team_id=team_id)
)
match tuple(patterns):
case (str(pattern),):
return pattern
case _:
return group
def _caller_may_probe_deployment(
deployment: Mapping[str, object], reachable: _ReachableDeployments, team_id: str | None, caller_is_admin: bool
) -> bool:
"""Same deployment visibility rule as routing: another team's deployment is never in scope, team-less callers included."""
if not caller_is_admin and not deployment_usable_by_team(deployment, team_id):
return False
if allowed_models is None:
return True
if llm_router is None:
return deployment.get("model_name") in allowed_models
model: Final = dict(deployment)
registry: Final = llm_router.registry()
return any(registry.includes(model, model_name=name, team_id=team_id) for name in allowed_models)
return reachable.include(deployment, team_id)
def _resolve_targeted_model_ids(
@ -1137,6 +1476,7 @@ def _filter_health_check_results_by_model_ids(results: dict, allowed_model_ids:
async def _perform_health_check_and_save(
model_list,
target_model,
recorded_model_name: str | None,
cli_model,
details,
prisma_client,
@ -1159,8 +1499,7 @@ async def _perform_health_check_and_save(
# Optionally save health check result to database (non-blocking)
if prisma_client is not None:
# For CLI model, use cli_model name; for router models, use target_model
model_name_for_db: Final = cli_model if cli_model is not None else target_model
model_name_for_db: Final = cli_model if cli_model is not None else recorded_model_name
if model_name_for_db is not None:
asyncio.create_task(
_save_health_check_to_db(
@ -1290,6 +1629,7 @@ async def health_endpoint(
cli_result: Final = await _perform_health_check_and_save(
model_list=[],
target_model=None,
recorded_model_name=None,
cli_model=user_model,
details=health_check_details,
prisma_client=prisma_client,
@ -1304,15 +1644,39 @@ async def health_endpoint(
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
detail={"error": "Model list not initialized"},
)
allowed_models: Final = _health_accessible_model_names(user_api_key_dict, llm_router)
restrict_to_allowed_models: Final = not is_admin or allowed_models is not None
restrict_to_allowed_models: Final = not is_admin or not _key_models_unrestricted(user_api_key_dict)
aliases: Final = await _caller_aliases(user_api_key_dict) if restrict_to_allowed_models else None
reachable: Final = (
await _deployments_key_may_reach(
_OBJECTS.validate_python(llm_model_list),
aliases,
user_api_key_dict,
await _key_owner(user_api_key_dict),
llm_router,
)
if aliases is not None
else _NOTHING_REACHABLE
)
requested_model: Final = (
_model_param_target(
model,
aliases,
_DEPLOYMENTS.validate_python(llm_model_list),
user_api_key_dict.team_id,
llm_router,
)
if model is not None and aliases is not None
else model
)
_llm_model_list: Final = [
m
for m in copy.deepcopy(llm_model_list)
if not restrict_to_allowed_models
or _caller_may_probe_deployment(m, allowed_models, llm_router, user_api_key_dict.team_id, is_admin)
or _caller_may_probe_deployment(m, reachable, user_api_key_dict.team_id, is_admin)
]
targeted_ids: Final = _resolve_targeted_model_ids(_llm_model_list, model, model_id, user_api_key_dict.team_id)
targeted_ids: Final = _resolve_targeted_model_ids(
_llm_model_list, requested_model, model_id, user_api_key_dict.team_id
)
if restrict_to_allowed_models and targeted_ids is not None and not targeted_ids:
raise HTTPException(
status_code=status.HTTP_403_FORBIDDEN,
@ -1370,7 +1734,8 @@ async def health_endpoint(
else:
router_result: Final = await _perform_health_check_and_save(
model_list=_llm_model_list,
target_model=target_model,
target_model=target_model if model_id else requested_model,
recorded_model_name=target_model,
cli_model=None,
details=health_check_details,
prisma_client=prisma_client,

View file

@ -16,6 +16,7 @@ from litellm.constants import BACKGROUND_INTERACTION_COST_POLLING_ENABLED, BILLE
from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.core_helpers import (
budget_reservation_from_metadata,
finite_cost,
get_litellm_metadata_from_kwargs,
get_metadata_variable_name_from_kwargs,
get_parent_otel_span_from_kwargs,
@ -703,7 +704,7 @@ class _ProxyDBLogger(CustomLogger):
raw_cost: Final = _str_keyed_fields(
sl_object if sl_object is not None else kwargs # pyright: ignore[reportUnknownArgumentType] # untyped success-callback payload
).get("response_cost")
priced_cost: Final = raw_cost if isinstance(raw_cost, (int, float)) else None
priced_cost: Final = finite_cost(raw_cost)
response_cost: Final = (
await apply_search_free_tier(
kwargs=kwargs, # pyright: ignore[reportUnknownArgumentType] # untyped success-callback payload

View file

@ -3188,9 +3188,9 @@ def _fallback_lists(
) -> tuple[Sequence[object], ...]:
"""Every fallback list the router can route this request through: the three
router-level lists, the same three keys from the request body (which replace
the router's when present, so the union is the safe superset), and the
key/team ``router_settings.fallbacks`` the proxy prefers over router-level
fallbacks in ``_configured_fallbacks``."""
the router's when present, so the union is the safe superset), and the same
three keys from key/team ``router_settings``, which the proxy forwards into
the request after this discovery runs."""
lists: Final[list[Sequence[object]]] = [] # mutable-ok: accumulates each fallback list found
configured: Final = llm_router.settings()
for router_list in (
@ -3205,11 +3205,12 @@ def _fallback_lists(
if isinstance(entries, list):
lists.append(cast("Sequence[object]", entries)) # cast-ok: request fallback lists hold mixed entry shapes
for settings in (router_settings, team_router_settings):
key_fallbacks: object = settings.get("fallbacks") if settings is not None else None
if isinstance(key_fallbacks, list):
lists.append(
cast("Sequence[object]", key_fallbacks) # cast-ok: settings fallback lists hold mixed entry shapes
)
for key in _REQUEST_FALLBACK_KEYS:
key_fallbacks: object = settings.get(key) if settings is not None else None
if isinstance(key_fallbacks, list):
lists.append(
cast("Sequence[object]", key_fallbacks) # cast-ok: settings fallback lists hold mixed entry shapes
)
return tuple(lists)

Some files were not shown because too many files have changed in this diff Show more