scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-11-11 08:38:24 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 359

Merge remote-tracking branch 'origin/main' into litellm_pr48393_head

Browse source 
# Conflicts:
#	tests/unit/litellm_core_utils/llm_response_utils/test_convert_dict_to_response.py
This commit is contained in:
mateo-berri 2026-10-30 11:10:32 -07:00
commit 08e0d4204a
194 changed files with 11597 additions and 1110 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

34
litellm-rust/crates/llms-types/src/providers/anthropic/beta.rs
View file

@ -199,7 +199,7 @@ impl AnthropicBeta {
&[BetaProvider::BedrockConverse, BetaProvider::Databricks]
}
Self::ContextManagement20250627 => &[BetaProvider::BedrockConverse],
Self::AdvisorTool20260301 | Self::Compact20260904 => &[
Self::AdvisorTool20260301 => &[
BetaProvider::AzureAi,
BetaProvider::BedrockConverse,
BetaProvider::Bedrock,
@ -207,6 +207,12 @@ impl AnthropicBeta {
BetaProvider::VertexAi,
BetaProvider::Databricks,
],
Self::Compact20260904 => &[
BetaProvider::BedrockConverse,
BetaProvider::Bedrock,
BetaProvider::BedrockMantle,
BetaProvider::Databricks,
],
Self::Bash20241022
| Self::Bash20250124
| Self::McpServers20251204
@ -256,14 +262,18 @@ impl AnthropicBeta {
Self::InterleavedThinking20250514 | Self::WebSearch20250305 => {
&[BetaProvider::BedrockConverse, BetaProvider::Bedrock]
}
Self::MidConversationOutputConfig20260701
| Self::MidConversationToolChanges20260701 => &[
Self::MidConversationOutputConfig20260701 => &[
BetaProvider::AzureAi,
BetaProvider::BedrockConverse,
BetaProvider::BedrockMantle,
BetaProvider::VertexAi,
BetaProvider::Databricks,
],
Self::MidConversationToolChanges20260701 => &[
BetaProvider::BedrockConverse,
BetaProvider::BedrockMantle,
BetaProvider::Databricks,
],
Self::Output128k20250219 | Self::TokenEfficientTools20250219 => &[
BetaProvider::AzureAi,
BetaProvider::BedrockConverse,
@ -483,7 +493,7 @@ mod tests {
let config = beta_headers_config();
let config_columns: Vec<String> = config
.keys()
.filter(|column| column.as_str() != "description")
.filter(|column| column.as_str() != "description" && !column.ends_with("_header_only"))
.cloned()
.collect();
let beta_provider_columns: Vec<String> = BetaProvider::VARIANTS
@ -502,12 +512,16 @@ mod tests {
let config = beta_headers_config();
for provider in BetaProvider::VARIANTS {
for beta in &AnthropicBeta::KNOWN {
let expected = config
.get(provider.as_ref())
.and_then(serde_json::Value::as_object)
.and_then(|column| column.get(beta.as_str()))
.and_then(serde_json::Value::as_str)
.map(str::to_string);
let cell = |column: &str| {
config
.get(column)
.and_then(serde_json::Value::as_object)
.and_then(|values| values.get(beta.as_str()))
.and_then(serde_json::Value::as_str)
.map(str::to_string)
};
let expected =
cell(&format!("{provider}_header_only")).or_else(|| cell(provider.as_ref()));
let actual = beta.on(*provider).map(|name| name.to_string());
assert_eq!(actual, expected, "provider {provider}, beta {beta}");
if matches!(beta, AnthropicBeta::AdvancedToolUse20251120)

99
litellm-rust/crates/llms/src/anthropic/messages/transformation.rs
View file

@ -1,7 +1,8 @@
use litellm_auth::CredentialPlacement;
use litellm_llms_types::{
formats::messages::{
ContextEdit, ContextManagement, Message, MessagesOptionalParams, MessagesRequest, Speed,
ContentBlock, ContentBlockType, ContextEdit, ContextManagement, Message,
MessagesOptionalParams, MessagesRequest, Speed, ThinkingConfig, ThinkingDisplay,
},
providers::anthropic::{AnthropicBeta, BetaProvider, BetaSet, DEFAULT_HEADERS},
recognized::Recognized,
@ -203,6 +204,10 @@ fn feature_betas(request: &MessagesRequest) -> BetaSet {
.then_some(AnthropicBeta::FastMode20260201),
messages_carry_output_config(&request.messages)
.then_some(AnthropicBeta::PerTurnControl20260701),
uses_thinking_display_updates(params)
.then_some(AnthropicBeta::ThinkingDisplayUpdates20260818),
messages_carry_tool_changes(&request.messages)
.then_some(AnthropicBeta::MidConversationToolChanges20260701),
has_advisor_tool(tools).then_some(AnthropicBeta::AdvisorTool20260301),
is_tool_search_used(tools).then_some(AnthropicBeta::AdvancedToolUse20251120),
]
@ -253,6 +258,36 @@ fn messages_carry_output_config(messages: &[Message]) -> bool {
.any(|message| message.extra.contains_key("output_config"))
}
fn uses_thinking_display_updates(params: &MessagesOptionalParams) -> bool {
let display = match params.thinking.as_ref().and_then(Recognized::known) {
Some(ThinkingConfig::Enabled(thinking)) => thinking.display.as_ref(),
Some(ThinkingConfig::Adaptive(thinking)) => thinking.display.as_ref(),
_ => None,
};
display.and_then(Recognized::known) == Some(&ThinkingDisplay::Updates)
}
fn is_tool_reference_change(block: &ContentBlock) -> bool {
matches!(
&block.block_type,
Some(ContentBlockType::Other(block_type))
if block_type == "tool_addition" || block_type == "tool_removal"
) && block
.extra
.get("tool")
.and_then(|tool| tool.get("type"))
.and_then(Value::as_str)
== Some("tool_reference")
}
fn messages_carry_tool_changes(messages: &[Message]) -> bool {
messages
.iter()
.filter(|message| message.role == "system")
.flat_map(Message::blocks)
.any(is_tool_reference_change)
}
fn unsupported_param(model: &str, param: &str, value: &str, hint: &str) -> Error {
Error::InvalidRequest(crate::ErrorDetail::UnsupportedParameter {
model: model.into(),
@ -1071,6 +1106,41 @@ mod tests {
json!({"safeguards": [{"type": "dangerous_tool_use", "classifier_context": {"v": 1, "permission_mode": "auto"}}]}),
&[]
)]
#[case::adaptive_display_updates(
json!({"thinking": {"type": "adaptive", "display": "updates"}}),
&["thinking-display-updates-2026-08-18"]
)]
#[case::enabled_display_updates(
json!({"thinking": {"type": "enabled", "budget_tokens": 2048, "display": "updates"}}),
&["thinking-display-updates-2026-08-18"]
)]
#[case::summarized_display(json!({"thinking": {"type": "adaptive", "display": "summarized"}}), &[])]
#[case::disabled_thinking(json!({"thinking": {"type": "disabled"}}), &[])]
#[case::system_tool_addition(
json!({"messages": [
{"role": "user", "content": "Use the ping tool"},
{"role": "system", "content": [{"type": "tool_addition", "tool": {"type": "tool_reference", "name": "ping"}}]},
]}),
&["mid-conversation-tool-changes-2026-07-01"]
)]
#[case::system_tool_removal(
json!({"messages": [
{"role": "system", "content": [{"type": "tool_removal", "tool": {"type": "tool_reference", "name": "ping"}}]},
]}),
&["mid-conversation-tool-changes-2026-07-01"]
)]
#[case::user_tool_addition(
json!({"messages": [
{"role": "user", "content": [{"type": "tool_addition", "tool": {"type": "tool_reference", "name": "ping"}}]},
]}),
&[]
)]
#[case::tool_addition_without_a_tool_reference(
json!({"messages": [
{"role": "system", "content": [{"type": "tool_addition", "tool": {"type": "tool_definition"}}]},
]}),
&[]
)]
fn feature_betas_follow_the_request(#[case] fields: Value, #[case] expected: &[&str]) {
assert_eq!(feature_betas(&request(fields)), betas(expected));
}
@ -1093,6 +1163,33 @@ mod tests {
);
}
#[rstest]
#[case::vertex_ai(
BetaProvider::VertexAi,
&["mid-conversation-tool-changes-2026-07-01", "thinking-display-updates-2026-08-18"]
)]
#[case::azure_ai(
BetaProvider::AzureAi,
&["mid-conversation-tool-changes-2026-07-01", "thinking-display-updates-2026-08-18"]
)]
#[case::bedrock_converse(BetaProvider::BedrockConverse, &["thinking-display-updates-2026-08-18"])]
#[case::databricks(BetaProvider::Databricks, &["thinking-display-updates-2026-08-18"])]
fn tool_change_and_display_betas_reach_the_providers_that_accept_them(
#[case] provider: BetaProvider,
#[case] expected: &[&str],
) {
let fields = json!({
"thinking": {"type": "adaptive", "display": "updates"},
"messages": [
{"role": "system", "content": [{"type": "tool_addition", "tool": {"type": "tool_reference", "name": "ping"}}]},
],
});
assert_eq!(
provider_feature_betas(&request(fields), provider),
betas(expected)
);
}
#[rstest]
fn hosts_without_a_beta_map_column_get_safeguards_without_a_derived_beta() {
let client = headers(&[("anthropic-beta", "claude-code-20250219")]);

5
litellm-rust/crates/llms/src/azure_ai/messages/transformation.rs
View file

@ -545,6 +545,11 @@ mod tests {
&[("anthropic-beta", "web-search-2025-03-05")],
&[("x-api-key", "k"), ("anthropic-beta", "structured-outputs-2025-11-13,web-search-2025-03-05")]
)]
#[case::compaction_needs_the_compact_2026_09_04_beta(
json!({"compaction": {"type": "summarize"}}),
&[],
&[("x-api-key", "k"), ("anthropic-beta", "compact-2026-09-04")]
)]
#[case::no_feature_needs_a_beta(json!({}), &[], &[("x-api-key", "k")])]
#[case::safeguards_need_the_dangerous_tool_use_beta(
json!({"safeguards": [{"type": "dangerous_tool_use"}]}),

12
litellm-rust/crates/llms/src/vertex_ai/messages/transformation.rs
View file

@ -798,6 +798,18 @@ mod tests {
]}}),
&["compact-2026-01-12", "context-management-2025-06-27"]
)]
#[case::compaction(json!({"compaction": {"type": "summarize"}}), &["compact-2026-09-04"])]
#[case::thinking_display_updates(
json!({"thinking": {"type": "adaptive", "display": "updates"}}),
&["thinking-display-updates-2026-08-18"]
)]
#[case::tool_addition(
json!({"messages": [
{"role": "user", "content": "Use the ping tool"},
{"role": "system", "content": [{"type": "tool_addition", "tool": {"type": "tool_reference", "name": "ping"}}]}
]}),
&["mid-conversation-tool-changes-2026-07-01"]
)]
#[case::advisor_tool_is_not_a_vertex_beta(
json!({"tools": [{"type": "advisor_20260301", "name": "advisor"}]}),
&[]

21
litellm/__init__.py
View file

@ -7,7 +7,6 @@ _apply_process_defaults()
### INIT VARIABLES #########################
import threading
import os
import datetime
from collections.abc import Mapping, Sequence
from typing import (
@ -550,6 +549,7 @@ _key_management_system: Optional["KeyManagementSystem"] = None
output_parse_pii: bool = False
#############################################
from litellm.litellm_core_utils.get_model_cost_map import get_model_cost_map, mark_litellm_import_complete
from litellm.litellm_core_utils.model_retirement import is_hidden_as_retired
model_cost = get_model_cost_map(url=model_cost_map_url)
cost_discount_config: Dict[str, float] = {} # Provider-specific cost discounts {"vertex_ai": 0.05} = 5% discount
@ -756,16 +756,6 @@ def is_openai_finetune_model(key: str) -> bool:
return key.startswith("ft:") and not key.count(":") > 1
def _is_past_deprecation_date(value: Mapping[str, object]) -> bool:
deprecation_date: Final = value.get("deprecation_date")
if not isinstance(deprecation_date, str):
return False
try:
return datetime.date.fromisoformat(deprecation_date) <= datetime.date.today()
except ValueError:
return False
def _populate_provider_model_sets(model_cost_map: Mapping[str, Mapping[str, object]]) -> None:
for key, value in tuple(model_cost_map.items()):
if value.get("litellm_provider") == "openai" and not is_openai_finetune_model(key):
@ -873,12 +863,12 @@ def _populate_provider_model_sets(model_cost_map: Mapping[str, Mapping[str, obje
if "-to-" not in key and "fireworks-ai-default" not in key:
fireworks_ai_models.add(key)
elif value.get("litellm_provider") == "together_ai":
if _is_past_deprecation_date(value):
if is_hidden_as_retired(value):
together_ai_models.discard(key)
elif not key.startswith("together-ai-"):
together_ai_models.add(key)
elif value.get("litellm_provider") == "baseten":
if _is_past_deprecation_date(value):
if is_hidden_as_retired(value):
baseten_models.discard(key)
else:
baseten_models.add(key)
@ -1051,7 +1041,10 @@ def _populate_provider_model_sets(model_cost_map: Mapping[str, Mapping[str, obje
elif value.get("litellm_provider") == "coralbricks":
coralbricks_models.add(key)
elif value.get("litellm_provider") == "aiand":
aiand_models.add(key)
if is_hidden_as_retired(value):
aiand_models.discard(key)
else:
aiand_models.add(key)
elif value.get("litellm_provider") == "reducto":
reducto_models.add(key)
elif value.get("litellm_provider") == "bedrock_mantle":

9
litellm/anthropic_beta_headers_config.json
View file

@ -1,5 +1,5 @@
{
"description": "Mapping of Anthropic beta headers for each provider. Keys are input header names, values are provider-specific header names (or null if unsupported). Only headers present in mapping keys with non-null values can be forwarded.",
"description": "Mapping of Anthropic beta headers for each provider. Keys are input header names, values are provider-specific header names (or null if unsupported). Only headers present in mapping keys with non-null values can be forwarded. A <provider>_header_only column lists betas that provider accepts only as the anthropic-beta HTTP header: they are forwarded there and dropped from the anthropic_beta body field. Proxies that predate these columns ignore them and read the provider column alone.",
"anthropic": {
"advisor-tool-2026-03-01": "advisor-tool-2026-03-01",
"advanced-tool-use-2025-11-20": "advanced-tool-use-2025-11-20",
@ -46,6 +46,7 @@
"bash_20250124": null,
"code-execution-2025-08-25": "code-execution-2025-08-25",
"compact-2026-01-12": null,
"compact-2026-09-04": "compact-2026-09-04",
"computer-use-2025-01-24": "computer-use-2025-01-24",
"computer-use-2025-11-24": "computer-use-2025-11-24",
"context-1m-2025-08-07": "context-1m-2025-08-07",
@ -59,6 +60,7 @@
"mcp-client-2025-11-20": "mcp-client-2025-11-20",
"mcp-client-2025-04-04": "mcp-client-2025-04-04",
"mcp-servers-2025-12-04": null,
"mid-conversation-tool-changes-2026-07-01": "mid-conversation-tool-changes-2026-07-01",
"output-128k-2025-02-19": null,
"structured-output-2024-03-01": null,
"per-turn-control-2026-07-01": "per-turn-control-2026-07-01",
@ -202,6 +204,7 @@
"mcp-client-2025-11-20": null,
"mcp-client-2025-04-04": null,
"mcp-servers-2025-12-04": null,
"mid-conversation-tool-changes-2026-07-01": "mid-conversation-tool-changes-2026-07-01",
"output-128k-2025-02-19": null,
"structured-output-2024-03-01": null,
"per-turn-control-2026-07-01": "per-turn-control-2026-07-01",
@ -218,6 +221,10 @@
"web-search-2025-03-05": "web-search-2025-03-05",
"inline-tools-2026-09-15": "inline-tools-2026-09-15"
},
"vertex_ai_header_only": {
"compact-2026-09-04": "compact-2026-09-04",
"inline-tools-2026-09-15": "inline-tools-2026-09-15"
},
"databricks": {
"advisor-tool-2026-03-01": null,
"advanced-tool-use-2025-11-20": null,

39
litellm/anthropic_beta_headers_manager.py
View file

@ -30,7 +30,7 @@ from collections.abc import Mapping
from concurrent.futures import ThreadPoolExecutor
from functools import cache
from importlib.resources import files
from typing import Final, TypeAlias
from typing import Final, Literal, TypeAlias
import httpx
from pydantic import TypeAdapter
@ -47,6 +47,9 @@ BetaHeadersConfig: TypeAlias = Mapping[str, str | Mapping[str, str | None]]
_BETA_HEADERS_CONFIG_ADAPTER: Final = TypeAdapter(BetaHeadersConfig)
_DECODED_JSON: Final = TypeAdapter(object)
BetaPlacement: TypeAlias = Literal["header", "body"]
_HEADER_ONLY_COLUMN_SUFFIX: Final = "_header_only"
class GetAnthropicBetaHeadersConfig:
"""
@ -241,6 +244,31 @@ def get_provider_name(provider: str) -> str:
return aliases.get(provider, provider)
def _beta_column(name: str) -> Mapping[str, str | None]:
column: Final[Mapping[str, str | None]] = _load_beta_headers_config().get(name, {})
return column
def _header_only_column(canonical_provider: str) -> Mapping[str, str | None]:
return _beta_column(f"{canonical_provider}{_HEADER_ONLY_COLUMN_SUFFIX}")
def _provider_beta_mapping(provider: str, placement: BetaPlacement) -> Mapping[str, str | None]:
canonical_provider: Final = get_provider_name(provider)
column: Final = _beta_column(canonical_provider)
header_only: Final = _header_only_column(canonical_provider)
match placement:
case "header":
return {**column, **header_only}
case "body":
return {beta: name for beta, name in column.items() if beta not in header_only}
def get_header_only_betas(provider: str) -> frozenset[str]:
header_only: Final = _header_only_column(get_provider_name(provider))
return frozenset(beta for beta, name in header_only.items() if name is not None)
def has_beta_headers_config(provider: str) -> bool:
config: Final = _bundled_beta_headers_config()
aliases: Final = config.get("provider_aliases", {})
@ -251,6 +279,7 @@ def has_beta_headers_config(provider: str) -> bool:
def filter_and_transform_beta_headers(
beta_headers: list[str],
provider: str,
placement: BetaPlacement = "header",
) -> list[str]:
"""
Filter and transform beta headers based on provider's mapping configuration.
@ -263,6 +292,7 @@ def filter_and_transform_beta_headers(
Args:
beta_headers: List of Anthropic beta header values
provider: Provider name (e.g., "anthropic", "bedrock", "vertex_ai")
placement: Whether the betas travel as the anthropic-beta header or the anthropic_beta body field
Returns:
List of filtered and transformed beta headers for the provider
@ -270,11 +300,7 @@ def filter_and_transform_beta_headers(
if not beta_headers:
return []
config: Final = _load_beta_headers_config()
provider = get_provider_name(provider)
# Get the header mapping for this provider
provider_mapping: Final = config.get(provider, {})
provider_mapping: Final = _provider_beta_mapping(provider, placement)
filtered_headers: Final[set[str]] = set()
@ -426,6 +452,7 @@ def update_request_with_filtered_beta(
filtered_body_betas: Final = filter_and_transform_beta_headers(
beta_headers=existing_body_betas,
provider=provider,
placement="body",
)
if filtered_body_betas:

53
litellm/integrations/prometheus.py
View file

@ -82,6 +82,7 @@ from litellm.types.proxy.carried_budget_state import (
UserBudgetSnapshot,
)
from litellm.types.utils import (
CallTypes,
StandardLoggingGuardrailInformation,
StandardLoggingPayload,
StandardLoggingZeroCostDiagnostic,
@ -279,6 +280,11 @@ def _logged_fields(value: object) -> Mapping[str, object]:
return _NO_LOGGED_FIELDS
_GOOGLE_PASS_THROUGH_PROVIDERS: Final = frozenset(
(litellm.LlmProviders.GEMINI.value, litellm.LlmProviders.VERTEX_AI.value)
)
def _is_llm_provider(name: str) -> bool:
routable: Final = (
name in litellm.provider_list or JSONProviderRegistry.exists(name) or is_registered_custom_provider(name)
@ -2942,6 +2948,22 @@ class PrometheusLogger(CustomLogger):
return False
@staticmethod
def _google_pass_through_provider(request_data: Mapping[str, object]) -> str | None:
declared_provider: Final = request_data.get("custom_llm_provider")
if request_data.get("call_type") != CallTypes.pass_through.value or not isinstance(declared_provider, str):
return None
return declared_provider if declared_provider in _GOOGLE_PASS_THROUGH_PROVIDERS else None
@staticmethod
def _failed_request_requested_model(request_data: Mapping[str, object]) -> object:
requested_model: Final = request_data.get("model", "")
if PrometheusLogger._google_pass_through_provider(request_data) is None:
return requested_model
request_metadata: Final = _logged_fields(_logged_fields(request_data.get("litellm_params")).get("metadata"))
model_group: Final = request_metadata.get("model_group")
return model_group if isinstance(model_group, str) and model_group else requested_model
@staticmethod
def _extract_api_provider_from_request_data(request_data: dict) -> str | None:
"""
@ -2950,22 +2972,27 @@ class PrometheusLogger(CustomLogger):
A request can fail before a deployment is resolved, so the provider is
not always known. Prefer the resolved ``custom_llm_provider`` on
``litellm_params``, then any provider recovered onto a partial
``standard_logging_object`` (e.g. a stream that broke mid-flight), and
finally infer it from the requested model name (e.g. ``gpt-4o-mini`` ->
``openai``) since the proxy's failure ``request_data`` usually carries
only the client-supplied model. Return ``None`` when it cannot be
determined so the label emits empty rather than a guess.
``standard_logging_object`` (e.g. a stream that broke mid-flight). On a
Gemini or Vertex pass-through failure only, next take the provider the
failure payload names (``gemini`` or ``vertex_ai``), so every other
pass-through route keeps its earlier label. Finally infer it from the
requested model name (e.g. ``gpt-4o-mini`` -> ``openai``) since the
proxy's failure ``request_data`` usually carries only the client-supplied
model. Return ``None`` when it cannot be determined so the label emits
empty rather than a guess.
"""
litellm_params: Final = request_data.get("litellm_params") or {}
provider = litellm_params.get("custom_llm_provider")
if provider:
return provider
standard_logging_object: Final = request_data.get("standard_logging_object") or {}
provider = standard_logging_object.get("custom_llm_provider")
if provider:
return provider
resolved_provider: Final = litellm_params.get("custom_llm_provider") or standard_logging_object.get(
"custom_llm_provider"
)
if resolved_provider:
return resolved_provider
google_provider: Final = PrometheusLogger._google_pass_through_provider(request_data)
if google_provider is not None:
return google_provider
model: Final = litellm_params.get("model") or request_data.get("model")
if not model:
if not isinstance(model, str) or not model:
return None
try:
return litellm.get_llm_provider(model=model)[1] or None
@ -3052,7 +3079,7 @@ class PrometheusLogger(CustomLogger):
team_alias=user_api_key_dict.team_alias,
org_id=user_api_key_dict.org_id,
org_alias=user_api_key_dict.organization_alias,
requested_model=_bounded_requested_model_label(request_data.get("model", "")),
requested_model=_bounded_requested_model_label(self._failed_request_requested_model(request_data)),
status_code=str(status_code),
exception_status=str(status_code),
exception_class=self._get_exception_class_name(original_exception),

28
litellm/litellm_core_utils/core_helpers.py
View file

@ -9,12 +9,13 @@ from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Final, Literal, Protocol, TypeVar, cast, runtime_checkable
import httpx
from pydantic import TypeAdapter, ValidationError
from pydantic import BaseModel, TypeAdapter, ValidationError
from typing_extensions import TypeIs # noqa: TID251 # narrows a duck-typed value to the Protocol its caller reads
import litellm
from litellm._logging import verbose_logger
from litellm.constants import KUBERNETES_POD_ROUTING_KEY
from litellm.litellm_core_utils.hidden_params import set_extra_attribute
from litellm.litellm_core_utils.llm_response_utils.get_headers import get_provider_request_id
from litellm.types.llms.openai import AllMessageValues, OpenAIChatCompletionFinishReason
@ -710,9 +711,9 @@ def preserve_upstream_non_openai_attributes(
_assign_attributes(model_response, original_chunk.model_dump(include=frozenset(candidate_keys)).items())
def _assign_attributes(target: object, values: Iterable[tuple[str, object]]) -> None:
def _assign_attributes(target: BaseModel, values: Iterable[tuple[str, object]]) -> None:
for key, value in values:
setattr(target, key, value)
set_extra_attribute(target, key, value)
def safe_deep_copy(data):
@ -982,25 +983,36 @@ def _usage_field(usage: object, name: str) -> object:
return getattr(usage, name, None)
def _cost_number(value: object) -> float | None:
if isinstance(value, bool) or not isinstance(value, int | float):
def finite_cost(value: object) -> float | None:
if isinstance(value, bool) or not isinstance(value, int | float) or not math.isfinite(value):
return None
return float(value)
def single_log_line(value: object) -> str:
return str(value).replace("\r", "").replace("\n", "")
def finite_reported_cost(value: object) -> float | None:
cost: Final = finite_cost(value)
if cost is None and isinstance(value, float):
verbose_logger.warning("Ignoring a non-finite provider-reported cost %s", value)
return cost
def _reported_cost_value(value: object) -> float | None:
"""A reported cost is a number, or a breakdown whose total lives under ``total_cost``."""
raw: Final[object] = value
if isinstance(value, Mapping):
return _cost_number(_usage_field(raw, "total_cost"))
return _cost_number(value)
return finite_reported_cost(_usage_field(raw, "total_cost"))
return finite_reported_cost(value)
def response_cost_in_hidden_params(response: _CarriesHiddenParams) -> float | None:
additional_headers: Final[object] = response.hidden_params.get("additional_headers")
if not isinstance(additional_headers, Mapping):
return None
return _cost_number(_USAGE_BLOCK_ADAPTER.validate_python(additional_headers).get(RESPONSE_COST_HEADER))
return finite_reported_cost(_USAGE_BLOCK_ADAPTER.validate_python(additional_headers).get(RESPONSE_COST_HEADER))
def override_stamped_response_cost(response: _CarriesHiddenParams, cost: float) -> None:

32
litellm/litellm_core_utils/exception_mapping_utils.py
View file

@ -18,6 +18,7 @@ from litellm.litellm_core_utils.bug_report import (
)
from litellm.litellm_core_utils.core_helpers import as_str_mapping, attribute_or_none
from litellm.litellm_core_utils.secret_redaction import redact_string as redact_secret_string
from litellm.llms.bedrock.common_utils import bedrock_model_reaches_converse_by_default
from litellm.llms.bigmodel.common_utils import raise_bigmodel_business_error
from litellm.types.utils import LlmProviders
@ -938,13 +939,22 @@ _BEDROCK_MANTLE_CONTEXT_WINDOW_GENERIC_MESSAGE: Final = (
)
def _bedrock_mantle_context_window_response(original_exception: _ProviderHTTPException) -> httpx.Response:
def _bedrock_response_with_status(original_exception: _ProviderHTTPException, status_code: int) -> httpx.Response:
original_response: Final = getattr(original_exception, "response", None)
if not isinstance(original_response, httpx.Response) or getattr(original_response, "_request", None) is None:
return httpx.Response(status_code=400, request=httpx.Request(method="POST", url="https://api.openai.com/v1/"))
if original_response.status_code == 400:
return httpx.Response(
status_code=status_code, request=httpx.Request(method="POST", url="https://api.openai.com/v1/")
)
if original_response.status_code == status_code:
return original_response
return httpx.Response(status_code=400, headers=original_response.headers, request=original_response.request)
return httpx.Response(status_code=status_code, headers=original_response.headers, request=original_response.request)
def _came_from_bedrock_converse(original_exception: _ProviderHTTPException) -> bool:
original_response: Final = getattr(original_exception, "response", None)
if not isinstance(original_response, httpx.Response) or getattr(original_response, "_request", None) is None:
return False
return original_response.request.url.path.endswith(("/converse", "/converse-stream"))
def _get_bedrock_mantle_context_window_message(error_str: str) -> str | None:
@ -980,7 +990,7 @@ def _map_bedrock_exception(
message=mantle_context_window_message,
model=model,
llm_provider=custom_llm_provider,
response=_bedrock_mantle_context_window_response(original_exception),
response=_bedrock_response_with_status(original_exception, 400),
)
if (
"too many tokens" in error_str
@ -1003,6 +1013,18 @@ def _map_bedrock_exception(
llm_provider="bedrock",
response=getattr(original_exception, "response", None),
)
elif (
"The provided model identifier is invalid" in error_str
and bool(model)
and _came_from_bedrock_converse(original_exception)
and bedrock_model_reaches_converse_by_default(model)
):
raise NotFoundError(
message=f"BedrockException - {error_str}",
model=model,
llm_provider="bedrock",
response=_bedrock_response_with_status(original_exception, 404),
)
elif "Malformed input request" in error_str:
raise BadRequestError(
message=f"BedrockException - {error_str}",

35
litellm/litellm_core_utils/hidden_params.py
View file

@ -1,8 +1,9 @@
from collections.abc import Iterator, Mapping, MutableMapping
from functools import cache, lru_cache
from types import MappingProxyType
from typing import Final, Protocol, cast
from pydantic import BaseModel
from pydantic import BaseModel, ConfigDict, create_model
from litellm.types.llms.base import HiddenParams
@ -60,6 +61,38 @@ def stored_attribute(obj: object, name: str) -> object:
return extras.get(name)
@cache
def _assigned_extras_join_fields_set() -> bool:
probe: Final = create_model("_ExtraAssignmentProbe", __config__=ConfigDict(extra="allow"))()
setattr(probe, "assigned", None)
return "assigned" in probe.model_fields_set
@lru_cache(maxsize=1024)
def _writes_plain_extra(model_class: type[BaseModel], name: str) -> bool:
config: Final = model_class.model_config
return (
model_class.__setattr__ is BaseModel.__setattr__
and config.get("extra") == "allow"
and not config.get("frozen")
and not config.get("validate_assignment")
and not name.startswith("_")
and name not in model_class.__pydantic_fields__
and name not in model_class.__class_vars__
and not hasattr(model_class, name)
and _assigned_extras_join_fields_set()
)
def set_extra_attribute(model: BaseModel, name: str, value: object) -> None:
extras: Final = model.__pydantic_extra__
if extras is None or not _writes_plain_extra(type(model), name):
setattr(model, name, value)
return
extras[name] = value
model.__pydantic_fields_set__.add(name)
def _get_hidden_params_storage(obj: object) -> object | None:
if isinstance(obj, dict):
return obj.get(_HIDDEN_PARAMS_ATTR)

6
litellm/litellm_core_utils/internal_call_metadata.py
View file

@ -35,6 +35,12 @@ from litellm.types.utils import (
BUDGET_RESERVATION_METADATA_KEYS: Final = frozenset({"user_api_key_budget_reservation"})
ITPM_RESERVED_KEY: Final = "_litellm_itpm_reserved"
OTPM_RESERVED_KEY: Final = "_litellm_otpm_reserved"
ITPM_CACHE_KEY: Final = "_litellm_itpm_cache_key"
OTPM_CACHE_KEY: Final = "_litellm_otpm_cache_key"
IO_TOKEN_RESERVATION_KEYS: Final = frozenset({ITPM_RESERVED_KEY, OTPM_RESERVED_KEY, ITPM_CACHE_KEY, OTPM_CACHE_KEY})
MODEL_ACCESS_GROUP_METADATA_KEY: Final = "user_api_key_matched_model_access_groups"
"""Where auth records the model access groups that authorized the request, for the spend writer.

62
litellm/litellm_core_utils/litellm_logging.py
View file

@ -76,6 +76,7 @@ from litellm.litellm_core_utils.classifier_logging import (
is_classifier_call,
)
from litellm.litellm_core_utils.core_helpers import (
finite_cost,
get_provider_response_headers_from_hidden_params,
is_expected_client_error,
native_messages_reported_cost,
@ -84,6 +85,7 @@ from litellm.litellm_core_utils.core_helpers import (
proxy_stamped_used_client_oauth_token,
reconstruct_model_name,
set_response_cost_in_hidden_params,
single_log_line,
)
from litellm.litellm_core_utils.error_normalization import normalize_error
from litellm.litellm_core_utils.get_litellm_params import get_litellm_params
@ -567,7 +569,13 @@ def _resolve_mantle_region_for_cost(
_PER_CALL_LITELLM_PARAM_DICTS: Final = frozenset({"metadata", "litellm_metadata"})
_LITELLM_PARAMS: Final = TypeAdapter(dict[str, object])
_SERVED_ANTHROPIC_BODY: Final = TypeAdapter(dict[str, object])
_REDACTED_RESULT: Final = TypeAdapter(object)
_GUARDRAIL_INFORMATION_KEY: Final = "standard_logging_guardrail_information"
def _without_guardrail_information(metadata: Mapping[str, object]) -> Mapping[str, object]:
return {key: value for key, value in metadata.items() if key != _GUARDRAIL_INFORMATION_KEY}
def _provider_response_id(source: object) -> str | None:
@ -2088,7 +2096,7 @@ class Logging(LiteLLMLoggingBaseClass):
litellm_model_name=litellm_model_name,
router_model_id=router_model_id or hidden_params.get("model_id"),
)
return hidden_params["response_cost"]
return self._recorded_response_cost(hidden_params["response_cost"])
elif router_model_id is None and "model_id" in hidden_params: # use model_id if not already set
router_model_id = hidden_params["model_id"]
@ -2188,7 +2196,7 @@ class Logging(LiteLLMLoggingBaseClass):
litellm_model_name=litellm_model_name,
router_model_id=router_model_id,
)
return total_response_cost
return self._recorded_response_cost(total_response_cost)
except Exception as e: # error calculating cost
debug_info = StandardLoggingModelCostFailureDebugInformation(
error_str=str(e),
@ -2214,6 +2222,15 @@ class Logging(LiteLLMLoggingBaseClass):
_response_cost_calculator = response_cost_calculator
def _recorded_response_cost(self, cost: float | None) -> float | None:
if cost is None or finite_cost(cost) is not None:
return cost
verbose_logger.warning(
"Recording a non-finite response_cost as 0 for litellm_call_id=%s", single_log_line(self.litellm_call_id)
)
self.cost_breakdown = None
return 0.0
def uncached_price(self, result: object) -> UncachedPrice:
previous_cost_breakdown: Final = self.cost_breakdown
previous_billed_token_rates: Final = self.billed_token_rates
@ -2566,6 +2583,23 @@ class Logging(LiteLLMLoggingBaseClass):
attempt.model_call_details = dict(self.model_call_details)
return attempt
def guardrail_scoped_copy(self) -> "Logging":
request_params: Final = _LITELLM_PARAMS.validate_python(
self.litellm_params or {} # pyright: ignore[reportUnknownMemberType] # Logging.litellm_params is untyped
)
litellm_params: Final = {
**request_params,
**{
key: _without_guardrail_information(_METADATA_MAPPING.validate_python(value))
for key, value in request_params.items()
if key in _PER_CALL_LITELLM_PARAM_DICTS and isinstance(value, dict)
},
}
scoped: Final = copy.copy(self)
scoped.litellm_params = litellm_params
scoped.model_call_details = {**self.model_call_details, "litellm_params": litellm_params}
return scoped
def dispatch_attempt_failure_handlers(
self, exception: Exception, traceback_exception: str
) -> Coroutine[None, None, None]:
@ -5034,13 +5068,15 @@ class Logging(LiteLLMLoggingBaseClass):
verbose_logger.warning("LiteLLM: the anthropic_messages stream assembled no response, logging an empty one")
return litellm.ModelResponse(model=self.model)
else:
from litellm.types.llms.anthropic import AnthropicResponse
pydantic_result: Final = AnthropicResponse.model_validate({"stop_sequence": None, **(served_body or {})})
import httpx
anthropic_body: Final = (
served_body
if served_body is not None
else _SERVED_ANTHROPIC_BODY.validate_python(
result.model_dump() if isinstance(result, BaseModel) else result
)
)
result = litellm.AnthropicConfig().transform_parsed_response(
completion_response=pydantic_result.model_dump(),
completion_response={"stop_sequence": None, **anthropic_body},
raw_response=httpx.Response(
status_code=200,
headers={},
@ -7560,7 +7596,15 @@ def get_standard_logging_object_payload(
)
custom_pricing: Final = False if spilled_over else logging_obj.custom_pricing_for(init_response_obj)
raw_response_cost: Final = kwargs.get("response_cost")
llm_response_cost: Final[float] = raw_response_cost or 0.0
numeric_response_cost: Final = raw_response_cost if isinstance(raw_response_cost, int | float) else None
priced_response_cost: Final = finite_cost(numeric_response_cost)
if numeric_response_cost is not None and priced_response_cost is None:
verbose_logger.warning(
"Logging a non-finite response_cost=%s as 0 for litellm_call_id=%s",
numeric_response_cost,
single_log_line(logging_obj.litellm_call_id),
)
llm_response_cost: Final[float] = priced_response_cost or 0.0
guardrail_cost: Final = guardrail_information_cost(metadata.get("standard_logging_guardrail_information"))
finalized_cost: Final = _finalized_cost_of(logging_obj)
response_cost: Final[float] = (

21
litellm/litellm_core_utils/llm_request_utils.py
View file

@ -1,4 +1,3 @@
import datetime
from collections.abc import Iterable, Iterator, Mapping
from dataclasses import dataclass
from typing import Final, cast
@ -9,6 +8,7 @@ from typing_extensions import NotRequired, ReadOnly, TypedDict
import litellm
from litellm.constants import DEFAULT_MAX_RECURSE_DEPTH
from litellm.litellm_core_utils.get_llm_provider_logic import inferred_provider
from litellm.litellm_core_utils.model_retirement import is_past_deprecation_date
def _form_field_value(value: object) -> str:
@ -168,7 +168,9 @@ class RankedChatModel:
def rank_chat_models_from_llm_provider(custom_llm_provider: str) -> tuple[RankedChatModel, ...]:
known_models: Final[Iterable[str]] = litellm.models_by_provider.get(custom_llm_provider, ())
from litellm.utils import bundled_provider_models
known_models: Final = bundled_provider_models(custom_llm_provider)
ranked_models: Final = (
RankedChatModel(
model=model, route_rank=route_rank, cost=cost, is_past_deprecation_date=is_past_deprecation_date
@ -187,18 +189,11 @@ _COST_MAP_DEPRECATION: Final = TypeAdapter(_CostMapDeprecation)
def _is_past_deprecation_date(cost_map_key: str) -> bool:
deprecation_date: Final = _COST_MAP_DEPRECATION.validate_python(
litellm.model_cost.get(cost_map_key) or {} # pyright: ignore[reportUnknownMemberType] # untyped cost map JSON
).get("deprecation_date")
if not isinstance(deprecation_date, str):
return False
try:
return (
datetime.date.fromisoformat(deprecation_date)
<= datetime.datetime.now(datetime.timezone.utc).astimezone().date()
return is_past_deprecation_date(
_COST_MAP_DEPRECATION.validate_python(
litellm.model_cost.get(cost_map_key) or {} # pyright: ignore[reportUnknownMemberType] # untyped cost map JSON
)
except ValueError:
return False
)
def _chat_row_costs(models: Iterable[str], custom_llm_provider: str) -> Iterator[tuple[str, float, bool]]:

19
litellm/litellm_core_utils/model_retirement.py Normal file
View file

@ -0,0 +1,19 @@
import datetime
from collections.abc import Mapping
from typing import Final
PROVIDERS_HIDING_RETIRED_MODELS: Final = frozenset({"aiand", "baseten", "together_ai"})
def is_past_deprecation_date(value: Mapping[str, object]) -> bool:
deprecation_date: Final = value.get("deprecation_date")
if not isinstance(deprecation_date, str):
return False
try:
return datetime.date.fromisoformat(deprecation_date) <= datetime.datetime.now(datetime.timezone.utc).date()
except ValueError:
return False
def is_hidden_as_retired(value: Mapping[str, object]) -> bool:
return value.get("litellm_provider") in PROVIDERS_HIDING_RETIRED_MODELS and is_past_deprecation_date(value)

77
litellm/litellm_core_utils/reasoning_effort_utils.py
View file

@ -1,6 +1,12 @@
import reprlib
from collections.abc import Collection, Mapping
from dataclasses import dataclass
from typing import Final, Literal
from pydantic import TypeAdapter
import litellm
from litellm._logging import verbose_logger
from litellm.constants import (
DEFAULT_REASONING_EFFORT_HIGH_THINKING_BUDGET,
DEFAULT_REASONING_EFFORT_LOW_THINKING_BUDGET,
@ -11,6 +17,54 @@ from litellm.litellm_core_utils.core_helpers import as_str_mapping
OpenAIStyleReasoningEffort = Literal["minimal", "low", "medium", "high"]
_DROP_WARNING: Final = "Dropping %s=%s for model=%r because %s"
_DROP_PARAMS_DROP_WARNING: Final = "Dropping %s=%s for model=%r (drop_params=True) because %s"
_NON_NUMERIC_BUDGET_REASON: Final = "it is not a number. Send budget_tokens as an integer to set a reasoning level."
_NON_STRING_EFFORT_REASON: Final = (
"it is not a string. Send reasoning_effort as a level name such as 'high' to set a reasoning level."
)
_REASONING_PARAMS: Final = ("reasoning", "reasoning_effort", "thinking")
OPAQUE_VALUE: Final = TypeAdapter(object)
def warn_dropped_reasoning(
param: str, value: object, model: str | None, reason: str, *, drop_params_gated: bool
) -> None:
verbose_logger.warning(
_DROP_PARAMS_DROP_WARNING if drop_params_gated else _DROP_WARNING, param, reprlib.repr(value), model, reason
)
def warn_non_string_reasoning_effort(value: object, model: str | None) -> None:
effort: Final = as_str_mapping(value)
requested_level: Final = effort.get("effort") if effort is not None else value
if requested_level is None or isinstance(value, str):
return
warn_dropped_reasoning("reasoning_effort", value, model, _NON_STRING_EFFORT_REASON, drop_params_gated=False)
def warn_dropped_reasoning_params(
dropped: Collection[object],
request: Mapping[str, object],
model: str | None,
reason: str,
*,
drop_params_gated: bool,
) -> None:
for param in _REASONING_PARAMS:
if param in dropped and param in request:
warn_dropped_reasoning(param, request[param], model, reason, drop_params_gated=drop_params_gated)
@dataclass(frozen=True, slots=True)
class DroppedThinkingBudget:
budget_tokens: object
def warn(self, model: str) -> None:
warn_dropped_reasoning(
"thinking.budget_tokens", self.budget_tokens, model, _NON_NUMERIC_BUDGET_REASON, drop_params_gated=True
)
def reasoning_effort_from_thinking_budget(
budget_tokens: float,
@ -29,7 +83,9 @@ def reasoning_effort_from_thinking_budget(
return "minimal"
def reasoning_effort_from_thinking(thinking: object, *, drop_params: bool) -> OpenAIStyleReasoningEffort | None:
def _effort_or_dropped_budget(
thinking: object, *, drop_params: bool
) -> OpenAIStyleReasoningEffort | DroppedThinkingBudget | None:
fields: Final = as_str_mapping(thinking)
if fields is None or fields.get("type") != "enabled":
return None
@ -37,10 +93,27 @@ def reasoning_effort_from_thinking(thinking: object, *, drop_params: bool) -> Op
if isinstance(budget, (int, float)) and not isinstance(budget, bool):
return reasoning_effort_from_thinking_budget(budget)
if drop_params or litellm.drop_params is True:
return None
return DroppedThinkingBudget(budget)
raise non_numeric_thinking_budget_error(budget)
def reasoning_effort_from_thinking(
thinking: object, *, model: str, drop_params: bool
) -> OpenAIStyleReasoningEffort | None:
effort: Final = _effort_or_dropped_budget(thinking, drop_params=drop_params)
if not isinstance(effort, DroppedThinkingBudget):
return effort
effort.warn(model)
return None
def reasoning_effort_from_thinking_silently(
thinking: object, *, drop_params: bool
) -> OpenAIStyleReasoningEffort | None:
effort: Final = _effort_or_dropped_budget(thinking, drop_params=drop_params)
return None if isinstance(effort, DroppedThinkingBudget) else effort
def non_numeric_thinking_budget_error(budget: object) -> UnsupportedParamsError:
return UnsupportedParamsError(
message=(

1
litellm/litellm_core_utils/redact_messages.py
View file

@ -265,6 +265,7 @@ def _redacted_provider_specific_fields(fields: Mapping[str, object]) -> Mapping[
"reasoningContentBlocks",
"citations",
"compaction_blocks",
"compaction_start",
"compaction_delta",
"token_ids",
)

23
litellm/litellm_core_utils/safe_json_dumps.py
View file

@ -1,4 +1,5 @@
import json
import math
from collections.abc import Callable
from typing import Any, Final
@ -14,16 +15,23 @@ def strip_null_bytes(value: str) -> str:
return value.replace("\x00", "")
def _finite_or_none(value: float | None) -> float | None:
return None if isinstance(value, float) and not math.isfinite(value) else value
def safe_json_structure(
data: object,
max_depth: int = DEFAULT_MAX_RECURSE_DEPTH,
value_transform: Callable[[str | None, str], str] | None = None,
key: str | None = None,
*,
non_finite_as_null: bool = False,
) -> object:
"""
Rebuild data out of JSON-native pieces while detecting circular references.
If a circular reference is detected then a marker string is returned.
NUL bytes are stripped from strings to prevent PostgreSQL 22P05 errors.
NUL bytes are stripped from strings to prevent PostgreSQL 22P05 errors. With
non_finite_as_null, NaN and infinite floats become null, since jsonb rejects them.
value_transform, when given, is applied to every string leaf (and to the
str() fallback for non-serializable objects) with the mapping key the leaf
@ -43,7 +51,7 @@ def safe_json_structure(
cleaned = obj.replace("\x00", "") if "\x00" in obj else obj
return _transform(key, cleaned)
if isinstance(obj, (int, float, bool, type(None))):
return obj
return _finite_or_none(obj) if non_finite_as_null else obj
# Check for circular reference.
if id(obj) in seen:
return "CircularReference Detected"
@ -65,8 +73,9 @@ def safe_json_structure(
seen.remove(id(obj))
return tuple_result
elif isinstance(obj, set):
sortable_values: Final[list[Any]] = [_serialize(item, seen, depth + 1, key) for item in obj]
set_result: Final = sorted(sortable_values)
members: Final[list[Any]] = [_serialize(item, seen, depth + 1, key) for item in obj]
present: Final = sorted(member for member in members if member is not None)
set_result: Final = present + [None] * members.count(None)
seen.remove(id(obj))
return set_result
elif isinstance(obj, BaseModel):
@ -88,6 +97,10 @@ def safe_dumps(
data: object,
max_depth: int = DEFAULT_MAX_RECURSE_DEPTH,
value_transform: Callable[[str | None, str], str] | None = None,
*,
non_finite_as_null: bool = False,
) -> str:
"""Serialize data to JSON text through safe_json_structure."""
return json.dumps(safe_json_structure(data, max_depth, value_transform), default=str)
return json.dumps(
safe_json_structure(data, max_depth, value_transform, non_finite_as_null=non_finite_as_null), default=str
)

9
litellm/litellm_core_utils/streaming_chunk_builder_utils.py
View file

@ -16,6 +16,7 @@ from litellm.constants import VLLM_CHOICE_FIELDS
from litellm.litellm_core_utils.core_helpers import (
PROVIDERS_BILLED_FROM_THEIR_OWN_USAGE_FIELDS,
chunk_usage,
finite_cost,
provider_reported_usage_cost,
stamp_litellm_bill_on_usage,
)
@ -1018,7 +1019,9 @@ class ChunkProcessor:
prompt_tokens_details, cache_creation_token_details
)
if usage_chunk_dict["cost"] is not None:
if Usage.dropped_non_finite_cost(usage_chunk):
cost = None
elif usage_chunk_dict["cost"] is not None:
cost = usage_chunk_dict["cost"]
prompt_tokens_details = attach_cache_creation_token_details(prompt_tokens_details, cache_creation_token_details)
@ -1449,8 +1452,8 @@ def _set_stream_builder_response_cost(
def _stamp_streaming_usage_cost(usage: Usage, response: ModelResponse, logging_obj: Optional["Logging"]) -> None:
if logging_obj is None:
return
if _logged_provider(logging_obj) in PROVIDERS_BILLED_FROM_THEIR_OWN_USAGE_FIELDS and isinstance(
getattr(usage, "cost", None), (int, float)
if _logged_provider(logging_obj) in PROVIDERS_BILLED_FROM_THEIR_OWN_USAGE_FIELDS and (
finite_cost(getattr(usage, "cost", None)) is not None
):
return
computed_cost: Final = logging_obj.response_cost_calculator(result=response)

52
litellm/litellm_core_utils/streaming_handler.py
View file

@ -29,6 +29,7 @@ from litellm.litellm_core_utils.core_helpers import chunk_usage, provider_report
from litellm.litellm_core_utils.hidden_params import (
HIDDEN_PARAMS_ATTR,
get_hidden_params,
set_extra_attribute,
set_hidden_param,
stored_attribute,
)
@ -1100,7 +1101,7 @@ class CustomStreamWrapper:
_initial_delta: Final = model_response.choices[0].delta.model_dump()
_initial_delta.pop("role", None)
model_response.choices[0].delta = Delta(**_initial_delta)
set_extra_attribute(model_response.choices[0], "delta", Delta(**_initial_delta))
return model_response
def _prebuilt_plain_delta(self, chunk: object) -> Delta | None:
@ -1126,13 +1127,13 @@ class CustomStreamWrapper:
def _set_role_on_fresh_delta(self, model_response: ModelResponseStream) -> None:
delta: Final = model_response.choices[0].delta
if self.sent_first_chunk is False:
delta.role = "assistant"
set_extra_attribute(delta, "role", "assistant")
self.sent_first_chunk = True
return
if not _holds_only_scalars(delta):
self.strip_role_from_delta(model_response)
return
delta.role = None
set_extra_attribute(delta, "role", None)
if "provider_specific_fields" not in delta.model_fields_set:
delta.provider_specific_fields = None
@ -1256,8 +1257,8 @@ class CustomStreamWrapper:
and _holds_only_scalars(upstream_delta)
)
if can_reuse_choice and upstream_choice is not None:
model_response.choices[0].finish_reason = None # rebind-ok: keep reused choice
model_response.choices[0].index = upstream_choice.index # rebind-ok: keep reused choice
set_extra_attribute(model_response.choices[0], "finish_reason", None)
set_extra_attribute(model_response.choices[0], "index", upstream_choice.index)
_prepend_upstream_choice_extras(
model_response.choices[0],
upstream_choice,
@ -1282,11 +1283,7 @@ class CustomStreamWrapper:
else:
return
model_response.system_fingerprint = upstream_chunk.system_fingerprint
setattr(
model_response,
"citations",
stored_attribute(upstream_chunk, "citations"),
)
set_extra_attribute(model_response, "citations", stored_attribute(upstream_chunk, "citations"))
preserve_upstream_non_openai_attributes(
model_response=model_response,
original_chunk=original_chunk,
@ -2662,6 +2659,25 @@ class CustomStreamWrapper:
def bill_failure_after_earlier_rounds(self, earlier: "BilledRoundsSpend") -> None:
self._earlier_rounds_spend = earlier
def partial_response(self) -> "ModelResponse | TextCompletionResponse | None":
partial: Final = (
litellm.stream_chunk_builder(
chunks=self.chunks,
messages=self.messages if isinstance(self.messages, list) else None,
logging_obj=self.logging_obj,
count_prompt_tokens=self.count_prompt_tokens,
)
if self.chunks
else None
)
usage: Final[object] = getattr(partial, "usage", None)
if partial is None or not isinstance(usage, Usage):
return None
if self.model:
partial.model = self.model
backfill_missing_cache_usage_fields(usage)
return partial
def _record_partial_usage_for_failure(self) -> None:
"""
A stream that breaks mid-flight still billed the provider for the chunks
@ -2671,23 +2687,15 @@ class CustomStreamWrapper:
moves it into terminal failure usage before opening the next attempt,
so a recovered attempt logs its own usage.
"""
if self.logging_obj is None or not self.chunks:
if self.logging_obj is None:
return
try:
partial_response: Final = litellm.stream_chunk_builder(
chunks=self.chunks,
messages=self.messages if isinstance(self.messages, list) else None,
logging_obj=self.logging_obj,
count_prompt_tokens=self.count_prompt_tokens,
)
partial_response: Final = self.partial_response()
if partial_response is None:
return
usage: Final = cast(Usage | None, getattr(partial_response, "usage", None))
if usage is None:
usage: Final[object] = getattr(partial_response, "usage", None)
if not isinstance(usage, Usage):
return
if self.model:
partial_response.model = self.model
backfill_missing_cache_usage_fields(usage)
partial_cost: Final = self.logging_obj.response_cost_calculator(result=partial_response) or 0.0
if self._earlier_rounds_spend is None:
self.logging_obj.model_call_details["combined_usage_object"] = usage

2
litellm/llms/anthropic/chat/transformation.py
View file

@ -48,6 +48,7 @@ from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import
place_mid_conversation_system,
split_leading_system_run,
)
from litellm.litellm_core_utils.reasoning_effort_utils import OPAQUE_VALUE, warn_non_string_reasoning_effort
from litellm.llms.base_llm.base_utils import (
ResponseFormatSchema,
as_json_object,
@ -1951,6 +1952,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
if isinstance(effort_value, dict):
effort_value = effort_value.get("effort")
if not isinstance(effort_value, str):
warn_non_string_reasoning_effort(OPAQUE_VALUE.validate_python(value), model)
continue
mapped_thinking = AnthropicConfig._map_reasoning_effort(
reasoning_effort=effort_value,

74
litellm/llms/anthropic/common_utils.py
View file

@ -37,7 +37,7 @@ from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import
message_field,
parts_of,
)
from litellm.litellm_core_utils.reasoning_effort_utils import non_numeric_thinking_budget_error
from litellm.litellm_core_utils.reasoning_effort_utils import DroppedThinkingBudget, non_numeric_thinking_budget_error
from litellm.litellm_core_utils.url_utils import join_api_base
from litellm.llms.anthropic.wif import (
aget_anthropic_wif_token,
@ -58,6 +58,7 @@ from litellm.types.llms.anthropic import (
ANTHROPIC_THINKING_BINDING_CONTROLS_BETA_HEADER,
ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER,
ANTHROPIC_TOOL_SEARCH_TOOL_TYPES,
LEGACY_THINKING_DISPLAY_UPDATES_PROVIDERS,
AllAnthropicToolsValues,
AnthropicMcpServerTool,
AnthropicMessagesToolChoice,
@ -126,6 +127,8 @@ _JSON_VALUE: Final = TypeAdapter(JsonValue)
_CLAUDE_CODE_USER_AGENT_PREFIXES: Final = ("claude-cli/", "claude-code/")
NATIVE_COMPACTION_PROVIDERS: Final = frozenset({"anthropic", "azure_ai", "vertex_ai"})
def is_claude_tool_target(custom_llm_provider: str | None, model: str) -> bool:
if custom_llm_provider == "anthropic":
@ -204,7 +207,7 @@ def requires_native_compaction_beta(
optional_params: Mapping[str, object],
messages: Sequence[object],
) -> bool:
return custom_llm_provider == "anthropic" and (
return custom_llm_provider in NATIVE_COMPACTION_PROVIDERS and (
optional_params.get("compaction") is not None or carries_signed_compaction_block(messages)
)
@ -246,6 +249,27 @@ def context_management_betas(context_management: JsonValue) -> tuple[str, ...]:
)
def count_tokens_feature_betas(
custom_llm_provider: str,
optional_params: Mapping[str, object],
messages: Sequence[object],
) -> tuple[str, ...]:
return tuple(
beta
for beta, used in (
(
ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_09_04.value,
requires_native_compaction_beta(custom_llm_provider, optional_params, messages),
),
(
ANTHROPIC_MID_CONVERSATION_TOOL_CHANGES_BETA_HEADER,
AnthropicModelInfo().is_mid_conversation_tool_change_used(messages),
),
)
if used
)
def supports_anthropic_cache_control(model: str, custom_llm_provider: str | None) -> bool:
from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
from litellm.utils import supports_prompt_caching
@ -610,11 +634,11 @@ def _numeric_budget_tokens(budget_tokens: object) -> int | None:
return None
def _legacy_budget_tokens(budget_tokens: object, *, drop_params: bool) -> int | None:
def _legacy_budget_tokens(budget_tokens: object, *, drop_params: bool) -> int | DroppedThinkingBudget:
if (numeric := _numeric_budget_tokens(budget_tokens)) is not None:
return numeric
if drop_params or litellm.drop_params is True:
return None
return DroppedThinkingBudget(budget_tokens)
raise non_numeric_thinking_budget_error(budget_tokens)
@ -645,10 +669,12 @@ class AnthropicModelInfo(BaseLLMModelInfo):
file_ids: Final = get_file_ids_from_messages(messages)
return len(file_ids) > 0
def is_thinking_display_updates_used(self, thinking: AnthropicThinkingParam | None) -> bool:
if not isinstance(thinking, dict):
return False
return thinking.get("type") in ("adaptive", "enabled") and thinking.get("display") == "updates"
def is_thinking_display_updates_used(self, thinking: object) -> bool:
match thinking:
case {"type": "adaptive" | "enabled", "display": "updates"}:
return True
case _:
return False
def thinking_betas(self, thinking: AnthropicThinkingParam | None) -> tuple[str, ...]:
return (
@ -1097,6 +1123,22 @@ class AnthropicModelInfo(BaseLLMModelInfo):
custom_llm_provider: str,
drop_params: bool,
) -> None:
dropped: Final = AnthropicModelInfo.translate_legacy_thinking_for_adaptive_model_silently(
model=model,
optional_params=optional_params,
custom_llm_provider=custom_llm_provider,
drop_params=drop_params,
)
if dropped is not None:
dropped.warn(model)
@staticmethod
def translate_legacy_thinking_for_adaptive_model_silently(
model: str,
optional_params: MutableMapping[str, object], # mutable-ok: in-place out-param like the sibling helpers
custom_llm_provider: str,
drop_params: bool,
) -> DroppedThinkingBudget | None:
"""Translate legacy ``thinking.type=enabled`` to adaptive for the
adaptive-thinking models that reject it (4.7+ and the 5 families).
Models flagged ``supports_legacy_thinking`` (the 4.6 family) accept the
@ -1105,22 +1147,25 @@ class AnthropicModelInfo(BaseLLMModelInfo):
``output_config.effort`` is never overridden.
"""
if not AnthropicModelInfo._is_adaptive_thinking_model(model, custom_llm_provider):
return
return None
if AnthropicModelInfo._supports_legacy_thinking(model, custom_llm_provider):
return
return None
thinking: Final = validated_object_mapping(optional_params.get("thinking"))
if thinking is None or thinking.get("type") != "enabled":
return
return None
budget_tokens: Final = _legacy_budget_tokens(thinking.get("budget_tokens", 0), drop_params=drop_params)
display: Final = thinking.get("display")
supported_display: Final = {"display": display} if display in ("summarized", "omitted") else {}
keeps_display: Final = display in ("summarized", "omitted") or (
display == "updates" and custom_llm_provider in LEGACY_THINKING_DISPLAY_UPDATES_PROVIDERS
)
supported_display: Final = {"display": display} if keeps_display else {}
carried_keys: Final = carried_thinking_keys(
thinking, rebuilt_keys=frozenset(("type", "budget_tokens", "display")), drop_params=drop_params
)
optional_params["thinking"] = {**carried_keys, "type": "adaptive", **supported_display}
if budget_tokens is None:
return
if isinstance(budget_tokens, DroppedThinkingBudget):
return budget_tokens
effort: Final = AnthropicModelInfo._legacy_budget_to_effort(
model=model, budget_tokens=budget_tokens, custom_llm_provider=custom_llm_provider
)
@ -1129,6 +1174,7 @@ class AnthropicModelInfo(BaseLLMModelInfo):
"effort": effort,
**(existing_output_config if isinstance(existing_output_config, dict) else MappingProxyType({})),
}
return None
@staticmethod
def _legacy_budget_to_effort(model: str, budget_tokens: int, custom_llm_provider: str) -> str:

18
litellm/llms/anthropic/messages/adapters/transformation.py
View file

@ -134,6 +134,7 @@ from litellm.litellm_core_utils.prompt_templates.factory import (
)
from litellm.litellm_core_utils.reasoning_effort_utils import (
reasoning_effort_from_thinking,
reasoning_effort_from_thinking_silently,
)
from litellm.llms.anthropic.common_utils import (
eager_input_streaming_flag,
@ -297,6 +298,7 @@ class AnthropicAdapter:
custom_llm_provider=custom_llm_provider,
api_base=api_base,
drop_params=drop_params,
report_reasoning_drops=True,
)
return translated_body, tool_name_mapping
@ -666,7 +668,9 @@ class LiteLLMAnthropicMessagesAdapter:
def translate_anthropic_thinking_to_reasoning_effort(
thinking: AnthropicThinkingParam,
*,
model: str,
drop_params: bool = True,
report_reasoning_drops: bool = False,
) -> str | None:
"""
Translate Anthropic's thinking parameter to OpenAI's reasoning_effort.
@ -685,7 +689,9 @@ class LiteLLMAnthropicMessagesAdapter:
if thinking_type == "disabled":
return "none"
elif thinking_type == "enabled":
return reasoning_effort_from_thinking(thinking, drop_params=drop_params)
if report_reasoning_drops:
return reasoning_effort_from_thinking(thinking, model=model, drop_params=drop_params)
return reasoning_effort_from_thinking_silently(thinking, drop_params=drop_params)
elif thinking_type == "adaptive":
# Adaptive thinking: effort is controlled by output_config.effort,
# not budget_tokens. Return a default; caller should override with
@ -752,7 +758,7 @@ class LiteLLMAnthropicMessagesAdapter:
return {"thinking": thinking}
else:
reasoning_effort: Final = LiteLLMAnthropicMessagesAdapter.translate_anthropic_thinking_to_reasoning_effort(
thinking
thinking, model=model
)
if reasoning_effort:
return {
@ -1147,6 +1153,7 @@ class LiteLLMAnthropicMessagesAdapter:
*,
custom_llm_provider: str | None = None,
drop_params: bool = True,
report_reasoning_drops: bool = False,
) -> None:
"""Translate Anthropic thinking to either thinking or reasoning_effort.
@ -1207,7 +1214,10 @@ class LiteLLMAnthropicMessagesAdapter:
return
reasoning_effort: Final = declared_effort or self.translate_anthropic_thinking_to_reasoning_effort(
cast(AnthropicThinkingParam, thinking), drop_params=drop_params
cast(AnthropicThinkingParam, thinking),
model=model,
drop_params=drop_params,
report_reasoning_drops=report_reasoning_drops,
)
if not reasoning_effort:
return
@ -1267,6 +1277,7 @@ class LiteLLMAnthropicMessagesAdapter:
api_base: str | None = None,
preserve_midturn_system: bool = False,
drop_params: bool = True,
report_reasoning_drops: bool = False,
) -> tuple[ChatCompletionRequest, dict[str, str]]:
"""
This is used by the beta Anthropic Adapter, for translating anthropic `/v1/messages` requests to the openai format.
@ -1321,6 +1332,7 @@ class LiteLLMAnthropicMessagesAdapter:
new_kwargs=new_kwargs,
custom_llm_provider=custom_llm_provider,
drop_params=drop_params,
report_reasoning_drops=report_reasoning_drops,
)
## CONVERT STOP_SEQUENCES
self._translate_stop_sequences_to_openai(

8
litellm/llms/anthropic/messages/native/handler.py
View file

@ -20,6 +20,7 @@ from litellm.litellm_core_utils.core_helpers import as_str_mapping
from litellm.litellm_core_utils.exception_mapping_utils import exception_type
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.llms.anthropic.common_utils import (
AnthropicModelInfo,
strip_provider_specific_fields_from_anthropic_messages,
tools_marked_for_a_tool_config_point,
)
@ -36,7 +37,6 @@ from litellm.router_utils.reasoning_effort_capability import (
keeps_effort,
model_group_from_metadata,
resolve_deployment_reasoning_effort,
with_adaptive_thinking_as_effort,
)
from litellm.types.llms.anthropic import AllAnthropicToolsValues
from litellm.types.llms.anthropic_messages.anthropic_request import AnthropicMetadata
@ -185,9 +185,11 @@ def _with_deployment_output_effort(
).value
def translate(source: Mapping[str, object]) -> Mapping[str, object]:
return with_adaptive_thinking_as_effort(
source, model=model, custom_llm_provider=custom_llm_provider, drop_params=drop_params
translated: Final = dict(source)
AnthropicModelInfo.translate_legacy_thinking_for_adaptive_model(
model=model, optional_params=translated, custom_llm_provider=custom_llm_provider, drop_params=drop_params
)
return translated
requested_output: Final = as_str_mapping(params.get("output_config"))
caller_output: Final = None if requested_output is None else resolve(requested_output)

14
litellm/llms/anthropic/messages/native/transformation.py
View file

@ -21,6 +21,7 @@ from litellm.constants import ANTHROPIC_BILLING_METADATA_PREFIX
from litellm.exceptions import AuthenticationError
from litellm.litellm_core_utils.core_helpers import drops_unsupported_params
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.litellm_core_utils.reasoning_effort_utils import OPAQUE_VALUE, warn_non_string_reasoning_effort
from litellm.litellm_core_utils.url_utils import ensure_api_route
from litellm.llms.anthropic.messages.convertible_blocks import drop_non_string_tool_result_texts
from litellm.llms.anthropic.messages.safeguards import proxy_serves_safeguards
@ -40,6 +41,7 @@ from litellm.types.llms.anthropic_tool_search import get_tool_search_beta_header
from litellm.types.router import GenericLiteLLMParams
from ...common_utils import (
NATIVE_COMPACTION_PROVIDERS,
OBJECT_LIST_ADAPTER,
AnthropicError,
AnthropicModelInfo,
@ -103,6 +105,7 @@ _MESSAGES_ADAPTER: Final = TypeAdapter(list[dict[str, object]])
class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
_workload_identity_eligible: ClassVar[bool] = True
_report_reasoning_drops: ClassVar[bool] = True
@property
def custom_llm_provider(self) -> str | None:
@ -126,7 +129,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
"tool_choice",
"thinking",
"context_management",
*(("compaction",) if self._resolved_provider == "anthropic" else ()),
*(("compaction",) if self._resolved_provider in NATIVE_COMPACTION_PROVIDERS else ()),
"output_format",
"inference_geo",
"speed",
@ -695,12 +698,17 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
):
anthropic_messages_optional_request_params.pop("output_format") # pyright: ignore[reportUnknownMemberType] # the request params arrive as a bare dict
requested_reasoning_effort: Final = OPAQUE_VALUE.validate_python(
anthropic_messages_optional_request_params.get("reasoning_effort") # pyright: ignore[reportUnknownMemberType] # the request params arrive as a bare dict
)
self._translate_reasoning_effort_to_anthropic(
model=model,
optional_params=anthropic_messages_optional_request_params,
max_tokens=max_tokens,
custom_llm_provider=self._resolved_provider,
)
if self._report_reasoning_drops:
warn_non_string_reasoning_effort(requested_reasoning_effort, model)
AnthropicModelInfo.maybe_drop_disabled_thinking(
model=model,
@ -708,12 +716,14 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
custom_llm_provider=self._resolved_provider,
)
AnthropicModelInfo.translate_legacy_thinking_for_adaptive_model(
dropped_budget: Final = AnthropicModelInfo.translate_legacy_thinking_for_adaptive_model_silently(
model=model,
optional_params=anthropic_messages_optional_request_params,
custom_llm_provider=self._resolved_provider,
drop_params=getattr(litellm_params, "drop_params", None) is True,
)
if dropped_budget is not None and self._report_reasoning_drops:
dropped_budget.warn(model)
self._translate_adaptive_effort_for_non_adaptive_model(
model=model,

4
litellm/llms/anthropic/messages/responses_adapters/transformation.py
View file

@ -534,6 +534,7 @@ class LiteLLMAnthropicToResponsesAPIAdapter:
output_config: dict[str, object] | None = None,
auto_summary_level: str = "detailed",
*,
model: str,
drop_params: bool = True,
) -> dict[str, object] | None:
"""
@ -555,7 +556,7 @@ class LiteLLMAnthropicToResponsesAPIAdapter:
if isinstance(output_config, dict) and output_config.get("effort"):
effort = output_config["effort"]
elif thinking_type == "enabled":
budget_effort: Final = reasoning_effort_from_thinking(thinking, drop_params=drop_params)
budget_effort: Final = reasoning_effort_from_thinking(thinking, model=model, drop_params=drop_params)
if budget_effort is None:
return None
effort = budget_effort
@ -667,6 +668,7 @@ class LiteLLMAnthropicToResponsesAPIAdapter:
thinking,
output_config=cast(dict[str, object] | None, output_config),
auto_summary_level=auto_summary_level,
model=model,
drop_params=drop_params,
)
if reasoning:

8
litellm/llms/anthropic/prompt_cache_prediction.py
View file

@ -9,7 +9,7 @@ from dataclasses import dataclass, field
from functools import reduce
from itertools import accumulate, groupby
from types import MappingProxyType
from typing import Annotated, Final, Literal, Protocol, TypeAlias
from typing import Annotated, ClassVar, Final, Literal, Protocol, TypeAlias
import httpx
from pydantic import ConfigDict, Field, JsonValue, StrictInt, TypeAdapter, ValidationError
@ -81,6 +81,10 @@ _DEPLOYMENT_OPTIONS: Final = frozenset(
)
class _BaselineAnthropicMessagesConfig(AnthropicMessagesConfig):
_report_reasoning_drops: ClassVar[bool] = False
class _StrictModel(LiteLLMBaseModel):
model_config = ConfigDict(extra="forbid", frozen=True, strict=True)
@ -661,7 +665,7 @@ def prepare_native_baseline_body(request: Mapping[str, object], model: str) -> M
TypeAdapter(tuple[str, ...]).validate_python(owned.get("additional_drop_params") or ()),
dict(options),
)
config: Final = AnthropicMessagesConfig()
config: Final = _BaselineAnthropicMessagesConfig()
body: Final = config.transform_anthropic_messages_request(
model=resolved_model,
messages=strip_provider_specific_fields_from_anthropic_messages(messages),

1
litellm/llms/azure_ai/anthropic/count_tokens/handler.py
View file

@ -84,6 +84,7 @@ class AzureAIAnthropicCountTokensHandler(AzureAIAnthropicCountTokensConfig):
headers: Final = self.get_required_headers(
api_key=api_key,
litellm_params=litellm_params,
messages=messages,
)
if not {"x-api-key", "Authorization"} & headers.keys():
raise AnthropicError(

8
litellm/llms/azure_ai/anthropic/count_tokens/transformation.py
View file

@ -4,11 +4,13 @@ Azure AI Anthropic CountTokens API transformation logic.
Extends the base Anthropic CountTokens transformation with Azure authentication.
"""
from collections.abc import Sequence
from typing import Final
from urllib.parse import urlsplit
from litellm.constants import ANTHROPIC_TOKEN_COUNTING_BETA_VERSION
from litellm.litellm_core_utils.url_utils import join_api_base
from litellm.llms.anthropic.common_utils import count_tokens_feature_betas, merge_anthropic_beta_headers
from litellm.llms.anthropic.count_tokens.transformation import (
AnthropicCountTokensConfig,
)
@ -28,6 +30,7 @@ class AzureAIAnthropicCountTokensConfig(AnthropicCountTokensConfig):
self,
api_key: str | None,
litellm_params: dict[str, object] | None = None,
messages: Sequence[object] = (),
) -> dict[str, str]:
"""
Get the required headers for the Azure AI Anthropic CountTokens API.
@ -38,6 +41,7 @@ class AzureAIAnthropicCountTokensConfig(AnthropicCountTokensConfig):
Args:
api_key: The Azure AI API key
litellm_params: Optional LiteLLM parameters for additional auth config
messages: The request messages, read for betas the counted features need
Returns:
Dictionary of required headers with both x-api-key and Azure authentication
@ -46,7 +50,9 @@ class AzureAIAnthropicCountTokensConfig(AnthropicCountTokensConfig):
headers: Final = {
"Content-Type": "application/json",
"anthropic-version": "2023-06-01",
"anthropic-beta": ANTHROPIC_TOKEN_COUNTING_BETA_VERSION,
"anthropic-beta": merge_anthropic_beta_headers(
ANTHROPIC_TOKEN_COUNTING_BETA_VERSION, count_tokens_feature_betas("azure_ai", {}, messages)
),
**({"x-api-key": api_key} if api_key else {}),
}

25
litellm/llms/azure_ai/chat/transformation.py
View file

@ -20,6 +20,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
prompt_cache_breakpoint_for_wire,
with_prompt_cache_breakpoint,
)
from litellm.litellm_core_utils.reasoning_effort_utils import warn_dropped_reasoning_params
from litellm.llms.azure.common_utils import BaseAzureLLM
from litellm.llms.azure_ai.common_utils import (
api_key_header_for_base,
@ -40,6 +41,14 @@ if TYPE_CHECKING:
from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
_REQUEST_BODY: Final = TypeAdapter(dict[str, object])
def _request_model(request: Mapping[str, object]) -> str | None:
model: Final = request.get("model")
return model if isinstance(model, str) else None
class AzureFoundryErrorStrings(str, enum.Enum):
SET_EXTRA_PARAMETERS_TO_PASS_THROUGH = "Set extra-parameters to 'pass-through'"
@ -510,7 +519,13 @@ class AzureAIStudioConfig(OpenAIConfig):
request_data = self._drop_extra_params_from_request_data(request_data, error_text)
if "Extra inputs are not permitted" in error_text and self._error_has_tool_level_extra_fields(error_text):
request_data = self._drop_tool_level_extra_fields(request_data, error_text)
data: Final = drop_params_from_unprocessable_entity_error(e=e, data=request_data)
data: Final = drop_params_from_unprocessable_entity_error(
e=e,
data=request_data,
model=_request_model(_REQUEST_BODY.validate_python(request_data)),
drop_params_gated="Extra inputs are not permitted" in error_text
and not self._error_has_tool_level_extra_fields(error_text),
)
return data
def _drop_tool_level_extra_fields(self, request_data: dict, error_text: str) -> dict:
@ -526,6 +541,14 @@ class AzureAIStudioConfig(OpenAIConfig):
def _drop_extra_params_from_request_data(self, request_data: dict, error_text: str) -> dict:
params_to_drop: Final = self._extract_params_to_drop_from_error_text(error_text)
if params_to_drop:
request: Final = _REQUEST_BODY.validate_python(request_data)
warn_dropped_reasoning_params(
params_to_drop,
request,
_request_model(request),
"Azure AI rejected it as an extra parameter its deployment does not pass through.",
drop_params_gated=False,
)
for param in params_to_drop:
if param in request_data:
request_data.pop(param, None)

17
litellm/llms/bedrock/chat/chat_completions/transformation.py
View file

@ -33,6 +33,7 @@ from litellm.litellm_core_utils.prompt_templates.image_handling import (
inline_remote_image_urls,
inline_remote_media,
)
from litellm.litellm_core_utils.reasoning_effort_utils import warn_dropped_reasoning
from litellm.litellm_core_utils.url_utils import ensure_api_route, join_api_base
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM, bedrock_bearer_token
@ -42,6 +43,7 @@ from litellm.llms.bedrock.common_utils import (
bedrock_rejects_stop_sequences,
bedrock_runtime_chat_completions_serves_reasoning_inline,
split_bedrock_region_path,
strip_bedrock_routing_prefix,
)
from litellm.llms.openai.chat.gpt_transformation import OpenAIChatCompletionStreamingHandler
from litellm.llms.openai_like.chat.transformation import OpenAILikeChatConfig
@ -121,6 +123,13 @@ def without_refused_reasoning_effort(model: str, params: Mapping[str, object]) -
effort: Final = params.get("reasoning_effort")
if not isinstance(effort, str) or effort not in chat_completions_reasoning_efforts_refused_for(model):
return params
warn_dropped_reasoning(
"reasoning_effort",
effort,
strip_bedrock_routing_prefix(model),
"AWS's Chat Completions endpoint rejects that level for this model family, so AWS applies its default effort.",
drop_params_gated=False,
)
return _without_params(params, frozenset(("reasoning_effort",)))
@ -445,6 +454,14 @@ class AmazonBedrockRuntimeChatCompletionsConfig(OpenAILikeChatConfig):
),
status_code=400,
)
if malformed_effort:
warn_dropped_reasoning(
"reasoning_effort",
raw_params["reasoning_effort"],
strip_bedrock_routing_prefix(model),
"Bedrock's Chat Completions endpoint takes it only as a string, so AWS applies its default effort.",
drop_params_gated=True,
)
rejected_stop: Final = frozenset(("stop",)) if bedrock_rejects_stop_sequences(model) else frozenset[str]()
return dict(
without_refused_reasoning_effort(

25
litellm/llms/bedrock/chat/converse_transformation.py
View file

@ -50,6 +50,7 @@ from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import
message_field,
parts_of,
)
from litellm.litellm_core_utils.reasoning_effort_utils import OPAQUE_VALUE, warn_dropped_reasoning
from litellm.llms.anthropic.chat.transformation import (
DROP_UNSUPPORTED_ADAPTIVE_THINKING_WARNING,
DROP_UNSUPPORTED_OUTPUT_CONFIG_WARNING,
@ -63,6 +64,7 @@ from litellm.llms.bedrock.chat.tool_result_images import place_tool_result_image
from litellm.llms.bedrock.common_utils import (
bedrock_chat_uses_native_structured_output,
bedrock_cleaned_output_schema,
bedrock_model_is_unknown,
bedrock_model_supports_regex_lookaround,
bedrock_native_output_format,
bedrock_native_output_is_opt_in,
@ -186,6 +188,8 @@ def _tools_the_model_accepts(
_OUTPUT_CONFIG_BLOCK: Final = TypeAdapter(OutputConfigBlock)
_DEEPSEEK_THINKING_REASON: Final = "DeepSeek on Bedrock reasons natively and takes no thinking field."
_DEEPSEEK_R1_EFFORT_REASON: Final = "DeepSeek R1 on Bedrock always reasons and rejects reasoning_effort."
def _has_native_text_format(output_config: object) -> bool:
@ -529,13 +533,14 @@ class AmazonConverseConfig(BaseConfig):
"""
Handle the reasoning_effort parameter based on the model type.
- GPT-OSS and DeepSeek V3 models: passed through unchanged via additionalModelRequestFields.
- GPT-OSS, DeepSeek V3 and unknown models (``bedrock_model_is_unknown``): passed through unchanged
via additionalModelRequestFields.
- OpenAI GPT-5.x and GPT-6 models: mapped to ``reasoning.effort`` via additionalModelRequestFields.
- Nova 2 models: transformed to reasoningConfig.
- Anthropic models: mapped to ``thinking`` (and ``output_config.effort`` on
adaptive Claude 4.6 / 4.7).
"""
if "gpt-oss" in model or "deepseek" in model:
if "gpt-oss" in model or "deepseek" in model or bedrock_model_is_unknown(model):
optional_params["reasoning_effort"] = reasoning_effort
elif self._is_openai_gpt_reasoning_model(model):
reasoning: Final[BedrockConverseGptReasoningEffortBlock] = {"effort": reasoning_effort}
@ -638,6 +643,10 @@ class AmazonConverseConfig(BaseConfig):
or supports_reasoning(model=base_model, custom_llm_provider=self.custom_llm_provider)
)
def _warn_native_reasoning_drop(self, param: str, value: object, model: str, reason: str) -> None:
if param in self.get_supported_openai_params(model):
warn_dropped_reasoning(param, value, model, reason, drop_params_gated=False)
def _model_rejects_reasoning_effort_param(self, model: str, base_model: str) -> bool:
"""Whether the model returns a 400 for every ``reasoning_effort`` shape on Converse.
@ -681,7 +690,7 @@ class AmazonConverseConfig(BaseConfig):
"parallel_tool_calls",
]
if "arn" in model: # we can't infer the model from the arn, so just add all params
if "arn" in model or bedrock_model_is_unknown(model):
supported_params.append("tools")
supported_params.append("tool_choice")
supported_params.append("thinking")
@ -1141,9 +1150,8 @@ class AmazonConverseConfig(BaseConfig):
"tool_choice": {"type": "auto", "disable_parallel_tool_use": not value}
}
if param == "thinking" and drop_thinking_param:
verbose_logger.debug(
"Dropping unsupported `thinking` param for Bedrock model=%s; it reasons natively.",
model,
self._warn_native_reasoning_drop(
param, OPAQUE_VALUE.validate_python(value), model, _DEEPSEEK_THINKING_REASON
)
elif param == "thinking" and not self._is_openai_gpt_reasoning_model(model):
if (
@ -1177,10 +1185,7 @@ class AmazonConverseConfig(BaseConfig):
drop_params=drop_params,
)
elif param == "reasoning_effort" and isinstance(value, str) and drop_reasoning_effort_param:
verbose_logger.debug(
"Dropping unsupported `reasoning_effort` param for Bedrock model=%s; it always reasons and rejects it.",
model,
)
self._warn_native_reasoning_drop(param, value, model, _DEEPSEEK_R1_EFFORT_REASON)
elif (
param == "reasoning_effort"
and isinstance(value, str)

62
litellm/llms/bedrock/chat/invoke_transformations/base_invoke_transformation.py
View file

@ -2,7 +2,7 @@ import copy
import json
import time
from collections.abc import Mapping, Sequence
from typing import TYPE_CHECKING, Any, Final, cast, get_args
from typing import TYPE_CHECKING, Any, Final
import httpx
from pydantic import ConfigDict, TypeAdapter, ValidationError, with_config
@ -74,10 +74,6 @@ _GUARDRAIL_CONFIG_EXPECTED_FORMAT: Final = (
"{'guardrailIdentifier': str, 'guardrailVersion': str, 'trace': 'enabled'|'disabled'|'enabled_full'}"
)
_INVOKE_PROVIDERS: Final = TypeAdapter(tuple[litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL, ...]).validate_python(
get_args(litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL)
)
def _response_header(headers: Mapping[str, str], name: str) -> str | None:
return headers.get(name, None)
@ -562,62 +558,6 @@ class AmazonInvokeConfig(BaseConfig, BaseAWSLLM):
"""
return False
@staticmethod
def get_bedrock_invoke_provider(
model: str,
) -> litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL | None:
"""
Helper function to get the bedrock provider from the model
handles 4 scenarios:
1. model=invoke/anthropic.claude-3-5-sonnet-20240620-v1:0 -> Returns `anthropic`
2. model=anthropic.claude-3-5-sonnet-20240620-v1:0 -> Returns `anthropic`
3. model=llama/arn:aws:bedrock:us-east-1:086734376398:imported-model/r4c4kewx2s0n -> Returns `llama`
4. model=us.amazon.nova-pro-v1:0 -> Returns `nova`
"""
if model.startswith("invoke/"):
model = model.replace("invoke/", "", 1)
# Special case: Check for "nova" in model name first (before "amazon")
# This handles amazon.nova-* models which would otherwise match "amazon" (Titan)
if "nova" in model.lower():
if "nova" in get_args(litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL):
return cast(litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL, "nova")
_split_model: Final = model.split(".")[0]
if _split_model in get_args(litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL):
return cast(litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL, _split_model)
# If not a known provider, check for pattern with two slashes
provider = AmazonInvokeConfig._get_provider_from_model_path(model)
if provider is not None:
return provider
for provider in _INVOKE_PROVIDERS:
if provider in model:
return provider
return None
@staticmethod
def _get_provider_from_model_path(
model_path: str,
) -> litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL | None:
"""
Helper function to get the provider from a model path with format: provider/model-name
Args:
model_path (str): The model path (e.g., 'llama/arn:aws:bedrock:us-east-1:086734376398:imported-model/r4c4kewx2s0n' or 'anthropic/model-name')
Returns:
Optional[str]: The provider name, or None if no valid provider found
"""
parts: Final = model_path.split("/")
if len(parts) >= 1:
provider: Final = parts[0]
if provider in get_args(litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL):
return cast(litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL, provider)
return None
def convert_messages_to_prompt(self, model: str, messages, provider, custom_prompt_dict) -> tuple[str, list | None]:
# handle anthropic prompts and amazon titan prompts
prompt = ""

68
litellm/llms/bedrock/common_utils.py
View file

@ -1098,6 +1098,11 @@ def _model_cost_entry(key: str) -> Mapping[str, object] | None:
return None if raw is None else _MODEL_COST_ENTRY_ADAPTER.validate_python(raw)
def _cost_map_row_names_a_provider(key: str) -> bool:
entry: Final = _model_cost_entry(key)
return entry is not None and entry.get("litellm_provider") is not None
def _bedrock_price_map_entries(model: str) -> tuple[Mapping[str, object] | None, ...]:
return tuple(
_model_cost_entry(key)
@ -1567,7 +1572,8 @@ def bedrock_model_supports_regex_lookaround(model: str, litellm_params: Mapping[
The deployment's ``model_info.supports_regex_lookaround`` wins, then the
``model_prices_and_context_window.json`` entry of its ``base_model``, then the
entry of ``model`` itself. A model nobody flagged keeps its schema as sent.
entry of ``model`` itself. A model nobody flagged keeps its schema as sent, unless
no candidate names a model LiteLLM knows, whatever route prefix it carries.
"""
params: Final = litellm_params or {}
deployment_flag: Final = deployment_model_info(params).get("supports_regex_lookaround")
@ -1576,7 +1582,41 @@ def bedrock_model_supports_regex_lookaround(model: str, litellm_params: Mapping[
base_model: Final = params.get("base_model")
candidates: Final = (*((base_model,) if isinstance(base_model, str) else ()), model)
flags: Final = (_bedrock_converse_model_flag(candidate, "supports_regex_lookaround") for candidate in candidates)
return next((flag for flag in flags if flag is not None), True)
flag: Final = next((flag for flag in flags if flag is not None), None)
if flag is not None:
return flag
return not all(bedrock_model_is_unknown(split_bedrock_region_path(candidate)[1]) for candidate in candidates)
def _bedrock_model_id_without_region(model: str) -> str:
routed_id: Final = model.removeprefix("bedrock/")
region, separator, regional_id = routed_id.partition("/")
return regional_id if separator and region in get_all_bedrock_regions() else routed_id
def bedrock_model_is_unknown(model: str) -> bool:
model_id: Final = _bedrock_model_id_without_region(model)
if "/" in model_id:
return False
base: Final = get_bedrock_base_model(model_id)
regional_ids: Final = (f"{region}.{base}" for region in get_bedrock_cross_region_inference_regions())
if any(_cost_map_row_names_a_provider(key) for key in (*_bedrock_model_info_candidates(model_id), *regional_ids)):
return False
from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM
return BaseAWSLLM.get_bedrock_invoke_provider(model_id) is None
def _names_no_invoke_provider(model: str) -> bool:
from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM
model_id: Final = _bedrock_model_id_without_region(model)
names_one_model: Final = bool(model_id) and ("/" not in model_id or model_id.startswith("arn:"))
return names_one_model and BaseAWSLLM.get_bedrock_invoke_provider(model) is None
def bedrock_model_reaches_converse_by_default(model: str) -> bool:
return BedrockModelInfo.named_bedrock_route(model, None) is None and _names_no_invoke_provider(model)
_BedrockConverseModelFlag: TypeAlias = Literal[
@ -1585,10 +1625,13 @@ _BedrockConverseModelFlag: TypeAlias = Literal[
]
def _bedrock_converse_model_flag(model: str, key: _BedrockConverseModelFlag) -> bool | None:
def _bedrock_model_info_candidates(model: str) -> tuple[str, ...]:
base: Final = get_bedrock_base_model(model)
candidates: Final = dict.fromkeys((model, base, _BEDROCK_MODEL_VERSION_SUFFIX_RE.sub("", base)))
for candidate in candidates:
return tuple(dict.fromkeys((model, base, _BEDROCK_MODEL_VERSION_SUFFIX_RE.sub("", base))))
def _bedrock_converse_model_flag(model: str, key: _BedrockConverseModelFlag) -> bool | None:
for candidate in _bedrock_model_info_candidates(model):
with contextlib.suppress(Exception):
model_info = get_cached_model_info()(
model=candidate,
@ -1766,8 +1809,19 @@ class BedrockModelInfo(BaseLLMModelInfo):
default (``bedrock_runtime_chat_completions_is_default``) and ``chat_completions/`` opts any other model in;
``request_params`` (the caller's chat params) sends such a request to Converse when it needs a
feature only Converse serves, and ``converse/`` pins a model to Converse. Every other OpenAI-family
model stays on Converse without the prefix.
model stays on Converse without the prefix. A model the cost map does not list as Converse goes to
Invoke only when its id names an Invoke provider, and to Converse otherwise.
"""
named_route: Final = BedrockModelInfo.named_bedrock_route(model, request_params)
if named_route is not None:
return named_route
return "converse" if _names_no_invoke_provider(model) else "invoke"
@staticmethod
def named_bedrock_route(
model: str,
request_params: Mapping[str, object] | None,
) -> BedrockRoute | None:
route_mappings: dict[
str,
Literal[
@ -1822,7 +1876,7 @@ class BedrockModelInfo(BaseLLMModelInfo):
return "converse"
if _OPENAI_FAMILY_MODEL_RE.search(base_model):
return "converse"
return "invoke"
return None
@staticmethod
def _explicit_converse_route(model: str) -> bool:

4
litellm/llms/custom_httpx/llm_http_handler.py
View file

@ -249,7 +249,7 @@ if TYPE_CHECKING:
FakeAnthropicMessagesStreamIterator,
)
from litellm.llms.base_llm.passthrough.transformation import BasePassthroughConfig
from litellm.responses.streaming_iterator import ResponsesWsFrameGuard
from litellm.responses.streaming_iterator import ResponsesWsFrameGuard, ResponsesWsTurnAdmission
from litellm.types.google_genai.main import GenerateContentResponse
from litellm.types.llms.openai_evals import (
CancelEvalResponse,
@ -7269,7 +7269,7 @@ class BaseLLMHTTPHandler:
custom_llm_provider: str | None = None,
first_message: str | None = None,
request_defaults: ResponsesWebSocketRequestDefaults | None = None,
turn_admission: Callable[[], Awaitable[None]] | None = None,
turn_admission: "ResponsesWsTurnAdmission | None" = None,
frame_guard: "ResponsesWsFrameGuard | None" = None,
**kwargs: object,
) -> Exception | None:

13
litellm/llms/deepseek/chat/transformation.py
View file

@ -6,10 +6,12 @@ from collections.abc import Coroutine, Mapping, Sequence
from typing import Final, Literal, cast, overload
import litellm
from litellm.litellm_core_utils.core_helpers import as_str_mapping
from litellm.litellm_core_utils.prompt_templates.common_utils import (
convert_content_list_to_str,
extract_search_results_text,
)
from litellm.litellm_core_utils.reasoning_effort_utils import OPAQUE_VALUE, warn_dropped_reasoning
from litellm.litellm_core_utils.url_utils import ensure_api_route
from litellm.secret_managers.main import get_secret_str
from litellm.types.llms.openai import AllMessageValues
@ -18,6 +20,16 @@ from litellm.utils import supports_reasoning, supports_vision
from ...openai.chat.gpt_transformation import OpenAIGPTConfig
SUPPORTED_REASONING_EFFORTS: Final = ("minimal", "low", "medium", "high", "xhigh", "max", "ultra")
_NO_BUDGET_REASON: Final = "DeepSeek takes thinking only as enabled or disabled, with no budget."
def _warn_on_dropped_budget(thinking: object, model: str) -> None:
fields: Final = as_str_mapping(thinking)
if fields is None or fields.get("type") != "enabled" or "budget_tokens" not in fields:
return
warn_dropped_reasoning(
"thinking.budget_tokens", fields["budget_tokens"], model, _NO_BUDGET_REASON, drop_params_gated=False
)
class DeepSeekChatConfig(OpenAIGPTConfig):
@ -53,6 +65,7 @@ class DeepSeekChatConfig(OpenAIGPTConfig):
# Then re-add only if valid for DeepSeek
thinking_value: Final = optional_params.pop("thinking", None)
reasoning_effort: Final = optional_params.pop("reasoning_effort", None)
_warn_on_dropped_budget(OPAQUE_VALUE.validate_python(thinking_value), model)
# Handle thinking parameter - accept both enabled and disabled, ignore budget_tokens
if isinstance(thinking_value, dict) and thinking_value.get("type") in ("enabled", "disabled"):

13
litellm/llms/fal_ai/chat/transformation.py
View file

@ -12,7 +12,9 @@ from typing import TYPE_CHECKING, Final
import httpx
from pydantic import ConfigDict, TypeAdapter
import litellm
from litellm.litellm_core_utils.core_helpers import map_finish_reason
from litellm.litellm_core_utils.reasoning_effort_utils import warn_dropped_reasoning
from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException
from litellm.secret_managers.main import get_secret_str
from litellm.types.llms.base import LiteLLMBaseModel
@ -29,6 +31,10 @@ PROVIDER_PREFIX: Final[str] = "fal_ai/"
PASSTHROUGH_PARAMS: Final[frozenset[str]] = frozenset(("reasoning", "temperature", "top_p"))
REASONING_DISABLED_EFFORTS: Final[frozenset[str]] = frozenset(("none", "minimal"))
REASONING_ENABLED_EFFORTS: Final[frozenset[str]] = frozenset(("low", "medium", "high"))
_UNSUPPORTED_EFFORT_REASON: Final = (
f"fal only turns reasoning off for {sorted(REASONING_DISABLED_EFFORTS)} "
f"and on for {sorted(REASONING_ENABLED_EFFORTS)}."
)
class _FalUsage(LiteLLMBaseModel):
@ -120,9 +126,10 @@ class FalAIChatConfig(BaseConfig):
return False
if isinstance(value, str) and value in REASONING_ENABLED_EFFORTS:
return True
if drop_params:
return None
raise FalAIError(status_code=400, message=f"Unsupported reasoning_effort {value!r} for {model}")
if not (drop_params or litellm.drop_params is True):
raise FalAIError(status_code=400, message=f"Unsupported reasoning_effort {value!r} for {model}")
warn_dropped_reasoning("reasoning_effort", value, model, _UNSUPPORTED_EFFORT_REASON, drop_params_gated=True)
return None
def _translate_param(self, param: str, value: object, model: str, drop_params: bool) -> tuple[str, object] | None:
if param in ("temperature", "top_p"):

14
litellm/llms/fireworks_ai/chat/transformation.py
View file

@ -17,6 +17,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
strip_litellm_response_only_fields,
unpack_legacy_defs,
)
from litellm.litellm_core_utils.reasoning_effort_utils import OPAQUE_VALUE, warn_dropped_reasoning
from litellm.litellm_core_utils.url_utils import join_api_base
from litellm.secret_managers.main import get_secret_str
from litellm.types.llms.openai import (
@ -97,6 +98,7 @@ def _json_schema_response_format(schema: object, name: str) -> Mapping[str, obje
EFFORT_KWARG_KEYS: Final = frozenset({"enable_thinking", "thinking", "reasoning_budget", "low_effort"})
_NOT_A_REASONING_MODEL_REASON: Final = "litellm's model map does not list it as a reasoning model."
def _bool_from_kwargs(kwargs: Mapping[str, object], keys: tuple[str, ...]) -> bool | None:
@ -107,6 +109,13 @@ def _bool_from_kwargs(kwargs: Mapping[str, object], keys: tuple[str, ...]) -> bo
return None
def warn_dropped_effort_kwargs(chat_template_kwargs: object, effort: object, model: str) -> None:
if effort != "none":
warn_dropped_reasoning(
"chat_template_kwargs", chat_template_kwargs, model, _NOT_A_REASONING_MODEL_REASON, drop_params_gated=False
)
def effort_from_chat_template_kwargs(kwargs: Mapping[str, object]) -> object:
enable_thinking: Final = _bool_from_kwargs(kwargs, ("enable_thinking", "thinking"))
if enable_thinking is False:
@ -438,10 +447,7 @@ class FireworksAIConfig(FireworksAIMixin, OpenAIGPTConfig):
if effort is None:
return ()
if not supports_reasoning(model=model, custom_llm_provider="fireworks_ai"):
verbose_logger.debug(
"fireworks_ai model %r does not support reasoning; dropping chat_template_kwargs effort keys.",
model,
)
warn_dropped_effort_kwargs(OPAQUE_VALUE.validate_python(chat_template_kwargs), effort, model)
return ()
return (("reasoning_effort", effort),)

7
litellm/llms/fireworks_ai/completion/transformation.py
View file

@ -2,6 +2,7 @@ from collections.abc import Mapping
from typing import Final
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.reasoning_effort_utils import OPAQUE_VALUE
from litellm.types.llms.openai import AllMessageValues, OpenAITextCompletionUserMessage
from litellm.utils import supports_reasoning
@ -12,6 +13,7 @@ from ..chat.transformation import (
NIM_VLLM_STRIP_PARAMS,
FireworksAIConfig,
effort_from_chat_template_kwargs,
warn_dropped_effort_kwargs,
)
from ..common_utils import FireworksAIMixin, resolve_fireworks_resource_name
@ -129,10 +131,7 @@ class FireworksAITextCompletionConfig(FireworksAIMixin, BaseTextCompletionConfig
)
return result
if not supports_reasoning(model=model, custom_llm_provider="fireworks_ai"):
verbose_logger.debug(
"fireworks_ai model %r does not support reasoning; dropping chat_template_kwargs effort keys.",
model,
)
warn_dropped_effort_kwargs(OPAQUE_VALUE.validate_python(chat_template_kwargs), effort, model)
return result
return {**result, "reasoning_effort": effort}

2
litellm/llms/hosted_vllm/chat/transformation.py
View file

@ -99,7 +99,7 @@ class HostedVLLMChatConfig(OpenAIGPTConfig):
thinking_effort: Final = (
None
if "reasoning_effort" in non_default_params
else reasoning_effort_from_thinking(thinking, drop_params=drop_params)
else reasoning_effort_from_thinking(thinking, model=model, drop_params=drop_params)
)
if thinking_effort is not None:
non_default_params["reasoning_effort"] = thinking_effort

25
litellm/llms/mistral/chat/transformation.py
View file

@ -25,6 +25,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
strip_litellm_internal_message_fields,
strip_none_values_from_message,
)
from litellm.litellm_core_utils.reasoning_effort_utils import OPAQUE_VALUE, warn_dropped_reasoning
from litellm.litellm_core_utils.url_utils import ensure_api_route
from litellm.llms.openai.chat.gpt_transformation import (
OpenAIChatCompletionStreamingHandler,
@ -74,6 +75,12 @@ _MISTRAL_REQUEST_ROUTING: Final = TypeAdapter(MistralRequestRouting)
_MISTRAL_CACHE_KEY_PARAMS: Final = TypeAdapter(MistralCacheKeyParams)
_MISTRAL_REASONING_EFFORT_PARAMS: Final = TypeAdapter(MistralReasoningEffortParams)
_MISTRAL_FILE_PART: Final = TypeAdapter(MistralFilePart)
_REASONING_PROMPT_REASON: Final = "litellm sends Magistral a reasoning system prompt instead of the requested level."
_NON_LEVEL_EFFORT_REASON: Final = (
"the Mistral API takes it only as a level such as 'high', so Mistral applies its default effort."
)
_DOCUMENTED_MAGISTRAL_EFFORTS: Final = frozenset({"low", "medium", "high"})
_REASONING_ON_THINKING_TYPES: Final = frozenset({"enabled", "adaptive"})
def _reasoning_level_and_summary(reasoning_effort: object) -> tuple[object, object]:
@ -83,6 +90,16 @@ def _reasoning_level_and_summary(reasoning_effort: object) -> tuple[object, obje
return requested.get("effort"), requested.get("summary")
def _warn_unless_reasoning_prompt_honors(param: str, value: object, model: str) -> None:
if param == "reasoning_effort" and isinstance(value, str) and value in _DOCUMENTED_MAGISTRAL_EFFORTS:
return
thinking: Final = as_str_mapping(value)
thinking_type: Final = thinking.get("type") if thinking is not None else None
if param == "thinking" and isinstance(thinking_type, str) and thinking_type in _REASONING_ON_THINKING_TYPES:
return
warn_dropped_reasoning(param, value, model, _REASONING_PROMPT_REASON, drop_params_gated=False)
def _is_json_array(value: object) -> TypeIs[Sequence[object]]: # guard-ok: a decoded JSON array is a list
return isinstance(value, list)
@ -253,6 +270,7 @@ class MistralConfig(OpenAIGPTConfig):
if param in ("reasoning_effort", "thinking") and "magistral" in model.lower():
# Flag that we need to add reasoning system prompt
optional_params["_add_reasoning_prompt"] = True
_warn_unless_reasoning_prompt_honors(param, OPAQUE_VALUE.validate_python(value), model)
if param in ("parallel_tool_calls", "prompt_cache_key"):
optional_params[param] = value
return optional_params
@ -260,13 +278,18 @@ class MistralConfig(OpenAIGPTConfig):
def _reasoning_level_to_send(
self, model: str, requested: MistralReasoningEffortParams, drop_params: bool
) -> str | None:
level, summary = _reasoning_level_and_summary(requested.get("reasoning_effort"))
requested_effort: Final = requested.get("reasoning_effort")
level, summary = _reasoning_level_and_summary(requested_effort)
sendable: Final = (
_accepted_reasoning_effort(model, level, self.custom_llm_provider) if isinstance(level, str) else None
)
if sendable is not None and summary is None:
return sendable
if drop_params or litellm.drop_params is True:
if sendable is None and level is not None:
warn_dropped_reasoning(
"reasoning_effort", requested_effort, model, _NON_LEVEL_EFFORT_REASON, drop_params_gated=True
)
return sendable
raise UnsupportedParamsError(
message=(

16
litellm/llms/oci/chat/transformation.py
View file

@ -21,6 +21,7 @@ from litellm._logging import verbose_logger
from litellm.constants import DEFAULT_OCI_CHAT_MAX_TOKENS
from litellm.litellm_core_utils.logging_utils import track_llm_api_timing
from litellm.litellm_core_utils.prompt_templates.common_utils import messages_with_file_parts_as_image_urls
from litellm.litellm_core_utils.reasoning_effort_utils import warn_dropped_reasoning
from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper
from litellm.litellm_core_utils.url_utils import join_api_base
from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException
@ -428,6 +429,13 @@ class OCIChatConfig(BaseConfig):
alias = param_map.get(key)
if key == "reasoning_effort" and alias and not _model_supports_reasoning_effort(model):
if drop_params or litellm.drop_params:
warn_dropped_reasoning(
key,
value,
model,
"the model map lists no reasoning_effort levels for it, so OCI would reject reasoningEffort.",
drop_params_gated=True,
)
continue
raise OCIError(
status_code=400,
@ -451,6 +459,14 @@ class OCIChatConfig(BaseConfig):
if key == "n" and (value is None or value == 1):
continue
if drop_params or litellm.drop_params:
if key == "reasoning_effort":
warn_dropped_reasoning(
key,
value,
model,
"OCI's Cohere chat API has no reasoning effort field.",
drop_params_gated=True,
)
continue
raise OCIError(
status_code=400,

8
litellm/llms/ollama/chat/transformation.py
View file

@ -14,6 +14,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
extract_images_from_message,
extract_reasoning_content,
)
from litellm.litellm_core_utils.reasoning_effort_utils import OPAQUE_VALUE
from litellm.litellm_core_utils.url_utils import ensure_api_route
from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator
from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException
@ -29,7 +30,7 @@ from litellm.types.llms.openai import (
)
from litellm.types.utils import ModelResponse, ModelResponseStream
from ..common_utils import OllamaError
from ..common_utils import OllamaError, ollama_think
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
@ -172,10 +173,7 @@ class OllamaChatConfig(BaseConfig):
if value.get("json_schema") and value["json_schema"].get("schema"):
optional_params["format"] = value["json_schema"]["schema"]
if param == "reasoning_effort" and value is not None:
if model.startswith("gpt-oss"):
optional_params["think"] = value
else:
optional_params["think"] = value in {"low", "medium", "high"}
optional_params["think"] = ollama_think(model, OPAQUE_VALUE.validate_python(value))
### FUNCTION CALLING LOGIC ###
# Ollama 0.4+ supports native tool calling - pass tools directly
# and let Ollama handle model capability detection

12
litellm/llms/ollama/common_utils.py
View file

@ -8,11 +8,14 @@ from pydantic import BaseModel, TypeAdapter
from litellm import verbose_logger
from litellm._uuid import uuid
from litellm.litellm_core_utils.reasoning_effort_utils import warn_dropped_reasoning
from litellm.litellm_core_utils.url_utils import join_api_base
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.types.utils import ModelResponse, ModelResponseStream
_PROVIDER_FIELDS: Final = TypeAdapter(dict[str, object])
_THINK_ON_EFFORTS: Final = frozenset({"low", "medium", "high"})
_THINK_OFF_REASON: Final = "Ollama takes think only as on or off, and litellm maps this level to off."
_DECODED_JSON: Final = TypeAdapter(object)
@ -296,3 +299,12 @@ class OllamaModelInfo(BaseLLMModelInfo):
Return the base model name for Ollama (no-op).
"""
return model
def ollama_think(model: str, reasoning_effort: object) -> object:
if model.startswith("gpt-oss"):
return reasoning_effort
think: Final = reasoning_effort in _THINK_ON_EFFORTS
if isinstance(reasoning_effort, str) and not think and reasoning_effort != "none":
warn_dropped_reasoning("reasoning_effort", reasoning_effort, model, _THINK_OFF_REASON, drop_params_gated=False)
return think

8
litellm/llms/ollama/completion/transformation.py
View file

@ -23,6 +23,7 @@ from litellm.litellm_core_utils.prompt_templates.image_handling import (
async_inline_remote_media,
inline_remote_image_urls,
)
from litellm.litellm_core_utils.reasoning_effort_utils import OPAQUE_VALUE
from litellm.litellm_core_utils.url_utils import ensure_api_route
from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator
from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException
@ -43,7 +44,7 @@ from litellm.types.utils import (
generate_id,
)
from ..common_utils import OllamaError, OllamaModelInfo, convert_image
from ..common_utils import OllamaError, OllamaModelInfo, convert_image, ollama_think
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
@ -249,10 +250,7 @@ class OllamaConfig(BaseConfig):
elif param == "stop":
optional_params["stop"] = value
elif param == "reasoning_effort" and value is not None:
if model.startswith("gpt-oss"):
optional_params["think"] = value
else:
optional_params["think"] = value in {"low", "medium", "high"}
optional_params["think"] = ollama_think(model, OPAQUE_VALUE.validate_python(value))
elif param == "response_format" and isinstance(value, dict):
if value["type"] == "json_object":
optional_params["format"] = "json"

22
litellm/llms/openai/common_utils.py
View file

@ -32,6 +32,7 @@ import litellm
from litellm.exceptions import error_response_without_credentials
from litellm.litellm_core_utils.completion_timeout import CompletionTimeout
from litellm.litellm_core_utils.core_helpers import as_str_mapping, decoded_json_object
from litellm.litellm_core_utils.reasoning_effort_utils import warn_dropped_reasoning_params
from litellm.litellm_core_utils.token_counter import token_counter, usage_tokenizer_model
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.llms.custom_httpx.http_handler import (
@ -165,17 +166,11 @@ _DECODED_JSON: Final = TypeAdapter(object)
def drop_params_from_unprocessable_entity_error(
e: openai.UnprocessableEntityError | httpx.HTTPStatusError,
data: Mapping[str, object],
*,
model: str | None,
drop_params_gated: bool,
) -> dict[str, object]:
"""
Helper function to read OpenAI UnprocessableEntityError and drop the params that raised an error from the error message.
Args:
e (UnprocessableEntityError): The UnprocessableEntityError exception
data (Dict[str, Any]): The original data dictionary containing all parameters
Returns:
Dict[str, Any]: A new dictionary with invalid parameters removed
"""
"""Return a copy of ``data`` without the params a 422 response names as invalid."""
invalid_params: Final[list[str]] = []
if isinstance(e, httpx.HTTPStatusError):
error_json: Final = decoded_json_object(_DECODED_JSON.validate_python(e.response.json()))
@ -203,6 +198,13 @@ def drop_params_from_unprocessable_entity_error(
invalid_params.append(error_dict["loc"][1])
new_data: Final = {k: v for k, v in data.items() if k not in invalid_params}
warn_dropped_reasoning_params(
invalid_params,
data,
model,
"the provider answered 422 naming it an invalid param.",
drop_params_gated=drop_params_gated,
)
return new_data

8
litellm/llms/openai/openai.py
View file

@ -940,7 +940,9 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
except openai.UnprocessableEntityError as e:
## check if body contains unprocessable params - related issue https://github.com/BerriAI/litellm/issues/4800
if litellm.drop_params is True or drop_params is True:
inference_params = drop_params_from_unprocessable_entity_error(e, inference_params)
inference_params = drop_params_from_unprocessable_entity_error(
e, inference_params, model=model, drop_params_gated=True
)
else:
raise e
# e.message
@ -1097,7 +1099,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
except openai.UnprocessableEntityError as e:
## check if body contains unprocessable params - related issue https://github.com/BerriAI/litellm/issues/4800
if litellm.drop_params is True or drop_params is True:
data = drop_params_from_unprocessable_entity_error(e, data)
data = drop_params_from_unprocessable_entity_error(e, data, model=model, drop_params_gated=True)
else:
raise e
# e.message
@ -1252,7 +1254,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
except openai.UnprocessableEntityError as e:
## check if body contains unprocessable params - related issue https://github.com/BerriAI/litellm/issues/4800
if litellm.drop_params is True or drop_params is True:
data = drop_params_from_unprocessable_entity_error(e, data)
data = drop_params_from_unprocessable_entity_error(e, data, model=model, drop_params_gated=True)
else:
raise e
except (

8
litellm/llms/openrouter/chat/transformation.py
View file

@ -13,6 +13,7 @@ from typing import TYPE_CHECKING, Final, cast
import httpx
from pydantic import ConfigDict, TypeAdapter
from litellm.litellm_core_utils.core_helpers import finite_cost
from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR, set_hidden_params
from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator
from litellm.llms.base_llm.chat.transformation import BaseLLMException
@ -225,8 +226,11 @@ class OpenrouterConfig(OpenAIGPTConfig):
additional_headers: Final = cast( # cast-ok: preserve mapping operations on response metadata
dict[str, object], hidden_params["additional_headers"]
)
if isinstance(response_cost, str | int | float):
additional_headers["llm_provider-x-litellm-response-cost"] = float(response_cost)
reported_cost: Final = (
finite_cost(float(response_cost)) if isinstance(response_cost, str | int | float) else None
)
if reported_cost is not None:
additional_headers["llm_provider-x-litellm-response-cost"] = reported_cost
except Exception:
# If we can't extract cost, continue without it - don't fail the response
pass

6
litellm/llms/pass_through/typesafe_logging_handler.py
View file

@ -6,6 +6,7 @@ import httpx
from pydantic import TypeAdapter, ValidationError
import litellm
from litellm.litellm_core_utils.core_helpers import finite_reported_cost
from litellm.litellm_core_utils.decisions_pricing import apply_decisions_pricing
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.litellm_core_utils.litellm_logging import (
@ -100,9 +101,10 @@ class TypeSafePassthroughLoggingHandler:
f"{custom_llm_provider}/{model}" for model in candidate_models if model is not None
)
pricing: Final = _pricing_for(candidate_model_keys)
reported_cost: Final = finite_reported_cost(usage.cost) if custom_llm_provider == "openrouter" else None
response_cost: Final = (
usage.cost
if custom_llm_provider == "openrouter" and usage.cost is not None
reported_cost
if reported_cost is not None
else input_tokens * pricing.input_cost_per_token + output_tokens * pricing.output_cost_per_token
)
usage_object: Final = Usage(

8
litellm/llms/perplexity/cost_calculator.py
View file

@ -6,6 +6,7 @@ Helper util for handling perplexity-specific cost calculation
from datetime import datetime
from typing import Final
from litellm.litellm_core_utils.core_helpers import finite_reported_cost
from litellm.litellm_core_utils.llm_cost_calc.utils import TokenRates, apply_off_peak_pricing
from litellm.types.utils import Usage
from litellm.utils import get_model_info
@ -34,10 +35,9 @@ def cost_per_token(model: str, usage: Usage, current_time: datetime | None = Non
total_cost = cost_info.get("total_cost")
elif isinstance(cost_info, (int, float)) and not isinstance(cost_info, bool):
total_cost = float(cost_info)
if total_cost is not None:
# Return total cost as completion_cost (prompt_cost=0) since Perplexity
# doesn't break down by input/output in their cost object
return (0.0, float(total_cost))
reported_cost: Final = None if total_cost is None else finite_reported_cost(float(total_cost))
if reported_cost is not None:
return (0.0, reported_cost)
## FALLBACK: Calculate cost manually if Perplexity doesn't provide it
## GET MODEL INFO

7
litellm/llms/tencent/chat/transformation.py
View file

@ -9,12 +9,15 @@ from typing import Final, TypedDict
from typing_extensions import ReadOnly
import litellm
from litellm.litellm_core_utils.reasoning_effort_utils import warn_dropped_reasoning
from litellm.litellm_core_utils.url_utils import ensure_api_route, join_api_base
from litellm.secret_managers.main import get_secret_str
from litellm.utils import supports_reasoning
from ...openai.chat.gpt_transformation import OpenAIGPTConfig
_ADAPTIVE_BUDGET_REASON: Final = "it is not an integer, so litellm sends adaptive thinking with no budget."
class ThinkingPayload(TypedDict, total=False):
"""Tencent TokenHub `thinking` object.
@ -102,6 +105,10 @@ class TencentChatConfig(OpenAIGPTConfig):
if isinstance(budget, int):
coerced_with_budget: Final[ThinkingPayload] = {"type": "adaptive", "budget_tokens": budget}
return coerced_with_budget
if budget is not None:
warn_dropped_reasoning(
"thinking.budget_tokens", budget, model, _ADAPTIVE_BUDGET_REASON, drop_params_gated=False
)
coerced: Final[ThinkingPayload] = {"type": "adaptive"}
return coerced

2
litellm/llms/vertex_ai/common_utils.py
View file

@ -1257,7 +1257,7 @@ def get_vertex_model_id_from_url(url: str) -> str | None:
`https://${LOCATION}-aiplatform.googleapis.com/v1/projects/${PROJECT_ID}/locations/${LOCATION}/publishers/google/models/${MODEL_ID}:streamGenerateContent`
"""
match: Final = re.search(r"/models/([^:]+)", url)
match: Final = re.search(r"/models/([^:?#]+)", url)
return match.group(1) if match else None

26
litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py
View file

@ -34,6 +34,7 @@ from litellm.litellm_core_utils.json_fragment_accumulator import JSONFragmentAcc
from litellm.litellm_core_utils.prompt_templates.factory import (
encode_tool_call_id_with_signature,
)
from litellm.litellm_core_utils.reasoning_effort_utils import warn_dropped_reasoning
from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException
from litellm.llms.custom_httpx.http_handler import (
AsyncHTTPHandler,
@ -201,6 +202,13 @@ def _unsupported_reasoning_effort(reasoning_effort: str) -> UnsupportedParamsErr
_SAMPLING_DEPRECATION_UNMAPPED_MODELS: Final = "models outside litellm's model map"
_NON_INTEGER_BUDGET_REASON: Final = (
"Gemini takes budget_tokens only as an integer, so it uses its default thinking budget."
)
_LEVEL_ONLY_BUDGET_REASON: Final = (
"Gemini 3 and later set reasoning by thinking level, not budget, so litellm does not send the budget. "
"Send reasoning_effort to pick a level."
)
_sampling_deprecation_lock: Final = threading.Lock()
@ -1466,6 +1474,14 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
params["includeThoughts"] = False
else:
params["includeThoughts"] = sends_thoughts
if thinking_budget is not None:
warn_dropped_reasoning(
"thinking.budget_tokens",
thinking_budget,
model,
_LEVEL_ONLY_BUDGET_REASON,
drop_params_gated=False,
)
# Follow provider defaults unless explicitly opted into legacy behavior.
if litellm.enable_gemini_default_thinking_level_low is True:
params["thinkingLevel"] = (
@ -1478,8 +1494,16 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
# For older Gemini models, use thinkingBudget
if thinking_enabled and not VertexGeminiConfig._is_thinking_budget_zero(thinking_budget):
params["includeThoughts"] = sends_thoughts
if thinking_budget is not None and isinstance(thinking_budget, int):
if isinstance(thinking_budget, int):
params["thinkingBudget"] = thinking_budget
elif thinking_budget is not None:
warn_dropped_reasoning(
"thinking.budget_tokens",
thinking_budget,
model,
_NON_INTEGER_BUDGET_REASON,
drop_params_gated=False,
)
return params

30
litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/messages/transformation.py
View file

@ -1,9 +1,15 @@
from collections.abc import Mapping, Sequence
from typing import Final
from pydantic import TypeAdapter
from litellm.exceptions import UnsupportedParamsError
from litellm.litellm_core_utils.core_helpers import drops_unsupported_params
from litellm.llms.anthropic.common_utils import AnthropicModelInfo, context_management_betas
from litellm.llms.anthropic.common_utils import (
AnthropicModelInfo,
context_management_betas,
requires_native_compaction_beta,
)
from litellm.llms.anthropic.messages.native.transformation import (
AnthropicMessagesConfig,
messages_carry_output_config,
@ -11,6 +17,7 @@ from litellm.llms.anthropic.messages.native.transformation import (
from litellm.types.llms.anthropic import (
ANTHROPIC_BETA_HEADER_VALUES,
ANTHROPIC_HOSTED_TOOLS,
ANTHROPIC_MID_CONVERSATION_TOOL_CHANGES_BETA_HEADER,
)
from litellm.types.llms.anthropic_tool_search import get_tool_search_beta_header
from litellm.types.llms.vertex_ai import VertexPartnerProvider
@ -19,6 +26,8 @@ from litellm.types.router import GenericLiteLLMParams
from ....vertex_llm_base import VertexBase
from ..output_params_utils import sanitize_vertex_anthropic_output_params
_OPTIONAL_PARAMS: Final = TypeAdapter(Mapping[str, object])
_VERSION_HEADER_VERTEX_READS_FROM_BODY: Final = "anthropic-version"
_VERTEX_DEFAULT_VERSION: Final = "vertex-2023-10-16"
_VERSIONS_SERVED_AS_VERTEX_DEFAULT: Final = frozenset({"", "2023-01-01", "2023-06-01"})
@ -190,14 +199,23 @@ class VertexAIPartnerModelsAnthropicMessagesConfig(AnthropicMessagesConfig, Vert
if messages_carry_output_config(messages):
beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.PER_TURN_CONTROL_2026_07_01.value)
beta_values.update(
typed_params: Final = _OPTIONAL_PARAMS.validate_python(optional_params)
if requires_native_compaction_beta("vertex_ai", typed_params, messages):
beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_09_04.value)
tool_change_betas: Final = (
(ANTHROPIC_MID_CONVERSATION_TOOL_CHANGES_BETA_HEADER,)
if anthropic_model_info.is_mid_conversation_tool_change_used(messages)
else ()
)
all_beta_values: Final = beta_values.union(
anthropic_model_info.thinking_betas(
optional_params.get("thinking"), # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # optional_params is the base config's bare dict
)
),
tool_change_betas,
)
if beta_values:
request_headers["anthropic-beta"] = ",".join(beta_values)
if all_beta_values:
request_headers["anthropic-beta"] = ",".join(sorted(all_beta_values))
return request_headers, api_base

82
litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/transformation.py
View file

@ -2,19 +2,23 @@
## Handler file for calling claude-3 on vertex ai
import base64
import json
from collections.abc import Mapping
from collections.abc import Mapping, Sequence
from typing import TYPE_CHECKING, Final
import httpx
from pydantic import TypeAdapter
from pydantic import TypeAdapter, ValidationError
import litellm
from litellm.anthropic_beta_headers_manager import get_header_only_betas
from litellm.litellm_core_utils.core_helpers import drops_unsupported_params
from litellm.litellm_core_utils.prompt_templates.image_handling import RemoteMedia, inline_remote_image_urls
from litellm.llms.base_llm.chat.transformation import LiteLLMLoggingObj
from litellm.types.llms.anthropic import ANTHROPIC_BETA_HEADER_VALUES
from litellm.types.llms.openai import AllMessageValues
from litellm.types.utils import ModelResponse
from ....anthropic.chat.transformation import AnthropicConfig
from ....anthropic.common_utils import AnthropicError, requires_native_compaction_beta
from .output_params_utils import sanitize_vertex_anthropic_output_params
if TYPE_CHECKING:
@ -36,6 +40,32 @@ def _vertex_labels_header(labels: object, header_names: Mapping[str, object]) ->
return base64.b64encode(json.dumps(string_labels).encode()).decode() if string_labels else None
_BETA_VALUE: Final[TypeAdapter[str | tuple[str, ...] | None]] = TypeAdapter(str | tuple[str, ...] | None)
_REQUEST_MESSAGES: Final = TypeAdapter(Sequence[object])
def _beta_names(value: object) -> frozenset[str] | None:
try:
parsed: Final = _BETA_VALUE.validate_python(value)
except ValidationError:
return None
names: Final = parsed.split(",") if isinstance(parsed, str) else parsed or ()
return frozenset(name.strip() for name in names if name.strip())
def _client_betas(value: object, field: str, drop_params: bool) -> frozenset[str]:
names: Final = _beta_names(value)
if names is not None:
return names
if drop_params:
return frozenset()
raise AnthropicError(
status_code=400,
message=f"{field} must be a string or a list of strings, got {type(value).__name__}",
)
class VertexAIError(Exception):
def __init__(self, status_code, message):
self.status_code = status_code
@ -157,21 +187,43 @@ class VertexAIAnthropicConfig(AnthropicConfig):
if context_management:
self._add_context_management_beta_headers(beta_set, context_management)
extra_headers: Final = optional_params.get("extra_headers") or {}
anthropic_beta_value: Final = extra_headers.get("anthropic-beta", "")
if isinstance(anthropic_beta_value, str) and anthropic_beta_value:
for beta in anthropic_beta_value.split(","):
beta = beta.strip()
if beta:
beta_set.add(beta)
elif isinstance(anthropic_beta_value, list):
beta_set.update(anthropic_beta_value)
request_params: Final = _NON_DEFAULT_PARAMS.validate_python(optional_params)
compaction_betas: Final = (
frozenset({ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_09_04.value})
if requires_native_compaction_beta(
"vertex_ai", request_params, _REQUEST_MESSAGES.validate_python(data["messages"])
)
else frozenset[str]()
)
drop_params: Final = drops_unsupported_params(litellm_params.get("drop_params"))
extra_headers: Final = _NON_DEFAULT_PARAMS.validate_python(request_params.get("extra_headers") or {})
data.pop("extra_headers", None)
injected_betas: Final = frozenset(beta_set) | _client_betas(
extra_headers.get("anthropic-beta"), "The anthropic-beta header", drop_params
)
client_body_betas: Final = _client_betas(
_NON_DEFAULT_PARAMS.validate_python(data).get("anthropic_beta"), "anthropic_beta", drop_params
)
if beta_set:
data["anthropic_beta"] = list(beta_set)
headers["anthropic-beta"] = ",".join(beta_set)
requested_betas: Final = injected_betas | client_body_betas
header_only_betas: Final = get_header_only_betas("vertex_ai") if requested_betas else frozenset[str]()
body_betas: Final = requested_betas - header_only_betas
if body_betas:
data["anthropic_beta"] = sorted(body_betas)
elif "anthropic_beta" in data:
del data["anthropic_beta"]
header_betas: Final = (
injected_betas
| compaction_betas
| (client_body_betas & header_only_betas)
| _client_betas(
_NON_DEFAULT_PARAMS.validate_python(headers).get("anthropic-beta"),
"The anthropic-beta header",
drop_params,
)
)
if header_betas:
headers["anthropic-beta"] = ",".join(sorted(header_betas))
return data

25
litellm/llms/vertex_ai/vertex_ai_partner_models/count_tokens/handler.py
View file

@ -6,12 +6,19 @@ Unlike Gemini models which use Google's token counting API, partner models use
their respective publisher-specific count-tokens endpoints.
"""
from collections.abc import Mapping
from collections.abc import Mapping, Sequence
from typing import Any, Final
from pydantic import TypeAdapter
from litellm.litellm_core_utils.core_helpers import drops_unsupported_params
from litellm.litellm_core_utils.url_utils import join_api_base
from litellm.llms.anthropic.common_utils import OBJECT_LIST_ADAPTER, is_claude_model_target, tools_without_custom_field
from litellm.llms.anthropic.common_utils import (
OBJECT_LIST_ADAPTER,
count_tokens_feature_betas,
is_claude_model_target,
tools_without_custom_field,
)
from litellm.llms.custom_httpx.http_handler import get_async_httpx_client
from litellm.llms.vertex_ai.common_utils import get_vertex_base_url
from litellm.llms.vertex_ai.vertex_llm_base import VertexBase
@ -23,6 +30,9 @@ _COUNT_TOKENS_PUBLISHERS: Final = (
)
_REQUEST_MESSAGES: Final = TypeAdapter(Sequence[object])
def count_tokens_request_body(
request_data: Mapping[str, object], model: str, drop_params: object
) -> Mapping[str, object]:
@ -43,6 +53,14 @@ def count_tokens_publisher(model: str) -> str | None:
return next((publisher for marker, publisher in _COUNT_TOKENS_PUBLISHERS if marker in model), None)
def count_tokens_headers(access_token: str, request_data: Mapping[str, object]) -> Mapping[str, str]:
authorization: Final = {"Authorization": f"Bearer {access_token}"}
betas: Final = count_tokens_feature_betas(
"vertex_ai", request_data, _REQUEST_MESSAGES.validate_python(request_data["messages"])
)
return {**authorization, "anthropic-beta": ",".join(betas)} if betas else authorization
class VertexAIPartnerModelsTokenCounter(VertexBase):
"""
Token counter for Vertex AI Partner Models.
@ -175,8 +193,7 @@ class VertexAIPartnerModelsTokenCounter(VertexBase):
api_base=litellm_params.get("api_base"),
)
# Prepare headers
headers: Final = {"Authorization": f"Bearer {access_token}"}
headers: Final = dict(count_tokens_headers(access_token, request_data))
# Get async HTTP client
from litellm import LlmProviders

20
litellm/llms/vllm/common_utils.py
View file

@ -1,3 +1,4 @@
from collections.abc import Mapping
from typing import Final
import httpx
@ -10,6 +11,14 @@ from litellm.types.llms.openai import AllMessageValues
from litellm.utils import add_path_to_api_base
def endpoint_under_api_base(endpoint: str, api_base: str | None) -> str:
return (
endpoint.lstrip("/").removeprefix("v1/")
if api_base is not None and api_base.rstrip("/").endswith("/v1")
else endpoint
)
class VLLMError(BaseLLMException):
def __init__(
self,
@ -39,9 +48,14 @@ class VLLMModelInfo(BaseLLMModelInfo):
api_key: str | None = None,
api_base: str | None = None,
) -> dict:
if api_key is not None:
headers["x-api-key"] = api_key
return headers
upstream_key: Final = self.upstream_api_key(api_key, api_base)
return {**headers, **self.upstream_auth_headers(upstream_key)} if upstream_key else headers
def upstream_api_key(self, api_key: str | None, api_base: str | None) -> str | None:
return api_key
def upstream_auth_headers(self, api_key: str) -> Mapping[str, str]:
return {"Authorization": f"Bearer {api_key}", "x-api-key": api_key}
@staticmethod
def get_api_base(api_base: str | None = None) -> str | None:

35
litellm/llms/vllm/passthrough/transformation.py
View file

@ -3,6 +3,7 @@ from typing import TYPE_CHECKING, Final
from httpx import Response
from litellm.constants import PLACEHOLDER_API_KEY
from litellm.litellm_core_utils.litellm_logging import Logging
from litellm.llms.base_llm.passthrough.transformation import BasePassthroughConfig
from litellm.llms.openai_like.passthrough.transformation import (
@ -12,8 +13,14 @@ from litellm.llms.openai_like.passthrough.transformation import (
logged_text_completion_stream,
relays_text_completion,
)
from litellm.secret_managers.main import (
GOOGLE_OIDC_SECRET_PREFIX,
UnresolvableSecretReference,
get_secret_str,
resolve_local_oidc_reference,
)
from ..common_utils import VLLMModelInfo
from ..common_utils import VLLMModelInfo, endpoint_under_api_base
if TYPE_CHECKING:
from httpx import URL
@ -22,6 +29,30 @@ if TYPE_CHECKING:
class VLLMPassthroughConfig(VLLMModelInfo, BasePassthroughConfig):
@staticmethod
def get_api_key(api_key: str | None = None) -> str | None:
return None if api_key == PLACEHOLDER_API_KEY else api_key
def upstream_api_key(self, api_key: str | None, api_base: str | None) -> str | None:
if api_key is not None or not self._is_vllm_api_base(api_base):
return api_key
vllm_api_key: Final = get_secret_str("VLLM_API_KEY")
if vllm_api_key is None or vllm_api_key.startswith(GOOGLE_OIDC_SECRET_PREFIX):
return None
match resolve_local_oidc_reference(vllm_api_key):
case UnresolvableSecretReference():
return None
case str() as resolved:
return resolved
@staticmethod
def _is_vllm_api_base(api_base: str | None) -> bool:
vllm_api_base: Final = get_secret_str("VLLM_API_BASE")
return api_base is not None and vllm_api_base is not None and api_base.rstrip("/") == vllm_api_base.rstrip("/")
def upstream_auth_headers(self, api_key: str) -> Mapping[str, str]:
return {"Authorization": f"Bearer {api_key}"}
def is_streaming_request(self, endpoint: str, request_data: dict) -> bool:
return "stream" in request_data
@ -40,7 +71,7 @@ class VLLMPassthroughConfig(VLLMModelInfo, BasePassthroughConfig):
raise Exception("VLLM api base not found")
return (
self.format_url(endpoint, base_target_url, request_query_params),
self.format_url(endpoint_under_api_base(endpoint, base_target_url), base_target_url, request_query_params),
base_target_url,
)

80
litellm/model_prices_and_context_window_backup.json
View file

@ -8715,7 +8715,7 @@
"search_context_size_low": 0.014,
"search_context_size_medium": 0.014
},
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'",
"source": "https://azure.microsoft.com/en-us/pricing/details/cognitive-services/openai-service/",
"supported_endpoints": [
"/v1/chat/completions",
"/v1/responses"
@ -8742,7 +8742,11 @@
"supports_tool_choice": true,
"supports_vision": true,
"supports_web_search": true,
"supports_xhigh_reasoning_effort": true
"supports_xhigh_reasoning_effort": true,
"cache_creation_input_token_cost_above_272k_tokens_flex": 1.25e-05,
"cache_read_input_token_cost_above_272k_tokens_flex": 1e-06,
"input_cost_per_token_above_272k_tokens_flex": 1e-05,
"output_cost_per_token_above_272k_tokens_flex": 3.75e-05
},
"azure/gpt-6-astra-2026-09-03": {
"cache_creation_input_token_cost": 1.25e-05,
@ -8763,7 +8767,7 @@
"search_context_size_low": 0.014,
"search_context_size_medium": 0.014
},
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'",
"source": "https://azure.microsoft.com/en-us/pricing/details/cognitive-services/openai-service/",
"supported_endpoints": [
"/v1/chat/completions",
"/v1/responses"
@ -8790,7 +8794,11 @@
"supports_tool_choice": true,
"supports_vision": true,
"supports_web_search": true,
"supports_xhigh_reasoning_effort": true
"supports_xhigh_reasoning_effort": true,
"cache_creation_input_token_cost_above_272k_tokens_flex": 1.25e-05,
"cache_read_input_token_cost_above_272k_tokens_flex": 1e-06,
"input_cost_per_token_above_272k_tokens_flex": 1e-05,
"output_cost_per_token_above_272k_tokens_flex": 3.75e-05
},
"azure/gpt-6-luna": {
"cache_creation_input_token_cost": 1.25e-07,
@ -30856,8 +30864,9 @@
"output_cost_per_token": 0,
"output_vector_size": 3072,
"rpm": 10000,
"source": "https://ai.google.dev/gemini-api/docs/embeddings#model-versions",
"tpm": 10000000
"source": "https://ai.google.dev/gemini-api/docs/pricing",
"tpm": 10000000,
"input_cost_per_token_batches": 7.5e-08
},
"gemini/gemini-embedding-2": {
"input_cost_per_audio_token": 6.5e-06,
@ -31713,7 +31722,10 @@
"input_cost_per_token_flex": 6.25e-07,
"output_cost_per_token_batches": 5e-06,
"output_cost_per_token_flex": 5e-06,
"supports_url_context": true
"supports_url_context": true,
"cache_read_input_token_cost_above_200k_tokens_batches": 2.5e-07,
"input_cost_per_token_above_200k_tokens_batches": 1.25e-06,
"output_cost_per_token_above_200k_tokens_batches": 7.5e-06
},
"gemini/gemini-2.5-computer-use-preview-10-2025": {
"deprecation_date": "2026-07-28",
@ -31871,7 +31883,8 @@
},
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"web_search_billing_unit": "per_query",
"google_maps_grounding_cost_per_query": 0.014
"google_maps_grounding_cost_per_query": 0.014,
"supports_computer_use": true
},
"gemini/gemini-3-flash-preview": {
"cache_read_input_audio_token_cost": 1e-07,
@ -31937,7 +31950,8 @@
"input_cost_per_token_flex": 2.5e-07,
"output_cost_per_token_batches": 1.5e-06,
"output_cost_per_token_flex": 1.5e-06,
"supports_audio_input": true
"supports_audio_input": true,
"supports_computer_use": true
},
"gemini/gemini-3.5-flash": {
"prompt_cache_min_tokens": 4096,
@ -32184,7 +32198,8 @@
},
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"web_search_billing_unit": "per_query",
"google_maps_grounding_cost_per_query": 0.014
"google_maps_grounding_cost_per_query": 0.014,
"supports_computer_use": true
},
"gemini/gemini-omni-flash-preview": {
"input_cost_per_audio_token": 1.5e-06,
@ -32280,7 +32295,10 @@
"web_search_billing_unit": "per_query",
"google_maps_grounding_cost_per_query": 0.014,
"input_cost_per_token_flex": 1e-06,
"output_cost_per_token_flex": 6e-06
"output_cost_per_token_flex": 6e-06,
"cache_read_input_token_cost_above_200k_tokens_batches": 4e-07,
"input_cost_per_token_above_200k_tokens_batches": 2e-06,
"output_cost_per_token_above_200k_tokens_batches": 9e-06
},
"gemini/gemini-3.1-pro-preview-customtools": {
"prompt_cache_min_tokens": 4096,
@ -32343,7 +32361,10 @@
"web_search_billing_unit": "per_query",
"google_maps_grounding_cost_per_query": 0.014,
"input_cost_per_token_flex": 1e-06,
"output_cost_per_token_flex": 6e-06
"output_cost_per_token_flex": 6e-06,
"cache_read_input_token_cost_above_200k_tokens_batches": 4e-07,
"input_cost_per_token_above_200k_tokens_batches": 2e-06,
"output_cost_per_token_above_200k_tokens_batches": 9e-06
},
"gemini-3-flash-preview": {
"cache_read_input_audio_token_cost": 1e-07,
@ -72228,6 +72249,7 @@
},
"aiand/deepseek-ai/deepseek-v4-flash": {
"cache_read_input_token_cost": 8e-08,
"deprecation_date": "2026-10-27",
"input_cost_per_token": 1.5e-07,
"litellm_provider": "aiand",
"max_input_tokens": 1048576,
@ -72257,6 +72279,22 @@
"supports_vision": false
},
"aiand/deepseek-ai/deepseek-v4.1-flash": {
"cache_read_input_token_cost": 2e-08,
"deprecation_date": "2026-11-05",
"input_cost_per_token": 3e-07,
"litellm_provider": "aiand",
"max_input_tokens": 1048576,
"max_output_tokens": 384000,
"mode": "chat",
"output_cost_per_token": 6e-07,
"source": "https://api.aiand.com/v1/models",
"supports_function_calling": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_tool_choice": true,
"supports_vision": true
},
"aiand/deepseek-ai/deepseek-v4.1-flash-1017": {
"cache_read_input_token_cost": 2e-08,
"input_cost_per_token": 3e-07,
"litellm_provider": "aiand",
@ -72286,8 +72324,24 @@
"supports_tool_choice": true,
"supports_vision": true
},
"aiand/meta/muse-spark-open": {
"cache_read_input_token_cost": 1.5e-07,
"input_cost_per_token": 1.25e-06,
"litellm_provider": "aiand",
"max_input_tokens": 1048576,
"max_output_tokens": 943718,
"mode": "chat",
"output_cost_per_token": 4.25e-06,
"source": "https://api.aiand.com/v1/models",
"supports_function_calling": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_tool_choice": true,
"supports_vision": true
},
"aiand/moonshotai/kimi-k2.7-code": {
"cache_read_input_token_cost": 2e-07,
"deprecation_date": "2026-10-27",
"input_cost_per_token": 7.5e-07,
"litellm_provider": "aiand",
"max_input_tokens": 262144,
@ -72378,6 +72432,7 @@
},
"aiand/zai-org/glm-5.2": {
"cache_read_input_token_cost": 3e-07,
"deprecation_date": "2026-10-27",
"input_cost_per_token": 1e-06,
"litellm_provider": "aiand",
"max_input_tokens": 1048576,
@ -76540,6 +76595,7 @@
],
"supports_audio_input": true,
"supports_audio_output": false,
"supports_computer_use": true,
"supports_function_calling": true,
"supports_minimal_reasoning_effort": false,
"supports_native_streaming": true,

17
litellm/proxy/_lazy_openapi_snapshot.json
View file

@ -3491,6 +3491,7 @@
"additionalProperties": {
"$ref": "#/components/schemas/GuardrailMetric"
},
"description": "Spend by guardrail ID, already included in key and model spend. Populated on user, team and organization aggregated activity only, without a model filter",
"title": "Guardrails",
"type": "object"
},
@ -3812,10 +3813,12 @@
"GuardrailKeyMetric": {
"properties": {
"metadata": {
"$ref": "#/components/schemas/KeyMetadata"
"$ref": "#/components/schemas/KeyMetadata",
"description": "API key alias and existence information"
},
"metrics": {
"$ref": "#/components/schemas/GuardrailSpendMetrics"
"$ref": "#/components/schemas/GuardrailSpendMetrics",
"description": "Guardrail spend and unpriced units for this API key"
}
},
"required": [
@ -3828,6 +3831,7 @@
"GuardrailMetadata": {
"properties": {
"guardrail_name": {
"description": "Guardrail display name, falling back to the guardrail ID",
"title": "Guardrail Name",
"type": "string"
}
@ -3844,14 +3848,17 @@
"additionalProperties": {
"$ref": "#/components/schemas/GuardrailKeyMetric"
},
"description": "Per-key guardrail spend and unpriced units, limited to keys in the day's API key breakdown",
"title": "Api Key Breakdown",
"type": "object"
},
"metadata": {
"$ref": "#/components/schemas/GuardrailMetadata"
"$ref": "#/components/schemas/GuardrailMetadata",
"description": "Guardrail name information"
},
"metrics": {
"$ref": "#/components/schemas/GuardrailSpendMetrics"
"$ref": "#/components/schemas/GuardrailSpendMetrics",
"description": "Guardrail spend and unpriced units across all matching API keys"
}
},
"required": [
@ -3865,11 +3872,13 @@
"properties": {
"spend": {
"default": 0.0,
"description": "Guardrail spend in USD, already included in key and model spend",
"title": "Spend",
"type": "number"
},
"untracked_units": {
"default": 0,
"description": "Guardrail units with no known price, excluded from spend",
"title": "Untracked Units",
"type": "integer"
}

84
litellm/proxy/auth/auth_checks.py
View file

@ -84,6 +84,7 @@ from litellm.proxy._types import (
UserAPIKeyAuth,
)
from litellm.proxy.agent_endpoints.auth.agent_access_groups import (
AgentAccessGroupCeiling,
CeilingResolver,
resolve_agent_access_group_ceiling,
)
@ -1684,6 +1685,7 @@ def get_actual_routes(allowed_routes: list) -> list:
KEY_END_USER_BUDGET_ID_METADATA_FIELD: Final = "end_user_budget_id"
_KEY_METADATA_ADAPTER: Final[TypeAdapter[Mapping[str, object] | None]] = TypeAdapter(Mapping[str, object] | None)
_OBJECTS_ADAPTER: Final[TypeAdapter[tuple[object, ...]]] = TypeAdapter(tuple[object, ...])
def get_key_end_user_budget_id(key_metadata: Mapping[str, object] | None) -> str | None:
@ -3455,28 +3457,35 @@ async def delete_cache_key_objects(
proxy_logging_obj: ProxyLogging | None,
) -> None:
"""
Evict a batch of key objects, for callers that delete keys in bulk rather than through
`/key/delete`. Auth resolves a cached key object without re-reading its team, so a key left
cached after its row is gone keeps buying access until its TTL expires.
Evicting locally only reaches this worker; `_delete_cache_key_object` itself broadcasts each
token, so a deleted key left in a peer worker's in-memory cache still authenticates there until
its TTL expires.
Evict a batch of deleted key objects here and on every other worker. Auth resolves a cached key
object without re-reading its team, so a key left cached after its row is gone keeps buying
access until its TTL expires.
Best-effort per key: the rows are already deleted by the time this runs, so an unreachable
cache backend must not abort the caller partway through its own cascade.
Every token is dropped from this worker's memory first, then from the auth cache and the
internal usage cache in one batch each, so a deleted key stops authenticating here the instant
its rows are gone instead of refilling from Redis on the next miss. Every other worker then
gets the eviction in one pipelined broadcast: a publish per key would pass the publisher's
pending cap on a bulk delete, and the dropped keys would keep authenticating there until their
TTL.
Every token is dropped from this worker's memory first, then from the auth cache's Redis layer,
so a deleted key stops authenticating here the instant its rows are gone instead of refilling
from Redis on the next miss. Every other worker then gets the eviction in one pipelined
broadcast: a publish per key would pass the publisher's pending cap on a bulk delete, and the
dropped keys would keep authenticating there until their TTL. The internal usage cache is not
an auth refill source, so its batch delete runs beside that chain and a hung Redis costs one
bounded wait there instead of another in line.
"""
evict_local(cache_keys=hashed_tokens, user_api_key_cache=user_api_key_cache)
usage_eviction: Final = (
()
if proxy_logging_obj is None
else (evict_shared(cache_keys=hashed_tokens, cache=proxy_logging_obj.internal_usage_cache.dual_cache),)
)
_ = await asyncio.gather(
_evict_auth_cache_then_broadcast(hashed_tokens=hashed_tokens, user_api_key_cache=user_api_key_cache),
*usage_eviction,
)
async def _evict_auth_cache_then_broadcast(hashed_tokens: Sequence[str], user_api_key_cache: UserApiKeyCache) -> None:
await evict_shared(cache_keys=hashed_tokens, cache=user_api_key_cache)
if proxy_logging_obj is not None:
await evict_shared(cache_keys=hashed_tokens, cache=proxy_logging_obj.internal_usage_cache.dual_cache)
await broadcast_evictions(cache_keys=hashed_tokens)
@ -5085,6 +5094,36 @@ async def can_agent_call_model(
)
async def _agent_access_group_ceilings(
valid_token: UserAPIKeyAuth, resolve_ceiling: CeilingResolver
) -> tuple[AgentAccessGroupCeiling, ...]:
if not valid_token.agent_id:
return ()
from litellm.proxy.agent_endpoints.auth.agent_access_groups import resolve_managed_agent_ceilings
managed: Final = managed_agent_policy(valid_token)
if managed is not None:
return await resolve_managed_agent_ceilings(managed)
unmanaged: Final = await resolve_ceiling(valid_token.agent_id)
return (unmanaged,) if unmanaged is not None else ()
def _managed_agent_model_grants(valid_token: UserAPIKeyAuth) -> frozenset[str]:
managed: Final = managed_agent_policy(valid_token)
granted: Final = (managed.object_permission or {}).get("models") if managed is not None else None
if not isinstance(granted, (list, tuple)):
return frozenset()
return frozenset(name for name in _OBJECTS_ADAPTER.validate_python(granted) if isinstance(name, str))
async def agent_model_grants(
valid_token: UserAPIKeyAuth, resolve_ceiling: CeilingResolver = resolve_agent_access_group_ceiling
) -> frozenset[str]:
ceilings: Final = await _agent_access_group_ceilings(valid_token, resolve_ceiling)
return _managed_agent_model_grants(valid_token).union(*(ceiling.models for ceiling in ceilings))
async def _check_agent_access_group_model_access(
model: str | list[str] | None, # mutable-ok: _can_object_call_model and the client message helper take list[str]
valid_token: UserAPIKeyAuth | None,
@ -5092,20 +5131,9 @@ async def _check_agent_access_group_model_access(
resolve_ceiling: CeilingResolver = resolve_agent_access_group_ceiling,
) -> Literal[True]:
"""Attached groups naming no model deny every model; the empty allowlist in ``_can_object_call_model`` allows."""
if not model or valid_token is None or not valid_token.agent_id:
if not model or valid_token is None:
return True
from litellm.proxy.agent_endpoints.auth.agent_access_groups import resolve_managed_agent_ceilings
managed: Final = managed_agent_policy(valid_token)
unmanaged: Final = await resolve_ceiling(valid_token.agent_id) if managed is None else None
ceilings: Final = (
await resolve_managed_agent_ceilings(managed)
if managed is not None
else (unmanaged,)
if unmanaged is not None
else ()
)
ceilings: Final = await _agent_access_group_ceilings(valid_token, resolve_ceiling)
dispatched: Final = _resolve_team_alias(
model, team_model_aliases_for_auth_check(valid_token), valid_token.team_id, llm_router
)

60
litellm/proxy/common_request_processing.py
View file

@ -60,6 +60,7 @@ from litellm.litellm_core_utils.bug_report import (
strip_bug_report_notice,
)
from litellm.litellm_core_utils.core_helpers import (
finite_cost,
get_or_create_metadata_bucket,
independent_snapshot,
is_expected_client_error,
@ -140,6 +141,7 @@ from litellm.proxy.utils import ( # noqa: F401 # legacy module exports
_check_and_merge_model_level_guardrails, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export
check_and_merge_model_level_guardrails,
)
from litellm.responses.mcp.parked_stream_dispatch import in_flight_round_biller, parked_success
from litellm.responses.streaming_iterator import partial_responses_incomplete_event_on_disconnect, spawn_logging_task
from litellm.router import Router
from litellm.router_api import RoutingContext
@ -535,6 +537,16 @@ async def _bill_partial_streamed_spend_on_disconnect(request_data: dict, respons
return await _dispatch_disconnect_billing_event(logging_obj, partial_response)
async def _bill_in_flight_round_on_disconnect(request_data: Mapping[str, object]) -> None:
bill_in_flight_round: Final = in_flight_round_biller(request_data.get("litellm_logging_obj"))
if bill_in_flight_round is None:
return
try:
await bill_in_flight_round()
except Exception as e: # noqa: BLE001 # disconnect billing is best-effort; never break stream teardown
verbose_proxy_logger.debug("Failed to bill the in-flight round after client disconnect: %s", e)
async def _dispatch_disconnect_billing_event(logging_obj: LiteLLMLoggingObj, partial_response: object) -> bool:
try:
await logging_obj.dispatch_success_handlers(
@ -1801,14 +1813,16 @@ def _get_cost_breakdown_from_logging_obj(
def _header_value(value: float | None) -> str | None:
return str(value) if value is not None else None
return str(value) if finite_cost(value) is not None else None
def _response_cost_header_value(hidden_params: Mapping[str, object]) -> float | str:
if hidden_params.get("cache_hit") is True:
return 0.0
response_cost: Final = hidden_params.get("response_cost")
return response_cost if isinstance(response_cost, (int, float, str)) and response_cost else ""
if isinstance(response_cost, str):
return response_cost
return response_cost if isinstance(response_cost, int | float) and finite_cost(response_cost) else ""
def cost_headers(
@ -2083,6 +2097,13 @@ class _ClientStreamFailureReport:
)
def _metadata_logged_on(logged_on: LiteLLMLoggingObj) -> Mapping[str, object]:
logged_params: Final = object_mapping(logged_on.model_call_details.get("litellm_params")) or MappingProxyType({})
return MappingProxyType(
{key: logged_params[key] for key in ("metadata", "litellm_metadata") if key in logged_params}
)
class ProxyBaseLLMRequestProcessing:
def __init__(self, data: dict, served_model: str | None = None):
self.data = data
@ -4151,11 +4172,21 @@ class ProxyBaseLLMRequestProcessing:
_captured_user_api_key_dict: Final = user_api_key_dict
_captured_logging_obj: Final = logging_obj
async def _on_deferred_stream_complete(assembled_response: object, cache_hit: object) -> None:
async def _on_deferred_stream_complete(
assembled_response: object, cache_hit: object, logged_on: LiteLLMLoggingObj | None = None
) -> None:
await ProxyBaseLLMRequestProcessing._run_deferred_stream_guardrails(
captured_data=_captured_data,
captured_data=_captured_data
if logged_on is None
else {
**_captured_data,
**{
key: value for key, value in _metadata_logged_on(logged_on).items() if key in _captured_data
},
"litellm_logging_obj": logged_on,
},
captured_user_api_key_dict=_captured_user_api_key_dict,
captured_logging_obj=_captured_logging_obj,
captured_logging_obj=_captured_logging_obj if logged_on is None else logged_on,
assembled_response=assembled_response,
cache_hit=cache_hit,
)
@ -4180,14 +4211,8 @@ class ProxyBaseLLMRequestProcessing:
GLOBAL_LOGGING_WORKER.ensure_initialized_and_enqueue(async_coroutine=logging_coroutine)
case (logging_coroutine,) if asyncio.iscoroutine(logging_coroutine):
GLOBAL_LOGGING_WORKER.ensure_initialized_and_enqueue(async_coroutine=logging_coroutine)
case (assembled_response, cache_hit):
await _as_success_dispatcher(_captured_native_logging_obj).dispatch_success_handlers(
assembled_response,
cache_hit=cache_hit,
start_time=None,
end_time=None,
prefer_async_handlers=True,
)
case _ if (success := parked_success(args)) is not None:
await success.dispatch(_captured_native_logging_obj)
case _:
verbose_proxy_logger.error(
"Deferred stream logging dropped: unexpected stored args shape %s",
@ -4487,7 +4512,7 @@ class ProxyBaseLLMRequestProcessing:
"""
if isinstance(chunk, dict):
# Use safe_dumps for proper JSON serialization with circular reference detection
chunk_str: Final = safe_dumps(chunk)
chunk_str: Final = safe_dumps(chunk, non_finite_as_null=True)
return f"{STREAM_SSE_DATA_PREFIX}{chunk_str}\n\n"
else:
return chunk
@ -4524,6 +4549,7 @@ class ProxyBaseLLMRequestProcessing:
client_disconnected,
)
if recorded_client_disconnect:
await _bill_in_flight_round_on_disconnect(_REQUEST_FIELDS.validate_python(request_data))
deferred_stream_logging_armed: Final = _deferred_stream_logging_is_armed(request_data)
ProxyLogging.fire_deferred_stream_logging(request_data)
# A disconnect-time success event (the deferred-guardrail flush
@ -4862,7 +4888,11 @@ class ProxyBaseLLMRequestProcessing:
obj, model_name, litellm_logging_obj
)
if maybe_modified is not None:
lines[idx] = "data: " + safe_dumps(maybe_modified) + ("\r" if ln.endswith("\r") else "")
lines[idx] = (
"data: "
+ safe_dumps(maybe_modified, non_finite_as_null=True)
+ ("\r" if ln.endswith("\r") else "")
)
return "\n".join(lines)
return None
except Exception:

11
litellm/proxy/common_utils/model_listing_utils.py
View file

@ -219,7 +219,7 @@ def caller_alias_maps(
def alias_map(aliases: object) -> Mapping[str, str]:
try:
entries: Final = _ALIAS_ENTRIES.validate_python(aliases, strict=True)
entries: Final = _ALIAS_ENTRIES.validate_python(aliases)
except ValidationError:
return _NO_ALIASES
return MappingProxyType(
@ -287,6 +287,15 @@ def _dispatch(model_id: str, layers: Sequence[_AliasLayer]) -> tuple[str, bool]:
return reduce(_through_layer, layers, (model_id, False))
def dispatched_model(model_id: str, aliases: CallerAliases) -> str:
target, _ = _dispatch(model_id, _alias_layers(aliases))
return target
def requestable_alias_names(aliases: CallerAliases) -> frozenset[str]:
return frozenset(chain.from_iterable(layer.aliases for layer in _alias_layers(aliases)))
def _lookup_id(name: str, lookup_by_response: Mapping[str, str], lookup_ids: Container[str]) -> str | None:
if name in lookup_by_response:
return lookup_by_response[name]

48
litellm/proxy/common_utils/openai_error_payload.py
View file

@ -28,6 +28,8 @@ _OPENAI_ERROR_TYPE_BY_STATUS: Final[Mapping[int, str]] = MappingProxyType(
_AUTH_FAILURE_STATUS_CODES: Final = frozenset({status.HTTP_401_UNAUTHORIZED, status.HTTP_403_FORBIDDEN})
_HTTP_ERROR_STATUSES: Final = range(status.HTTP_400_BAD_REQUEST, 600)
_ERROR_BODY: Final = TypeAdapter(dict[str, object])
_FAILOVER_SQLSTATES: Final = frozenset({"25006", "57P01", "57P02", "57P03"})
@ -47,13 +49,26 @@ def attribute_of(value: object, name: str, default: object = None) -> object:
def error_status_code(exc: object, default: int) -> int:
"""The HTTP status an exception carries as ``status_code`` or, the way ``ProxyException``
stores it, as a stringified ``code``; ``default`` when it carries neither."""
carried: Final = attribute_of(exc, "status_code")
if isinstance(carried, int) and not isinstance(carried, bool):
"""The 4xx or 5xx HTTP status an exception carries as ``status_code`` (an int or a decimal string) or,
the way ``ProxyException`` stores it, as a stringified ``code``; ``default`` when it carries neither."""
carried: Final = _http_error_status(attribute_of(exc, "status_code"))
if carried is not None:
return carried
stringified: Final = attribute_of(exc, "code")
return int(stringified) if isinstance(stringified, str) and stringified.isdecimal() else default
from_code: Final = _http_error_status(stringified) if isinstance(stringified, str) else None
return default if from_code is None else from_code
def _http_error_status(value: object) -> int | None:
match value:
case bool():
return None
case int() if value in _HTTP_ERROR_STATUSES:
return value
case str() if len(value) == 3 and value.isascii() and value.isdigit() and int(value) in _HTTP_ERROR_STATUSES:
return int(value)
case _:
return None
def openai_error_type(exc: object, status_code: int) -> str:
@ -105,9 +120,9 @@ def proxy_exception_for(exc: Exception, default_status_code: int) -> ProxyExcept
code=status.HTTP_503_SERVICE_UNAVAILABLE,
)
case HTTPException(status_code=status_code, detail=detail):
return _openai_shaped_proxy_exception(exc, str(detail), status_code)
return openai_shaped_proxy_exception(exc, str(detail), status_code)
case _:
return _openai_shaped_proxy_exception(
return openai_shaped_proxy_exception(
exc, str(exc) or type(exc).__name__, error_status_code(exc, default_status_code)
)
@ -176,14 +191,19 @@ def _route_failure_error_type(exc: Exception, raised_status: int, answered_statu
return openai_error_type(exc, answered_status)
def _openai_shaped_proxy_exception(exc: Exception, message: str, status_code: int) -> ProxyException:
def openai_shaped_proxy_exception(exc: Exception, message: str, status_code: int) -> ProxyException:
answered_status: Final = answered_status_code(message, status_code)
return ProxyException(
message=message,
type=_catch_all_error_type(exc, answered_status),
param=openai_error_param(exc),
code=answered_status,
)
return _proxy_exception(exc, message, answered_status, _catch_all_error_type(exc, answered_status))
def status_typed_proxy_exception(exc: HTTPException) -> ProxyException:
message: Final = str(exc.detail)
answered_status: Final = answered_status_code(message, exc.status_code)
return _proxy_exception(exc, message, answered_status, openai_error_type(exc, answered_status))
def _proxy_exception(exc: Exception, message: str, answered_status: int, error_type: str) -> ProxyException:
return ProxyException(message=message, type=error_type, param=openai_error_param(exc), code=answered_status)
def _catch_all_error_type(exc: Exception, status_code: int) -> str:

12
litellm/proxy/db/db_spend_update_writer.py
View file

@ -34,6 +34,7 @@ from litellm.constants import (
DB_SPEND_UPDATE_JOB_NAME,
INTERNAL_CALL_ORIGIN_METADATA_KEY,
)
from litellm.litellm_core_utils.core_helpers import finite_cost, single_log_line
from litellm.litellm_core_utils.litellm_logging import coerce_model_access_groups
from litellm.litellm_core_utils.safe_json_loads import safe_json_loads
from litellm.proxy._types import (
@ -652,7 +653,14 @@ class DBSpendUpdateWriter:
end_time=end_time,
llm_router=llm_router,
)
payload["spend"] = response_cost or 0.0
billable_cost: Final = finite_cost(response_cost)
if response_cost is not None and billable_cost is None:
verbose_proxy_logger.warning(
"Spend tracking - recording a non-finite response_cost=%s as 0 for request_id=%s",
response_cost,
single_log_line(payload["request_id"]),
)
payload["spend"] = billable_cost or 0.0
if isinstance(payload["startTime"], datetime):
payload["startTime"] = payload["startTime"].isoformat()
if isinstance(payload["endTime"], datetime):
@ -702,7 +710,7 @@ class DBSpendUpdateWriter:
spawn_logging_task(
asyncio.get_running_loop(),
self._batch_database_updates(
response_cost=response_cost,
response_cost=billable_cost,
user_id=user_id,
hashed_token=hashed_token,
team_id=team_id,

24
litellm/proxy/db/db_transaction_queue/redis_update_buffer.py
View file

@ -8,7 +8,7 @@ import asyncio
import json
from collections.abc import Mapping, Sequence
from datetime import datetime
from functools import reduce
from functools import partial, reduce
from itertools import chain
from typing import TYPE_CHECKING, Any, Final, Literal, TypeAlias, TypeVar, cast
@ -109,6 +109,16 @@ _ValueT = TypeVar("_ValueT")
_DECODED_JSON: Final = TypeAdapter(object)
def _zero_for_non_finite_constant(constant: str) -> float:
verbose_proxy_logger.warning(
"Redis spend buffer - reading a parked %s spend as 0, the spend it held is lost", constant
)
return 0.0
_load_buffered_transaction: Final = partial(json.loads, parse_constant=_zero_for_non_finite_constant)
def _spend_log_json_default(value: object) -> str:
return value.isoformat() if isinstance(value, datetime) else str(value)
@ -118,7 +128,7 @@ def _encode_spend_log_row(row: SpendLogRow) -> str:
def _decode_spend_log_row(encoded: str) -> dict[str, object] | None:
decoded: Final = _DECODED_JSON.validate_python(json.loads(encoded))
decoded: Final = _DECODED_JSON.validate_python(_load_buffered_transaction(encoded))
return decoded if isinstance(decoded, dict) else None
@ -825,13 +835,13 @@ class RedisUpdateBuffer:
if slot_result is None:
daily_results.append(None)
else:
list_of_daily = [json.loads(t) for t in slot_result]
list_of_daily = [_load_buffered_transaction(t) for t in slot_result]
aggregated = DailySpendUpdateQueue.get_aggregated_daily_spend_update_transactions(list_of_daily)
daily_results.append(aggregated)
window_spend: Final = (
WindowSpendUpdateQueue.get_aggregated_window_spend_transactions(
tuple(json.loads(transaction) for transaction in raw_results[6])
tuple(_load_buffered_transaction(transaction) for transaction in raw_results[6])
)
if raw_results[6] is not None
else None
@ -880,7 +890,7 @@ class RedisUpdateBuffer:
)
if list_of_transactions is None:
return None
return [json.loads(transaction) for transaction in list_of_transactions]
return [_load_buffered_transaction(transaction) for transaction in list_of_transactions]
async def get_all_daily_spend_update_transactions_from_redis_buffer(
self,
@ -1010,9 +1020,9 @@ class RedisUpdateBuffer:
Parses the list of transactions from Redis
"""
if isinstance(list_of_transactions, list):
return [json.loads(transaction) for transaction in list_of_transactions]
return [_load_buffered_transaction(transaction) for transaction in list_of_transactions]
else:
return [json.loads(list_of_transactions)]
return [_load_buffered_transaction(list_of_transactions)]
@staticmethod
def _combine_list_of_transactions(

3
litellm/proxy/guardrails/guardrail_endpoints.py
View file

@ -22,6 +22,7 @@ from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
from litellm.proxy.common_utils.callback_utils import CALLBACK_VAR_ENCRYPTED_PREFIX
from litellm.proxy.common_utils.openai_error_payload import status_typed_proxy_exception
from litellm.proxy.common_utils.path_utils import is_within, safe_join
from litellm.proxy.guardrails.content_filter_data import CATEGORIES_DIR, DATA_ROOTS, category_dirs, find_category_file
from litellm.proxy.guardrails.guardrail_hooks.custom_code.bounded_execution import (
@ -2533,7 +2534,7 @@ async def apply_guardrail(
e = transformed_exception
except Exception:
verbose_proxy_logger.exception("apply_guardrail: post_call_failure_hook failed")
raise handle_exception_on_proxy(e)
raise handle_exception_on_proxy(status_typed_proxy_exception(e) if isinstance(e, HTTPException) else e)
# Success logging outside except so a hook error never triggers failure handlers.
response = await _emit_guardrail_success_logs(

427
litellm/proxy/health_endpoints/_health_endpoints.py
View file

@ -7,13 +7,15 @@ import secrets
import time
import traceback
from collections.abc import Iterable, Mapping, Sequence
from dataclasses import dataclass
from datetime import datetime, timedelta, timezone
from itertools import chain, groupby
from types import MappingProxyType
from typing import Final, Literal, TypedDict
import fastapi
from fastapi import APIRouter, Depends, HTTPException, Request, Response, status
from pydantic import TypeAdapter
from pydantic import BaseModel, ConfigDict, TypeAdapter
from typing_extensions import ReadOnly
import litellm
@ -31,7 +33,9 @@ from litellm.proxy._types import (
CallInfo,
EnterpriseLicenseData,
Litellm_EntityType,
LiteLLM_UserTable,
LitellmUserRoles,
ModelAccessDeniedProxyException,
ProxyException,
SpecialModelNames,
UserAPIKeyAuth,
@ -39,13 +43,28 @@ from litellm.proxy._types import (
)
from litellm.proxy.auth.auth_checks import (
_resolve_key_models_for_auth_check, # pyright: ignore[reportPrivateUsage] # the auth layer's sentinel resolution, reused so /health scopes exactly like a request
agent_model_grants,
can_agent_call_model,
can_key_call_resolved_model,
can_user_call_model,
get_user_object,
key_model_aliases_for_auth_check,
model_matches_any_wildcard_pattern_in_list,
team_model_aliases_for_auth_check,
)
from litellm.proxy.auth.auth_exception_handler import (
_as_proxy_exception, # pyright: ignore[reportPrivateUsage] # one auth-failure mapping, shared with user_api_key_auth
)
from litellm.proxy.auth.auth_utils import (
_BANNED_REQUEST_BODY_PARAMS, # pyright: ignore[reportPrivateUsage] # one canonical list, shared with the request-body check
)
from litellm.proxy.auth.model_checks import get_key_models
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
from litellm.proxy.common_utils import http_parsing_utils
from litellm.proxy.common_utils.model_listing_utils import (
CallerAliases,
dispatched_model,
requestable_alias_names,
)
from litellm.proxy.common_utils.openai_error_payload import proxy_exception_for
from litellm.proxy.db.db_lookup_gate import db_lookup_stall_tracker
from litellm.proxy.db.exception_handler import PrismaDBExceptionHandler
@ -79,6 +98,7 @@ from litellm.router_utils.clientside_credential_handler import (
_ADMIN_CONFIG_FIELDS_TO_CLEAR_ON_BASE_OVERRIDE, # pyright: ignore[reportPrivateUsage] # one canonical list, shared with the router path
clientside_credential_keys,
)
from litellm.router_utils.common_utils import resolve_model_group_alias
from litellm.secret_managers.main import get_secret_bool
from litellm.types.proxy.litellm_pre_call_utils import RedactedDict, SecretFields
from litellm.types.router import reject_server_owned_wif_fields
@ -253,6 +273,28 @@ def get_callback_identifier(callback):
router: Final = APIRouter()
_OBJECT_MAPPING: Final = TypeAdapter(Mapping[str, object])
_OBJECTS: Final = TypeAdapter(tuple[object, ...])
_DEPLOYMENTS: Final = TypeAdapter(tuple[Mapping[str, object], ...])
class _ModelGrants(BaseModel):
model_config = ConfigDict(frozen=True, from_attributes=True)
models: tuple[object, ...] = ()
class _ModelInfoAccessGroups(BaseModel):
model_config = ConfigDict(frozen=True)
access_groups: tuple[object, ...]
class _DeploymentAccessGroups(BaseModel):
model_config = ConfigDict(frozen=True)
model_info: _ModelInfoAccessGroups
services = (
Literal[
"slack_budget_alerts",
@ -1045,42 +1087,339 @@ def _strip_admin_only_fields_from_health_result(result: dict) -> dict:
return out
def _health_accessible_model_names(
user_api_key_dict: UserAPIKeyAuth, llm_router: Router | None
) -> frozenset[str] | None:
"""Model names the caller may health-check, or None when the key is unrestricted."""
def _key_models_unrestricted(user_api_key_dict: UserAPIKeyAuth) -> bool:
granted_models: Final = _resolve_key_models_for_auth_check(user_api_key_dict)
if not granted_models or SpecialModelNames.all_proxy_models.value in granted_models:
return not granted_models or SpecialModelNames.all_proxy_models.value in granted_models
def _own_team_names(deployment: object, team_id: str | None) -> tuple[str, str] | None:
match deployment:
case {
"model_name": str(internal_name),
"model_info": {"team_id": str(owner_team_id), "team_public_model_name": str(public_name)},
} if owner_team_id == team_id:
return (internal_name, public_name)
case _:
return None
def _names_caller_may_request(deployment: object, team_id: str | None) -> frozenset[str]:
team_names: Final = _own_team_names(deployment, team_id)
if team_names is not None:
return frozenset(team_names)
match deployment:
case {"model_name": str(name)}:
return frozenset((name,))
case _:
return frozenset()
def _public_names_by_internal_name(llm_model_list: Sequence[object], team_id: str | None) -> Mapping[str, str]:
team_names: Final = (_own_team_names(deployment, team_id) for deployment in llm_model_list)
return MappingProxyType(dict(names for names in team_names if names is not None))
async def _key_owner(user_api_key_dict: UserAPIKeyAuth) -> LiteLLM_UserTable | None:
if user_api_key_dict.team_id is not None or user_api_key_dict.user_id is None:
return None
from litellm.proxy.proxy_server import prisma_client, proxy_logging_obj, user_api_key_cache
try:
return await get_user_object(
user_id=user_api_key_dict.user_id,
prisma_client=prisma_client,
user_api_key_cache=user_api_key_cache,
user_id_upsert=False,
parent_otel_span=user_api_key_dict.parent_otel_span,
proxy_logging_obj=proxy_logging_obj,
)
except Exception: # noqa: BLE001 # auth reads an unloadable user as no user limit, so /health does too
return None
async def _key_may_call_model(
model_name: str,
user_api_key_dict: UserAPIKeyAuth,
owner: LiteLLM_UserTable | None,
llm_router: Router | None,
) -> bool:
from litellm.proxy.proxy_server import prisma_client, proxy_logging_obj, user_api_key_cache
await asyncio.sleep(0)
try:
await can_key_call_resolved_model(
model=model_name, llm_model_list=None, valid_token=user_api_key_dict, llm_router=llm_router
)
await can_agent_call_model(
model=model_name,
valid_token=user_api_key_dict,
llm_router=llm_router,
prisma_client=prisma_client,
user_api_key_cache=user_api_key_cache,
proxy_logging_obj=proxy_logging_obj,
)
await can_user_call_model(
model=model_name,
llm_router=llm_router,
user_object=owner,
key_model_aliases=key_model_aliases_for_auth_check(user_api_key_dict),
)
except ModelAccessDeniedProxyException:
return False
except ProxyException:
raise
except Exception as failure:
if isinstance(failure, HTTPException) and failure.status_code == status.HTTP_403_FORBIDDEN:
return False
raise _as_proxy_exception(failure) from failure
return True
async def _caller_aliases(user_api_key_dict: UserAPIKeyAuth) -> CallerAliases:
from litellm.proxy.proxy_server import prisma_client, proxy_config, proxy_logging_obj
router_settings: Final = await proxy_config.get_hierarchical_router_settings(
user_api_key_dict, prisma_client, proxy_logging_obj
)
return CallerAliases(
team_model_aliases_for_auth_check(user_api_key_dict),
key_model_aliases_for_auth_check(user_api_key_dict),
router_settings.get("model_group_alias") if router_settings is not None else None,
)
def _model_auth_checks(model_name: str, aliases: CallerAliases) -> str:
return resolve_model_group_alias(aliases.router, model_name) or model_name
def _deployment_id(deployment: object) -> str | None:
match deployment:
case {"model_info": {"id": str(model_id)}}:
return model_id
case _:
return None
def _granted_names(user_api_key_dict: UserAPIKeyAuth) -> frozenset[str]:
grants: Final = (
*_ModelGrants.model_validate(user_api_key_dict).models,
*_OBJECTS.validate_python(user_api_key_dict.team_models),
)
return frozenset(name for name in grants if isinstance(name, str))
async def _names_any_limit_grants(user_api_key_dict: UserAPIKeyAuth, owner: LiteLLM_UserTable | None) -> frozenset[str]:
owner_grants: Final = _ModelGrants.model_validate(owner).models if owner is not None else ()
return (
_granted_names(user_api_key_dict)
| frozenset(name for name in owner_grants if isinstance(name, str))
| await agent_model_grants(user_api_key_dict)
)
def _alias_names_caller_may_request(aliases: CallerAliases, llm_router: Router | None) -> frozenset[str]:
router_aliases: Final = frozenset(llm_router.registry().aliases()) if llm_router is not None else frozenset[str]()
return requestable_alias_names(aliases) | router_aliases
def _served_group(model_name: str, aliases: CallerAliases, llm_router: Router | None) -> str:
dispatched: Final = dispatched_model(model_name, aliases)
return llm_router.registry().resolve(dispatched) if llm_router is not None else dispatched
def _name_id_pairs(deployment: object, team_id: str | None) -> tuple[tuple[str, str], ...]:
model_id: Final = _deployment_id(deployment)
if model_id is None:
return ()
return tuple((name, model_id) for name in _names_caller_may_request(deployment, team_id) | {model_id})
def _ids_by_requested_name(llm_model_list: Sequence[object], team_id: str | None) -> Mapping[str, frozenset[str]]:
pairs: Final = sorted(chain.from_iterable(_name_id_pairs(deployment, team_id) for deployment in llm_model_list))
return MappingProxyType(
{name: frozenset(model_id for _, model_id in group) for name, group in groupby(pairs, key=lambda pair: pair[0])}
)
def _ids_of(deployments: Iterable[object]) -> frozenset[str]:
return frozenset(model_id for model_id in map(_deployment_id, deployments) if model_id is not None)
def _access_groups_of(deployment: object) -> frozenset[str]:
match deployment:
case {"model_info": {"access_groups": list()}}:
access_groups: Final = _DeploymentAccessGroups.model_validate(deployment).model_info.access_groups
return frozenset(access_group for access_group in access_groups if isinstance(access_group, str))
case _:
return frozenset()
@dataclass(frozen=True, slots=True)
class _AccessGroupGrants:
granted_names: frozenset[str]
ids_in_granted_groups: frozenset[str]
def requested_name(self, name: str, public_names: Mapping[str, str], admitted_names: frozenset[str]) -> str:
public_name: Final = public_names.get(name)
if public_name is None or public_name not in admitted_names or self._grants(name):
return name
return public_name
def _grants(self, name: str) -> bool:
return name in self.granted_names or model_matches_any_wildcard_pattern_in_list(name, tuple(self.granted_names))
def narrow(self, serving_ids: frozenset[str], served_names: frozenset[str]) -> frozenset[str]:
in_granted_groups: Final = serving_ids & self.ids_in_granted_groups
granted_directly: Final = not self.granted_names.isdisjoint(served_names)
return serving_ids if granted_directly or not in_granted_groups else in_granted_groups
def _access_group_grants(llm_model_list: Sequence[object], user_api_key_dict: UserAPIKeyAuth) -> _AccessGroupGrants:
granted: Final = _granted_names(user_api_key_dict)
in_granted_groups: Final = (
deployment for deployment in llm_model_list if not granted.isdisjoint(_access_groups_of(deployment))
)
return _AccessGroupGrants(granted, _ids_of(in_granted_groups))
def _ids_serving_group(
group: str,
ids_by_name: Mapping[str, frozenset[str]],
user_api_key_dict: UserAPIKeyAuth,
llm_router: Router | None,
access_group_grants: _AccessGroupGrants,
) -> frozenset[str]:
named: Final = ids_by_name.get(group)
if named is not None:
return access_group_grants.narrow(named, frozenset((group,)))
if llm_router is None:
return frozenset(granted_models)
registry: Final = llm_router.registry()
return frozenset()
team_id: Final = user_api_key_dict.team_id
wildcard_served: Final = llm_router.registry().deployments(group, source="wildcard", team_id=team_id)
served_names: Final = frozenset((group,)).union(
*(_names_caller_may_request(deployment, team_id) for deployment in wildcard_served)
)
return access_group_grants.narrow(_ids_of(wildcard_served), served_names)
async def _names_key_may_request(
names: Iterable[str],
aliases: CallerAliases,
user_api_key_dict: UserAPIKeyAuth,
owner: LiteLLM_UserTable | None,
llm_router: Router | None,
) -> frozenset[str]:
return frozenset(
get_key_models(
user_api_key_dict=user_api_key_dict,
proxy_model_list=registry.names(team_id=user_api_key_dict.team_id),
model_access_groups=registry.access_groups(),
[
name
for name in names
if await _key_may_call_model(_model_auth_checks(name, aliases), user_api_key_dict, owner, llm_router)
]
)
def _ids_serving(
groups: Iterable[str],
ids_by_name: Mapping[str, frozenset[str]],
user_api_key_dict: UserAPIKeyAuth,
llm_router: Router | None,
access_group_grants: _AccessGroupGrants,
) -> frozenset[str]:
return frozenset[str]().union(
*(
_ids_serving_group(group, ids_by_name, user_api_key_dict, llm_router, access_group_grants)
for group in groups
)
)
def _caller_may_probe_deployment(
deployment: Mapping[str, object],
allowed_models: frozenset[str] | None,
def _shadowed_ids(
llm_model_list: Sequence[object], admitted_names: frozenset[str], reached_ids: frozenset[str], team_id: str | None
) -> frozenset[str]:
ids_of_admitted_names: Final = (
_deployment_id(deployment)
for deployment in llm_model_list
if not admitted_names.isdisjoint(_names_caller_may_request(deployment, team_id))
)
return frozenset(model_id for model_id in ids_of_admitted_names if model_id is not None) - reached_ids
@dataclass(frozen=True, slots=True)
class _ReachableDeployments:
model_ids: frozenset[str]
groups: frozenset[str]
def include(self, deployment: object, team_id: str | None) -> bool:
model_id: Final = _deployment_id(deployment)
if model_id is None:
return not self.groups.isdisjoint(_names_caller_may_request(deployment, team_id))
return model_id in self.model_ids
_NOTHING_REACHABLE: Final = _ReachableDeployments(frozenset(), frozenset())
async def _deployments_key_may_reach(
llm_model_list: Sequence[object],
aliases: CallerAliases,
user_api_key_dict: UserAPIKeyAuth,
owner: LiteLLM_UserTable | None,
llm_router: Router | None,
) -> _ReachableDeployments:
candidate_names: Final = (await _names_any_limit_grants(user_api_key_dict, owner)).union(
_alias_names_caller_may_request(aliases, llm_router),
*(_names_caller_may_request(deployment, user_api_key_dict.team_id) for deployment in llm_model_list),
)
admitted_names: Final = await _names_key_may_request(candidate_names, aliases, user_api_key_dict, owner, llm_router)
access_group_grants: Final = _access_group_grants(llm_model_list, user_api_key_dict)
public_names: Final = _public_names_by_internal_name(llm_model_list, user_api_key_dict.team_id)
groups: Final = frozenset(
_served_group(access_group_grants.requested_name(name, public_names, admitted_names), aliases, llm_router)
for name in admitted_names
)
ids_by_name: Final = _ids_by_requested_name(llm_model_list, user_api_key_dict.team_id)
reached_ids: Final = _ids_serving(groups, ids_by_name, user_api_key_dict, llm_router, access_group_grants)
admitted_ids: Final = await _names_key_may_request(
_shadowed_ids(llm_model_list, admitted_names, reached_ids, user_api_key_dict.team_id),
aliases,
user_api_key_dict,
owner,
llm_router,
)
id_groups: Final = frozenset(_served_group(model_id, aliases, llm_router) for model_id in admitted_ids)
return _ReachableDeployments(
reached_ids | _ids_serving(id_groups, ids_by_name, user_api_key_dict, llm_router, access_group_grants), groups
)
def _model_param_target(
model: str,
aliases: CallerAliases,
llm_model_list: Sequence[Mapping[str, object]],
team_id: str | None,
caller_is_admin: bool,
llm_router: Router | None,
) -> str:
group: Final = _served_group(model, aliases, llm_router)
if llm_router is None or deployments_targeted_by_name(llm_model_list, group, team_id):
return group
patterns: Final = frozenset(
deployment.get("model_name")
for deployment in llm_router.registry().deployments(group, source="wildcard", team_id=team_id)
)
match tuple(patterns):
case (str(pattern),):
return pattern
case _:
return group
def _caller_may_probe_deployment(
deployment: Mapping[str, object], reachable: _ReachableDeployments, team_id: str | None, caller_is_admin: bool
) -> bool:
"""Same deployment visibility rule as routing: another team's deployment is never in scope, team-less callers included."""
if not caller_is_admin and not deployment_usable_by_team(deployment, team_id):
return False
if allowed_models is None:
return True
if llm_router is None:
return deployment.get("model_name") in allowed_models
model: Final = dict(deployment)
registry: Final = llm_router.registry()
return any(registry.includes(model, model_name=name, team_id=team_id) for name in allowed_models)
return reachable.include(deployment, team_id)
def _resolve_targeted_model_ids(
@ -1137,6 +1476,7 @@ def _filter_health_check_results_by_model_ids(results: dict, allowed_model_ids:
async def _perform_health_check_and_save(
model_list,
target_model,
recorded_model_name: str | None,
cli_model,
details,
prisma_client,
@ -1159,8 +1499,7 @@ async def _perform_health_check_and_save(
# Optionally save health check result to database (non-blocking)
if prisma_client is not None:
# For CLI model, use cli_model name; for router models, use target_model
model_name_for_db: Final = cli_model if cli_model is not None else target_model
model_name_for_db: Final = cli_model if cli_model is not None else recorded_model_name
if model_name_for_db is not None:
asyncio.create_task(
_save_health_check_to_db(
@ -1290,6 +1629,7 @@ async def health_endpoint(
cli_result: Final = await _perform_health_check_and_save(
model_list=[],
target_model=None,
recorded_model_name=None,
cli_model=user_model,
details=health_check_details,
prisma_client=prisma_client,
@ -1304,15 +1644,39 @@ async def health_endpoint(
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
detail={"error": "Model list not initialized"},
)
allowed_models: Final = _health_accessible_model_names(user_api_key_dict, llm_router)
restrict_to_allowed_models: Final = not is_admin or allowed_models is not None
restrict_to_allowed_models: Final = not is_admin or not _key_models_unrestricted(user_api_key_dict)
aliases: Final = await _caller_aliases(user_api_key_dict) if restrict_to_allowed_models else None
reachable: Final = (
await _deployments_key_may_reach(
_OBJECTS.validate_python(llm_model_list),
aliases,
user_api_key_dict,
await _key_owner(user_api_key_dict),
llm_router,
)
if aliases is not None
else _NOTHING_REACHABLE
)
requested_model: Final = (
_model_param_target(
model,
aliases,
_DEPLOYMENTS.validate_python(llm_model_list),
user_api_key_dict.team_id,
llm_router,
)
if model is not None and aliases is not None
else model
)
_llm_model_list: Final = [
m
for m in copy.deepcopy(llm_model_list)
if not restrict_to_allowed_models
or _caller_may_probe_deployment(m, allowed_models, llm_router, user_api_key_dict.team_id, is_admin)
or _caller_may_probe_deployment(m, reachable, user_api_key_dict.team_id, is_admin)
]
targeted_ids: Final = _resolve_targeted_model_ids(_llm_model_list, model, model_id, user_api_key_dict.team_id)
targeted_ids: Final = _resolve_targeted_model_ids(
_llm_model_list, requested_model, model_id, user_api_key_dict.team_id
)
if restrict_to_allowed_models and targeted_ids is not None and not targeted_ids:
raise HTTPException(
status_code=status.HTTP_403_FORBIDDEN,
@ -1370,7 +1734,8 @@ async def health_endpoint(
else:
router_result: Final = await _perform_health_check_and_save(
model_list=_llm_model_list,
target_model=target_model,
target_model=target_model if model_id else requested_model,
recorded_model_name=target_model,
cli_model=None,
details=health_check_details,
prisma_client=prisma_client,

3
litellm/proxy/hooks/proxy_track_cost_callback.py
View file

@ -16,6 +16,7 @@ from litellm.constants import BACKGROUND_INTERACTION_COST_POLLING_ENABLED, BILLE
from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.core_helpers import (
budget_reservation_from_metadata,
finite_cost,
get_litellm_metadata_from_kwargs,
get_metadata_variable_name_from_kwargs,
get_parent_otel_span_from_kwargs,
@ -703,7 +704,7 @@ class _ProxyDBLogger(CustomLogger):
raw_cost: Final = _str_keyed_fields(
sl_object if sl_object is not None else kwargs # pyright: ignore[reportUnknownArgumentType] # untyped success-callback payload
).get("response_cost")
priced_cost: Final = raw_cost if isinstance(raw_cost, (int, float)) else None
priced_cost: Final = finite_cost(raw_cost)
response_cost: Final = (
await apply_search_free_tier(
kwargs=kwargs, # pyright: ignore[reportUnknownArgumentType] # untyped success-callback payload

6
litellm/proxy/management_endpoints/daily_activity_scopes.py
View file

@ -216,7 +216,11 @@ def _user_query(
start_date: str | None = Query(default=None, description="Start date in YYYY-MM-DD format"),
end_date: str | None = Query(default=None, description="End date in YYYY-MM-DD format"),
model: str | None = Query(default=None, description="Filter by specific model"),
api_key: str | None = Query(default=None, description="Filter by specific API key"),
api_key: str | None = Query(
default=None,
description="Filter by reporting API key ID. Missing keys are reported as unattributed. "
"For a literal key unattributed or a key starting with raw:, add an extra raw: prefix",
),
user_id: str | None = Query(
default=None,
description="Filter by specific user ID. Admins can filter by any user or omit for global view. "

3
litellm/proxy/management_endpoints/internal_user_endpoints.py
View file

@ -2922,7 +2922,8 @@ async def get_user_daily_activity(
),
api_key: str | None = fastapi.Query(
default=None,
description="Filter by specific API key",
description="Filter by reporting API key ID. Missing keys are reported as unattributed. "
"For a literal key unattributed or a key starting with raw:, add an extra raw: prefix",
),
user_id: str | None = fastapi.Query(
default=None,

15
litellm/proxy/management_endpoints/key_management_endpoints.py
View file

@ -50,6 +50,7 @@ from litellm.proxy.auth.auth_checks import ( # noqa: F401 # legacy module expo
_delete_cache_key_object, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export
can_team_access_model,
delete_cache_key_object,
delete_cache_key_objects,
get_jwt_key_mapping_cache_keys_for_token,
get_key_end_user_budget_id,
get_org_object,
@ -78,7 +79,7 @@ from litellm.proxy.common_utils.config_sync_pubsub import (
from litellm.proxy.common_utils.openai_error_payload import openai_error_param, proxy_exception_for
from litellm.proxy.common_utils.rbac_utils import check_org_admin_can_generate_keys
from litellm.proxy.common_utils.timezone_utils import get_budget_reset_time
from litellm.proxy.common_utils.user_api_key_cache import AUTH_OBJECTS_TARGET, UserApiKeyCache
from litellm.proxy.common_utils.user_api_key_cache import AUTH_OBJECTS_TARGET, UserApiKeyCache, is_user_key_cache_key
from litellm.proxy.hooks.key_management_event_hooks import KeyManagementEventHooks
from litellm.proxy.hooks.model_max_budget_limiter import build_model_max_budget_usage
from litellm.proxy.management.teams.authz import TEAM_ADMIN_ONLY, TEAM_OR_ORG_ADMIN, is_team_admin
@ -5483,7 +5484,7 @@ async def delete_verification_tokens(
- List of keys being deleted, this contains information about the key_alias, token, and user_id being deleted,
this is passed down to the KeyManagementEventHooks to delete the keys from the secret manager and handle audit logs
"""
from litellm.proxy.proxy_server import prisma_client
from litellm.proxy.proxy_server import prisma_client, proxy_logging_obj
failed_tokens: list = []
try:
@ -5567,11 +5568,11 @@ async def delete_verification_tokens(
verbose_proxy_logger.debug(traceback.format_exc())
raise e
for key in tokens:
user_api_key_cache.delete_cache(key)
# remove hash token from cache
hashed_token = hash_token(cast(str, key))
user_api_key_cache.delete_cache(hashed_token)
await delete_cache_key_objects(
hashed_tokens=tuple(token for token in hashed_tokens if is_user_key_cache_key(token)),
user_api_key_cache=user_api_key_cache,
proxy_logging_obj=proxy_logging_obj,
)
# After credential invalidation, so a failure here can never keep a deleted key alive.
for deleted_key in authorized_keys:

15
litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py
View file

@ -32,6 +32,7 @@ from typing_extensions import ReadOnly, TypedDict
import litellm
from litellm import get_llm_provider
from litellm._logging import verbose_proxy_logger
from litellm._uuid import uuid
from litellm.constants import (
ALLOWED_VERTEX_AI_PASSTHROUGH_HEADERS,
AWS_ERROR_TYPE_HEADER,
@ -72,6 +73,7 @@ from litellm.llms.openai.common_utils import is_openai_agents_api_path, openai_a
from litellm.llms.openai.workload_identity import get_workload_identity_bearer_token_for_api_base
from litellm.llms.oss_decision import OssDecisionProvider, oss_connection, validate_oss_request
from litellm.llms.vertex_ai.vertex_llm_base import VertexBase
from litellm.llms.vllm.common_utils import endpoint_under_api_base
from litellm.passthrough.main import AsyncPassthroughStreamingResponse, is_provider_error
from litellm.proxy._types import *
from litellm.proxy.auth.auth_checks import enforced_model_allowlists
@ -148,7 +150,7 @@ from litellm.types.passthrough_endpoints.tinyfish import (
)
from litellm.types.passthrough_endpoints.vertex_ai import VertexPassThroughCredentials
from litellm.types.router import LiteLLMParamsTypedDict
from litellm.types.utils import LlmProviders
from litellm.types.utils import CallTypes, LlmProviders
from litellm.types.vector_stores import LiteLLM_ManagedVectorStore
from litellm.utils import ProviderConfigManager
@ -600,7 +602,7 @@ async def vllm_proxy_route(
)
return await llm_passthrough_factory_proxy_route(
endpoint=endpoint,
endpoint=endpoint_under_api_base(endpoint, get_secret_str("VLLM_API_BASE")),
request=request,
fastapi_response=fastapi_response,
user_api_key_dict=user_api_key_dict,
@ -4104,7 +4106,12 @@ async def vertex_ai_live_websocket_passthrough(
await proxy_logging_obj.post_call_failure_hook(
user_api_key_dict=user_api_key_dict,
original_exception=e,
request_data={},
request_data={
"model": model or "",
"custom_llm_provider": litellm.LlmProviders.VERTEX_AI.value,
"call_type": CallTypes.pass_through.value,
"litellm_call_id": str(uuid.uuid4()),
},
)
if websocket.client_state != WebSocketState.DISCONNECTED:
await websocket.close(code=1011, reason=VERTEX_LIVE_UNCONFIGURED_CLOSE_REASON)
@ -4139,6 +4146,8 @@ async def vertex_ai_live_websocket_passthrough(
forward_headers=False,
endpoint="/vertex_ai/live",
accept_websocket=False,
builtin_provider_route=True,
model=model,
setup_model_rewriter=_build_vertex_live_setup_model_rewriter(
vertex_project=resolved_project,
vertex_location=resolved_location,

8
litellm/proxy/pass_through_endpoints/llm_provider_handlers/gemini_passthrough_logging_handler.py
View file

@ -1,4 +1,3 @@
import re
from collections.abc import Sequence
from datetime import datetime
from typing import TYPE_CHECKING, Any, Final
@ -9,6 +8,7 @@ import litellm
from litellm._logging import verbose_proxy_logger
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.llms.gemini.videos.transformation import GeminiVideoConfig
from litellm.llms.vertex_ai.common_utils import get_vertex_model_id_from_url
from litellm.proxy._types import PassThroughEndpointLoggingTypedDict
from litellm.proxy.pass_through_endpoints.llm_provider_handlers.vertex_passthrough_logging_handler import (
GeminiSseChunkParser,
@ -207,11 +207,7 @@ class GeminiPassthroughLoggingHandler:
@staticmethod
def extract_model_from_url(url: str) -> str:
pattern: Final = r"/models/([^:]+)"
match: Final = re.search(pattern, url)
if match:
return match.group(1)
return "unknown"
return get_vertex_model_id_from_url(url) or "unknown"
@staticmethod
def _create_gemini_response_logging_payload_for_generate_content(

63
litellm/proxy/pass_through_endpoints/llm_provider_handlers/vertex_passthrough_logging_handler.py
View file

@ -3,7 +3,7 @@ import re
from collections.abc import Mapping, Sequence
from datetime import datetime
from typing import TYPE_CHECKING, Any, Final, Literal, Protocol, cast
from urllib.parse import urlparse
from urllib.parse import parse_qs, urlparse
import httpx
from pydantic import ConfigDict, TypeAdapter
@ -19,6 +19,7 @@ from litellm.litellm_core_utils.llm_cost_calc.usage_object_transformation import
from litellm.llms.vertex_ai.common_utils import (
get_vertex_ai_lyria_generation_cost,
get_vertex_location_from_url,
get_vertex_model_id_from_url,
)
from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import (
ModelResponseIterator as VertexModelResponseIterator,
@ -47,6 +48,19 @@ from litellm.types.utils import (
)
vertex_search_api_config: Final = VertexSearchAPIVectorStoreConfig()
GEMINI_API_HOST_SUFFIX: Final = "generativelanguage.googleapis.com"
GOOGLE_AI_HOST_SUFFIXES: Final = (
GEMINI_API_HOST_SUFFIX,
"aiplatform.googleapis.com",
"discoveryengine.googleapis.com",
)
_VERTEX_MULTI_REGION_HOST: Final = re.compile(r"aiplatform\.[a-z0-9-]+\.rep\.googleapis\.com")
def _is_host_or_subdomain(hostname: str, suffix: str) -> bool:
return hostname == suffix or hostname.endswith((f".{suffix}", f"-{suffix}"))
if TYPE_CHECKING:
from litellm.types.utils import LiteLLMBatch
@ -684,11 +698,18 @@ class VertexPassthroughLoggingHandler:
@staticmethod
def extract_model_from_url(url: str) -> str:
pattern: Final = r"/models/([^:]+)"
match: Final = re.search(pattern, url)
if match:
return match.group(1)
return "unknown"
return get_vertex_model_id_from_url(url) or "unknown"
@staticmethod
def model_from_url_route(url_route: str) -> str | None:
parsed_url: Final = urlparse(url_route)
model: Final = get_vertex_model_id_from_url(parsed_url.path)
if model is None:
return "vertex_ai/search_api" if ":search" in parsed_url.path else None
streamed: Final = "sse" in parse_qs(parsed_url.query).get("alt", ())
if parsed_url.path.endswith((":rawPredict", ":streamRawPredict")) and not streamed:
return f"vertex_ai/{model}"
return model
@staticmethod
def extract_model_name_from_vertex_path(vertex_model_path: str) -> str:
@ -734,18 +755,32 @@ class VertexPassthroughLoggingHandler:
return None
@staticmethod
def _get_custom_llm_provider(url: str, logging_obj: LiteLLMLoggingObj) -> str:
def is_google_ai_url(url: str) -> bool:
hostname: Final = urlparse(url).hostname
if logging_obj.model_call_details.get("custom_llm_provider") == litellm.LlmProviders.GEMINI.value or (
hostname is not None
and (
hostname == "generativelanguage.googleapis.com"
or hostname.endswith(".generativelanguage.googleapis.com")
)
):
return hostname is not None and (
any(_is_host_or_subdomain(hostname, suffix) for suffix in GOOGLE_AI_HOST_SUFFIXES)
or _VERTEX_MULTI_REGION_HOST.fullmatch(hostname) is not None
)
@staticmethod
def is_gemini_api_host(url: str) -> bool:
hostname: Final = urlparse(url).hostname
return hostname is not None and (
hostname == GEMINI_API_HOST_SUFFIX or hostname.endswith(f".{GEMINI_API_HOST_SUFFIX}")
)
@staticmethod
def custom_llm_provider_from_url(url: str) -> str:
if VertexPassthroughLoggingHandler.is_gemini_api_host(url):
return litellm.LlmProviders.GEMINI.value
return litellm.LlmProviders.VERTEX_AI.value
@staticmethod
def _get_custom_llm_provider(url: str, logging_obj: LiteLLMLoggingObj) -> str:
if logging_obj.model_call_details.get("custom_llm_provider") == litellm.LlmProviders.GEMINI.value:
return litellm.LlmProviders.GEMINI.value
return VertexPassthroughLoggingHandler.custom_llm_provider_from_url(url)
@staticmethod
def _is_multimodal_embedding_response(json_response: dict) -> bool:
"""

136
litellm/proxy/pass_through_endpoints/pass_through_endpoints.py
View file

@ -3,6 +3,7 @@ import asyncio
import copy
import json
import posixpath
import re
import traceback
from base64 import b64encode
from collections.abc import AsyncGenerator, AsyncIterator, Callable, Iterable, Mapping, Sequence
@ -77,6 +78,7 @@ from litellm.llms.base_llm.managed_resources.utils import (
from litellm.llms.custom_httpx.http_handler import get_async_httpx_client
from litellm.llms.eu_jev.common_utils import eu_jev_scoped_model, eu_jev_upstream_body
from litellm.llms.oss_decision import validate_oss_request
from litellm.llms.vertex_ai.common_utils import get_vertex_model_id_from_url
from litellm.passthrough import BasePassthroughUtils
from litellm.proxy._lazy_features import lazy_owned_routes
from litellm.proxy._types import (
@ -147,11 +149,12 @@ from litellm.types.passthrough_endpoints.pass_through_endpoints import (
EndpointType,
PassthroughStandardLoggingPayload,
)
from litellm.types.utils import TRUSTED_CALLBACK_VARS_FIELD, Usage
from litellm.types.utils import TRUSTED_CALLBACK_VARS_FIELD, CallTypes, Usage
from .llm_provider_handlers.tinyfish_passthrough_logging_handler import (
is_tinyfish_agent_url,
)
from .llm_provider_handlers.vertex_passthrough_logging_handler import VertexPassthroughLoggingHandler
from .streaming_handler import PassThroughStreamingHandler
from .success_handler import PassThroughEndpointLogging
from .upstream_usage_headers import (
@ -874,13 +877,45 @@ def _carry_guardrail_logging_info(request_data: dict, guardrail_data: dict | Non
metadata.setdefault("standard_logging_guardrail_information", list(entries))
_GOOGLE_PASSTHROUGH_PROVIDERS: Final = frozenset(
(litellm.LlmProviders.GEMINI.value, litellm.LlmProviders.VERTEX_AI.value)
)
def _passthrough_google_provider(url_route: str, custom_llm_provider: str | None) -> str | None:
if custom_llm_provider is not None:
return custom_llm_provider if custom_llm_provider in _GOOGLE_PASSTHROUGH_PROVIDERS else None
if VertexPassthroughLoggingHandler.is_google_ai_url(url_route):
return VertexPassthroughLoggingHandler.custom_llm_provider_from_url(url_route)
return None
_OPERATION_POLL_SUFFIX: Final = re.compile(r"/operations/[^/]+$")
def _without_operation_poll_suffix(model: str | None) -> str | None:
return _OPERATION_POLL_SUFFIX.sub("", model) if model else model
def _passthrough_google_model(url_route: str, google_provider: str | None) -> str | None:
match google_provider:
case "gemini":
return _without_operation_poll_suffix(get_vertex_model_id_from_url(urlparse(url_route).path))
case "vertex_ai":
return _without_operation_poll_suffix(VertexPassthroughLoggingHandler.model_from_url_route(url_route))
case _:
return None
def _build_passthrough_failure_request_payload(
parsed_body: dict | None,
kwargs: dict | None,
logging_obj: LiteLLMLoggingObj | None,
custom_llm_provider: str | None,
upstream_usage: UpstreamReportedUsage | None = None,
url: httpx.URL | None = None,
upstream_model: str | None = None,
builtin_provider_route: bool = False,
) -> dict:
"""Build the ``request_data`` dict passed to ``post_call_failure_hook``.
@ -893,16 +928,25 @@ def _build_passthrough_failure_request_payload(
comes paired with a usage object, so both keys are written together.
``upstream_model`` is the model name the upstream reported, which replaces
the body's ``model`` on the failure row just as it does on a success row.
``builtin_provider_route`` lets a Gemini or Vertex URL name the failure's
model and provider. A user-defined route never gets that, since its URL
names an upstream model rather than a LiteLLM-managed one.
"""
request_payload: Final[dict] = dict(cast(Mapping[str, object], parsed_body or {})) # cast-ok: json body
if kwargs:
request_payload.update(kwargs)
if logging_obj is not None:
request_payload["litellm_logging_obj"] = logging_obj
if "model" not in request_payload and parsed_body and isinstance(parsed_body, dict):
request_payload["model"] = parsed_body.get("model", "")
if "custom_llm_provider" not in request_payload and custom_llm_provider:
request_payload["custom_llm_provider"] = custom_llm_provider
url_route: Final = str(url) if url is not None else ""
google_provider: Final = (
_passthrough_google_provider(url_route, custom_llm_provider) if builtin_provider_route else None
)
if "model" not in request_payload:
request_payload["model"] = _passthrough_google_model(url_route, google_provider) or ""
resolved_provider: Final = custom_llm_provider or google_provider
request_payload.pop("custom_llm_provider", None)
if resolved_provider:
request_payload["custom_llm_provider"] = resolved_provider
if upstream_usage is not None:
request_payload["response_cost"] = upstream_usage.response_cost or 0.0
request_payload["combined_usage_object"] = Usage(total_tokens=upstream_usage.total_tokens or 0)
@ -1203,6 +1247,7 @@ async def pass_through_request(
# Initialize variables
#########################################################
litellm_call_id: Final = str(uuid.uuid4())
builtin_provider_route: Final = not request_dispatched_to_pass_through_endpoint(request)
url: httpx.URL | None = None
# parsed request body
@ -1612,6 +1657,8 @@ async def pass_through_request(
logging_obj=logging_obj,
custom_llm_provider=custom_llm_provider,
upstream_usage=upstream_usage,
url=url,
builtin_provider_route=builtin_provider_route,
upstream_model=upstream_model,
),
logging_obj=logging_obj,
@ -1651,6 +1698,8 @@ async def pass_through_request(
kwargs=kwargs,
logging_obj=logging_obj,
custom_llm_provider=custom_llm_provider,
url=url,
builtin_provider_route=builtin_provider_route,
),
),
managed_id_provider=_managed_id_provider,
@ -1710,6 +1759,8 @@ async def pass_through_request(
logging_obj=logging_obj,
custom_llm_provider=custom_llm_provider,
upstream_usage=upstream_usage,
url=url,
builtin_provider_route=builtin_provider_route,
upstream_model=upstream_model,
),
logging_obj=logging_obj,
@ -1749,6 +1800,8 @@ async def pass_through_request(
kwargs=kwargs,
logging_obj=logging_obj,
custom_llm_provider=custom_llm_provider,
url=url,
builtin_provider_route=builtin_provider_route,
),
),
managed_id_provider=_managed_id_provider,
@ -1811,6 +1864,8 @@ async def pass_through_request(
logging_obj=logging_obj,
custom_llm_provider=custom_llm_provider,
upstream_usage=upstream_usage,
url=url,
builtin_provider_route=builtin_provider_route,
upstream_model=upstream_model,
)
failure_request_payload["response_body"] = response_body
@ -1991,7 +2046,14 @@ async def pass_through_request(
await proxy_logging_obj.post_call_failure_hook(
user_api_key_dict=user_api_key_dict,
original_exception=e,
request_data=e.request_data,
request_data=_build_passthrough_failure_request_payload(
parsed_body=e.request_data,
kwargs={"call_type": CallTypes.pass_through.value, "litellm_call_id": litellm_call_id},
logging_obj=None,
custom_llm_provider=custom_llm_provider,
url=url,
builtin_provider_route=builtin_provider_route,
),
)
except Exception:
verbose_proxy_logger.warning(
@ -2010,6 +2072,7 @@ async def pass_through_request(
content=json.dumps(error_body),
status_code=200,
media_type="application/json",
headers={"x-litellm-call-id": litellm_call_id},
)
except Exception as e:
custom_headers = ProxyBaseLLMRequestProcessing.get_custom_headers(
@ -2033,19 +2096,14 @@ async def pass_through_request(
# Monitoring: Trigger post_call_failure_hook
# for pass through endpoint failure
#########################################################
request_payload: Final[dict] = _parsed_body or {}
# add user_api_key_dict, litellm_call_id, passthrough_logging_payloa for logging
if kwargs:
for key, value in kwargs.items():
request_payload[key] = value
if logging_obj is not None:
request_payload["litellm_logging_obj"] = logging_obj
if "model" not in request_payload and _parsed_body and isinstance(_parsed_body, dict):
request_payload["model"] = _parsed_body.get("model", "")
if "custom_llm_provider" not in request_payload and custom_llm_provider:
request_payload["custom_llm_provider"] = custom_llm_provider
request_payload: Final = _build_passthrough_failure_request_payload(
parsed_body=_parsed_body,
kwargs=kwargs,
logging_obj=logging_obj,
custom_llm_provider=custom_llm_provider,
url=url,
builtin_provider_route=builtin_provider_route,
)
_carry_guardrail_logging_info(request_payload, post_call_guardrail_data)
await proxy_logging_obj.post_call_failure_hook(
@ -2547,6 +2605,8 @@ async def websocket_passthrough_request(
cost_per_request: float | None = None,
accept_websocket: bool = True,
setup_model_rewriter: Callable[[str], str] | None = None,
builtin_provider_route: bool = False,
model: str | None = None,
):
"""
WebSocket passthrough request handler.
@ -2560,6 +2620,8 @@ async def websocket_passthrough_request(
endpoint: The endpoint path (for logging purposes)
cost_per_request: Optional field - cost per request to the target endpoint
setup_model_rewriter: Optional rewrite of the setup frame's model before it reaches the upstream
builtin_provider_route: Whether a built-in provider route serves the connection, so its URL names the provider
model: The model the caller named before connecting, logged on a failure before the setup frame arrives
"""
from litellm.litellm_core_utils.litellm_logging import Logging
from litellm.proxy.proxy_server import proxy_config, proxy_logging_obj
@ -2912,19 +2974,17 @@ async def websocket_passthrough_request(
except InvalidStatus as exc:
verbose_proxy_logger.exception("WebSocket passthrough (%s): upstream rejected WebSocket connection", endpoint)
# Prepare request payload for logging
request_payload = {}
if kwargs:
for key, value in kwargs.items():
request_payload[key] = value
if logging_obj is not None:
request_payload["litellm_logging_obj"] = logging_obj
# Log the connection failure using the same pattern as HTTP
await proxy_logging_obj.post_call_failure_hook(
user_api_key_dict=user_api_key_dict,
original_exception=exc,
request_data=request_payload,
request_data=_build_passthrough_failure_request_payload(
parsed_body=None if model is None else {"model": model},
kwargs=kwargs,
logging_obj=logging_obj,
custom_llm_provider=None,
url=httpx.URL(target),
builtin_provider_route=builtin_provider_route,
),
traceback_str=traceback.format_exc(
limit=MAXIMUM_TRACEBACK_LINES_TO_LOG,
),
@ -2940,19 +3000,17 @@ async def websocket_passthrough_request(
"WebSocket passthrough (%s): unexpected error while proxying WebSocket", endpoint
)
# Prepare request payload for logging
request_payload = {}
if kwargs:
for key, value in kwargs.items():
request_payload[key] = value
if logging_obj is not None:
request_payload["litellm_logging_obj"] = logging_obj
# Log the unexpected error using the same pattern as HTTP
await proxy_logging_obj.post_call_failure_hook(
user_api_key_dict=user_api_key_dict,
original_exception=e,
request_data=request_payload,
request_data=_build_passthrough_failure_request_payload(
parsed_body=None if model is None else {"model": model},
kwargs=kwargs,
logging_obj=logging_obj,
custom_llm_provider=None,
url=httpx.URL(target),
builtin_provider_route=builtin_provider_route,
),
traceback_str=traceback.format_exc(
limit=MAXIMUM_TRACEBACK_LINES_TO_LOG,
),

6
litellm/proxy/spend_tracking/savings.py
View file

@ -689,9 +689,13 @@ def compute_savings_spend(
cost_breakdown=cost_breakdown,
)
)
return SavingsSpend(
savings: Final = SavingsSpend(
compression=compression,
prompt_caching=prompt_caching,
autorouter=0.0 if autorouter is None else autorouter,
gateway_injected_caching=gateway_injected_caching,
)
if all(isfinite(value) for value in savings):
return savings
verbose_proxy_logger.warning("Spend tracking - recording non-finite savings %s as 0 for model=%s", savings, model)
return SavingsSpend(*(value if isfinite(value) else 0.0 for value in savings))

11
litellm/proxy/spend_tracking/spend_tracking_utils.py
View file

@ -912,7 +912,7 @@ def get_logging_payload(
user=metadata.get("user_api_key_user_id", "") or "",
team_id=metadata.get("user_api_key_team_id", "") or "",
organization_id=metadata.get("user_api_key_org_id") or "",
metadata=safe_dumps(clean_metadata),
metadata=safe_dumps(clean_metadata, non_finite_as_null=True),
cache_key=cache_key,
spend=kwargs.get("response_cost", 0),
total_tokens=usage.get("total_tokens", standard_logging_total_tokens),
@ -1224,7 +1224,7 @@ def _get_messages_for_spend_logs_payload(
messages: Final = standard_logging_payload.get("messages")
if messages is not None:
try:
return safe_dumps(messages)
return safe_dumps(messages, non_finite_as_null=True)
except Exception:
return "{}"
return "{}"
@ -1919,7 +1919,8 @@ def _get_proxy_server_request_for_spend_logs_payload(
_request_body_json_str: Final = safe_dumps(
_stored_request_without_requester_ip_address(stored_request_body)
if drop_requester_ip_address
else stored_request_body
else stored_request_body,
non_finite_as_null=True,
)
if LITELLM_TRUNCATED_PAYLOAD_FIELD in _request_body_json_str:
verbose_proxy_logger.info(
@ -2000,7 +2001,7 @@ def _get_response_for_spend_logs_payload(
if isinstance(sanitized_response, str):
result_str = strip_null_bytes(sanitized_response)
else:
result_str = safe_dumps(sanitized_response)
result_str = safe_dumps(sanitized_response, non_finite_as_null=True)
if LITELLM_TRUNCATED_PAYLOAD_FIELD in result_str:
verbose_proxy_logger.info(
"Spend Log: response was truncated before storing in DB. %s",
@ -2071,7 +2072,7 @@ def spend_log_row_with_retained_metadata(
return row
metadata: Final = _SPEND_LOGS_METADATA_ADAPTER.validate_json(metadata_json)
retained: Final = {name: value for name, value in metadata.items() if fields.keeps(name)}
return {**row, "metadata": safe_dumps(retained)}
return {**row, "metadata": safe_dumps(retained, non_finite_as_null=True)}
def _get_status_for_spend_log(

38
litellm/proxy/utils.py
View file

@ -86,12 +86,13 @@ from litellm.proxy._types import (
from litellm.proxy.bug_report_config import build_proxy_bug_report
from litellm.proxy.common_utils.openai_error_payload import (
attribute_of,
litellm_call_id_headers,
openai_error_param,
error_status_code,
openai_shaped_proxy_exception,
with_litellm_call_id,
)
from litellm.proxy.spend_tracking.spend_log_error_logger import spend_log_error
from litellm.responses.litellm_completion_transformation.compaction_items import messages_since_newest_compaction
from litellm.responses.mcp.parked_stream_dispatch import held_round_spend
from litellm.types.guardrails import GuardrailEventHooks
from litellm.types.proxy.model_listing import ModelInfoResponse
from litellm.types.utils import MCP_GUARDRAIL_CALL_TYPES, CallTypes, CallTypesLiteral, ModelInfo, Usage
@ -4988,6 +4989,8 @@ class ProxyLogging:
if not isinstance(error, _PROXY_ONLY_LLM_API_ERRORS):
return False
match _args:
case (ModelResponse(), *_) if (held := held_round_spend(logging_obj)) is not None:
logging_obj.record_billed_rounds_for_failure(held)
case (ModelResponse() as assembled, *_):
logging_obj.record_assembled_response_for_failure(assembled)
case (BilledRoundsDispatch() as billed,):
@ -9231,39 +9234,30 @@ def _recreate_writer_on_read_only_transaction(prisma_client: "PrismaClient | Non
asyncio.create_task(prisma_client.recreate_read_only_writer(reason="postgres_read_only_transaction"))
def _client_message(e: Exception) -> str:
status_is_none: Final = hasattr(e, "status_code") and attribute_of(e, "status_code") is None
return "Internal server error" if status_is_none else strip_bug_report_notice(str(e))
def handle_exception_on_proxy(e: Exception, litellm_call_id: str | None = None) -> ProxyException:
"""
Returns an Exception as ProxyException, this ensures all exceptions are OpenAI API compatible
"""
from fastapi import status
verbose_proxy_logger.exception("Exception: %s", e)
if PrismaDBExceptionHandler.is_read_only_transaction_error(e):
from litellm.proxy.proxy_server import prisma_client
_recreate_writer_on_read_only_transaction(prisma_client)
headers: Final = litellm_call_id_headers(litellm_call_id)
if isinstance(e, HTTPException):
return ProxyException(
message=getattr(e, "detail", f"error({e})"),
type=ProxyErrorTypes.internal_server_error,
param=openai_error_param(e),
headers=headers,
code=getattr(e, "status_code", status.HTTP_500_INTERNAL_SERVER_ERROR),
)
elif isinstance(e, ProxyException):
if isinstance(e, ProxyException):
return with_litellm_call_id(e, litellm_call_id)
_status_code: Final = getattr(e, "status_code", status.HTTP_500_INTERNAL_SERVER_ERROR)
if isinstance(e, HTTPException):
return with_litellm_call_id(openai_shaped_proxy_exception(e, str(e.detail), e.status_code), litellm_call_id)
if should_report_bug(e):
verbose_proxy_logger.error(bug_report_notice(build_proxy_bug_report(e)))
return ProxyException(
message=strip_bug_report_notice(str(e)),
type=ProxyErrorTypes.internal_server_error,
param=openai_error_param(e),
headers=headers,
code=_status_code,
)
message: Final = _client_message(e)
status_code: Final = error_status_code(e, status.HTTP_500_INTERNAL_SERVER_ERROR)
return with_litellm_call_id(openai_shaped_proxy_exception(e, message, status_code), litellm_call_id)
def require_enterprise_license(feature: str | None = None) -> None:

10
litellm/responses/litellm_completion_transformation/transformation.py
View file

@ -398,23 +398,25 @@ def tool_output_to_chat_content(
if single_media is not None:
return [single_media]
if isinstance(output, list):
parts: Final = cast(Sequence[object], output) # cast-ok: checked list shape, elements remain unvalidated
normalized_blocks: Final[list[ToolMessageContentPart]] = []
text_acc: Final[list[str]] = []
for part in output:
for part in parts:
if not isinstance(part, dict):
continue
part_type = part.get("type")
if part_type in ("input_text", "output_text", "text"):
txt = part.get("text")
if isinstance(txt, str) and txt:
if isinstance(txt, str):
text_acc.append(txt)
normalized_blocks.append({"type": "text", "text": txt})
if txt:
normalized_blocks.append({"type": "text", "text": txt})
elif (media := decode_responses_media(part)) is not None:
normalized_blocks.append(media)
if any(b.get("type") != "text" for b in normalized_blocks):
return normalized_blocks
if text_acc:
if any(text_acc) or len(text_acc) == len(parts):
return "".join(text_acc)
try:
return json.dumps(output)

6
litellm/responses/main.py
View file

@ -1,7 +1,7 @@
import asyncio
import contextvars
import json
from collections.abc import Awaitable, Callable, Coroutine, Generator, Iterable, Mapping, Sequence
from collections.abc import Coroutine, Generator, Iterable, Mapping, Sequence
from contextlib import contextmanager
from dataclasses import dataclass
from functools import partial
@ -81,7 +81,7 @@ from litellm.types.llms.openai import (
# Handle ResponseText import with fallback
if TYPE_CHECKING:
from litellm.responses.streaming_iterator import ResponsesWsFrameGuard
from litellm.responses.streaming_iterator import ResponsesWsFrameGuard, ResponsesWsTurnAdmission
from litellm.types.llms.openai import ResponseText
else:
ResponseText = str # Fallback for ResponseText import
@ -2867,7 +2867,7 @@ async def _aresponses_websocket(
api_base: str | None = None,
api_key: str | None = None,
timeout: float | None = None,
turn_admission: Callable[[], Awaitable[None]] | None = None,
turn_admission: "ResponsesWsTurnAdmission | None" = None,
responses_ws_frame_guard: "ResponsesWsFrameGuard | None" = None,
**kwargs,
) -> Exception | None:

101
litellm/responses/mcp/chat_completions_handler.py
View file

@ -1,15 +1,18 @@
"""Helpers for handling MCP-aware `/chat/completions` requests."""
import asyncio
import logging
from types import MappingProxyType
from typing import TYPE_CHECKING, Final, cast
import anyio
from pydantic import TypeAdapter
from typing_extensions import TypedDict, Unpack
from litellm._internal_context import internal_sub_call
from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR, set_hidden_params
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.responses.mcp.chat_stream_round_billing import ChatStreamRoundBilling, engaged_round_billing
from litellm.responses.mcp.litellm_proxy_mcp_handler import (
LiteLLM_Proxy_MCP_Handler,
)
@ -19,7 +22,7 @@ from litellm.responses.mcp.round_billing import (
first_round_billed_on_failure,
summed_cost_breakdown,
)
from litellm.types.utils import Message, ModelResponse
from litellm.types.utils import Message, ModelResponse, Usage
from litellm.utils import CustomStreamWrapper
if TYPE_CHECKING:
@ -258,10 +261,20 @@ async def acompletion_with_mcp(
self.follow_up_stream = None
self.follow_up_iterator = None
self.follow_up_exhausted = False
self._round_billing: ChatStreamRoundBilling | None = engaged_round_billing(initial_stream.logging_obj)
self._unsent_usage_chunk: ModelResponseStream | None = None
def __aiter__(self):
return self
def __del__(self) -> None:
self.release_abandoned()
def release_abandoned(self) -> None:
round_billing: Final[object] = getattr(self, "_round_billing", None)
if isinstance(round_billing, ChatStreamRoundBilling):
round_billing.release_abandoned()
def _add_mcp_list_tools_to_chunk(self, chunk: ModelResponseStream) -> ModelResponseStream:
"""Add mcp_list_tools to the first chunk."""
from litellm.types.utils import (
@ -321,18 +334,54 @@ async def acompletion_with_mcp(
return chunk
async def _drain_inner_stream(self):
async def _drain_inner_stream(self) -> tuple[ModelResponseStream, ...]:
try:
while True:
await self._stream_iterator.__anext__()
except StopAsyncIteration:
pass
return tuple([chunk async for chunk in initial_stream])
except Exception:
logging.getLogger("LiteLLM").exception(
"Error draining inner MCP stream after final chunk; spend logging may be incomplete"
)
return ()
async def __anext__(self):
async def _complete_first_round(self) -> ModelResponseStream | None:
drained: Final = await self._drain_inner_stream()
if self._round_billing is not None:
self._round_billing.complete_first_round()
usage_chunks: Final = tuple(
chunk for chunk in drained if isinstance(getattr(chunk, "usage", None), Usage)
)
return usage_chunks[-1] if usage_chunks else None
async def _follow_up_or_release_first_round(self, usage_chunk: ModelResponseStream | None) -> None:
if self.tool_results and self.complete_response:
await self._prepare_follow_up_call()
if self.follow_up_stream is not None:
return
self._unsent_usage_chunk = usage_chunk
if self._round_billing is not None:
self._round_billing.finish()
async def __anext__(self) -> ModelResponseStream:
round_billing: Final = self._round_billing
if round_billing is None:
return await self._next_chunk()
with round_billing.armed_while_reading():
try:
return await self._next_chunk()
except StopAsyncIteration:
usage_chunk: Final = round_billing.finish()
if usage_chunk is None:
raise
return usage_chunk
except asyncio.CancelledError:
with anyio.CancelScope(shield=True):
await round_billing.bill_in_flight_round()
raise
except Exception:
round_billing.fail()
raise
async def _next_chunk(self) -> ModelResponseStream:
# Phase 1: Collect and yield initial stream chunks
if not self.stream_exhausted:
# Get the iterator from the stream wrapper
@ -362,29 +411,22 @@ async def acompletion_with_mcp(
if is_final:
self.stream_exhausted = True
usage_chunk: Final = await self._complete_first_round()
await self._process_tool_calls()
chunk = self._add_mcp_tool_metadata_to_final_chunk(chunk)
if self.tool_results and self.complete_response:
await self._prepare_follow_up_call()
# Drain inner stream so CustomStreamWrapper fires its
# end-of-stream handler (dispatch_success_handlers →
# _ProxyDBLogger → LiteLLM_SpendLogs). The CSW may
# yield one usage chunk before raising StopAsyncIteration.
await self._drain_inner_stream()
await self._follow_up_or_release_first_round(usage_chunk)
return chunk
except StopAsyncIteration:
self.stream_exhausted = True
drained_usage_chunk: Final = await self._complete_first_round()
# Process tool calls after stream is exhausted
await self._process_tool_calls()
# If we have chunks, yield the final one with metadata
if self.collected_chunks:
final_chunk = self.collected_chunks[-1]
final_chunk = self._add_mcp_tool_metadata_to_final_chunk(final_chunk)
# If we have tool results, prepare follow-up call
if self.tool_results and self.complete_response:
await self._prepare_follow_up_call()
await self._drain_inner_stream()
await self._follow_up_or_release_first_round(drained_usage_chunk)
return final_chunk
# Phase 2: Yield follow-up stream chunks if available
@ -400,7 +442,6 @@ async def acompletion_with_mcp(
from litellm._logging import verbose_logger
verbose_logger.debug("Follow-up chunk yielded: %s", chunk)
return chunk
except StopAsyncIteration:
self.follow_up_exhausted = True
from litellm._logging import verbose_logger
@ -408,6 +449,10 @@ async def acompletion_with_mcp(
verbose_logger.debug("Follow-up stream exhausted")
# After follow-up stream is exhausted, check if we need to raise StopAsyncIteration
raise StopAsyncIteration
client_chunk: Final = (
chunk if self._round_billing is None else self._round_billing.client_chunk(chunk)
)
return await self._next_chunk() if client_chunk is None else client_chunk
# If we're here and follow_up_stream is None but we expected it, log a warning
if (
@ -420,7 +465,11 @@ async def acompletion_with_mcp(
verbose_logger.warning("Follow-up stream was not created despite having tool results")
raise StopAsyncIteration
unsent_usage_chunk: Final = self._unsent_usage_chunk
if unsent_usage_chunk is None:
raise StopAsyncIteration
self._unsent_usage_chunk = None
return unsent_usage_chunk
async def _process_tool_calls(self):
"""Process tool calls after streaming completes."""
@ -488,10 +537,16 @@ async def acompletion_with_mcp(
# This ensures the patch works correctly in tests
import litellm
earlier_rounds: Final = None if self._round_billing is None else self._round_billing.bank_first_round()
follow_up_response: Final = await litellm.acompletion(**follow_up_call_args)
# Ensure follow-up response is a CustomStreamWrapper
if isinstance(follow_up_response, CustomStreamWrapper):
if earlier_rounds is not None:
follow_up_response.bill_failure_after_earlier_rounds(earlier_rounds)
follow_up_response.messages = follow_up_messages
if self._round_billing is not None:
self._round_billing.follow_up_started(follow_up_response.partial_response)
self.follow_up_stream = follow_up_response
from litellm._logging import verbose_logger
@ -548,11 +603,13 @@ async def acompletion_with_mcp(
def __aiter__(self):
return self._custom_iterator
async def aclose(self) -> None:
self._custom_iterator.release_abandoned()
await super().aclose()
def __iter__(self):
# For synchronous iteration, create a sync wrapper
if self._sync_iterator is None:
import asyncio
try:
self._sync_loop = asyncio.get_event_loop()
except RuntimeError:

281
litellm/responses/mcp/chat_stream_round_billing.py Normal file
View file

@ -0,0 +1,281 @@
import asyncio
from collections.abc import Callable, Generator
from contextlib import contextmanager
from dataclasses import dataclass, replace
from types import MappingProxyType
from typing import Final, TypeAlias
from litellm.litellm_core_utils.asyncify import asyncify, is_event_loop_running
from litellm.litellm_core_utils.hidden_params import get_hidden_params, served_from_cache, set_hidden_params
from litellm.litellm_core_utils.litellm_logging import BilledRoundsSpend, UncachedPrice
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.responses.mcp.parked_stream_dispatch import (
DISPATCH_RELEASE,
HELD_ROUND_SPEND,
IN_FLIGHT_ROUND,
PARKED_DISPATCH,
SERVED_FROM_CACHE,
DeferredStreamRelease,
parked_success,
)
from litellm.responses.mcp.round_billing import billed_for_every_round, merged_chat_usage, summed_cost_breakdown
from litellm.responses.streaming_iterator import spawn_logging_task
from litellm.types.utils import ModelResponse, ModelResponseStream, Usage
Parked: TypeAlias = tuple[object, ...]
PartialAnswer: TypeAlias = Callable[[], object]
@dataclass(frozen=True, slots=True)
class _AssembledRound:
response: ModelResponse
usage: Usage
cache_hit: bool
parked: Parked
@dataclass(frozen=True, slots=True)
class _PricedRound:
response: ModelResponse
usage: Usage
price: UncachedPrice
cache_hit: bool
@property
def spend(self) -> BilledRoundsSpend:
return BilledRoundsSpend(self.usage, self.price.cost or 0.0, self.price.cost_breakdown)
@dataclass(frozen=True, slots=True)
class _HeldUsageChunk:
chunk: ModelResponseStream
usage: Usage
@dataclass(frozen=True, slots=True)
class _FirstRound:
priced: _PricedRound
parked: Parked
logged_alone: Parked
banked: LiteLLMLoggingObj | None = None
usage_chunk: _HeldUsageChunk | None = None
def client_usage_chunk(self, billed: ModelResponse | None) -> ModelResponseStream | None:
if self.usage_chunk is None:
return None
usage: Final = (
self.priced.usage.model_copy(update=MappingProxyType({"cost": self.priced.price.cost}))
if billed is None
else merged_chat_usage(self.priced.usage, self.usage_chunk.usage).model_copy(
update=MappingProxyType({"cost": _cost(billed)})
)
)
return self.usage_chunk.chunk.model_copy(
update=MappingProxyType({"id": self.priced.response.id, "usage": usage})
)
def _cost(response: ModelResponse) -> object:
hidden_params: Final = get_hidden_params(response)
return None if hidden_params is None else hidden_params.get("response_cost")
def _assembled_round(parked: Parked | None) -> _AssembledRound | None:
match parked:
case (ModelResponse() as response, cache_hit) if isinstance(usage := getattr(response, "usage", None), Usage):
return _AssembledRound(response, usage, cache_hit is True, parked)
case _:
return None
def engaged_round_billing(logging_obj: object) -> "ChatStreamRoundBilling | None":
return ChatStreamRoundBilling(logging_obj) if isinstance(logging_obj, LiteLLMLoggingObj) else None
class ChatStreamRoundBilling:
def __init__(self, logging_obj: LiteLLMLoggingObj) -> None:
self._logging_obj: Final = logging_obj
self._first: _FirstRound | None = None
self._partial_follow_up: PartialAnswer | None = None
self._reading_loop: asyncio.AbstractEventLoop | None = None
@contextmanager
def armed_while_reading(self) -> Generator[None]:
armed: Final = getattr(self._logging_obj, DISPATCH_RELEASE, None) is None
if armed:
setattr(self._logging_obj, DISPATCH_RELEASE, self._log_success)
self._reading_loop = asyncio.get_running_loop()
try:
yield
finally:
if armed and self._peek_parked() is None:
self._disarm()
def complete_first_round(self) -> None:
assembled: Final = _assembled_round(self._peek_parked())
if assembled is not None:
self._first = _FirstRound(self._priced(assembled), assembled.parked, assembled.parked)
setattr(self._logging_obj, HELD_ROUND_SPEND, self._held_round_one_spend)
def bank_first_round(self) -> BilledRoundsSpend | None:
self._bank_apart_from_the_follow_up()
return None if self._first is None else self._first.priced.spend
def client_chunk(self, chunk: ModelResponseStream) -> ModelResponseStream | None:
usage: Final[object] = getattr(chunk, "usage", None)
if self._first is None or not isinstance(usage, Usage):
return chunk
self._first = replace(self._first, usage_chunk=_HeldUsageChunk(chunk, usage))
return None
def finish(self) -> ModelResponseStream | None:
first: Final = self._first
billed: Final = self._release_rounds()
return None if first is None else first.client_usage_chunk(billed)
def fail(self) -> None:
self._bank_apart_from_the_follow_up()
first: Final = self._first
if first is None or first.banked is None or not self._holds_round_one_alone(first):
self.finish()
return
self._first = None
setattr(self._logging_obj, PARKED_DISPATCH, None)
self._log_through_the_release((*first.parked, first.banked.guardrail_scoped_copy()))
def follow_up_started(self, partial_follow_up: PartialAnswer) -> None:
self._partial_follow_up = partial_follow_up
setattr(self._logging_obj, IN_FLIGHT_ROUND, self.bill_in_flight_round)
async def bill_in_flight_round(self) -> None:
partial_follow_up: Final = self._partial_follow_up
if partial_follow_up is None or self._first_round_held_alone() is None:
return
final_round: Final = _assembled_round((await asyncify(partial_follow_up)(), False))
first: Final = self._first_round_held_alone()
if final_round is None or first is None:
return
self._first = None
setattr(
self._logging_obj, PARKED_DISPATCH, self._billed_for_both_rounds(first.priced, self._priced(final_round))
)
def release_abandoned(self) -> None:
if not self._owns_release():
return
if is_event_loop_running() or self._idle_reading_loop() is not None:
self.finish()
return
setattr(self._logging_obj, PARKED_DISPATCH, None)
self._first = None
self._disarm()
def _bank_apart_from_the_follow_up(self) -> None:
first: Final = self._first
if first is None or first.banked is not None or self._peek_parked() is not first.parked:
return
banked: Final = self._logging_obj.attempt_scoped_copy()
if not self._owns_release():
self._first = replace(first, banked=banked)
return
logged_alone: Final = (*first.parked, banked)
self._first = replace(first, logged_alone=logged_alone, banked=banked)
setattr(self._logging_obj, PARKED_DISPATCH, logged_alone)
def _release_rounds(self) -> ModelResponse | None:
parked: Final = self._take_parked()
first: Final = self._first
self._first = None
if parked is None:
self._disarm()
return None
if first is None or parked is first.logged_alone:
self._release(parked)
return None
final_round: Final = _assembled_round(parked)
if final_round is None:
self._release(first.logged_alone)
return None
billed, cache_hit = self._billed_for_both_rounds(first.priced, self._priced(final_round))
self._release((billed, cache_hit))
return billed
def _billed_for_both_rounds(self, first: _PricedRound, final: _PricedRound) -> tuple[ModelResponse, bool]:
first_breakdown: Final = first.price.cost_breakdown
final_breakdown: Final = final.price.cost_breakdown
rounds: Final = billed_for_every_round(first.response, first_breakdown, final.response, final_breakdown)
usage: Final[object] = getattr(rounds, "usage", None)
charged_usage: Final = (
usage.model_copy(update=MappingProxyType({"cost": _cost(rounds)})) if isinstance(usage, Usage) else usage
)
billed: Final = rounds.model_copy(update=MappingProxyType({"id": first.response.id, "usage": charged_usage}))
cache_hit: Final = first.cache_hit and final.cache_hit
set_hidden_params(billed, {**(get_hidden_params(rounds) or {}), "cache_hit": cache_hit})
self._logging_obj.cost_breakdown = summed_cost_breakdown(
first.response, first_breakdown, final.response, final_breakdown
)
return billed, cache_hit
def _priced(self, assembled: _AssembledRound) -> _PricedRound:
served: Final = assembled.cache_hit or served_from_cache(assembled.response)
price: Final = SERVED_FROM_CACHE if served else self._logging_obj.uncached_price(assembled.response)
priced: Final = assembled.response.model_copy()
set_hidden_params(
priced,
{**(get_hidden_params(assembled.response) or {}), "response_cost": price.cost, "cache_hit": served},
)
return _PricedRound(priced, assembled.usage, price, served)
def _peek_parked(self) -> Parked | None:
parked: Final[Parked | None] = getattr(self._logging_obj, PARKED_DISPATCH, None)
return parked if isinstance(parked, tuple) else None
def _take_parked(self) -> Parked | None:
parked: Final = self._peek_parked()
setattr(self._logging_obj, PARKED_DISPATCH, None)
return parked
def _first_round_held_alone(self) -> _FirstRound | None:
first: Final = self._first
return first if first is not None and self._holds_round_one_alone(first) else None
def _held_round_one_spend(self) -> BilledRoundsSpend | None:
first: Final = self._first_round_held_alone()
return None if first is None else first.priced.spend
def _holds_round_one_alone(self, first: _FirstRound) -> bool:
parked: Final = self._peek_parked()
return parked is first.parked or parked is first.logged_alone
def _owns_release(self) -> bool:
release: Final[object] = getattr(self._logging_obj, DISPATCH_RELEASE, None)
return release == self._log_success
def _release(self, parked: Parked) -> None:
if getattr(self._logging_obj, DISPATCH_RELEASE, None) is not None and not self._owns_release():
setattr(self._logging_obj, PARKED_DISPATCH, parked)
return
self._log_through_the_release(parked)
def _log_through_the_release(self, parked: Parked) -> None:
armed: Final[DeferredStreamRelease | None] = getattr(self._logging_obj, DISPATCH_RELEASE, None)
setattr(self._logging_obj, DISPATCH_RELEASE, None)
logged: Final = (self._log_success if armed is None else armed)(*parked)
idle_reading_loop: Final = None if is_event_loop_running() else self._idle_reading_loop()
if idle_reading_loop is not None:
idle_reading_loop.run_until_complete(logged)
return
spawn_logging_task(asyncio.get_running_loop(), logged, task_name="Chat completions MCP stream success log")
def _idle_reading_loop(self) -> asyncio.AbstractEventLoop | None:
loop: Final = self._reading_loop
return loop if loop is not None and not loop.is_closed() and not loop.is_running() else None
def _disarm(self) -> None:
if self._owns_release():
setattr(self._logging_obj, DISPATCH_RELEASE, None)
async def _log_success(self, *parked: object) -> None:
success: Final = parked_success(parked)
if success is not None:
await success.dispatch(self._logging_obj)

72
litellm/responses/mcp/parked_stream_dispatch.py Normal file
View file

@ -0,0 +1,72 @@
from collections.abc import Coroutine
from dataclasses import dataclass
from typing import Final, Protocol
from litellm.litellm_core_utils.litellm_logging import BilledRoundsSpend, UncachedPrice
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
PARKED_DISPATCH: Final = "_deferred_stream_complete_args"
DISPATCH_RELEASE: Final = "_on_deferred_stream_complete"
IN_FLIGHT_ROUND: Final = "_bill_in_flight_round_on_disconnect"
HELD_ROUND_SPEND: Final = "_held_round_spend_for_failure"
SERVED_FROM_CACHE: Final = UncachedPrice(cost=0.0, cost_breakdown=None)
class DispatchesSuccessHandlers(Protocol):
async def dispatch_success_handlers(
self,
result: object = None,
start_time: object = None,
end_time: object = None,
cache_hit: object = None,
prefer_async_handlers: bool = False,
) -> None: ...
class DeferredStreamRelease(Protocol):
def __call__(self, *parked: object) -> Coroutine[object, object, None]: ...
class InFlightRoundBiller(Protocol):
def __call__(self) -> Coroutine[object, object, None]: ...
class HeldRoundSpend(Protocol):
def __call__(self) -> BilledRoundsSpend | None: ...
def success_dispatcher(logger: LiteLLMLoggingObj) -> DispatchesSuccessHandlers:
return logger
@dataclass(frozen=True, slots=True)
class ParkedSuccess:
assembled: object
cache_hit: object
logged_on: LiteLLMLoggingObj | None = None
async def dispatch(self, request_logger: LiteLLMLoggingObj) -> None:
logger: Final = request_logger if self.logged_on is None else self.logged_on
await success_dispatcher(logger).dispatch_success_handlers(
self.assembled, cache_hit=self.cache_hit, start_time=None, end_time=None, prefer_async_handlers=True
)
def in_flight_round_biller(logging_obj: object) -> InFlightRoundBiller | None:
biller: Final[InFlightRoundBiller | None] = getattr(logging_obj, IN_FLIGHT_ROUND, None)
return biller if isinstance(logging_obj, LiteLLMLoggingObj) and callable(biller) else None
def held_round_spend(logging_obj: object) -> BilledRoundsSpend | None:
held: Final[HeldRoundSpend | None] = getattr(logging_obj, HELD_ROUND_SPEND, None)
return held() if isinstance(logging_obj, LiteLLMLoggingObj) and callable(held) else None
def parked_success(parked: tuple[object, ...]) -> ParkedSuccess | None:
match parked:
case (assembled, cache_hit):
return ParkedSuccess(assembled, cache_hit)
case (assembled, cache_hit, LiteLLMLoggingObj() as logged_on):
return ParkedSuccess(assembled, cache_hit, logged_on)
case _:
return None

58
litellm/responses/mcp/stream_round_billing.py
View file

@ -2,11 +2,18 @@ import asyncio
from collections.abc import Callable, Coroutine, Sequence
from dataclasses import dataclass
from datetime import datetime
from typing import Final, Protocol, TypeAlias
from typing import Final, TypeAlias
from litellm.litellm_core_utils.hidden_params import get_hidden_params, served_from_cache
from litellm.litellm_core_utils.litellm_logging import BilledRoundsDispatch, BilledRoundsSpend, UncachedPrice
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.responses.mcp.parked_stream_dispatch import (
DISPATCH_RELEASE,
PARKED_DISPATCH,
SERVED_FROM_CACHE,
parked_success,
success_dispatcher,
)
from litellm.responses.mcp.round_billing import billed_for_every_round, merged_chat_usage, summed_cost_breakdown
from litellm.responses.streaming_iterator import BaseResponsesAPIStreamingIterator, spawn_logging_task
from litellm.responses.utils import ResponseAPILoggingUtils
@ -16,25 +23,6 @@ from litellm.types.utils import CostBreakdown, ModelResponse, Usage
Parked: TypeAlias = tuple[object, ...]
RestoresHeaders: TypeAlias = Callable[[ResponsesAPIStreamingResponse], None]
_PARKED_DISPATCH: Final = "_deferred_stream_complete_args"
_DISPATCH_RELEASE: Final = "_on_deferred_stream_complete"
_SERVED_FROM_CACHE: Final = UncachedPrice(cost=0.0, cost_breakdown=None)
class _DispatchesSuccessHandlers(Protocol):
async def dispatch_success_handlers(
self,
result: object = None,
start_time: object = None,
end_time: object = None,
cache_hit: object = None,
prefer_async_handlers: bool = False,
) -> None: ...
def _success_dispatcher(logger: LiteLLMLoggingObj) -> _DispatchesSuccessHandlers:
return logger
def _event_loop_running() -> bool:
try:
@ -198,8 +186,8 @@ class StreamedRoundBilling:
self._billed: _BilledRounds | None = None
self._banked: Parked | None = None
self._banked_row: _RoundRow | None = None
if getattr(logging_obj, _DISPATCH_RELEASE, None) is None:
setattr(logging_obj, _DISPATCH_RELEASE, self._run_parked)
if getattr(logging_obj, DISPATCH_RELEASE, None) is None:
setattr(logging_obj, DISPATCH_RELEASE, self._run_parked)
def complete_round(
self,
@ -286,7 +274,7 @@ class StreamedRoundBilling:
def _price(self, response: ResponsesAPIResponse, round_parked: Parked | None) -> UncachedPrice:
if served_from_cache(response):
return _SERVED_FROM_CACHE
return SERVED_FROM_CACHE
match round_parked:
case (ModelResponse() as assembled, bool()):
return self._logging_obj.uncached_price(assembled)
@ -297,7 +285,7 @@ class StreamedRoundBilling:
self._banked = parked
self._banked_row = row
if parked is not None:
setattr(self._logging_obj, _PARKED_DISPATCH, parked)
setattr(self._logging_obj, PARKED_DISPATCH, parked)
def _bank_apart_from_later_rounds(self, billed: _BilledRounds) -> None:
row: Final = self._banked_row
@ -320,7 +308,7 @@ class StreamedRoundBilling:
@staticmethod
async def _log_success(success: _SuccessLog) -> None:
success.carry_bridged_usage()
await _success_dispatcher(success.logger).dispatch_success_handlers(
await success_dispatcher(success.logger).dispatch_success_handlers(
success.row.logged,
start_time=success.logger.start_time,
end_time=success.row.end_time,
@ -329,10 +317,10 @@ class StreamedRoundBilling:
)
def _take_parked(self) -> Parked | None:
parked: Final[Parked | None] = getattr(self._logging_obj, _PARKED_DISPATCH, None)
parked: Final[Parked | None] = getattr(self._logging_obj, PARKED_DISPATCH, None)
if not isinstance(parked, tuple):
return None
setattr(self._logging_obj, _PARKED_DISPATCH, None)
setattr(self._logging_obj, PARKED_DISPATCH, None)
return parked
async def _run_parked(self, *parked: object) -> None:
@ -341,29 +329,27 @@ class StreamedRoundBilling:
await dispatch
case (dispatch,) if asyncio.iscoroutine(dispatch):
await dispatch
case (assembled, cache_hit):
await _success_dispatcher(self._logging_obj).dispatch_success_handlers(
assembled, cache_hit=cache_hit, start_time=None, end_time=None, prefer_async_handlers=True
)
case _ if (success := parked_success(parked)) is not None:
await success.dispatch(self._logging_obj)
case _:
return
def _owns_release(self) -> bool:
release: Final[object] = getattr(self._logging_obj, _DISPATCH_RELEASE, None)
release: Final[object] = getattr(self._logging_obj, DISPATCH_RELEASE, None)
return release == self._run_parked
def _release(self, parked: Parked) -> None:
if getattr(self._logging_obj, _DISPATCH_RELEASE, None) is not None and not self._owns_release():
setattr(self._logging_obj, _PARKED_DISPATCH, parked)
if getattr(self._logging_obj, DISPATCH_RELEASE, None) is not None and not self._owns_release():
setattr(self._logging_obj, PARKED_DISPATCH, parked)
return
setattr(self._logging_obj, _DISPATCH_RELEASE, None)
setattr(self._logging_obj, DISPATCH_RELEASE, None)
spawn_logging_task(
asyncio.get_running_loop(), self._run_parked(*parked), task_name="Responses MCP stream success logging"
)
def _disarm(self) -> None:
if self._owns_release():
setattr(self._logging_obj, _DISPATCH_RELEASE, None)
setattr(self._logging_obj, DISPATCH_RELEASE, None)
@staticmethod
def _close(parked: Parked | None) -> None:

103
litellm/responses/streaming_iterator.py
View file

@ -2145,18 +2145,32 @@ async def _enforce_frame_project_quota(
)
async def _admit_frame_turn(turn_admission: Callable[[], Awaitable[None]] | None, raw_message: str) -> Exception | None:
@dataclass(frozen=True, slots=True)
class _AdmittedTurn:
reservation: Mapping[str, object] = field(default_factory=lambda: MappingProxyType({}))
@dataclass(frozen=True, slots=True)
class _RefusedTurn:
rejection: Exception
ResponsesWsTurnAdmission: TypeAlias = Callable[[], Awaitable[Mapping[str, object]]]
async def _admit_frame_turn(
turn_admission: ResponsesWsTurnAdmission | None, raw_message: str
) -> _AdmittedTurn | _RefusedTurn:
frame: Final = _json_object_or_none(raw_message)
if turn_admission is None or frame is None or frame.get("type") != "response.create":
return None
return _AdmittedTurn()
try:
await turn_admission()
return _AdmittedTurn(await turn_admission())
except RateLimitError as refusal:
return refusal
return _RefusedTurn(refusal)
except Exception as e:
verbose_logger.exception("Responses WS turn admission failed: %s", e)
return e
return None
return _RefusedTurn(e)
def _turn_refusal_frame(refusal: RateLimitError) -> str:
@ -2191,6 +2205,7 @@ RESPONSES_WS_MASKABLE_TEXT_BLOCK_TYPES: Final = frozenset({"input_text", "output
_RESPONSES_WS_FAILURE_EVENT_TYPES: Final = frozenset({"error", "response.failed"})
_PER_TURN_METADATA_KEYS: Final = frozenset({"metadata", "litellm_metadata"})
_NO_IO_TOKEN_RESERVATION: Final[Mapping[str, object]] = MappingProxyType({})
_GUARDRAIL_INFORMATION_KEY: Final = "standard_logging_guardrail_information"
@ -2318,6 +2333,17 @@ def _set_turn_guardrail_information(
_replace_guardrail_information(litellm_params.get(key), guardrail_information)
def _set_turn_io_token_reservation(turn_logging_obj: LiteLLMLoggingObj, reservation: Mapping[str, object]) -> None:
litellm_params: Final = _json_object_or_none(
turn_logging_obj.litellm_params # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # Logging.litellm_params is untyped
)
if litellm_params is None or not reservation:
return
for bucket in (litellm_params.get(key) for key in _PER_TURN_METADATA_KEYS):
if _is_json_object(bucket):
bucket.update(reservation)
def _replace_guardrail_information(bucket: object, guardrail_information: tuple[Mapping[str, object], ...]) -> None:
if not _is_json_object(bucket):
return
@ -2458,7 +2484,7 @@ class ResponsesWebSocketStreaming:
authorized_model: str | None = None,
custom_llm_provider: str | None = None,
request_defaults: ResponsesWebSocketRequestDefaults | None = None,
turn_admission: Callable[[], Awaitable[None]] | None = None,
turn_admission: ResponsesWsTurnAdmission | None = None,
frame_guard: ResponsesWsFrameGuard | None = None,
):
self.websocket = websocket
@ -2480,10 +2506,13 @@ class ResponsesWebSocketStreaming:
# response.create frame to prevent deployment-substitution attacks.
self.authorized_model: str | None = authorized_model
self.request_defaults: ResponsesWebSocketRequestDefaults | None = request_defaults
self.turn_admission: Callable[[], Awaitable[None]] | None = turn_admission
self.turn_admission: ResponsesWsTurnAdmission | None = turn_admission
def _start_turn(
self, client_frame: str | None, guardrail_information: tuple[Mapping[str, object], ...] = ()
self,
client_frame: str | None,
guardrail_information: tuple[Mapping[str, object], ...] = (),
io_token_reservation: Mapping[str, object] = _NO_IO_TOKEN_RESERVATION,
) -> None:
start_time: Final = datetime.now() # noqa: DTZ005 # naive to match the logging pipeline start_time
turn_logging_obj: Final = self.logging_obj.copy_for_new_call(
@ -2491,6 +2520,7 @@ class ResponsesWebSocketStreaming:
)
self._detach_budget_reservation()
_set_turn_guardrail_information(turn_logging_obj, guardrail_information)
_set_turn_io_token_reservation(turn_logging_obj, io_token_reservation)
turn_logging_obj.model_call_details["messages"] = list(_logged_user_messages(client_frame))
turn_logging_obj.pre_call(input=client_frame, api_key="")
self._open_turns = (*self._open_turns, _ResponsesWsTurn(logging_obj=turn_logging_obj))
@ -2510,11 +2540,14 @@ class ResponsesWebSocketStreaming:
}
def _record_client_frame(
self, client_frame: str, guardrail_information: tuple[Mapping[str, object], ...] = ()
self,
client_frame: str,
guardrail_information: tuple[Mapping[str, object], ...] = (),
io_token_reservation: Mapping[str, object] = _NO_IO_TOKEN_RESERVATION,
) -> None:
frame: Final = _json_object_or_none(client_frame)
if frame is not None and frame.get("type") == "response.create":
self._start_turn(client_frame, guardrail_information)
self._start_turn(client_frame, guardrail_information, io_token_reservation)
async def _guard_frame(self, message: str) -> GuardedResponsesWsFrame | None:
"""Run the proxy's pre_call guardrails on one ``response.create`` frame.
@ -2533,15 +2566,18 @@ class ResponsesWebSocketStreaming:
return None
async def _forward_guarded_frame(
self, message: str, turn_admission: Callable[[], Awaitable[None]] | None = None
self, message: str, turn_admission: ResponsesWsTurnAdmission | None = None
) -> None:
guarded: Final = await self._guard_frame(message)
if guarded is None or not await self._admit_or_reject_turn(guarded, message, turn_admission):
if guarded is None:
return
admitted: Final = await self._admit_or_reject_turn(guarded, message, turn_admission)
if admitted is None:
return
if guarded.pii_tokens:
self._remember_pii_tokens(guarded.pii_tokens)
masked: Final = await self._mask_response_create(guarded.frame)
self._record_client_frame(masked, guarded.guardrail_information)
self._record_client_frame(masked, guarded.guardrail_information, admitted.reservation)
await self.backend_ws.send(masked)
def _remember_pii_tokens(self, pii_tokens: Mapping[str, str]) -> None:
@ -2941,15 +2977,17 @@ class ResponsesWebSocketStreaming:
return True
async def _admit_or_reject_turn(
self, guarded: GuardedResponsesWsFrame, message: str, turn_admission: Callable[[], Awaitable[None]] | None
) -> bool:
rejection: Final = await _admit_frame_turn(turn_admission, message)
if rejection is None:
return True
if self._failure is None:
self._failure = rejection
await _reject_guarded_turn(self.websocket, guarded, rejection)
return False
self, guarded: GuardedResponsesWsFrame, message: str, turn_admission: ResponsesWsTurnAdmission | None
) -> _AdmittedTurn | None:
verdict: Final = await _admit_frame_turn(turn_admission, message)
match verdict:
case _AdmittedTurn():
return verdict
case _RefusedTurn(rejection=rejection):
if self._failure is None:
self._failure = rejection
await _reject_guarded_turn(self.websocket, guarded, rejection)
return None
async def _send_turn_refusal(self, refusal: RateLimitError) -> None:
try:
@ -3052,7 +3090,7 @@ class ManagedResponsesWebSocketHandler:
custom_llm_provider: str | None = None,
first_message: str | None = None,
quota_callbacks: Sequence[ProjectQuotaCallback] | None = None,
turn_admission: Callable[[], Awaitable[None]] | None = None,
turn_admission: ResponsesWsTurnAdmission | None = None,
frame_guard: ResponsesWsFrameGuard | None = None,
guardrail_callbacks: Sequence[PresidioGuardrailCallback] | None = None,
**kwargs: object,
@ -3076,7 +3114,7 @@ class ManagedResponsesWebSocketHandler:
self._connection_provider = self._resolve_provider(model) or custom_llm_provider
self.first_message = first_message
self.quota_callbacks: tuple[ProjectQuotaCallback, ...] = tuple(quota_callbacks) if quota_callbacks else ()
self.turn_admission: Callable[[], Awaitable[None]] | None = turn_admission
self.turn_admission: ResponsesWsTurnAdmission | None = turn_admission
self._failure: Exception | None = None
# Carry through safe pass-through kwargs (e.g. extra_headers)
self.extra_kwargs: dict[str, object] = {k: v for k, v in kwargs.items() if k not in _MANAGED_WS_SKIP_KWARGS}
@ -3473,7 +3511,7 @@ class ManagedResponsesWebSocketHandler:
# ------------------------------------------------------------------
async def _process_response_create(
self, raw_message: str, turn_admission: Callable[[], Awaitable[None]] | None = None
self, raw_message: str, turn_admission: ResponsesWsTurnAdmission | None = None
) -> None:
"""
Parse one ``response.create`` event, call ``litellm.aresponses(stream=True)``,
@ -3520,11 +3558,11 @@ class ManagedResponsesWebSocketHandler:
guarded: Final = await self._guard_frame(raw_message)
if guarded is None:
return
rejection: Final = await _admit_frame_turn(turn_admission, raw_message)
if rejection is not None:
verdict: Final = await _admit_frame_turn(turn_admission, raw_message)
if isinstance(verdict, _RefusedTurn):
if self._failure is None:
self._failure = rejection
await _reject_guarded_turn(self.websocket, guarded, rejection)
self._failure = verdict.rejection
await _reject_guarded_turn(self.websocket, guarded, verdict.rejection)
return
if guarded.pii_tokens:
self._pii_tokens = {**self._pii_tokens, **guarded.pii_tokens}
@ -3555,6 +3593,11 @@ class ManagedResponsesWebSocketHandler:
)
self._update_proxy_request(call_kwargs, requested_model or self.model_group or model)
call_kwargs.update(self.extra_kwargs)
turn_litellm_metadata: Final = call_kwargs.get("litellm_metadata")
call_kwargs["litellm_metadata"] = {
**(turn_litellm_metadata if _is_json_object(turn_litellm_metadata) else {}),
**verdict.reservation,
}
try:
terminal_event: Final = await self._stream_and_forward(model, call_kwargs)

26
litellm/router.py
View file

@ -378,6 +378,7 @@ from litellm.router_utils.pre_call_checks.encrypted_content_affinity_check impor
EncryptedContentAffinityCheck,
)
from litellm.router_utils.pre_call_checks.io_token_rate_limit_check import (
async_admit_io_token_turn,
async_io_token_refund_failure,
build_io_token_rate_limit_headers,
deployment_has_io_token_limits,
@ -5113,6 +5114,7 @@ class Router(DeprecatedRouterMembers):
kwargs: dict,
function_name: str | None = None,
splits_stream_timeout: bool = False,
opens_responses_socket: bool = False,
) -> None:
"""
5 jobs:
@ -5202,7 +5204,12 @@ class Router(DeprecatedRouterMembers):
# here before it's wiped below, instead of relying on that attempt's
# (possibly still-pending) failure event to do it.
refund_stale_reservation_before_retry(self._core_cache, kwargs)
set_io_token_rate_limit_request_kwargs(kwargs, store_in_context=deployment_has_io_token_limits(deployment))
set_io_token_rate_limit_request_kwargs(
kwargs,
store_in_context=deployment_has_io_token_limits(deployment),
opens_responses_socket=opens_responses_socket,
metadata_channel=metadata_variable_name,
)
kwargs[metadata_variable_name].setdefault(
ROUTING_REQUEST_TAGS_METADATA_KEY,
@ -6615,6 +6622,7 @@ class Router(DeprecatedRouterMembers):
kwargs=kwargs,
function_name=function_name,
splits_stream_timeout=handler_name in _STREAM_TIMEOUT_AWARE_GENERIC_HANDLERS,
opens_responses_socket=handler_name == "_aresponses_websocket",
)
data: Final = deployment["litellm_params"].copy()
@ -10465,9 +10473,20 @@ class Router(DeprecatedRouterMembers):
return deployment if current is None else current.model_dump(exclude_none=True)
async def async_admit_deployment_turn(
self, model_group: str, opened_on: Mapping[str, object], request_kwargs: Mapping[str, object]
) -> None:
self,
model_group: str,
opened_on: Mapping[str, object],
request_kwargs: Mapping[str, object],
) -> Mapping[str, object]:
deployment: Final = self._deployment_as_configured_now(opened_on)
rate_limit_check: Final = next(
(callback for callback in self._routing_callbacks() if isinstance(callback, ModelRateLimitingCheck)), None
)
io_token_admission: Final[Mapping[str, object]] = (
MappingProxyType({})
if rate_limit_check is None
else await async_admit_io_token_turn(rate_limit_check.dual_cache, deployment, get_utc_datetime())
)
budget_limiter: Final = self._core_get_router_deployment_budget_limiter()
if budget_limiter is not None:
try:
@ -10488,6 +10507,7 @@ class Router(DeprecatedRouterMembers):
await self.async_routing_strategy_pre_call_checks( # pyright: ignore[reportUnknownMemberType] # legacy signature takes an untyped deployment dict
deployment=dict(without_io_token_limits(deployment)), parent_otel_span=None
)
return io_token_admission
async def _admit_turn_through_routing_strategy(
self, model_group: str, deployment: Mapping[str, object], request_kwargs: Mapping[str, object]

209
litellm/router_utils/pre_call_checks/io_token_rate_limit_check.py
View file

@ -14,6 +14,9 @@ from __future__ import annotations
import contextlib
import contextvars
from collections.abc import Mapping, MutableMapping
from dataclasses import dataclass
from datetime import datetime
from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Final
import httpx
@ -23,7 +26,15 @@ from litellm import token_counter
from litellm._internal_context import with_service_target
from litellm._logging import verbose_router_logger
from litellm.caching.dual_cache import DualCache
from litellm.caching.redis_cache import RedisCache, RedisCircuitBreakerOpenError
from litellm.constants import NO_MAX_TOKENS_TPM_RESERVATION_FRACTION
from litellm.litellm_core_utils.internal_call_metadata import (
IO_TOKEN_RESERVATION_KEYS,
ITPM_CACHE_KEY,
ITPM_RESERVED_KEY,
OTPM_CACHE_KEY,
OTPM_RESERVED_KEY,
)
from litellm.litellm_core_utils.token_counter import deployment_openai_model, offload_token_count
from litellm.llms.azure.ptu_capacity import is_str_keyed_mapping
from litellm.router_utils.routing_read_batch import ROUTER_USAGE_TARGET
@ -39,18 +50,27 @@ else:
RoutingArgsTTL: Final = 60
_io_token_rate_limit_request_kwargs: Final[contextvars.ContextVar[dict[str, object] | None]] = contextvars.ContextVar(
"io_token_rate_limit_request_kwargs",
@dataclass(frozen=True, slots=True)
class _IoTokenRateLimitRequest:
kwargs: dict[str, object] | None
opens_responses_socket: bool
metadata_channel: str
_io_token_rate_limit_request: Final[contextvars.ContextVar[_IoTokenRateLimitRequest | None]] = contextvars.ContextVar(
"io_token_rate_limit_request",
default=None,
)
ITPM_RESERVED_KEY: Final = "_litellm_itpm_reserved"
OTPM_RESERVED_KEY: Final = "_litellm_otpm_reserved"
ITPM_CACHE_KEY: Final = "_litellm_itpm_cache_key"
OTPM_CACHE_KEY: Final = "_litellm_otpm_cache_key"
def set_io_token_rate_limit_request_kwargs(kwargs: dict[str, object] | None, store_in_context: bool = True) -> None:
def set_io_token_rate_limit_request_kwargs(
kwargs: dict[str, object] | None,
store_in_context: bool = True,
*,
opens_responses_socket: bool = False,
metadata_channel: str,
) -> None:
# The reservation sentinels are server-only, but `metadata` is caller
# controlled on proxy requests. Strip any client-supplied copies here (this
# runs before the router stashes its own reservation) so a forged
@ -64,11 +84,14 @@ def set_io_token_rate_limit_request_kwargs(kwargs: dict[str, object] | None, sto
# every other deployment overwrite the slot with None instead of the
# kwargs; overwriting (rather than skipping) also releases a previous
# request's kwargs when a context is reused.
_io_token_rate_limit_request_kwargs.set(kwargs if store_in_context else None)
_io_token_rate_limit_request.set(
_IoTokenRateLimitRequest(kwargs, opens_responses_socket, metadata_channel) if store_in_context else None
)
def get_io_token_rate_limit_request_kwargs() -> dict[str, object] | None:
return _io_token_rate_limit_request_kwargs.get()
request: Final = _io_token_rate_limit_request.get()
return None if request is None else request.kwargs
def seconds_until_minute_reset() -> int:
@ -224,27 +247,32 @@ def _otpm_reservation(
return default if from_deployment is None else min(from_deployment, default)
def _usage_field(container: object, name: str) -> object:
if is_str_keyed_mapping(container):
return container.get(name)
return getattr(container, name, None)
def _cached_prompt_tokens(usage: object) -> int:
reported: Final = (
_usage_field(_usage_field(usage, "prompt_tokens_details"), "cached_tokens"),
_usage_field(_usage_field(usage, "input_tokens_details"), "cached_tokens"),
_usage_field(usage, "cache_read_input_tokens"),
)
return next((int(tokens) for tokens in reported if isinstance(tokens, int | float) and tokens), 0)
def _get_usage_tokens(usage: object) -> tuple[int, int, int]:
if usage is None:
return 0, 0, 0
if hasattr(usage, "prompt_tokens") or hasattr(usage, "input_tokens"):
prompt = int(getattr(usage, "prompt_tokens", None) or getattr(usage, "input_tokens", 0) or 0)
completion = int(getattr(usage, "completion_tokens", None) or getattr(usage, "output_tokens", 0) or 0)
cached = 0
details: object = getattr(usage, "prompt_tokens_details", None)
if details is not None:
cached = int(getattr(details, "cached_tokens", 0) or 0)
if not cached:
cached = int(getattr(usage, "cache_read_input_tokens", 0) or 0)
return prompt, completion, cached
return prompt, completion, _cached_prompt_tokens(usage)
if isinstance(usage, dict):
prompt = int(usage.get("prompt_tokens") or usage.get("input_tokens") or 0)
completion = int(usage.get("completion_tokens") or usage.get("output_tokens") or 0)
details = usage.get("prompt_tokens_details") or {}
cached = int(details.get("cached_tokens", 0) or 0) if isinstance(details, dict) else 0
if not cached:
cached = int(usage.get("cache_read_input_tokens") or 0)
return prompt, completion, cached
return prompt, completion, _cached_prompt_tokens(usage)
return 0, 0, 0
@ -298,19 +326,36 @@ def _resolve_reconcile_usage_tokens(
if isinstance(standard_logging_object, dict):
prompt_tokens = int(standard_logging_object.get("prompt_tokens") or 0)
completion_tokens = int(standard_logging_object.get("completion_tokens") or 0)
cached_tokens = 0
metadata: Final = standard_logging_object.get("metadata")
if isinstance(metadata, dict):
cached_tokens = int(metadata.get("cache_read_input_tokens") or 0)
logged_usage: Final = _usage_field(standard_logging_object.get("metadata"), "usage_object")
logged_cached_tokens: Final = _cached_prompt_tokens(logged_usage)
# Same rationale as _usage_is_present: a bare total_tokens with no
# prompt/completion breakdown can't be split, so it isn't treated
# as resolved usage - the reservation is kept instead of refunded.
if prompt_tokens or completion_tokens:
return max(0, prompt_tokens - cached_tokens), completion_tokens, True
return max(0, prompt_tokens - logged_cached_tokens), completion_tokens, True
return 0, 0, False
def _reservation(
itpm_reserved: int, otpm_reserved: int, itpm_cache_key: str | None, otpm_cache_key: str | None
) -> Mapping[str, object]:
return MappingProxyType(
{
ITPM_RESERVED_KEY: itpm_reserved,
OTPM_RESERVED_KEY: otpm_reserved,
ITPM_CACHE_KEY: itpm_cache_key,
OTPM_CACHE_KEY: otpm_cache_key,
}
)
def _stash_reservation_in_metadata(request: _IoTokenRateLimitRequest | None, reservation: Mapping[str, object]) -> None:
metadata: Final = None if request is None or not request.kwargs else request.kwargs.get(request.metadata_channel)
if isinstance(metadata, dict):
metadata.update(reservation)
def _key_for_this_minute(reserved_key: str) -> str:
counter, _, _ = reserved_key.rpartition(":")
return f"{counter}:{get_utc_datetime().strftime('%H-%M')}"
@ -326,30 +371,6 @@ def _reconcile_increments(
)
def _stash_reservation_in_metadata(
request_kwargs: dict[str, object] | None,
*,
itpm_reserved: int,
otpm_reserved: int,
itpm_cache_key: str | None,
otpm_cache_key: str | None,
) -> None:
if not request_kwargs:
return
reservation: Final = {
ITPM_RESERVED_KEY: itpm_reserved,
OTPM_RESERVED_KEY: otpm_reserved,
ITPM_CACHE_KEY: itpm_cache_key,
OTPM_CACHE_KEY: otpm_cache_key,
}
for channel in ("metadata", "litellm_metadata"):
existing = request_kwargs.get(channel)
if isinstance(existing, dict):
existing.update(reservation)
elif channel == "metadata":
request_kwargs[channel] = dict(reservation)
def _extract_reservation(reservation: Mapping[str, int | str | None]) -> tuple[int, int, str | None, str | None]:
itpm_cache_key: Final = reservation.get(ITPM_CACHE_KEY)
otpm_cache_key: Final = reservation.get(OTPM_CACHE_KEY)
@ -399,7 +420,7 @@ def _clear_reservation_from_kwargs(kwargs: Mapping[str, object] | None) -> None:
for channel in _reservation_channels(kwargs):
channel_dict = _as_mutable_mapping(channel)
if channel_dict is not None:
for key in (ITPM_RESERVED_KEY, OTPM_RESERVED_KEY, ITPM_CACHE_KEY, OTPM_CACHE_KEY):
for key in IO_TOKEN_RESERVATION_KEYS:
channel_dict.pop(key, None)
@ -499,7 +520,8 @@ def io_token_pre_call_check(
if itpm_limit is None and otpm_limit is None:
return deployment
request_kwargs: Final = get_io_token_rate_limit_request_kwargs()
request: Final = _io_token_rate_limit_request.get()
request_kwargs: Final = None if request is None else request.kwargs
_model, served_model = _tokenizer_and_served_models(deployment)
estimated_input: Final = _estimate_input_tokens(request_kwargs, model=_model)
otpm_to_reserve: Final = (
@ -546,11 +568,13 @@ def io_token_pre_call_check(
raise
_stash_reservation_in_metadata(
request_kwargs,
itpm_reserved=itpm_reserved,
otpm_reserved=otpm_reserved,
itpm_cache_key=itpm_key if itpm_limit is not None else None,
otpm_cache_key=otpm_key if otpm_limit is not None else None,
request,
_reservation(
itpm_reserved,
otpm_reserved,
itpm_key if itpm_limit is not None else None,
otpm_key if otpm_limit is not None else None,
),
)
return deployment
@ -565,7 +589,14 @@ async def async_io_token_pre_call_check(
if itpm_limit is None and otpm_limit is None:
return deployment
request_kwargs: Final = get_io_token_rate_limit_request_kwargs()
request: Final = _io_token_rate_limit_request.get()
request_kwargs: Final = None if request is None else request.kwargs
if request is not None and request.opens_responses_socket:
_stash_reservation_in_metadata(
request,
await async_admit_io_token_turn(dual_cache, deployment, get_utc_datetime(), parent_otel_span),
)
return deployment
_model, served_model = _tokenizer_and_served_models(deployment)
estimated_input: Final = await offload_token_count(_estimate_input_tokens)(request_kwargs, model=_model)
otpm_to_reserve: Final = (
@ -618,15 +649,65 @@ async def async_io_token_pre_call_check(
raise
_stash_reservation_in_metadata(
request_kwargs,
itpm_reserved=itpm_reserved,
otpm_reserved=otpm_reserved,
itpm_cache_key=itpm_key if itpm_limit is not None else None,
otpm_cache_key=otpm_key if otpm_limit is not None else None,
request,
_reservation(
itpm_reserved,
otpm_reserved,
itpm_key if itpm_limit is not None else None,
otpm_key if otpm_limit is not None else None,
),
)
return deployment
async def _async_redis_usage(redis_cache: RedisCache | None, key: str, parent_otel_span: Span | None) -> int | None:
if redis_cache is None:
return None
with contextlib.suppress(RedisCircuitBreakerOpenError):
return await redis_cache.async_get_cache(key=key, parent_otel_span=parent_otel_span)
return None
@with_service_target(ROUTER_USAGE_TARGET)
async def _async_shared_usage(dual_cache: DualCache, key: str, parent_otel_span: Span | None) -> int | None:
shared: Final = await _async_redis_usage(dual_cache.redis_cache, key, parent_otel_span)
if shared is not None:
return shared
return await dual_cache.async_get_cache(key=key, local_only=True)
async def _raise_if_at_limit(
dual_cache: DualCache, key: str, limit: int | None, limit_label: str, parent_otel_span: Span | None
) -> None:
if limit is None:
return
usage: Final = await _async_shared_usage(dual_cache, key, parent_otel_span)
if usage is not None and usage >= limit:
raise _rate_limit_error(limit_label, limit, usage)
@with_service_target(ROUTER_USAGE_TARGET)
async def async_admit_io_token_turn(
dual_cache: DualCache,
deployment: Mapping[str, object],
now: datetime,
parent_otel_span: Span | None = None,
) -> Mapping[str, object]:
configured: Final = dict(deployment)
itpm_limit, otpm_limit = get_deployment_io_token_limits(configured)
if itpm_limit is None and otpm_limit is None:
return MappingProxyType({})
cache_keys: Final = _get_cache_keys(configured, now.strftime("%H-%M"))
if cache_keys is None:
return MappingProxyType({})
itpm_key, otpm_key = cache_keys
await _raise_if_at_limit(dual_cache, itpm_key, itpm_limit, "ITPM", parent_otel_span)
await _raise_if_at_limit(dual_cache, otpm_key, otpm_limit, "OTPM", parent_otel_span)
return _reservation(
0, 0, itpm_key if itpm_limit is not None else None, otpm_key if otpm_limit is not None else None
)
@with_service_target(ROUTER_USAGE_TARGET)
def io_token_reconcile_success(
dual_cache: DualCache,

8
litellm/router_utils/reasoning_effort_capability.py
View file

@ -38,7 +38,7 @@ from litellm.litellm_core_utils.reasoning_effort_strength import (
ReasoningEffortRounding,
nearest_declared_reasoning_effort,
)
from litellm.litellm_core_utils.reasoning_effort_utils import reasoning_effort_from_thinking
from litellm.litellm_core_utils.reasoning_effort_utils import reasoning_effort_from_thinking_silently
from litellm.llms.anthropic.common_utils import AnthropicModelInfo, validated_object_mapping
from litellm.types.llms.anthropic import AnthropicThinkingParam
from litellm.types.llms.openai import REASONING_EFFORT
@ -143,7 +143,7 @@ def with_adaptive_thinking_as_effort(
params: Mapping[str, object], *, model: str, custom_llm_provider: str, drop_params: bool
) -> Mapping[str, object]:
translated: Final = dict(params)
AnthropicModelInfo.translate_legacy_thinking_for_adaptive_model(
AnthropicModelInfo.translate_legacy_thinking_for_adaptive_model_silently(
model=model, optional_params=translated, custom_llm_provider=custom_llm_provider, drop_params=drop_params
)
return translated
@ -151,7 +151,7 @@ def with_adaptive_thinking_as_effort(
def thinking_budget_effort(thinking: object, *, model: str, custom_llm_provider: str, drop_params: bool) -> str | None:
if custom_llm_provider in _PROVIDERS_SENDING_THINKING_AS_EFFORT:
return reasoning_effort_from_thinking(thinking, drop_params=drop_params)
return reasoning_effort_from_thinking_silently(thinking, drop_params=drop_params)
adaptive_config: Final = as_str_mapping(
with_adaptive_thinking_as_effort(
{"thinking": thinking}, model=model, custom_llm_provider=custom_llm_provider, drop_params=drop_params
@ -162,7 +162,7 @@ def thinking_budget_effort(thinking: object, *, model: str, custom_llm_provider:
return adaptive_effort if isinstance(adaptive_effort, str) else None
if "thinking" in (get_supported_openai_params(model=model, custom_llm_provider=custom_llm_provider) or ()):
return None
return reasoning_effort_from_thinking(thinking, drop_params=drop_params)
return reasoning_effort_from_thinking_silently(thinking, drop_params=drop_params)
def provider_reads_thinking(thinking: object, *, model: str, custom_llm_provider: str, drop_params: bool) -> bool:

15
litellm/types/llms/anthropic.py
View file

@ -713,6 +713,14 @@ class AnthropicResponseContentBlockRedactedThinking(LiteLLMBaseModel):
data: str
class AnthropicResponseContentBlockCompaction(LiteLLMBaseModel):
model_config = ConfigDict(frozen=True)
type: Literal["compaction"]
content: str | None = None
signature: str | None = None
class AnthropicResponseContentBlockServerToolUse(LiteLLMBaseModel):
model_config = ConfigDict(frozen=True)
@ -743,7 +751,7 @@ class AnthropicOutputTokensDetails(LiteLLMBaseModel):
thinking_tokens: int | None = None
AnthropicFinishReason = Literal["end_turn", "max_tokens", "stop_sequence", "tool_use", "refusal"]
AnthropicFinishReason = Literal["end_turn", "max_tokens", "stop_sequence", "tool_use", "refusal", "compaction"]
class AnthropicResponse(LiteLLMBaseModel):
@ -761,6 +769,7 @@ class AnthropicResponse(LiteLLMBaseModel):
| AnthropicResponseContentBlockToolUse
| AnthropicResponseContentBlockThinking
| AnthropicResponseContentBlockRedactedThinking
| AnthropicResponseContentBlockCompaction
| AnthropicResponseContentBlockServerToolUse
| AnthropicResponseContentBlockWebSearchToolResult
]
@ -808,6 +817,9 @@ class AnthropicThinkingParam(TypedDict, total=False):
block_binding: ReadOnly[AnthropicThinkingBlockBinding]
LEGACY_THINKING_DISPLAY_UPDATES_PROVIDERS: Final = frozenset(("anthropic", "azure_ai", "vertex_ai"))
class ANTHROPIC_HOSTED_TOOLS(str, Enum):
WEB_SEARCH = "web_search"
BASH = "bash"
@ -828,6 +840,7 @@ class ANTHROPIC_BETA_HEADER_VALUES(str, Enum):
CONTEXT_MANAGEMENT_2025_06_27 = "context-management-2025-06-27"
COMPACT_2026_01_12 = "compact-2026-01-12"
COMPACT_2026_09_04 = "compact-2026-09-04"
INLINE_TOOLS_2026_09_15 = "inline-tools-2026-09-15"
STRUCTURED_OUTPUT_2025_09_25 = "structured-outputs-2025-11-13"
ADVANCED_TOOL_USE_2025_11_20 = "advanced-tool-use-2025-11-20"
FAST_MODE_2026_02_01 = "fast-mode-2026-02-01"

10
litellm/types/llms/openai.py
View file

@ -1,4 +1,5 @@
import builtins
import math
from collections.abc import Iterable, Mapping, Sequence
from enum import Enum
from os import PathLike
@ -83,6 +84,7 @@ from typing_extensions import (
override,
)
from litellm._logging import verbose_logger
from litellm.types.llms.base import BaseLiteLLMOpenAIResponseObject, LiteLLMBaseModel
from litellm.types.responses.main import (
CustomToolCallOutputItem,
@ -1446,9 +1448,11 @@ class ResponseAPIUsage(BaseLiteLLMOpenAIResponseObject):
@classmethod
def parse_cost(cls, v: object) -> object:
"""Normalise cost: accept either a float or a dict with a ``total_cost`` key."""
if isinstance(v, dict):
return v.get("total_cost")
return v
cost: Final = v.get("total_cost") if isinstance(v, dict) else v
if isinstance(cost, float) and not math.isfinite(cost):
verbose_logger.warning("Dropping a non-finite provider usage cost")
return None
return cost
model_config = ConfigDict(extra="allow")

27
litellm/types/proxy/management_endpoints/common_daily_activity.py
View file

@ -68,29 +68,34 @@ class MetricWithMetadata(MetricBase):
class GuardrailSpendMetrics(LiteLLMBaseModel):
model_config = ConfigDict(frozen=True)
spend: float = 0.0
untracked_units: int = 0
spend: float = Field(default=0.0, description="Guardrail spend in USD, already included in key and model spend")
untracked_units: int = Field(default=0, description="Guardrail units with no known price, excluded from spend")
class GuardrailKeyMetric(LiteLLMBaseModel):
model_config = ConfigDict(frozen=True)
metrics: GuardrailSpendMetrics
metadata: KeyMetadata
metrics: GuardrailSpendMetrics = Field(description="Guardrail spend and unpriced units for this API key")
metadata: KeyMetadata = Field(description="API key alias and existence information")
class GuardrailMetadata(LiteLLMBaseModel):
model_config = ConfigDict(frozen=True)
guardrail_name: str
guardrail_name: str = Field(description="Guardrail display name, falling back to the guardrail ID")
class GuardrailMetric(LiteLLMBaseModel):
model_config = ConfigDict(frozen=True)
metrics: GuardrailSpendMetrics
metadata: GuardrailMetadata
api_key_breakdown: Mapping[str, GuardrailKeyMetric] = Field(default_factory=dict)
metrics: GuardrailSpendMetrics = Field(
description="Guardrail spend and unpriced units across all matching API keys"
)
metadata: GuardrailMetadata = Field(description="Guardrail name information")
api_key_breakdown: Mapping[str, GuardrailKeyMetric] = Field(
default_factory=dict,
description="Per-key guardrail spend and unpriced units, limited to keys in the day's API key breakdown",
)
class BreakdownMetrics(LiteLLMBaseModel):
@ -103,7 +108,11 @@ class BreakdownMetrics(LiteLLMBaseModel):
endpoints: dict[str, MetricWithMetadata] = Field(default_factory=dict) # endpoint -> {metrics, metadata}
api_keys: dict[str, KeyMetricWithMetadata] = Field(default_factory=dict) # api_key -> {metrics, metadata}
entities: dict[str, MetricWithMetadata] = Field(default_factory=dict) # entity -> {metrics, metadata}
guardrails: Mapping[str, GuardrailMetric] = Field(default_factory=dict) # guardrail_id -> spend
guardrails: Mapping[str, GuardrailMetric] = Field(
default_factory=dict,
description="Spend by guardrail ID, already included in key and model spend. "
"Populated on user, team and organization aggregated activity only, without a model filter",
)
class DailySpendData(LiteLLMBaseModel):

62
litellm/types/utils.py
View file

@ -1,4 +1,5 @@
import json
import math
import re
import time
from collections.abc import Collection, Mapping, Sequence
@ -74,6 +75,7 @@ from litellm.types.workload_identity import (
)
from ..litellm_core_utils.core_helpers import map_finish_reason, process_response_headers
from ..litellm_core_utils.hidden_params import set_extra_attribute
from . import litellm_params as _litellm_params
from .agents import LiteLLMSendMessageResponse
from .guardrails import GuardrailEventHooks
@ -1971,6 +1973,19 @@ class ServerToolUse(LiteLLMBaseModel):
return getattr(self, key)
_COST_BREAKDOWN_ADAPTER: Final = TypeAdapter(Mapping[object, object])
def _is_non_finite_float(value: object) -> bool:
return isinstance(value, float) and not math.isfinite(value)
def _carries_non_finite_number(cost: object) -> bool:
if not isinstance(cost, Mapping):
return _is_non_finite_float(cost)
return any(_is_non_finite_float(member) for member in _COST_BREAKDOWN_ADAPTER.validate_python(cost).values())
class Usage(SafeAttributeModel, CompletionUsage):
_cache_creation_input_tokens: int = PrivateAttr(
0
@ -1978,6 +1993,7 @@ class Usage(SafeAttributeModel, CompletionUsage):
_cache_read_input_tokens: int = PrivateAttr(
0
) # hidden param for prompt caching. Might change, once openai introduces their equivalent.
_dropped_non_finite_cost: bool = PrivateAttr(False)
server_tool_use: ServerToolUse | None = None
cost: float | None = None
@ -2084,10 +2100,14 @@ class Usage(SafeAttributeModel, CompletionUsage):
else: # maintain openai compatibility in usage object if possible
del self.server_tool_use
if cost is not None:
self.cost = cost
else:
dropped_non_finite_cost: Final = cost is not None and _carries_non_finite_number(cost)
if cost is None:
del self.cost
elif dropped_non_finite_cost:
verbose_logger.warning("Dropping a non-finite provider usage cost")
del self.cost
else:
self.cost = cost
## ANTHROPIC MAPPING ##
if "cache_creation_input_tokens" in params and isinstance(params["cache_creation_input_tokens"], int):
@ -2102,6 +2122,11 @@ class Usage(SafeAttributeModel, CompletionUsage):
for k, v in params.items():
setattr(self, k, v)
self._dropped_non_finite_cost = dropped_non_finite_cost
@staticmethod
def dropped_non_finite_cost(usage: "Usage") -> bool:
return usage._dropped_non_finite_cost
def __contains__(self, key) -> bool:
# Define custom behavior for the 'in' operator
@ -2125,6 +2150,13 @@ class UnreportedUsage(Usage):
class StreamingChoices(OpenAIObject):
if TYPE_CHECKING:
finish_reason: str | None
index: int
delta: Delta
enhancements: object
logprobs: ChoiceLogprobs | JsonValue
def __init__(
self,
finish_reason=None,
@ -2138,25 +2170,22 @@ class StreamingChoices(OpenAIObject):
# https://github.com/BerriAI/litellm/issues/8455
params.pop("message", None)
super().__init__(**params)
if finish_reason:
self.finish_reason = map_finish_reason(finish_reason)
else:
self.finish_reason = None
self.index = index
set_extra_attribute(self, "finish_reason", map_finish_reason(finish_reason) if finish_reason else None)
set_extra_attribute(self, "index", index)
if delta is not None:
if isinstance(delta, Delta):
self.delta = delta
set_extra_attribute(self, "delta", delta)
elif isinstance(delta, dict):
self.delta = Delta(**delta)
set_extra_attribute(self, "delta", Delta(**delta))
else:
self.delta = Delta()
set_extra_attribute(self, "delta", Delta())
if enhancements is not None:
self.enhancements = enhancements
set_extra_attribute(self, "enhancements", enhancements)
if isinstance(logprobs, (dict, BaseModel)):
self.logprobs = CHOICE_LOGPROBS_OR_JSON.validate_python(logprobs)
set_extra_attribute(self, "logprobs", CHOICE_LOGPROBS_OR_JSON.validate_python(logprobs))
else:
self.logprobs = logprobs
set_extra_attribute(self, "logprobs", logprobs)
def __contains__(self, key) -> bool:
# Define custom behavior for the 'in' operator
@ -2210,7 +2239,10 @@ class ModelResponseBase(OpenAIObject):
@property
def hidden_params(self) -> dict[str, object]: # mutable-ok: API requires mutation
return self._hidden_params
private: Final = self.__pydantic_private__
if private is None or "_hidden_params" not in private:
return self._hidden_params
return private["_hidden_params"]
@hidden_params.setter
def hidden_params(self, hidden_params: dict[str, object]) -> None: # mutable-ok: API requires mutation

10
litellm/utils.py
View file

@ -94,6 +94,7 @@ from litellm.litellm_core_utils.fallback_generalizations import (
)
from litellm.litellm_core_utils.get_model_cost_map import AliasedCostEntry, GetModelCostMap
from litellm.litellm_core_utils.model_param_helper import peek_reasoning_summary_aliases
from litellm.litellm_core_utils.model_retirement import is_hidden_as_retired
from litellm.litellm_core_utils.offline_mode import is_offline_mode
from litellm.litellm_core_utils.sensitive_data_masker import redact_credentials_in_payload
from litellm.litellm_core_utils.tokenizer import strip_special_tokens
@ -8806,6 +8807,12 @@ def _get_valid_models_from_provider_api(
return []
def bundled_provider_models(provider: str) -> tuple[str, ...]:
bundled: Final[tuple[str, ...]] = tuple(litellm.models_by_provider.get(provider, ()))
cost_map: Final = _builtin_cost_map()
return tuple(model for model in bundled if not is_hidden_as_retired(cost_map.get(model, {})))
def get_valid_models(
check_provider_endpoint: bool | None = None,
custom_llm_provider: str | None = None,
@ -8871,8 +8878,7 @@ def get_valid_models(
)
)
else:
models_for_provider = copy.deepcopy(litellm.models_by_provider.get(provider, []))
valid_models.extend(models_for_provider)
valid_models.extend(bundled_provider_models(provider))
return valid_models
except Exception as e:

80
model_prices_and_context_window.json
View file

@ -8715,7 +8715,7 @@
"search_context_size_low": 0.014,
"search_context_size_medium": 0.014
},
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'",
"source": "https://azure.microsoft.com/en-us/pricing/details/cognitive-services/openai-service/",
"supported_endpoints": [
"/v1/chat/completions",
"/v1/responses"
@ -8742,7 +8742,11 @@
"supports_tool_choice": true,
"supports_vision": true,
"supports_web_search": true,
"supports_xhigh_reasoning_effort": true
"supports_xhigh_reasoning_effort": true,
"cache_creation_input_token_cost_above_272k_tokens_flex": 1.25e-05,
"cache_read_input_token_cost_above_272k_tokens_flex": 1e-06,
"input_cost_per_token_above_272k_tokens_flex": 1e-05,
"output_cost_per_token_above_272k_tokens_flex": 3.75e-05
},
"azure/gpt-6-astra-2026-09-03": {
"cache_creation_input_token_cost": 1.25e-05,
@ -8763,7 +8767,7 @@
"search_context_size_low": 0.014,
"search_context_size_medium": 0.014
},
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'",
"source": "https://azure.microsoft.com/en-us/pricing/details/cognitive-services/openai-service/",
"supported_endpoints": [
"/v1/chat/completions",
"/v1/responses"
@ -8790,7 +8794,11 @@
"supports_tool_choice": true,
"supports_vision": true,
"supports_web_search": true,
"supports_xhigh_reasoning_effort": true
"supports_xhigh_reasoning_effort": true,
"cache_creation_input_token_cost_above_272k_tokens_flex": 1.25e-05,
"cache_read_input_token_cost_above_272k_tokens_flex": 1e-06,
"input_cost_per_token_above_272k_tokens_flex": 1e-05,
"output_cost_per_token_above_272k_tokens_flex": 3.75e-05
},
"azure/gpt-6-luna": {
"cache_creation_input_token_cost": 1.25e-07,
@ -30856,8 +30864,9 @@
"output_cost_per_token": 0,
"output_vector_size": 3072,
"rpm": 10000,
"source": "https://ai.google.dev/gemini-api/docs/embeddings#model-versions",
"tpm": 10000000
"source": "https://ai.google.dev/gemini-api/docs/pricing",
"tpm": 10000000,
"input_cost_per_token_batches": 7.5e-08
},
"gemini/gemini-embedding-2": {
"input_cost_per_audio_token": 6.5e-06,
@ -31713,7 +31722,10 @@
"input_cost_per_token_flex": 6.25e-07,
"output_cost_per_token_batches": 5e-06,
"output_cost_per_token_flex": 5e-06,
"supports_url_context": true
"supports_url_context": true,
"cache_read_input_token_cost_above_200k_tokens_batches": 2.5e-07,
"input_cost_per_token_above_200k_tokens_batches": 1.25e-06,
"output_cost_per_token_above_200k_tokens_batches": 7.5e-06
},
"gemini/gemini-2.5-computer-use-preview-10-2025": {
"deprecation_date": "2026-07-28",
@ -31871,7 +31883,8 @@
},
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"web_search_billing_unit": "per_query",
"google_maps_grounding_cost_per_query": 0.014
"google_maps_grounding_cost_per_query": 0.014,
"supports_computer_use": true
},
"gemini/gemini-3-flash-preview": {
"cache_read_input_audio_token_cost": 1e-07,
@ -31937,7 +31950,8 @@
"input_cost_per_token_flex": 2.5e-07,
"output_cost_per_token_batches": 1.5e-06,
"output_cost_per_token_flex": 1.5e-06,
"supports_audio_input": true
"supports_audio_input": true,
"supports_computer_use": true
},
"gemini/gemini-3.5-flash": {
"prompt_cache_min_tokens": 4096,
@ -32184,7 +32198,8 @@
},
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"web_search_billing_unit": "per_query",
"google_maps_grounding_cost_per_query": 0.014
"google_maps_grounding_cost_per_query": 0.014,
"supports_computer_use": true
},
"gemini/gemini-omni-flash-preview": {
"input_cost_per_audio_token": 1.5e-06,
@ -32280,7 +32295,10 @@
"web_search_billing_unit": "per_query",
"google_maps_grounding_cost_per_query": 0.014,
"input_cost_per_token_flex": 1e-06,
"output_cost_per_token_flex": 6e-06
"output_cost_per_token_flex": 6e-06,
"cache_read_input_token_cost_above_200k_tokens_batches": 4e-07,
"input_cost_per_token_above_200k_tokens_batches": 2e-06,
"output_cost_per_token_above_200k_tokens_batches": 9e-06
},
"gemini/gemini-3.1-pro-preview-customtools": {
"prompt_cache_min_tokens": 4096,
@ -32343,7 +32361,10 @@
"web_search_billing_unit": "per_query",
"google_maps_grounding_cost_per_query": 0.014,
"input_cost_per_token_flex": 1e-06,
"output_cost_per_token_flex": 6e-06
"output_cost_per_token_flex": 6e-06,
"cache_read_input_token_cost_above_200k_tokens_batches": 4e-07,
"input_cost_per_token_above_200k_tokens_batches": 2e-06,
"output_cost_per_token_above_200k_tokens_batches": 9e-06
},
"gemini-3-flash-preview": {
"cache_read_input_audio_token_cost": 1e-07,
@ -72228,6 +72249,7 @@
},
"aiand/deepseek-ai/deepseek-v4-flash": {
"cache_read_input_token_cost": 8e-08,
"deprecation_date": "2026-10-27",
"input_cost_per_token": 1.5e-07,
"litellm_provider": "aiand",
"max_input_tokens": 1048576,
@ -72257,6 +72279,22 @@
"supports_vision": false
},
"aiand/deepseek-ai/deepseek-v4.1-flash": {
"cache_read_input_token_cost": 2e-08,
"deprecation_date": "2026-11-05",
"input_cost_per_token": 3e-07,
"litellm_provider": "aiand",
"max_input_tokens": 1048576,
"max_output_tokens": 384000,
"mode": "chat",
"output_cost_per_token": 6e-07,
"source": "https://api.aiand.com/v1/models",
"supports_function_calling": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_tool_choice": true,
"supports_vision": true
},
"aiand/deepseek-ai/deepseek-v4.1-flash-1017": {
"cache_read_input_token_cost": 2e-08,
"input_cost_per_token": 3e-07,
"litellm_provider": "aiand",
@ -72286,8 +72324,24 @@
"supports_tool_choice": true,
"supports_vision": true
},
"aiand/meta/muse-spark-open": {
"cache_read_input_token_cost": 1.5e-07,
"input_cost_per_token": 1.25e-06,
"litellm_provider": "aiand",
"max_input_tokens": 1048576,
"max_output_tokens": 943718,
"mode": "chat",
"output_cost_per_token": 4.25e-06,
"source": "https://api.aiand.com/v1/models",
"supports_function_calling": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_tool_choice": true,
"supports_vision": true
},
"aiand/moonshotai/kimi-k2.7-code": {
"cache_read_input_token_cost": 2e-07,
"deprecation_date": "2026-10-27",
"input_cost_per_token": 7.5e-07,
"litellm_provider": "aiand",
"max_input_tokens": 262144,
@ -72378,6 +72432,7 @@
},
"aiand/zai-org/glm-5.2": {
"cache_read_input_token_cost": 3e-07,
"deprecation_date": "2026-10-27",
"input_cost_per_token": 1e-06,
"litellm_provider": "aiand",
"max_input_tokens": 1048576,
@ -76540,6 +76595,7 @@
],
"supports_audio_input": true,
"supports_audio_output": false,
"supports_computer_use": true,
"supports_function_calling": true,
"supports_minimal_reasoning_effort": false,
"supports_native_streaming": true,

18
tests/integration/_support/hook_count_probe.py
View file

@ -75,6 +75,24 @@ class BlockingPostCallGuardrail(CustomGuardrail):
raise HTTPException(status_code=400, detail="blocked by the hook-count guardrail")
class UsageRecordingPostCallGuardrail(CustomGuardrail):
"""Records the token usage of the response it is handed in the request's guardrail information, so the spend row
shows which response the guardrail saw."""
async def async_post_call_success_hook(
self, data: dict[str, object], user_api_key_dict: object, response: object
) -> None:
usage: Final = getattr(response, "usage", None)
self.add_standard_logging_guardrail_information_to_request_data(
guardrail_json_response={
"prompt_tokens": getattr(usage, "prompt_tokens", None),
"completion_tokens": getattr(usage, "completion_tokens", None),
},
request_data=data,
guardrail_status="success",
)
probe: Final = HookCountProbe()
sync_only_probe: Final = SyncOnlyHookCountProbe()

4
tests/integration/authorization/test_guardrail_submissions.py
View file

@ -432,7 +432,7 @@ def test_team_guardrail_submissions_require_admin_approval(
f"Guardrail '{first_name}' not found. "
"Please ensure the guardrail is configured in your LiteLLM proxy."
),
"type": "internal_server_error",
"type": "invalid_request_error",
"param": None,
"code": "404",
}
@ -481,7 +481,7 @@ def test_team_guardrail_submissions_require_admin_approval(
f"Guardrail '{second_name}' not found. "
"Please ensure the guardrail is configured in your LiteLLM proxy."
),
"type": "internal_server_error",
"type": "invalid_request_error",
"param": None,
"code": "404",
}

2
tests/integration/authorization/test_user_self_update_guard.py
View file

@ -279,7 +279,7 @@ def test_new_user_with_admin_role_requires_proxy_admin(gateway: Gateway, role: s
assert refused.json() == {
"error": {
"message": f"Only proxy admins can create administrative users (proxy_admin, proxy_admin_viewer). Attempted to create user with role: {role}. Your role: internal_user",
"type": "internal_server_error",
"type": "auth_error",
"param": None,
"code": "403",
}

15
tests/integration/management/test_key_delete_revocation.py
View file

@ -171,11 +171,8 @@ def test_deleting_a_key_revokes_it(gateway: Gateway, spelling: str) -> None:
assert deleted_rows == (_DeletedKeyRow(token=key_hash),)
def test_deleting_a_key_revokes_it_on_a_warmed_peer(gateway: Gateway, peer: Gateway) -> None:
pytest.skip(
"BUG: POST /key/delete does not broadcast the key eviction, so a warmed peer keeps answering 200 for about "
"60 seconds until its in-memory cache TTL expires"
)
@pytest.mark.parametrize("spelling", ("keys", "hashed_keys", "key_aliases"))
def test_deleting_a_key_revokes_it_on_a_warmed_peer(gateway: Gateway, peer: Gateway, spelling: str) -> None:
backend: Final = "integration-keys-auth-delete"
prompts: Final = (
"delete warm primary " + uuid.uuid4().hex,
@ -208,9 +205,13 @@ def test_deleting_a_key_revokes_it_on_a_warmed_peer(gateway: Gateway, peer: Gate
)
assert len(warm_responses) == 2
assert tuple((request.method, request.target) for request in wire.drain()) == (_EXPECTED_CHAT,) * 2
deleted: Final = gateway.request("POST", "/key/delete", {"keys": [key]})
identifier: Final = {"keys": key, "hashed_keys": sha256(key.encode()).hexdigest(), "key_aliases": alias}[
spelling
]
delete_body: Final = {"key_aliases" if spelling == "key_aliases" else "keys": [identifier]}
deleted: Final = gateway.request("POST", "/key/delete", delete_body)
assert deleted.status_code == 200, deleted.text
assert _DeleteResponse.model_validate_json(deleted.content).deleted_keys == (key,), deleted.text
assert _DeleteResponse.model_validate_json(deleted.content).deleted_keys == (identifier,), deleted.text
peer_refusal: Final = eventually(
lambda: _chat(peer, model, key, prompts[2]),
lambda response: response.status_code == 401,

4
tests/integration/management/test_user_delete_revokes_keys.py
View file

@ -170,7 +170,7 @@ def test_user_delete_revokes_keys_and_memberships(gateway: Gateway, peer: Gatewa
assert info.json() == {
"error": {
"message": f"User {user_id} not found",
"type": "internal_server_error",
"type": "invalid_request_error",
"param": None,
"code": "404",
}
@ -337,7 +337,7 @@ def test_scim_user_delete_revokes_keys(gateway: Gateway, peer: Gateway, monkeypa
expected_info: Final = {
"error": {
"message": f"User {user_id} not found",
"type": "internal_server_error",
"type": "invalid_request_error",
"param": None,
"code": "404",
}

Some files were not shown because too many files have changed in this diff Show more