mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
Merge remote-tracking branch 'origin/main' into litellm_mistral_magistral_reasoning_effort
Some checks are pending
Cost map pins recheck / rerun-cost-map-guard (push) Waiting to run
LiteLLM Rust DB / db-queries (push) Waiting to run
LiteLLM Rust / rust-lint (push) Waiting to run
LiteLLM Rust / rust-test (push) Waiting to run
LiteLLM Rust / rust-wheel (push) Waiting to run
Terraform Provider / gofmt, vet, build, test (push) Waiting to run
Terraform Provider / Provider endpoints vs proxy OpenAPI schema (push) Waiting to run
Some checks are pending
Cost map pins recheck / rerun-cost-map-guard (push) Waiting to run
LiteLLM Rust DB / db-queries (push) Waiting to run
LiteLLM Rust / rust-lint (push) Waiting to run
LiteLLM Rust / rust-test (push) Waiting to run
LiteLLM Rust / rust-wheel (push) Waiting to run
Terraform Provider / gofmt, vet, build, test (push) Waiting to run
Terraform Provider / Provider endpoints vs proxy OpenAPI schema (push) Waiting to run
# Conflicts: # litellm/llms/mistral/chat/transformation.py
This commit is contained in:
commit
08e01d3dbe
293 changed files with 20579 additions and 1677 deletions
1
.github/workflows/test-unit.yml
vendored
1
.github/workflows/test-unit.yml
vendored
|
|
@ -227,6 +227,7 @@ jobs:
|
|||
tests/unit/llms/test_oss_decision.py
|
||||
tests/unit/llms/test_polling_url_origin_match.py
|
||||
tests/unit/llms/test_predibase_transformation.py
|
||||
tests/unit/llms/test_subscription_accounts.py
|
||||
tests/unit/llms/a2a
|
||||
tests/unit/llms/aiml
|
||||
tests/unit/llms/aiohttp_openai
|
||||
|
|
|
|||
|
|
@ -55,6 +55,7 @@ BACKEND_PATH_PREFIXES: tuple[str, ...] = (
|
|||
"/cache_settings",
|
||||
"/coordination_redis/",
|
||||
"/cost_tracking",
|
||||
"/subscription_accounts/",
|
||||
"/cost_optimization/",
|
||||
"/cost/",
|
||||
"/credentials",
|
||||
|
|
|
|||
|
|
@ -0,0 +1,21 @@
|
|||
SET LOCAL lock_timeout = '10s';
|
||||
|
||||
-- CreateTable
|
||||
CREATE TABLE IF NOT EXISTS "LiteLLM_SubscriptionAccountTable" (
|
||||
"subscription_account_id" TEXT NOT NULL,
|
||||
"custom_llm_provider" TEXT NOT NULL,
|
||||
"account_id" TEXT NOT NULL,
|
||||
"label" TEXT,
|
||||
"monthly_fee" DOUBLE PRECISION NOT NULL,
|
||||
"currency" TEXT NOT NULL,
|
||||
"billing_period_start" TEXT NOT NULL,
|
||||
"created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
"created_by" TEXT NOT NULL,
|
||||
"updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
"updated_by" TEXT NOT NULL,
|
||||
|
||||
CONSTRAINT "LiteLLM_SubscriptionAccountTable_pkey" PRIMARY KEY ("subscription_account_id")
|
||||
);
|
||||
|
||||
-- CreateIndex
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS "LiteLLM_SubscriptionAccountTable_provider_account_key" ON "LiteLLM_SubscriptionAccountTable"("custom_llm_provider", "account_id");
|
||||
|
|
@ -2131,3 +2131,19 @@ model LiteLLM_AccountingDelivery {
|
|||
@@id([event_id, target, version])
|
||||
@@index([target, state, next_attempt_at])
|
||||
}
|
||||
|
||||
model LiteLLM_SubscriptionAccountTable {
|
||||
subscription_account_id String @id @default(uuid())
|
||||
custom_llm_provider String
|
||||
account_id String
|
||||
label String?
|
||||
monthly_fee Float
|
||||
currency String
|
||||
billing_period_start String
|
||||
created_at DateTime @default(now()) @map("created_at")
|
||||
created_by String
|
||||
updated_at DateTime @default(now()) @updatedAt @map("updated_at")
|
||||
updated_by String
|
||||
|
||||
@@unique([custom_llm_provider, account_id], map: "LiteLLM_SubscriptionAccountTable_provider_account_key")
|
||||
}
|
||||
|
|
|
|||
|
|
@ -199,7 +199,7 @@ impl AnthropicBeta {
|
|||
&[BetaProvider::BedrockConverse, BetaProvider::Databricks]
|
||||
}
|
||||
Self::ContextManagement20250627 => &[BetaProvider::BedrockConverse],
|
||||
Self::AdvisorTool20260301 | Self::Compact20260904 => &[
|
||||
Self::AdvisorTool20260301 => &[
|
||||
BetaProvider::AzureAi,
|
||||
BetaProvider::BedrockConverse,
|
||||
BetaProvider::Bedrock,
|
||||
|
|
@ -207,6 +207,12 @@ impl AnthropicBeta {
|
|||
BetaProvider::VertexAi,
|
||||
BetaProvider::Databricks,
|
||||
],
|
||||
Self::Compact20260904 => &[
|
||||
BetaProvider::BedrockConverse,
|
||||
BetaProvider::Bedrock,
|
||||
BetaProvider::BedrockMantle,
|
||||
BetaProvider::Databricks,
|
||||
],
|
||||
Self::Bash20241022
|
||||
| Self::Bash20250124
|
||||
| Self::McpServers20251204
|
||||
|
|
@ -256,14 +262,18 @@ impl AnthropicBeta {
|
|||
Self::InterleavedThinking20250514 | Self::WebSearch20250305 => {
|
||||
&[BetaProvider::BedrockConverse, BetaProvider::Bedrock]
|
||||
}
|
||||
Self::MidConversationOutputConfig20260701
|
||||
| Self::MidConversationToolChanges20260701 => &[
|
||||
Self::MidConversationOutputConfig20260701 => &[
|
||||
BetaProvider::AzureAi,
|
||||
BetaProvider::BedrockConverse,
|
||||
BetaProvider::BedrockMantle,
|
||||
BetaProvider::VertexAi,
|
||||
BetaProvider::Databricks,
|
||||
],
|
||||
Self::MidConversationToolChanges20260701 => &[
|
||||
BetaProvider::BedrockConverse,
|
||||
BetaProvider::BedrockMantle,
|
||||
BetaProvider::Databricks,
|
||||
],
|
||||
Self::Output128k20250219 | Self::TokenEfficientTools20250219 => &[
|
||||
BetaProvider::AzureAi,
|
||||
BetaProvider::BedrockConverse,
|
||||
|
|
@ -483,7 +493,7 @@ mod tests {
|
|||
let config = beta_headers_config();
|
||||
let config_columns: Vec<String> = config
|
||||
.keys()
|
||||
.filter(|column| column.as_str() != "description")
|
||||
.filter(|column| column.as_str() != "description" && !column.ends_with("_header_only"))
|
||||
.cloned()
|
||||
.collect();
|
||||
let beta_provider_columns: Vec<String> = BetaProvider::VARIANTS
|
||||
|
|
@ -502,12 +512,16 @@ mod tests {
|
|||
let config = beta_headers_config();
|
||||
for provider in BetaProvider::VARIANTS {
|
||||
for beta in &AnthropicBeta::KNOWN {
|
||||
let expected = config
|
||||
.get(provider.as_ref())
|
||||
.and_then(serde_json::Value::as_object)
|
||||
.and_then(|column| column.get(beta.as_str()))
|
||||
.and_then(serde_json::Value::as_str)
|
||||
.map(str::to_string);
|
||||
let cell = |column: &str| {
|
||||
config
|
||||
.get(column)
|
||||
.and_then(serde_json::Value::as_object)
|
||||
.and_then(|values| values.get(beta.as_str()))
|
||||
.and_then(serde_json::Value::as_str)
|
||||
.map(str::to_string)
|
||||
};
|
||||
let expected =
|
||||
cell(&format!("{provider}_header_only")).or_else(|| cell(provider.as_ref()));
|
||||
let actual = beta.on(*provider).map(|name| name.to_string());
|
||||
assert_eq!(actual, expected, "provider {provider}, beta {beta}");
|
||||
if matches!(beta, AnthropicBeta::AdvancedToolUse20251120)
|
||||
|
|
|
|||
|
|
@ -1,7 +1,8 @@
|
|||
use litellm_auth::CredentialPlacement;
|
||||
use litellm_llms_types::{
|
||||
formats::messages::{
|
||||
ContextEdit, ContextManagement, Message, MessagesOptionalParams, MessagesRequest, Speed,
|
||||
ContentBlock, ContentBlockType, ContextEdit, ContextManagement, Message,
|
||||
MessagesOptionalParams, MessagesRequest, Speed, ThinkingConfig, ThinkingDisplay,
|
||||
},
|
||||
providers::anthropic::{AnthropicBeta, BetaProvider, BetaSet, DEFAULT_HEADERS},
|
||||
recognized::Recognized,
|
||||
|
|
@ -203,6 +204,10 @@ fn feature_betas(request: &MessagesRequest) -> BetaSet {
|
|||
.then_some(AnthropicBeta::FastMode20260201),
|
||||
messages_carry_output_config(&request.messages)
|
||||
.then_some(AnthropicBeta::PerTurnControl20260701),
|
||||
uses_thinking_display_updates(params)
|
||||
.then_some(AnthropicBeta::ThinkingDisplayUpdates20260818),
|
||||
messages_carry_tool_changes(&request.messages)
|
||||
.then_some(AnthropicBeta::MidConversationToolChanges20260701),
|
||||
has_advisor_tool(tools).then_some(AnthropicBeta::AdvisorTool20260301),
|
||||
is_tool_search_used(tools).then_some(AnthropicBeta::AdvancedToolUse20251120),
|
||||
]
|
||||
|
|
@ -253,6 +258,36 @@ fn messages_carry_output_config(messages: &[Message]) -> bool {
|
|||
.any(|message| message.extra.contains_key("output_config"))
|
||||
}
|
||||
|
||||
fn uses_thinking_display_updates(params: &MessagesOptionalParams) -> bool {
|
||||
let display = match params.thinking.as_ref().and_then(Recognized::known) {
|
||||
Some(ThinkingConfig::Enabled(thinking)) => thinking.display.as_ref(),
|
||||
Some(ThinkingConfig::Adaptive(thinking)) => thinking.display.as_ref(),
|
||||
_ => None,
|
||||
};
|
||||
display.and_then(Recognized::known) == Some(&ThinkingDisplay::Updates)
|
||||
}
|
||||
|
||||
fn is_tool_reference_change(block: &ContentBlock) -> bool {
|
||||
matches!(
|
||||
&block.block_type,
|
||||
Some(ContentBlockType::Other(block_type))
|
||||
if block_type == "tool_addition" || block_type == "tool_removal"
|
||||
) && block
|
||||
.extra
|
||||
.get("tool")
|
||||
.and_then(|tool| tool.get("type"))
|
||||
.and_then(Value::as_str)
|
||||
== Some("tool_reference")
|
||||
}
|
||||
|
||||
fn messages_carry_tool_changes(messages: &[Message]) -> bool {
|
||||
messages
|
||||
.iter()
|
||||
.filter(|message| message.role == "system")
|
||||
.flat_map(Message::blocks)
|
||||
.any(is_tool_reference_change)
|
||||
}
|
||||
|
||||
fn unsupported_param(model: &str, param: &str, value: &str, hint: &str) -> Error {
|
||||
Error::InvalidRequest(crate::ErrorDetail::UnsupportedParameter {
|
||||
model: model.into(),
|
||||
|
|
@ -1071,6 +1106,41 @@ mod tests {
|
|||
json!({"safeguards": [{"type": "dangerous_tool_use", "classifier_context": {"v": 1, "permission_mode": "auto"}}]}),
|
||||
&[]
|
||||
)]
|
||||
#[case::adaptive_display_updates(
|
||||
json!({"thinking": {"type": "adaptive", "display": "updates"}}),
|
||||
&["thinking-display-updates-2026-08-18"]
|
||||
)]
|
||||
#[case::enabled_display_updates(
|
||||
json!({"thinking": {"type": "enabled", "budget_tokens": 2048, "display": "updates"}}),
|
||||
&["thinking-display-updates-2026-08-18"]
|
||||
)]
|
||||
#[case::summarized_display(json!({"thinking": {"type": "adaptive", "display": "summarized"}}), &[])]
|
||||
#[case::disabled_thinking(json!({"thinking": {"type": "disabled"}}), &[])]
|
||||
#[case::system_tool_addition(
|
||||
json!({"messages": [
|
||||
{"role": "user", "content": "Use the ping tool"},
|
||||
{"role": "system", "content": [{"type": "tool_addition", "tool": {"type": "tool_reference", "name": "ping"}}]},
|
||||
]}),
|
||||
&["mid-conversation-tool-changes-2026-07-01"]
|
||||
)]
|
||||
#[case::system_tool_removal(
|
||||
json!({"messages": [
|
||||
{"role": "system", "content": [{"type": "tool_removal", "tool": {"type": "tool_reference", "name": "ping"}}]},
|
||||
]}),
|
||||
&["mid-conversation-tool-changes-2026-07-01"]
|
||||
)]
|
||||
#[case::user_tool_addition(
|
||||
json!({"messages": [
|
||||
{"role": "user", "content": [{"type": "tool_addition", "tool": {"type": "tool_reference", "name": "ping"}}]},
|
||||
]}),
|
||||
&[]
|
||||
)]
|
||||
#[case::tool_addition_without_a_tool_reference(
|
||||
json!({"messages": [
|
||||
{"role": "system", "content": [{"type": "tool_addition", "tool": {"type": "tool_definition"}}]},
|
||||
]}),
|
||||
&[]
|
||||
)]
|
||||
fn feature_betas_follow_the_request(#[case] fields: Value, #[case] expected: &[&str]) {
|
||||
assert_eq!(feature_betas(&request(fields)), betas(expected));
|
||||
}
|
||||
|
|
@ -1093,6 +1163,33 @@ mod tests {
|
|||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::vertex_ai(
|
||||
BetaProvider::VertexAi,
|
||||
&["mid-conversation-tool-changes-2026-07-01", "thinking-display-updates-2026-08-18"]
|
||||
)]
|
||||
#[case::azure_ai(
|
||||
BetaProvider::AzureAi,
|
||||
&["mid-conversation-tool-changes-2026-07-01", "thinking-display-updates-2026-08-18"]
|
||||
)]
|
||||
#[case::bedrock_converse(BetaProvider::BedrockConverse, &["thinking-display-updates-2026-08-18"])]
|
||||
#[case::databricks(BetaProvider::Databricks, &["thinking-display-updates-2026-08-18"])]
|
||||
fn tool_change_and_display_betas_reach_the_providers_that_accept_them(
|
||||
#[case] provider: BetaProvider,
|
||||
#[case] expected: &[&str],
|
||||
) {
|
||||
let fields = json!({
|
||||
"thinking": {"type": "adaptive", "display": "updates"},
|
||||
"messages": [
|
||||
{"role": "system", "content": [{"type": "tool_addition", "tool": {"type": "tool_reference", "name": "ping"}}]},
|
||||
],
|
||||
});
|
||||
assert_eq!(
|
||||
provider_feature_betas(&request(fields), provider),
|
||||
betas(expected)
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn hosts_without_a_beta_map_column_get_safeguards_without_a_derived_beta() {
|
||||
let client = headers(&[("anthropic-beta", "claude-code-20250219")]);
|
||||
|
|
|
|||
|
|
@ -545,6 +545,11 @@ mod tests {
|
|||
&[("anthropic-beta", "web-search-2025-03-05")],
|
||||
&[("x-api-key", "k"), ("anthropic-beta", "structured-outputs-2025-11-13,web-search-2025-03-05")]
|
||||
)]
|
||||
#[case::compaction_needs_the_compact_2026_09_04_beta(
|
||||
json!({"compaction": {"type": "summarize"}}),
|
||||
&[],
|
||||
&[("x-api-key", "k"), ("anthropic-beta", "compact-2026-09-04")]
|
||||
)]
|
||||
#[case::no_feature_needs_a_beta(json!({}), &[], &[("x-api-key", "k")])]
|
||||
#[case::safeguards_need_the_dangerous_tool_use_beta(
|
||||
json!({"safeguards": [{"type": "dangerous_tool_use"}]}),
|
||||
|
|
|
|||
|
|
@ -798,6 +798,18 @@ mod tests {
|
|||
]}}),
|
||||
&["compact-2026-01-12", "context-management-2025-06-27"]
|
||||
)]
|
||||
#[case::compaction(json!({"compaction": {"type": "summarize"}}), &["compact-2026-09-04"])]
|
||||
#[case::thinking_display_updates(
|
||||
json!({"thinking": {"type": "adaptive", "display": "updates"}}),
|
||||
&["thinking-display-updates-2026-08-18"]
|
||||
)]
|
||||
#[case::tool_addition(
|
||||
json!({"messages": [
|
||||
{"role": "user", "content": "Use the ping tool"},
|
||||
{"role": "system", "content": [{"type": "tool_addition", "tool": {"type": "tool_reference", "name": "ping"}}]}
|
||||
]}),
|
||||
&["mid-conversation-tool-changes-2026-07-01"]
|
||||
)]
|
||||
#[case::advisor_tool_is_not_a_vertex_beta(
|
||||
json!({"tools": [{"type": "advisor_20260301", "name": "advisor"}]}),
|
||||
&[]
|
||||
|
|
|
|||
|
|
@ -7,7 +7,6 @@ _apply_process_defaults()
|
|||
### INIT VARIABLES #########################
|
||||
import threading
|
||||
import os
|
||||
import datetime
|
||||
|
||||
from collections.abc import Mapping, Sequence
|
||||
from typing import (
|
||||
|
|
@ -550,6 +549,7 @@ _key_management_system: Optional["KeyManagementSystem"] = None
|
|||
output_parse_pii: bool = False
|
||||
#############################################
|
||||
from litellm.litellm_core_utils.get_model_cost_map import get_model_cost_map, mark_litellm_import_complete
|
||||
from litellm.litellm_core_utils.model_retirement import is_hidden_as_retired
|
||||
|
||||
model_cost = get_model_cost_map(url=model_cost_map_url)
|
||||
cost_discount_config: Dict[str, float] = {} # Provider-specific cost discounts {"vertex_ai": 0.05} = 5% discount
|
||||
|
|
@ -756,16 +756,6 @@ def is_openai_finetune_model(key: str) -> bool:
|
|||
return key.startswith("ft:") and not key.count(":") > 1
|
||||
|
||||
|
||||
def _is_past_deprecation_date(value: Mapping[str, object]) -> bool:
|
||||
deprecation_date: Final = value.get("deprecation_date")
|
||||
if not isinstance(deprecation_date, str):
|
||||
return False
|
||||
try:
|
||||
return datetime.date.fromisoformat(deprecation_date) <= datetime.date.today()
|
||||
except ValueError:
|
||||
return False
|
||||
|
||||
|
||||
def _populate_provider_model_sets(model_cost_map: Mapping[str, Mapping[str, object]]) -> None:
|
||||
for key, value in tuple(model_cost_map.items()):
|
||||
if value.get("litellm_provider") == "openai" and not is_openai_finetune_model(key):
|
||||
|
|
@ -873,12 +863,12 @@ def _populate_provider_model_sets(model_cost_map: Mapping[str, Mapping[str, obje
|
|||
if "-to-" not in key and "fireworks-ai-default" not in key:
|
||||
fireworks_ai_models.add(key)
|
||||
elif value.get("litellm_provider") == "together_ai":
|
||||
if _is_past_deprecation_date(value):
|
||||
if is_hidden_as_retired(value):
|
||||
together_ai_models.discard(key)
|
||||
elif not key.startswith("together-ai-"):
|
||||
together_ai_models.add(key)
|
||||
elif value.get("litellm_provider") == "baseten":
|
||||
if _is_past_deprecation_date(value):
|
||||
if is_hidden_as_retired(value):
|
||||
baseten_models.discard(key)
|
||||
else:
|
||||
baseten_models.add(key)
|
||||
|
|
@ -1051,7 +1041,10 @@ def _populate_provider_model_sets(model_cost_map: Mapping[str, Mapping[str, obje
|
|||
elif value.get("litellm_provider") == "coralbricks":
|
||||
coralbricks_models.add(key)
|
||||
elif value.get("litellm_provider") == "aiand":
|
||||
aiand_models.add(key)
|
||||
if is_hidden_as_retired(value):
|
||||
aiand_models.discard(key)
|
||||
else:
|
||||
aiand_models.add(key)
|
||||
elif value.get("litellm_provider") == "reducto":
|
||||
reducto_models.add(key)
|
||||
elif value.get("litellm_provider") == "bedrock_mantle":
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
{
|
||||
"description": "Mapping of Anthropic beta headers for each provider. Keys are input header names, values are provider-specific header names (or null if unsupported). Only headers present in mapping keys with non-null values can be forwarded.",
|
||||
"description": "Mapping of Anthropic beta headers for each provider. Keys are input header names, values are provider-specific header names (or null if unsupported). Only headers present in mapping keys with non-null values can be forwarded. A <provider>_header_only column lists betas that provider accepts only as the anthropic-beta HTTP header: they are forwarded there and dropped from the anthropic_beta body field. Proxies that predate these columns ignore them and read the provider column alone.",
|
||||
"anthropic": {
|
||||
"advisor-tool-2026-03-01": "advisor-tool-2026-03-01",
|
||||
"advanced-tool-use-2025-11-20": "advanced-tool-use-2025-11-20",
|
||||
|
|
@ -46,6 +46,7 @@
|
|||
"bash_20250124": null,
|
||||
"code-execution-2025-08-25": "code-execution-2025-08-25",
|
||||
"compact-2026-01-12": null,
|
||||
"compact-2026-09-04": "compact-2026-09-04",
|
||||
"computer-use-2025-01-24": "computer-use-2025-01-24",
|
||||
"computer-use-2025-11-24": "computer-use-2025-11-24",
|
||||
"context-1m-2025-08-07": "context-1m-2025-08-07",
|
||||
|
|
@ -59,6 +60,7 @@
|
|||
"mcp-client-2025-11-20": "mcp-client-2025-11-20",
|
||||
"mcp-client-2025-04-04": "mcp-client-2025-04-04",
|
||||
"mcp-servers-2025-12-04": null,
|
||||
"mid-conversation-tool-changes-2026-07-01": "mid-conversation-tool-changes-2026-07-01",
|
||||
"output-128k-2025-02-19": null,
|
||||
"structured-output-2024-03-01": null,
|
||||
"per-turn-control-2026-07-01": "per-turn-control-2026-07-01",
|
||||
|
|
@ -202,6 +204,7 @@
|
|||
"mcp-client-2025-11-20": null,
|
||||
"mcp-client-2025-04-04": null,
|
||||
"mcp-servers-2025-12-04": null,
|
||||
"mid-conversation-tool-changes-2026-07-01": "mid-conversation-tool-changes-2026-07-01",
|
||||
"output-128k-2025-02-19": null,
|
||||
"structured-output-2024-03-01": null,
|
||||
"per-turn-control-2026-07-01": "per-turn-control-2026-07-01",
|
||||
|
|
@ -218,6 +221,10 @@
|
|||
"web-search-2025-03-05": "web-search-2025-03-05",
|
||||
"inline-tools-2026-09-15": "inline-tools-2026-09-15"
|
||||
},
|
||||
"vertex_ai_header_only": {
|
||||
"compact-2026-09-04": "compact-2026-09-04",
|
||||
"inline-tools-2026-09-15": "inline-tools-2026-09-15"
|
||||
},
|
||||
"databricks": {
|
||||
"advisor-tool-2026-03-01": null,
|
||||
"advanced-tool-use-2025-11-20": null,
|
||||
|
|
|
|||
|
|
@ -30,7 +30,7 @@ from collections.abc import Mapping
|
|||
from concurrent.futures import ThreadPoolExecutor
|
||||
from functools import cache
|
||||
from importlib.resources import files
|
||||
from typing import Final, TypeAlias
|
||||
from typing import Final, Literal, TypeAlias
|
||||
|
||||
import httpx
|
||||
from pydantic import TypeAdapter
|
||||
|
|
@ -47,6 +47,9 @@ BetaHeadersConfig: TypeAlias = Mapping[str, str | Mapping[str, str | None]]
|
|||
_BETA_HEADERS_CONFIG_ADAPTER: Final = TypeAdapter(BetaHeadersConfig)
|
||||
_DECODED_JSON: Final = TypeAdapter(object)
|
||||
|
||||
BetaPlacement: TypeAlias = Literal["header", "body"]
|
||||
_HEADER_ONLY_COLUMN_SUFFIX: Final = "_header_only"
|
||||
|
||||
|
||||
class GetAnthropicBetaHeadersConfig:
|
||||
"""
|
||||
|
|
@ -241,6 +244,31 @@ def get_provider_name(provider: str) -> str:
|
|||
return aliases.get(provider, provider)
|
||||
|
||||
|
||||
def _beta_column(name: str) -> Mapping[str, str | None]:
|
||||
column: Final[Mapping[str, str | None]] = _load_beta_headers_config().get(name, {})
|
||||
return column
|
||||
|
||||
|
||||
def _header_only_column(canonical_provider: str) -> Mapping[str, str | None]:
|
||||
return _beta_column(f"{canonical_provider}{_HEADER_ONLY_COLUMN_SUFFIX}")
|
||||
|
||||
|
||||
def _provider_beta_mapping(provider: str, placement: BetaPlacement) -> Mapping[str, str | None]:
|
||||
canonical_provider: Final = get_provider_name(provider)
|
||||
column: Final = _beta_column(canonical_provider)
|
||||
header_only: Final = _header_only_column(canonical_provider)
|
||||
match placement:
|
||||
case "header":
|
||||
return {**column, **header_only}
|
||||
case "body":
|
||||
return {beta: name for beta, name in column.items() if beta not in header_only}
|
||||
|
||||
|
||||
def get_header_only_betas(provider: str) -> frozenset[str]:
|
||||
header_only: Final = _header_only_column(get_provider_name(provider))
|
||||
return frozenset(beta for beta, name in header_only.items() if name is not None)
|
||||
|
||||
|
||||
def has_beta_headers_config(provider: str) -> bool:
|
||||
config: Final = _bundled_beta_headers_config()
|
||||
aliases: Final = config.get("provider_aliases", {})
|
||||
|
|
@ -251,6 +279,7 @@ def has_beta_headers_config(provider: str) -> bool:
|
|||
def filter_and_transform_beta_headers(
|
||||
beta_headers: list[str],
|
||||
provider: str,
|
||||
placement: BetaPlacement = "header",
|
||||
) -> list[str]:
|
||||
"""
|
||||
Filter and transform beta headers based on provider's mapping configuration.
|
||||
|
|
@ -263,6 +292,7 @@ def filter_and_transform_beta_headers(
|
|||
Args:
|
||||
beta_headers: List of Anthropic beta header values
|
||||
provider: Provider name (e.g., "anthropic", "bedrock", "vertex_ai")
|
||||
placement: Whether the betas travel as the anthropic-beta header or the anthropic_beta body field
|
||||
|
||||
Returns:
|
||||
List of filtered and transformed beta headers for the provider
|
||||
|
|
@ -270,11 +300,7 @@ def filter_and_transform_beta_headers(
|
|||
if not beta_headers:
|
||||
return []
|
||||
|
||||
config: Final = _load_beta_headers_config()
|
||||
provider = get_provider_name(provider)
|
||||
|
||||
# Get the header mapping for this provider
|
||||
provider_mapping: Final = config.get(provider, {})
|
||||
provider_mapping: Final = _provider_beta_mapping(provider, placement)
|
||||
|
||||
filtered_headers: Final[set[str]] = set()
|
||||
|
||||
|
|
@ -426,6 +452,7 @@ def update_request_with_filtered_beta(
|
|||
filtered_body_betas: Final = filter_and_transform_beta_headers(
|
||||
beta_headers=existing_body_betas,
|
||||
provider=provider,
|
||||
placement="body",
|
||||
)
|
||||
|
||||
if filtered_body_betas:
|
||||
|
|
|
|||
|
|
@ -68,6 +68,7 @@ RUNTIME_UPDATABLE_ROUTER_SETTINGS: Final[frozenset[str]] = frozenset(
|
|||
"retry_after",
|
||||
"fallbacks",
|
||||
"context_window_fallbacks",
|
||||
"content_policy_fallbacks",
|
||||
"retry_policy",
|
||||
"model_group_retry_policy",
|
||||
"model_group_alias",
|
||||
|
|
@ -899,6 +900,7 @@ PROVIDERS_THAT_AUTHENTICATE_ON_PROVIDER_INFO: Final = frozenset(
|
|||
}
|
||||
)
|
||||
|
||||
SUBSCRIPTION_BACKED_PROVIDERS: Final = frozenset({"chatgpt"})
|
||||
A2A_CHAT_COMPLETION_BRIDGE_PROVIDERS: Final = frozenset({"databricks_agent"})
|
||||
|
||||
LITELLM_EMBEDDING_PROVIDERS_SUPPORTING_INPUT_ARRAY_OF_TOKENS: Final = [
|
||||
|
|
|
|||
|
|
@ -116,12 +116,12 @@ def configured_injection_points(value: object) -> Sequence[CacheControlInjection
|
|||
return tuple(cast(CacheControlInjectionPoint, entry) for entry in value if isinstance(entry, dict))
|
||||
|
||||
|
||||
def _served_provider_for_model(model: str) -> str | None:
|
||||
def _served_provider_for_model(model: str, api_base: object = None) -> str | None:
|
||||
from litellm.exceptions import BadRequestError
|
||||
from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
|
||||
|
||||
try:
|
||||
_, provider, _, _ = get_llm_provider(model=model)
|
||||
_, provider, _, _ = get_llm_provider(model=model, api_base=api_base if isinstance(api_base, str) else None)
|
||||
except BadRequestError:
|
||||
return None
|
||||
return provider
|
||||
|
|
@ -536,7 +536,7 @@ class AnthropicCacheControlHook(CustomPromptManagement):
|
|||
if model is None:
|
||||
return False
|
||||
hosted_flag: Final = _hosted_openai_dialect_flag(
|
||||
model, custom_llm_provider, AnthropicCacheControlHook._resolve_provider
|
||||
model, custom_llm_provider, lambda name: AnthropicCacheControlHook._resolve_provider(name, api_base)
|
||||
)
|
||||
if hosted_flag is not None:
|
||||
return hosted_flag
|
||||
|
|
@ -559,8 +559,8 @@ class AnthropicCacheControlHook(CustomPromptManagement):
|
|||
)
|
||||
|
||||
@staticmethod
|
||||
def _resolve_provider(model: str) -> str | None:
|
||||
return _served_provider_for_model(model)
|
||||
def _resolve_provider(model: str, api_base: object = None) -> str | None:
|
||||
return _served_provider_for_model(model, api_base)
|
||||
|
||||
@staticmethod
|
||||
def count_request_cache_breakpoints(
|
||||
|
|
|
|||
|
|
@ -361,6 +361,7 @@ _PRISMA_MODELS: Final[frozenset[str]] = frozenset(
|
|||
"LiteLLM_LensReview",
|
||||
"LiteLLM_LensIngestionKey",
|
||||
"LiteLLM_LensWorker",
|
||||
"LiteLLM_SubscriptionAccountTable",
|
||||
"LiteLLM_LensSignalConfig",
|
||||
"LiteLLM_LensTraceSignal",
|
||||
)
|
||||
|
|
|
|||
|
|
@ -82,6 +82,7 @@ from litellm.types.proxy.carried_budget_state import (
|
|||
UserBudgetSnapshot,
|
||||
)
|
||||
from litellm.types.utils import (
|
||||
CallTypes,
|
||||
StandardLoggingGuardrailInformation,
|
||||
StandardLoggingPayload,
|
||||
StandardLoggingZeroCostDiagnostic,
|
||||
|
|
@ -279,6 +280,11 @@ def _logged_fields(value: object) -> Mapping[str, object]:
|
|||
return _NO_LOGGED_FIELDS
|
||||
|
||||
|
||||
_GOOGLE_PASS_THROUGH_PROVIDERS: Final = frozenset(
|
||||
(litellm.LlmProviders.GEMINI.value, litellm.LlmProviders.VERTEX_AI.value)
|
||||
)
|
||||
|
||||
|
||||
def _is_llm_provider(name: str) -> bool:
|
||||
routable: Final = (
|
||||
name in litellm.provider_list or JSONProviderRegistry.exists(name) or is_registered_custom_provider(name)
|
||||
|
|
@ -2942,6 +2948,22 @@ class PrometheusLogger(CustomLogger):
|
|||
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
def _google_pass_through_provider(request_data: Mapping[str, object]) -> str | None:
|
||||
declared_provider: Final = request_data.get("custom_llm_provider")
|
||||
if request_data.get("call_type") != CallTypes.pass_through.value or not isinstance(declared_provider, str):
|
||||
return None
|
||||
return declared_provider if declared_provider in _GOOGLE_PASS_THROUGH_PROVIDERS else None
|
||||
|
||||
@staticmethod
|
||||
def _failed_request_requested_model(request_data: Mapping[str, object]) -> object:
|
||||
requested_model: Final = request_data.get("model", "")
|
||||
if PrometheusLogger._google_pass_through_provider(request_data) is None:
|
||||
return requested_model
|
||||
request_metadata: Final = _logged_fields(_logged_fields(request_data.get("litellm_params")).get("metadata"))
|
||||
model_group: Final = request_metadata.get("model_group")
|
||||
return model_group if isinstance(model_group, str) and model_group else requested_model
|
||||
|
||||
@staticmethod
|
||||
def _extract_api_provider_from_request_data(request_data: dict) -> str | None:
|
||||
"""
|
||||
|
|
@ -2950,22 +2972,27 @@ class PrometheusLogger(CustomLogger):
|
|||
A request can fail before a deployment is resolved, so the provider is
|
||||
not always known. Prefer the resolved ``custom_llm_provider`` on
|
||||
``litellm_params``, then any provider recovered onto a partial
|
||||
``standard_logging_object`` (e.g. a stream that broke mid-flight), and
|
||||
finally infer it from the requested model name (e.g. ``gpt-4o-mini`` ->
|
||||
``openai``) since the proxy's failure ``request_data`` usually carries
|
||||
only the client-supplied model. Return ``None`` when it cannot be
|
||||
determined so the label emits empty rather than a guess.
|
||||
``standard_logging_object`` (e.g. a stream that broke mid-flight). On a
|
||||
Gemini or Vertex pass-through failure only, next take the provider the
|
||||
failure payload names (``gemini`` or ``vertex_ai``), so every other
|
||||
pass-through route keeps its earlier label. Finally infer it from the
|
||||
requested model name (e.g. ``gpt-4o-mini`` -> ``openai``) since the
|
||||
proxy's failure ``request_data`` usually carries only the client-supplied
|
||||
model. Return ``None`` when it cannot be determined so the label emits
|
||||
empty rather than a guess.
|
||||
"""
|
||||
litellm_params: Final = request_data.get("litellm_params") or {}
|
||||
provider = litellm_params.get("custom_llm_provider")
|
||||
if provider:
|
||||
return provider
|
||||
standard_logging_object: Final = request_data.get("standard_logging_object") or {}
|
||||
provider = standard_logging_object.get("custom_llm_provider")
|
||||
if provider:
|
||||
return provider
|
||||
resolved_provider: Final = litellm_params.get("custom_llm_provider") or standard_logging_object.get(
|
||||
"custom_llm_provider"
|
||||
)
|
||||
if resolved_provider:
|
||||
return resolved_provider
|
||||
google_provider: Final = PrometheusLogger._google_pass_through_provider(request_data)
|
||||
if google_provider is not None:
|
||||
return google_provider
|
||||
model: Final = litellm_params.get("model") or request_data.get("model")
|
||||
if not model:
|
||||
if not isinstance(model, str) or not model:
|
||||
return None
|
||||
try:
|
||||
return litellm.get_llm_provider(model=model)[1] or None
|
||||
|
|
@ -3052,7 +3079,7 @@ class PrometheusLogger(CustomLogger):
|
|||
team_alias=user_api_key_dict.team_alias,
|
||||
org_id=user_api_key_dict.org_id,
|
||||
org_alias=user_api_key_dict.organization_alias,
|
||||
requested_model=_bounded_requested_model_label(request_data.get("model", "")),
|
||||
requested_model=_bounded_requested_model_label(self._failed_request_requested_model(request_data)),
|
||||
status_code=str(status_code),
|
||||
exception_status=str(status_code),
|
||||
exception_class=self._get_exception_class_name(original_exception),
|
||||
|
|
|
|||
|
|
@ -9,12 +9,13 @@ from types import MappingProxyType
|
|||
from typing import TYPE_CHECKING, Any, Final, Literal, Protocol, TypeVar, cast, runtime_checkable
|
||||
|
||||
import httpx
|
||||
from pydantic import TypeAdapter, ValidationError
|
||||
from pydantic import BaseModel, TypeAdapter, ValidationError
|
||||
from typing_extensions import TypeIs # noqa: TID251 # narrows a duck-typed value to the Protocol its caller reads
|
||||
|
||||
import litellm
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.constants import KUBERNETES_POD_ROUTING_KEY
|
||||
from litellm.litellm_core_utils.hidden_params import set_extra_attribute
|
||||
from litellm.litellm_core_utils.llm_response_utils.get_headers import get_provider_request_id
|
||||
from litellm.types.llms.openai import AllMessageValues, OpenAIChatCompletionFinishReason
|
||||
|
||||
|
|
@ -710,9 +711,9 @@ def preserve_upstream_non_openai_attributes(
|
|||
_assign_attributes(model_response, original_chunk.model_dump(include=frozenset(candidate_keys)).items())
|
||||
|
||||
|
||||
def _assign_attributes(target: object, values: Iterable[tuple[str, object]]) -> None:
|
||||
def _assign_attributes(target: BaseModel, values: Iterable[tuple[str, object]]) -> None:
|
||||
for key, value in values:
|
||||
setattr(target, key, value)
|
||||
set_extra_attribute(target, key, value)
|
||||
|
||||
|
||||
def safe_deep_copy(data):
|
||||
|
|
@ -982,25 +983,36 @@ def _usage_field(usage: object, name: str) -> object:
|
|||
return getattr(usage, name, None)
|
||||
|
||||
|
||||
def _cost_number(value: object) -> float | None:
|
||||
if isinstance(value, bool) or not isinstance(value, int | float):
|
||||
def finite_cost(value: object) -> float | None:
|
||||
if isinstance(value, bool) or not isinstance(value, int | float) or not math.isfinite(value):
|
||||
return None
|
||||
return float(value)
|
||||
|
||||
|
||||
def single_log_line(value: object) -> str:
|
||||
return str(value).replace("\r", "").replace("\n", "")
|
||||
|
||||
|
||||
def finite_reported_cost(value: object) -> float | None:
|
||||
cost: Final = finite_cost(value)
|
||||
if cost is None and isinstance(value, float):
|
||||
verbose_logger.warning("Ignoring a non-finite provider-reported cost %s", value)
|
||||
return cost
|
||||
|
||||
|
||||
def _reported_cost_value(value: object) -> float | None:
|
||||
"""A reported cost is a number, or a breakdown whose total lives under ``total_cost``."""
|
||||
raw: Final[object] = value
|
||||
if isinstance(value, Mapping):
|
||||
return _cost_number(_usage_field(raw, "total_cost"))
|
||||
return _cost_number(value)
|
||||
return finite_reported_cost(_usage_field(raw, "total_cost"))
|
||||
return finite_reported_cost(value)
|
||||
|
||||
|
||||
def response_cost_in_hidden_params(response: _CarriesHiddenParams) -> float | None:
|
||||
additional_headers: Final[object] = response.hidden_params.get("additional_headers")
|
||||
if not isinstance(additional_headers, Mapping):
|
||||
return None
|
||||
return _cost_number(_USAGE_BLOCK_ADAPTER.validate_python(additional_headers).get(RESPONSE_COST_HEADER))
|
||||
return finite_reported_cost(_USAGE_BLOCK_ADAPTER.validate_python(additional_headers).get(RESPONSE_COST_HEADER))
|
||||
|
||||
|
||||
def override_stamped_response_cost(response: _CarriesHiddenParams, cost: float) -> None:
|
||||
|
|
|
|||
|
|
@ -18,6 +18,7 @@ from litellm.litellm_core_utils.bug_report import (
|
|||
)
|
||||
from litellm.litellm_core_utils.core_helpers import as_str_mapping, attribute_or_none
|
||||
from litellm.litellm_core_utils.secret_redaction import redact_string as redact_secret_string
|
||||
from litellm.llms.bedrock.common_utils import bedrock_model_reaches_converse_by_default
|
||||
from litellm.llms.bigmodel.common_utils import raise_bigmodel_business_error
|
||||
from litellm.types.utils import LlmProviders
|
||||
|
||||
|
|
@ -938,13 +939,22 @@ _BEDROCK_MANTLE_CONTEXT_WINDOW_GENERIC_MESSAGE: Final = (
|
|||
)
|
||||
|
||||
|
||||
def _bedrock_mantle_context_window_response(original_exception: _ProviderHTTPException) -> httpx.Response:
|
||||
def _bedrock_response_with_status(original_exception: _ProviderHTTPException, status_code: int) -> httpx.Response:
|
||||
original_response: Final = getattr(original_exception, "response", None)
|
||||
if not isinstance(original_response, httpx.Response) or getattr(original_response, "_request", None) is None:
|
||||
return httpx.Response(status_code=400, request=httpx.Request(method="POST", url="https://api.openai.com/v1/"))
|
||||
if original_response.status_code == 400:
|
||||
return httpx.Response(
|
||||
status_code=status_code, request=httpx.Request(method="POST", url="https://api.openai.com/v1/")
|
||||
)
|
||||
if original_response.status_code == status_code:
|
||||
return original_response
|
||||
return httpx.Response(status_code=400, headers=original_response.headers, request=original_response.request)
|
||||
return httpx.Response(status_code=status_code, headers=original_response.headers, request=original_response.request)
|
||||
|
||||
|
||||
def _came_from_bedrock_converse(original_exception: _ProviderHTTPException) -> bool:
|
||||
original_response: Final = getattr(original_exception, "response", None)
|
||||
if not isinstance(original_response, httpx.Response) or getattr(original_response, "_request", None) is None:
|
||||
return False
|
||||
return original_response.request.url.path.endswith(("/converse", "/converse-stream"))
|
||||
|
||||
|
||||
def _get_bedrock_mantle_context_window_message(error_str: str) -> str | None:
|
||||
|
|
@ -980,7 +990,7 @@ def _map_bedrock_exception(
|
|||
message=mantle_context_window_message,
|
||||
model=model,
|
||||
llm_provider=custom_llm_provider,
|
||||
response=_bedrock_mantle_context_window_response(original_exception),
|
||||
response=_bedrock_response_with_status(original_exception, 400),
|
||||
)
|
||||
if (
|
||||
"too many tokens" in error_str
|
||||
|
|
@ -1003,6 +1013,18 @@ def _map_bedrock_exception(
|
|||
llm_provider="bedrock",
|
||||
response=getattr(original_exception, "response", None),
|
||||
)
|
||||
elif (
|
||||
"The provided model identifier is invalid" in error_str
|
||||
and bool(model)
|
||||
and _came_from_bedrock_converse(original_exception)
|
||||
and bedrock_model_reaches_converse_by_default(model)
|
||||
):
|
||||
raise NotFoundError(
|
||||
message=f"BedrockException - {error_str}",
|
||||
model=model,
|
||||
llm_provider="bedrock",
|
||||
response=_bedrock_response_with_status(original_exception, 404),
|
||||
)
|
||||
elif "Malformed input request" in error_str:
|
||||
raise BadRequestError(
|
||||
message=f"BedrockException - {error_str}",
|
||||
|
|
|
|||
|
|
@ -1,8 +1,9 @@
|
|||
from collections.abc import Iterator, Mapping, MutableMapping
|
||||
from functools import cache, lru_cache
|
||||
from types import MappingProxyType
|
||||
from typing import Final, Protocol, cast
|
||||
|
||||
from pydantic import BaseModel
|
||||
from pydantic import BaseModel, ConfigDict, create_model
|
||||
|
||||
from litellm.types.llms.base import HiddenParams
|
||||
|
||||
|
|
@ -60,6 +61,38 @@ def stored_attribute(obj: object, name: str) -> object:
|
|||
return extras.get(name)
|
||||
|
||||
|
||||
@cache
|
||||
def _assigned_extras_join_fields_set() -> bool:
|
||||
probe: Final = create_model("_ExtraAssignmentProbe", __config__=ConfigDict(extra="allow"))()
|
||||
setattr(probe, "assigned", None)
|
||||
return "assigned" in probe.model_fields_set
|
||||
|
||||
|
||||
@lru_cache(maxsize=1024)
|
||||
def _writes_plain_extra(model_class: type[BaseModel], name: str) -> bool:
|
||||
config: Final = model_class.model_config
|
||||
return (
|
||||
model_class.__setattr__ is BaseModel.__setattr__
|
||||
and config.get("extra") == "allow"
|
||||
and not config.get("frozen")
|
||||
and not config.get("validate_assignment")
|
||||
and not name.startswith("_")
|
||||
and name not in model_class.__pydantic_fields__
|
||||
and name not in model_class.__class_vars__
|
||||
and not hasattr(model_class, name)
|
||||
and _assigned_extras_join_fields_set()
|
||||
)
|
||||
|
||||
|
||||
def set_extra_attribute(model: BaseModel, name: str, value: object) -> None:
|
||||
extras: Final = model.__pydantic_extra__
|
||||
if extras is None or not _writes_plain_extra(type(model), name):
|
||||
setattr(model, name, value)
|
||||
return
|
||||
extras[name] = value
|
||||
model.__pydantic_fields_set__.add(name)
|
||||
|
||||
|
||||
def _get_hidden_params_storage(obj: object) -> object | None:
|
||||
if isinstance(obj, dict):
|
||||
return obj.get(_HIDDEN_PARAMS_ATTR)
|
||||
|
|
|
|||
|
|
@ -35,6 +35,12 @@ from litellm.types.utils import (
|
|||
|
||||
BUDGET_RESERVATION_METADATA_KEYS: Final = frozenset({"user_api_key_budget_reservation"})
|
||||
|
||||
ITPM_RESERVED_KEY: Final = "_litellm_itpm_reserved"
|
||||
OTPM_RESERVED_KEY: Final = "_litellm_otpm_reserved"
|
||||
ITPM_CACHE_KEY: Final = "_litellm_itpm_cache_key"
|
||||
OTPM_CACHE_KEY: Final = "_litellm_otpm_cache_key"
|
||||
IO_TOKEN_RESERVATION_KEYS: Final = frozenset({ITPM_RESERVED_KEY, OTPM_RESERVED_KEY, ITPM_CACHE_KEY, OTPM_CACHE_KEY})
|
||||
|
||||
MODEL_ACCESS_GROUP_METADATA_KEY: Final = "user_api_key_matched_model_access_groups"
|
||||
"""Where auth records the model access groups that authorized the request, for the spend writer.
|
||||
|
||||
|
|
|
|||
|
|
@ -76,6 +76,7 @@ from litellm.litellm_core_utils.classifier_logging import (
|
|||
is_classifier_call,
|
||||
)
|
||||
from litellm.litellm_core_utils.core_helpers import (
|
||||
finite_cost,
|
||||
get_provider_response_headers_from_hidden_params,
|
||||
is_expected_client_error,
|
||||
native_messages_reported_cost,
|
||||
|
|
@ -84,6 +85,7 @@ from litellm.litellm_core_utils.core_helpers import (
|
|||
proxy_stamped_used_client_oauth_token,
|
||||
reconstruct_model_name,
|
||||
set_response_cost_in_hidden_params,
|
||||
single_log_line,
|
||||
)
|
||||
from litellm.litellm_core_utils.error_normalization import normalize_error
|
||||
from litellm.litellm_core_utils.get_litellm_params import get_litellm_params
|
||||
|
|
@ -567,7 +569,13 @@ def _resolve_mantle_region_for_cost(
|
|||
|
||||
_PER_CALL_LITELLM_PARAM_DICTS: Final = frozenset({"metadata", "litellm_metadata"})
|
||||
_LITELLM_PARAMS: Final = TypeAdapter(dict[str, object])
|
||||
_SERVED_ANTHROPIC_BODY: Final = TypeAdapter(dict[str, object])
|
||||
_REDACTED_RESULT: Final = TypeAdapter(object)
|
||||
_GUARDRAIL_INFORMATION_KEY: Final = "standard_logging_guardrail_information"
|
||||
|
||||
|
||||
def _without_guardrail_information(metadata: Mapping[str, object]) -> Mapping[str, object]:
|
||||
return {key: value for key, value in metadata.items() if key != _GUARDRAIL_INFORMATION_KEY}
|
||||
|
||||
|
||||
def _provider_response_id(source: object) -> str | None:
|
||||
|
|
@ -2088,7 +2096,7 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
litellm_model_name=litellm_model_name,
|
||||
router_model_id=router_model_id or hidden_params.get("model_id"),
|
||||
)
|
||||
return hidden_params["response_cost"]
|
||||
return self._recorded_response_cost(hidden_params["response_cost"])
|
||||
elif router_model_id is None and "model_id" in hidden_params: # use model_id if not already set
|
||||
router_model_id = hidden_params["model_id"]
|
||||
|
||||
|
|
@ -2188,7 +2196,7 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
litellm_model_name=litellm_model_name,
|
||||
router_model_id=router_model_id,
|
||||
)
|
||||
return total_response_cost
|
||||
return self._recorded_response_cost(total_response_cost)
|
||||
except Exception as e: # error calculating cost
|
||||
debug_info = StandardLoggingModelCostFailureDebugInformation(
|
||||
error_str=str(e),
|
||||
|
|
@ -2214,6 +2222,15 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
|
||||
_response_cost_calculator = response_cost_calculator
|
||||
|
||||
def _recorded_response_cost(self, cost: float | None) -> float | None:
|
||||
if cost is None or finite_cost(cost) is not None:
|
||||
return cost
|
||||
verbose_logger.warning(
|
||||
"Recording a non-finite response_cost as 0 for litellm_call_id=%s", single_log_line(self.litellm_call_id)
|
||||
)
|
||||
self.cost_breakdown = None
|
||||
return 0.0
|
||||
|
||||
def uncached_price(self, result: object) -> UncachedPrice:
|
||||
previous_cost_breakdown: Final = self.cost_breakdown
|
||||
previous_billed_token_rates: Final = self.billed_token_rates
|
||||
|
|
@ -2566,6 +2583,23 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
attempt.model_call_details = dict(self.model_call_details)
|
||||
return attempt
|
||||
|
||||
def guardrail_scoped_copy(self) -> "Logging":
|
||||
request_params: Final = _LITELLM_PARAMS.validate_python(
|
||||
self.litellm_params or {} # pyright: ignore[reportUnknownMemberType] # Logging.litellm_params is untyped
|
||||
)
|
||||
litellm_params: Final = {
|
||||
**request_params,
|
||||
**{
|
||||
key: _without_guardrail_information(_METADATA_MAPPING.validate_python(value))
|
||||
for key, value in request_params.items()
|
||||
if key in _PER_CALL_LITELLM_PARAM_DICTS and isinstance(value, dict)
|
||||
},
|
||||
}
|
||||
scoped: Final = copy.copy(self)
|
||||
scoped.litellm_params = litellm_params
|
||||
scoped.model_call_details = {**self.model_call_details, "litellm_params": litellm_params}
|
||||
return scoped
|
||||
|
||||
def dispatch_attempt_failure_handlers(
|
||||
self, exception: Exception, traceback_exception: str
|
||||
) -> Coroutine[None, None, None]:
|
||||
|
|
@ -5034,13 +5068,15 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
verbose_logger.warning("LiteLLM: the anthropic_messages stream assembled no response, logging an empty one")
|
||||
return litellm.ModelResponse(model=self.model)
|
||||
else:
|
||||
from litellm.types.llms.anthropic import AnthropicResponse
|
||||
|
||||
pydantic_result: Final = AnthropicResponse.model_validate({"stop_sequence": None, **(served_body or {})})
|
||||
import httpx
|
||||
|
||||
anthropic_body: Final = (
|
||||
served_body
|
||||
if served_body is not None
|
||||
else _SERVED_ANTHROPIC_BODY.validate_python(
|
||||
result.model_dump() if isinstance(result, BaseModel) else result
|
||||
)
|
||||
)
|
||||
result = litellm.AnthropicConfig().transform_parsed_response(
|
||||
completion_response=pydantic_result.model_dump(),
|
||||
completion_response={"stop_sequence": None, **anthropic_body},
|
||||
raw_response=httpx.Response(
|
||||
status_code=200,
|
||||
headers={},
|
||||
|
|
@ -7560,7 +7596,15 @@ def get_standard_logging_object_payload(
|
|||
)
|
||||
custom_pricing: Final = False if spilled_over else logging_obj.custom_pricing_for(init_response_obj)
|
||||
raw_response_cost: Final = kwargs.get("response_cost")
|
||||
llm_response_cost: Final[float] = raw_response_cost or 0.0
|
||||
numeric_response_cost: Final = raw_response_cost if isinstance(raw_response_cost, int | float) else None
|
||||
priced_response_cost: Final = finite_cost(numeric_response_cost)
|
||||
if numeric_response_cost is not None and priced_response_cost is None:
|
||||
verbose_logger.warning(
|
||||
"Logging a non-finite response_cost=%s as 0 for litellm_call_id=%s",
|
||||
numeric_response_cost,
|
||||
single_log_line(logging_obj.litellm_call_id),
|
||||
)
|
||||
llm_response_cost: Final[float] = priced_response_cost or 0.0
|
||||
guardrail_cost: Final = guardrail_information_cost(metadata.get("standard_logging_guardrail_information"))
|
||||
finalized_cost: Final = _finalized_cost_of(logging_obj)
|
||||
response_cost: Final[float] = (
|
||||
|
|
|
|||
|
|
@ -1,4 +1,3 @@
|
|||
import datetime
|
||||
from collections.abc import Iterable, Iterator, Mapping
|
||||
from dataclasses import dataclass
|
||||
from typing import Final, cast
|
||||
|
|
@ -9,6 +8,7 @@ from typing_extensions import NotRequired, ReadOnly, TypedDict
|
|||
import litellm
|
||||
from litellm.constants import DEFAULT_MAX_RECURSE_DEPTH
|
||||
from litellm.litellm_core_utils.get_llm_provider_logic import inferred_provider
|
||||
from litellm.litellm_core_utils.model_retirement import is_past_deprecation_date
|
||||
|
||||
|
||||
def _form_field_value(value: object) -> str:
|
||||
|
|
@ -168,7 +168,9 @@ class RankedChatModel:
|
|||
|
||||
|
||||
def rank_chat_models_from_llm_provider(custom_llm_provider: str) -> tuple[RankedChatModel, ...]:
|
||||
known_models: Final[Iterable[str]] = litellm.models_by_provider.get(custom_llm_provider, ())
|
||||
from litellm.utils import bundled_provider_models
|
||||
|
||||
known_models: Final = bundled_provider_models(custom_llm_provider)
|
||||
ranked_models: Final = (
|
||||
RankedChatModel(
|
||||
model=model, route_rank=route_rank, cost=cost, is_past_deprecation_date=is_past_deprecation_date
|
||||
|
|
@ -187,18 +189,11 @@ _COST_MAP_DEPRECATION: Final = TypeAdapter(_CostMapDeprecation)
|
|||
|
||||
|
||||
def _is_past_deprecation_date(cost_map_key: str) -> bool:
|
||||
deprecation_date: Final = _COST_MAP_DEPRECATION.validate_python(
|
||||
litellm.model_cost.get(cost_map_key) or {} # pyright: ignore[reportUnknownMemberType] # untyped cost map JSON
|
||||
).get("deprecation_date")
|
||||
if not isinstance(deprecation_date, str):
|
||||
return False
|
||||
try:
|
||||
return (
|
||||
datetime.date.fromisoformat(deprecation_date)
|
||||
<= datetime.datetime.now(datetime.timezone.utc).astimezone().date()
|
||||
return is_past_deprecation_date(
|
||||
_COST_MAP_DEPRECATION.validate_python(
|
||||
litellm.model_cost.get(cost_map_key) or {} # pyright: ignore[reportUnknownMemberType] # untyped cost map JSON
|
||||
)
|
||||
except ValueError:
|
||||
return False
|
||||
)
|
||||
|
||||
|
||||
def _chat_row_costs(models: Iterable[str], custom_llm_provider: str) -> Iterator[tuple[str, float, bool]]:
|
||||
|
|
|
|||
19
litellm/litellm_core_utils/model_retirement.py
Normal file
19
litellm/litellm_core_utils/model_retirement.py
Normal file
|
|
@ -0,0 +1,19 @@
|
|||
import datetime
|
||||
from collections.abc import Mapping
|
||||
from typing import Final
|
||||
|
||||
PROVIDERS_HIDING_RETIRED_MODELS: Final = frozenset({"aiand", "baseten", "together_ai"})
|
||||
|
||||
|
||||
def is_past_deprecation_date(value: Mapping[str, object]) -> bool:
|
||||
deprecation_date: Final = value.get("deprecation_date")
|
||||
if not isinstance(deprecation_date, str):
|
||||
return False
|
||||
try:
|
||||
return datetime.date.fromisoformat(deprecation_date) <= datetime.datetime.now(datetime.timezone.utc).date()
|
||||
except ValueError:
|
||||
return False
|
||||
|
||||
|
||||
def is_hidden_as_retired(value: Mapping[str, object]) -> bool:
|
||||
return value.get("litellm_provider") in PROVIDERS_HIDING_RETIRED_MODELS and is_past_deprecation_date(value)
|
||||
|
|
@ -2023,12 +2023,14 @@ def infer_content_type_from_url_and_content(
|
|||
"""
|
||||
from litellm.litellm_core_utils.token_counter import get_image_type
|
||||
|
||||
media_type: Final = current_content_type.split(";", 1)[0].strip().lower() if current_content_type else None
|
||||
|
||||
# If we have a valid content type that's not generic, use it
|
||||
if current_content_type and current_content_type not in [
|
||||
if media_type and media_type not in [
|
||||
"binary/octet-stream",
|
||||
"application/octet-stream",
|
||||
]:
|
||||
return current_content_type
|
||||
return media_type
|
||||
|
||||
# Extension to MIME type mapping
|
||||
# Supports images, documents, and other common file types
|
||||
|
|
|
|||
|
|
@ -912,9 +912,9 @@ def create_anthropic_image_param(
|
|||
image_url_input: str | dict,
|
||||
format: str | None = None,
|
||||
is_bedrock_invoke: bool = False,
|
||||
) -> AnthropicMessagesImageParam:
|
||||
) -> AnthropicMessagesImageParam | AnthropicMessagesDocumentParam:
|
||||
"""
|
||||
Create an AnthropicMessagesImageParam from an image URL input.
|
||||
Create an Anthropic content block from an image URL input.
|
||||
|
||||
Supports both URL references (for HTTP/HTTPS URLs) and base64 encoding.
|
||||
"""
|
||||
|
|
@ -933,14 +933,7 @@ def create_anthropic_image_param(
|
|||
if is_bedrock_invoke or image_url.startswith("http://"):
|
||||
base64_url: Final = convert_url_to_base64(url=image_url)
|
||||
image_chunk = convert_to_anthropic_image_obj(openai_image_url=base64_url, format=format)
|
||||
return AnthropicMessagesImageParam(
|
||||
type="image",
|
||||
source=AnthropicContentParamSource(
|
||||
type="base64",
|
||||
media_type=image_chunk["media_type"],
|
||||
data=image_chunk["data"],
|
||||
),
|
||||
)
|
||||
return _anthropic_content_element_factory(image_chunk)
|
||||
else:
|
||||
# HTTPS URL - pass directly for regular Anthropic
|
||||
return AnthropicMessagesImageParam(
|
||||
|
|
@ -953,14 +946,7 @@ def create_anthropic_image_param(
|
|||
else:
|
||||
# Convert to base64 for data URIs or other formats
|
||||
image_chunk = convert_to_anthropic_image_obj(openai_image_url=image_url, format=format)
|
||||
return AnthropicMessagesImageParam(
|
||||
type="image",
|
||||
source=AnthropicContentParamSource(
|
||||
type="base64",
|
||||
media_type=image_chunk["media_type"],
|
||||
data=image_chunk["data"],
|
||||
),
|
||||
)
|
||||
return _anthropic_content_element_factory(image_chunk)
|
||||
|
||||
|
||||
# The following XML functions will be deprecated once JSON schema support is available on Bedrock and Vertex
|
||||
|
|
@ -1910,7 +1896,12 @@ def convert_to_anthropic_tool_result(
|
|||
anthropic_content_element=_anthropic_image_param,
|
||||
original_content_element=content,
|
||||
)
|
||||
anthropic_content_list.append(cast(AnthropicMessagesImageParam, _anthropic_image_param))
|
||||
anthropic_content_list.append(
|
||||
cast(
|
||||
AnthropicMessagesImageParam | AnthropicMessagesDocumentParam,
|
||||
_anthropic_image_param,
|
||||
)
|
||||
)
|
||||
elif content["type"] == "tool_reference":
|
||||
anthropic_content_list.append(ToolReference(type="tool_reference", tool_name=content["tool_name"]))
|
||||
elif content["type"] == "file":
|
||||
|
|
|
|||
|
|
@ -1,6 +1,12 @@
|
|||
import reprlib
|
||||
from collections.abc import Collection, Mapping
|
||||
from dataclasses import dataclass
|
||||
from typing import Final, Literal
|
||||
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
import litellm
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.constants import (
|
||||
DEFAULT_REASONING_EFFORT_HIGH_THINKING_BUDGET,
|
||||
DEFAULT_REASONING_EFFORT_LOW_THINKING_BUDGET,
|
||||
|
|
@ -11,6 +17,54 @@ from litellm.litellm_core_utils.core_helpers import as_str_mapping
|
|||
|
||||
OpenAIStyleReasoningEffort = Literal["minimal", "low", "medium", "high"]
|
||||
|
||||
_DROP_WARNING: Final = "Dropping %s=%s for model=%r because %s"
|
||||
_DROP_PARAMS_DROP_WARNING: Final = "Dropping %s=%s for model=%r (drop_params=True) because %s"
|
||||
_NON_NUMERIC_BUDGET_REASON: Final = "it is not a number. Send budget_tokens as an integer to set a reasoning level."
|
||||
_NON_STRING_EFFORT_REASON: Final = (
|
||||
"it is not a string. Send reasoning_effort as a level name such as 'high' to set a reasoning level."
|
||||
)
|
||||
_REASONING_PARAMS: Final = ("reasoning", "reasoning_effort", "thinking")
|
||||
OPAQUE_VALUE: Final = TypeAdapter(object)
|
||||
|
||||
|
||||
def warn_dropped_reasoning(
|
||||
param: str, value: object, model: str | None, reason: str, *, drop_params_gated: bool
|
||||
) -> None:
|
||||
verbose_logger.warning(
|
||||
_DROP_PARAMS_DROP_WARNING if drop_params_gated else _DROP_WARNING, param, reprlib.repr(value), model, reason
|
||||
)
|
||||
|
||||
|
||||
def warn_non_string_reasoning_effort(value: object, model: str | None) -> None:
|
||||
effort: Final = as_str_mapping(value)
|
||||
requested_level: Final = effort.get("effort") if effort is not None else value
|
||||
if requested_level is None or isinstance(value, str):
|
||||
return
|
||||
warn_dropped_reasoning("reasoning_effort", value, model, _NON_STRING_EFFORT_REASON, drop_params_gated=False)
|
||||
|
||||
|
||||
def warn_dropped_reasoning_params(
|
||||
dropped: Collection[object],
|
||||
request: Mapping[str, object],
|
||||
model: str | None,
|
||||
reason: str,
|
||||
*,
|
||||
drop_params_gated: bool,
|
||||
) -> None:
|
||||
for param in _REASONING_PARAMS:
|
||||
if param in dropped and param in request:
|
||||
warn_dropped_reasoning(param, request[param], model, reason, drop_params_gated=drop_params_gated)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class DroppedThinkingBudget:
|
||||
budget_tokens: object
|
||||
|
||||
def warn(self, model: str) -> None:
|
||||
warn_dropped_reasoning(
|
||||
"thinking.budget_tokens", self.budget_tokens, model, _NON_NUMERIC_BUDGET_REASON, drop_params_gated=True
|
||||
)
|
||||
|
||||
|
||||
def reasoning_effort_from_thinking_budget(
|
||||
budget_tokens: float,
|
||||
|
|
@ -29,7 +83,9 @@ def reasoning_effort_from_thinking_budget(
|
|||
return "minimal"
|
||||
|
||||
|
||||
def reasoning_effort_from_thinking(thinking: object, *, drop_params: bool) -> OpenAIStyleReasoningEffort | None:
|
||||
def _effort_or_dropped_budget(
|
||||
thinking: object, *, drop_params: bool
|
||||
) -> OpenAIStyleReasoningEffort | DroppedThinkingBudget | None:
|
||||
fields: Final = as_str_mapping(thinking)
|
||||
if fields is None or fields.get("type") != "enabled":
|
||||
return None
|
||||
|
|
@ -37,10 +93,27 @@ def reasoning_effort_from_thinking(thinking: object, *, drop_params: bool) -> Op
|
|||
if isinstance(budget, (int, float)) and not isinstance(budget, bool):
|
||||
return reasoning_effort_from_thinking_budget(budget)
|
||||
if drop_params or litellm.drop_params is True:
|
||||
return None
|
||||
return DroppedThinkingBudget(budget)
|
||||
raise non_numeric_thinking_budget_error(budget)
|
||||
|
||||
|
||||
def reasoning_effort_from_thinking(
|
||||
thinking: object, *, model: str, drop_params: bool
|
||||
) -> OpenAIStyleReasoningEffort | None:
|
||||
effort: Final = _effort_or_dropped_budget(thinking, drop_params=drop_params)
|
||||
if not isinstance(effort, DroppedThinkingBudget):
|
||||
return effort
|
||||
effort.warn(model)
|
||||
return None
|
||||
|
||||
|
||||
def reasoning_effort_from_thinking_silently(
|
||||
thinking: object, *, drop_params: bool
|
||||
) -> OpenAIStyleReasoningEffort | None:
|
||||
effort: Final = _effort_or_dropped_budget(thinking, drop_params=drop_params)
|
||||
return None if isinstance(effort, DroppedThinkingBudget) else effort
|
||||
|
||||
|
||||
def non_numeric_thinking_budget_error(budget: object) -> UnsupportedParamsError:
|
||||
return UnsupportedParamsError(
|
||||
message=(
|
||||
|
|
|
|||
|
|
@ -249,6 +249,7 @@ def _redacted_provider_specific_fields(fields: Mapping[str, object]) -> Mapping[
|
|||
"reasoningContentBlocks",
|
||||
"citations",
|
||||
"compaction_blocks",
|
||||
"compaction_start",
|
||||
"compaction_delta",
|
||||
)
|
||||
if key in fields
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
import json
|
||||
import math
|
||||
from collections.abc import Callable
|
||||
from typing import Any, Final
|
||||
|
||||
|
|
@ -14,16 +15,23 @@ def strip_null_bytes(value: str) -> str:
|
|||
return value.replace("\x00", "")
|
||||
|
||||
|
||||
def _finite_or_none(value: float | None) -> float | None:
|
||||
return None if isinstance(value, float) and not math.isfinite(value) else value
|
||||
|
||||
|
||||
def safe_json_structure(
|
||||
data: object,
|
||||
max_depth: int = DEFAULT_MAX_RECURSE_DEPTH,
|
||||
value_transform: Callable[[str | None, str], str] | None = None,
|
||||
key: str | None = None,
|
||||
*,
|
||||
non_finite_as_null: bool = False,
|
||||
) -> object:
|
||||
"""
|
||||
Rebuild data out of JSON-native pieces while detecting circular references.
|
||||
If a circular reference is detected then a marker string is returned.
|
||||
NUL bytes are stripped from strings to prevent PostgreSQL 22P05 errors.
|
||||
NUL bytes are stripped from strings to prevent PostgreSQL 22P05 errors. With
|
||||
non_finite_as_null, NaN and infinite floats become null, since jsonb rejects them.
|
||||
|
||||
value_transform, when given, is applied to every string leaf (and to the
|
||||
str() fallback for non-serializable objects) with the mapping key the leaf
|
||||
|
|
@ -43,7 +51,7 @@ def safe_json_structure(
|
|||
cleaned = obj.replace("\x00", "") if "\x00" in obj else obj
|
||||
return _transform(key, cleaned)
|
||||
if isinstance(obj, (int, float, bool, type(None))):
|
||||
return obj
|
||||
return _finite_or_none(obj) if non_finite_as_null else obj
|
||||
# Check for circular reference.
|
||||
if id(obj) in seen:
|
||||
return "CircularReference Detected"
|
||||
|
|
@ -65,8 +73,9 @@ def safe_json_structure(
|
|||
seen.remove(id(obj))
|
||||
return tuple_result
|
||||
elif isinstance(obj, set):
|
||||
sortable_values: Final[list[Any]] = [_serialize(item, seen, depth + 1, key) for item in obj]
|
||||
set_result: Final = sorted(sortable_values)
|
||||
members: Final[list[Any]] = [_serialize(item, seen, depth + 1, key) for item in obj]
|
||||
present: Final = sorted(member for member in members if member is not None)
|
||||
set_result: Final = present + [None] * members.count(None)
|
||||
seen.remove(id(obj))
|
||||
return set_result
|
||||
elif isinstance(obj, BaseModel):
|
||||
|
|
@ -88,6 +97,10 @@ def safe_dumps(
|
|||
data: object,
|
||||
max_depth: int = DEFAULT_MAX_RECURSE_DEPTH,
|
||||
value_transform: Callable[[str | None, str], str] | None = None,
|
||||
*,
|
||||
non_finite_as_null: bool = False,
|
||||
) -> str:
|
||||
"""Serialize data to JSON text through safe_json_structure."""
|
||||
return json.dumps(safe_json_structure(data, max_depth, value_transform), default=str)
|
||||
return json.dumps(
|
||||
safe_json_structure(data, max_depth, value_transform, non_finite_as_null=non_finite_as_null), default=str
|
||||
)
|
||||
|
|
|
|||
|
|
@ -14,11 +14,18 @@ import litellm
|
|||
from litellm._logging import verbose_logger
|
||||
from litellm.litellm_core_utils.core_helpers import (
|
||||
PROVIDERS_BILLED_FROM_THEIR_OWN_USAGE_FIELDS,
|
||||
UPSTREAM_USAGE_COST,
|
||||
chunk_usage,
|
||||
finite_cost,
|
||||
provider_reported_usage_cost,
|
||||
stamp_litellm_bill_on_usage,
|
||||
)
|
||||
from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR, set_hidden_params
|
||||
from litellm.litellm_core_utils.hidden_params import (
|
||||
HIDDEN_PARAMS_ATTR,
|
||||
get_hidden_params,
|
||||
set_hidden_param,
|
||||
set_hidden_params,
|
||||
)
|
||||
from litellm.litellm_core_utils.llm_cost_calc.guardrail_cost import BEDROCK_TRACE_FIELDS, object_items, object_mapping
|
||||
from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import partition_annotations
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
||||
|
|
@ -1014,7 +1021,9 @@ class ChunkProcessor:
|
|||
prompt_tokens_details, cache_creation_token_details
|
||||
)
|
||||
|
||||
if usage_chunk_dict["cost"] is not None:
|
||||
if Usage.dropped_non_finite_cost(usage_chunk):
|
||||
cost = None
|
||||
elif usage_chunk_dict["cost"] is not None:
|
||||
cost = usage_chunk_dict["cost"]
|
||||
|
||||
prompt_tokens_details = attach_cache_creation_token_details(prompt_tokens_details, cache_creation_token_details)
|
||||
|
|
@ -1441,11 +1450,19 @@ def _set_stream_builder_response_cost(
|
|||
_set_hidden_response_cost(response, response_cost)
|
||||
|
||||
|
||||
def _carry_upstream_usage_cost(response: ModelResponse, chunks: Sequence[object]) -> None:
|
||||
chunk_hidden_params: Final = (get_hidden_params(chunk) for chunk in reversed(chunks))
|
||||
stamped: Final = (hidden for hidden in chunk_hidden_params if hidden is not None and UPSTREAM_USAGE_COST in hidden)
|
||||
for hidden in stamped:
|
||||
set_hidden_param(response, UPSTREAM_USAGE_COST, hidden[UPSTREAM_USAGE_COST])
|
||||
return
|
||||
|
||||
|
||||
def _stamp_streaming_usage_cost(usage: Usage, response: ModelResponse, logging_obj: Optional["Logging"]) -> None:
|
||||
if logging_obj is None:
|
||||
return
|
||||
if _logged_provider(logging_obj) in PROVIDERS_BILLED_FROM_THEIR_OWN_USAGE_FIELDS and isinstance(
|
||||
getattr(usage, "cost", None), (int, float)
|
||||
if _logged_provider(logging_obj) in PROVIDERS_BILLED_FROM_THEIR_OWN_USAGE_FIELDS and (
|
||||
finite_cost(getattr(usage, "cost", None)) is not None
|
||||
):
|
||||
return
|
||||
computed_cost: Final = logging_obj.response_cost_calculator(result=response)
|
||||
|
|
@ -1791,6 +1808,7 @@ def stream_chunk_builder(
|
|||
response,
|
||||
logging_obj,
|
||||
)
|
||||
_carry_upstream_usage_cost(response, chunks) # pyright: ignore[reportUnknownArgumentType] # chunks is an untyped list param
|
||||
_stamp_streaming_usage_cost(usage, response, logging_obj)
|
||||
_set_stream_builder_response_cost(
|
||||
response,
|
||||
|
|
@ -1833,6 +1851,7 @@ def stream_chunk_builder(
|
|||
)
|
||||
break
|
||||
|
||||
_carry_upstream_usage_cost(response, chunks) # pyright: ignore[reportUnknownArgumentType] # chunks is an untyped list param
|
||||
_stamp_streaming_usage_cost(usage, response, logging_obj)
|
||||
_set_stream_builder_response_cost(
|
||||
response,
|
||||
|
|
|
|||
|
|
@ -24,10 +24,16 @@ import litellm
|
|||
from litellm import verbose_logger
|
||||
from litellm._uuid import uuid
|
||||
from litellm.litellm_core_utils.asyncify import asyncify
|
||||
from litellm.litellm_core_utils.core_helpers import chunk_usage, provider_reported_usage_cost
|
||||
from litellm.litellm_core_utils.core_helpers import (
|
||||
UPSTREAM_USAGE_COST,
|
||||
chunk_usage,
|
||||
provider_reported_usage_cost,
|
||||
stamp_litellm_bill_on_usage,
|
||||
)
|
||||
from litellm.litellm_core_utils.hidden_params import (
|
||||
HIDDEN_PARAMS_ATTR,
|
||||
get_hidden_params,
|
||||
set_extra_attribute,
|
||||
set_hidden_param,
|
||||
stored_attribute,
|
||||
)
|
||||
|
|
@ -40,6 +46,7 @@ from litellm.llms.ollama.common_utils import OllamaResponseIdNormalizer
|
|||
from litellm.types.llms.base import LiteLLMBaseModel
|
||||
from litellm.types.router import GenericLiteLLMParams
|
||||
from litellm.types.utils import (
|
||||
CHOICE_LOGPROBS_OR_JSON,
|
||||
CacheCreationTokenDetails,
|
||||
CompletionTokensDetailsWrapper,
|
||||
Delta,
|
||||
|
|
@ -96,12 +103,17 @@ def _stamp_usage_cost(
|
|||
usage_chunk: ModelResponse | TextCompletionResponse | ModelResponseStream,
|
||||
complete_response: ModelResponse | TextCompletionResponse | ModelResponseStream,
|
||||
) -> None:
|
||||
bill: Final = (get_hidden_params(complete_response) or {}).get("response_cost")
|
||||
complete_hidden: Final = get_hidden_params(complete_response) or {}
|
||||
bill: Final = complete_hidden.get("response_cost")
|
||||
usage: Final[object] = getattr(usage_chunk, "usage", None)
|
||||
if usage is None or isinstance(bill, bool) or not isinstance(bill, (int, float)):
|
||||
return
|
||||
if bill > 0 or isinstance(getattr(usage, "cost", None), (int, float)):
|
||||
setattr(usage, "cost", float(bill)) # noqa: B010 # cost is an optional extra on the usage model
|
||||
if bill <= 0 and not isinstance(getattr(usage, "cost", None), (int, float)):
|
||||
return
|
||||
chunk_hidden: Final = get_hidden_params(usage_chunk)
|
||||
if chunk_hidden is not None and UPSTREAM_USAGE_COST in complete_hidden:
|
||||
chunk_hidden[UPSTREAM_USAGE_COST] = complete_hidden[UPSTREAM_USAGE_COST]
|
||||
stamp_litellm_bill_on_usage(usage_chunk, usage, float(bill))
|
||||
|
||||
|
||||
def _stamp_served_service_tier(response: ModelResponseStream, complete_streaming_response: ModelResponse) -> None:
|
||||
|
|
@ -1034,7 +1046,7 @@ class CustomStreamWrapper:
|
|||
_initial_delta: Final = model_response.choices[0].delta.model_dump()
|
||||
|
||||
_initial_delta.pop("role", None)
|
||||
model_response.choices[0].delta = Delta(**_initial_delta)
|
||||
set_extra_attribute(model_response.choices[0], "delta", Delta(**_initial_delta))
|
||||
return model_response
|
||||
|
||||
def _prebuilt_plain_delta(self, chunk: object) -> Delta | None:
|
||||
|
|
@ -1060,13 +1072,13 @@ class CustomStreamWrapper:
|
|||
def _set_role_on_fresh_delta(self, model_response: ModelResponseStream) -> None:
|
||||
delta: Final = model_response.choices[0].delta
|
||||
if self.sent_first_chunk is False:
|
||||
delta.role = "assistant"
|
||||
set_extra_attribute(delta, "role", "assistant")
|
||||
self.sent_first_chunk = True
|
||||
return
|
||||
if not _holds_only_scalars(delta):
|
||||
self.strip_role_from_delta(model_response)
|
||||
return
|
||||
delta.role = None
|
||||
set_extra_attribute(delta, "role", None)
|
||||
if "provider_specific_fields" not in delta.model_fields_set:
|
||||
delta.provider_specific_fields = None
|
||||
|
||||
|
|
@ -1185,8 +1197,8 @@ class CustomStreamWrapper:
|
|||
and _holds_only_scalars(upstream_delta)
|
||||
)
|
||||
if can_reuse_choice and upstream_choice is not None:
|
||||
model_response.choices[0].finish_reason = None # rebind-ok: keep reused choice
|
||||
model_response.choices[0].index = upstream_choice.index # rebind-ok: keep reused choice
|
||||
set_extra_attribute(model_response.choices[0], "finish_reason", None)
|
||||
set_extra_attribute(model_response.choices[0], "index", upstream_choice.index)
|
||||
_prepend_upstream_choice_extras(
|
||||
model_response.choices[0],
|
||||
upstream_choice,
|
||||
|
|
@ -1201,18 +1213,17 @@ class CustomStreamWrapper:
|
|||
choice_json.pop(
|
||||
"finish_reason", None
|
||||
) # for mistral etc. which return a value in their last chunk (not-openai compatible).
|
||||
choices.append(StreamingChoices(**choice_json))
|
||||
choice_json.pop("logprobs", None)
|
||||
choices.append(
|
||||
StreamingChoices(**choice_json, logprobs=getattr(choice, "logprobs", None))
|
||||
)
|
||||
except Exception:
|
||||
choices.append(StreamingChoices())
|
||||
setattr(model_response, "choices", choices)
|
||||
else:
|
||||
return
|
||||
model_response.system_fingerprint = upstream_chunk.system_fingerprint
|
||||
setattr(
|
||||
model_response,
|
||||
"citations",
|
||||
stored_attribute(upstream_chunk, "citations"),
|
||||
)
|
||||
set_extra_attribute(model_response, "citations", stored_attribute(upstream_chunk, "citations"))
|
||||
preserve_upstream_non_openai_attributes(
|
||||
model_response=model_response,
|
||||
original_chunk=original_chunk,
|
||||
|
|
@ -1605,7 +1616,7 @@ class CustomStreamWrapper:
|
|||
model_response.system_fingerprint = response_obj["original_chunk"].system_fingerprint
|
||||
self.system_fingerprint = response_obj["original_chunk"].system_fingerprint
|
||||
if response_obj["logprobs"] is not None:
|
||||
model_response.choices[0].logprobs = response_obj["logprobs"]
|
||||
model_response.choices[0].logprobs = CHOICE_LOGPROBS_OR_JSON.validate_python(response_obj["logprobs"])
|
||||
|
||||
if response_obj["usage"] is not None:
|
||||
if isinstance(response_obj["usage"], dict):
|
||||
|
|
@ -2482,6 +2493,25 @@ class CustomStreamWrapper:
|
|||
def bill_failure_after_earlier_rounds(self, earlier: "BilledRoundsSpend") -> None:
|
||||
self._earlier_rounds_spend = earlier
|
||||
|
||||
def partial_response(self) -> "ModelResponse | TextCompletionResponse | None":
|
||||
partial: Final = (
|
||||
litellm.stream_chunk_builder(
|
||||
chunks=self.chunks,
|
||||
messages=self.messages if isinstance(self.messages, list) else None,
|
||||
logging_obj=self.logging_obj,
|
||||
count_prompt_tokens=self.count_prompt_tokens,
|
||||
)
|
||||
if self.chunks
|
||||
else None
|
||||
)
|
||||
usage: Final[object] = getattr(partial, "usage", None)
|
||||
if partial is None or not isinstance(usage, Usage):
|
||||
return None
|
||||
if self.model:
|
||||
partial.model = self.model
|
||||
backfill_missing_cache_usage_fields(usage)
|
||||
return partial
|
||||
|
||||
def _record_partial_usage_for_failure(self) -> None:
|
||||
"""
|
||||
A stream that breaks mid-flight still billed the provider for the chunks
|
||||
|
|
@ -2491,23 +2521,15 @@ class CustomStreamWrapper:
|
|||
moves it into terminal failure usage before opening the next attempt,
|
||||
so a recovered attempt logs its own usage.
|
||||
"""
|
||||
if self.logging_obj is None or not self.chunks:
|
||||
if self.logging_obj is None:
|
||||
return
|
||||
try:
|
||||
partial_response: Final = litellm.stream_chunk_builder(
|
||||
chunks=self.chunks,
|
||||
messages=self.messages if isinstance(self.messages, list) else None,
|
||||
logging_obj=self.logging_obj,
|
||||
count_prompt_tokens=self.count_prompt_tokens,
|
||||
)
|
||||
partial_response: Final = self.partial_response()
|
||||
if partial_response is None:
|
||||
return
|
||||
usage: Final = cast(Usage | None, getattr(partial_response, "usage", None))
|
||||
if usage is None:
|
||||
usage: Final[object] = getattr(partial_response, "usage", None)
|
||||
if not isinstance(usage, Usage):
|
||||
return
|
||||
if self.model:
|
||||
partial_response.model = self.model
|
||||
backfill_missing_cache_usage_fields(usage)
|
||||
partial_cost: Final = self.logging_obj.response_cost_calculator(result=partial_response) or 0.0
|
||||
if self._earlier_rounds_spend is None:
|
||||
self.logging_obj.model_call_details["combined_usage_object"] = usage
|
||||
|
|
|
|||
|
|
@ -48,6 +48,7 @@ from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import
|
|||
place_mid_conversation_system,
|
||||
split_leading_system_run,
|
||||
)
|
||||
from litellm.litellm_core_utils.reasoning_effort_utils import OPAQUE_VALUE, warn_non_string_reasoning_effort
|
||||
from litellm.llms.base_llm.base_utils import (
|
||||
ResponseFormatSchema,
|
||||
as_json_object,
|
||||
|
|
@ -1951,6 +1952,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
|
|||
if isinstance(effort_value, dict):
|
||||
effort_value = effort_value.get("effort")
|
||||
if not isinstance(effort_value, str):
|
||||
warn_non_string_reasoning_effort(OPAQUE_VALUE.validate_python(value), model)
|
||||
continue
|
||||
mapped_thinking = AnthropicConfig._map_reasoning_effort(
|
||||
reasoning_effort=effort_value,
|
||||
|
|
|
|||
|
|
@ -37,7 +37,7 @@ from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import
|
|||
message_field,
|
||||
parts_of,
|
||||
)
|
||||
from litellm.litellm_core_utils.reasoning_effort_utils import non_numeric_thinking_budget_error
|
||||
from litellm.litellm_core_utils.reasoning_effort_utils import DroppedThinkingBudget, non_numeric_thinking_budget_error
|
||||
from litellm.litellm_core_utils.url_utils import join_api_base
|
||||
from litellm.llms.anthropic.wif import (
|
||||
aget_anthropic_wif_token,
|
||||
|
|
@ -58,6 +58,7 @@ from litellm.types.llms.anthropic import (
|
|||
ANTHROPIC_THINKING_BINDING_CONTROLS_BETA_HEADER,
|
||||
ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER,
|
||||
ANTHROPIC_TOOL_SEARCH_TOOL_TYPES,
|
||||
LEGACY_THINKING_DISPLAY_UPDATES_PROVIDERS,
|
||||
AllAnthropicToolsValues,
|
||||
AnthropicMcpServerTool,
|
||||
AnthropicMessagesToolChoice,
|
||||
|
|
@ -126,6 +127,8 @@ _JSON_VALUE: Final = TypeAdapter(JsonValue)
|
|||
|
||||
_CLAUDE_CODE_USER_AGENT_PREFIXES: Final = ("claude-cli/", "claude-code/")
|
||||
|
||||
NATIVE_COMPACTION_PROVIDERS: Final = frozenset({"anthropic", "azure_ai", "vertex_ai"})
|
||||
|
||||
|
||||
def is_claude_tool_target(custom_llm_provider: str | None, model: str) -> bool:
|
||||
if custom_llm_provider == "anthropic":
|
||||
|
|
@ -204,7 +207,7 @@ def requires_native_compaction_beta(
|
|||
optional_params: Mapping[str, object],
|
||||
messages: Sequence[object],
|
||||
) -> bool:
|
||||
return custom_llm_provider == "anthropic" and (
|
||||
return custom_llm_provider in NATIVE_COMPACTION_PROVIDERS and (
|
||||
optional_params.get("compaction") is not None or carries_signed_compaction_block(messages)
|
||||
)
|
||||
|
||||
|
|
@ -246,6 +249,27 @@ def context_management_betas(context_management: JsonValue) -> tuple[str, ...]:
|
|||
)
|
||||
|
||||
|
||||
def count_tokens_feature_betas(
|
||||
custom_llm_provider: str,
|
||||
optional_params: Mapping[str, object],
|
||||
messages: Sequence[object],
|
||||
) -> tuple[str, ...]:
|
||||
return tuple(
|
||||
beta
|
||||
for beta, used in (
|
||||
(
|
||||
ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_09_04.value,
|
||||
requires_native_compaction_beta(custom_llm_provider, optional_params, messages),
|
||||
),
|
||||
(
|
||||
ANTHROPIC_MID_CONVERSATION_TOOL_CHANGES_BETA_HEADER,
|
||||
AnthropicModelInfo().is_mid_conversation_tool_change_used(messages),
|
||||
),
|
||||
)
|
||||
if used
|
||||
)
|
||||
|
||||
|
||||
def supports_anthropic_cache_control(model: str, custom_llm_provider: str | None) -> bool:
|
||||
from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
|
||||
from litellm.utils import supports_prompt_caching
|
||||
|
|
@ -610,11 +634,11 @@ def _numeric_budget_tokens(budget_tokens: object) -> int | None:
|
|||
return None
|
||||
|
||||
|
||||
def _legacy_budget_tokens(budget_tokens: object, *, drop_params: bool) -> int | None:
|
||||
def _legacy_budget_tokens(budget_tokens: object, *, drop_params: bool) -> int | DroppedThinkingBudget:
|
||||
if (numeric := _numeric_budget_tokens(budget_tokens)) is not None:
|
||||
return numeric
|
||||
if drop_params or litellm.drop_params is True:
|
||||
return None
|
||||
return DroppedThinkingBudget(budget_tokens)
|
||||
raise non_numeric_thinking_budget_error(budget_tokens)
|
||||
|
||||
|
||||
|
|
@ -645,10 +669,12 @@ class AnthropicModelInfo(BaseLLMModelInfo):
|
|||
file_ids: Final = get_file_ids_from_messages(messages)
|
||||
return len(file_ids) > 0
|
||||
|
||||
def is_thinking_display_updates_used(self, thinking: AnthropicThinkingParam | None) -> bool:
|
||||
if not isinstance(thinking, dict):
|
||||
return False
|
||||
return thinking.get("type") in ("adaptive", "enabled") and thinking.get("display") == "updates"
|
||||
def is_thinking_display_updates_used(self, thinking: object) -> bool:
|
||||
match thinking:
|
||||
case {"type": "adaptive" | "enabled", "display": "updates"}:
|
||||
return True
|
||||
case _:
|
||||
return False
|
||||
|
||||
def thinking_betas(self, thinking: AnthropicThinkingParam | None) -> tuple[str, ...]:
|
||||
return (
|
||||
|
|
@ -1097,6 +1123,22 @@ class AnthropicModelInfo(BaseLLMModelInfo):
|
|||
custom_llm_provider: str,
|
||||
drop_params: bool,
|
||||
) -> None:
|
||||
dropped: Final = AnthropicModelInfo.translate_legacy_thinking_for_adaptive_model_silently(
|
||||
model=model,
|
||||
optional_params=optional_params,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
drop_params=drop_params,
|
||||
)
|
||||
if dropped is not None:
|
||||
dropped.warn(model)
|
||||
|
||||
@staticmethod
|
||||
def translate_legacy_thinking_for_adaptive_model_silently(
|
||||
model: str,
|
||||
optional_params: MutableMapping[str, object], # mutable-ok: in-place out-param like the sibling helpers
|
||||
custom_llm_provider: str,
|
||||
drop_params: bool,
|
||||
) -> DroppedThinkingBudget | None:
|
||||
"""Translate legacy ``thinking.type=enabled`` to adaptive for the
|
||||
adaptive-thinking models that reject it (4.7+ and the 5 families).
|
||||
Models flagged ``supports_legacy_thinking`` (the 4.6 family) accept the
|
||||
|
|
@ -1105,22 +1147,25 @@ class AnthropicModelInfo(BaseLLMModelInfo):
|
|||
``output_config.effort`` is never overridden.
|
||||
"""
|
||||
if not AnthropicModelInfo._is_adaptive_thinking_model(model, custom_llm_provider):
|
||||
return
|
||||
return None
|
||||
if AnthropicModelInfo._supports_legacy_thinking(model, custom_llm_provider):
|
||||
return
|
||||
return None
|
||||
thinking: Final = validated_object_mapping(optional_params.get("thinking"))
|
||||
if thinking is None or thinking.get("type") != "enabled":
|
||||
return
|
||||
return None
|
||||
|
||||
budget_tokens: Final = _legacy_budget_tokens(thinking.get("budget_tokens", 0), drop_params=drop_params)
|
||||
display: Final = thinking.get("display")
|
||||
supported_display: Final = {"display": display} if display in ("summarized", "omitted") else {}
|
||||
keeps_display: Final = display in ("summarized", "omitted") or (
|
||||
display == "updates" and custom_llm_provider in LEGACY_THINKING_DISPLAY_UPDATES_PROVIDERS
|
||||
)
|
||||
supported_display: Final = {"display": display} if keeps_display else {}
|
||||
carried_keys: Final = carried_thinking_keys(
|
||||
thinking, rebuilt_keys=frozenset(("type", "budget_tokens", "display")), drop_params=drop_params
|
||||
)
|
||||
optional_params["thinking"] = {**carried_keys, "type": "adaptive", **supported_display}
|
||||
if budget_tokens is None:
|
||||
return
|
||||
if isinstance(budget_tokens, DroppedThinkingBudget):
|
||||
return budget_tokens
|
||||
effort: Final = AnthropicModelInfo._legacy_budget_to_effort(
|
||||
model=model, budget_tokens=budget_tokens, custom_llm_provider=custom_llm_provider
|
||||
)
|
||||
|
|
@ -1129,6 +1174,7 @@ class AnthropicModelInfo(BaseLLMModelInfo):
|
|||
"effort": effort,
|
||||
**(existing_output_config if isinstance(existing_output_config, dict) else MappingProxyType({})),
|
||||
}
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _legacy_budget_to_effort(model: str, budget_tokens: int, custom_llm_provider: str) -> str:
|
||||
|
|
|
|||
|
|
@ -134,6 +134,7 @@ from litellm.litellm_core_utils.prompt_templates.factory import (
|
|||
)
|
||||
from litellm.litellm_core_utils.reasoning_effort_utils import (
|
||||
reasoning_effort_from_thinking,
|
||||
reasoning_effort_from_thinking_silently,
|
||||
)
|
||||
from litellm.llms.anthropic.common_utils import (
|
||||
eager_input_streaming_flag,
|
||||
|
|
@ -297,6 +298,7 @@ class AnthropicAdapter:
|
|||
custom_llm_provider=custom_llm_provider,
|
||||
api_base=api_base,
|
||||
drop_params=drop_params,
|
||||
report_reasoning_drops=True,
|
||||
)
|
||||
|
||||
return translated_body, tool_name_mapping
|
||||
|
|
@ -666,7 +668,9 @@ class LiteLLMAnthropicMessagesAdapter:
|
|||
def translate_anthropic_thinking_to_reasoning_effort(
|
||||
thinking: AnthropicThinkingParam,
|
||||
*,
|
||||
model: str,
|
||||
drop_params: bool = True,
|
||||
report_reasoning_drops: bool = False,
|
||||
) -> str | None:
|
||||
"""
|
||||
Translate Anthropic's thinking parameter to OpenAI's reasoning_effort.
|
||||
|
|
@ -685,7 +689,9 @@ class LiteLLMAnthropicMessagesAdapter:
|
|||
if thinking_type == "disabled":
|
||||
return "none"
|
||||
elif thinking_type == "enabled":
|
||||
return reasoning_effort_from_thinking(thinking, drop_params=drop_params)
|
||||
if report_reasoning_drops:
|
||||
return reasoning_effort_from_thinking(thinking, model=model, drop_params=drop_params)
|
||||
return reasoning_effort_from_thinking_silently(thinking, drop_params=drop_params)
|
||||
elif thinking_type == "adaptive":
|
||||
# Adaptive thinking: effort is controlled by output_config.effort,
|
||||
# not budget_tokens. Return a default; caller should override with
|
||||
|
|
@ -752,7 +758,7 @@ class LiteLLMAnthropicMessagesAdapter:
|
|||
return {"thinking": thinking}
|
||||
else:
|
||||
reasoning_effort: Final = LiteLLMAnthropicMessagesAdapter.translate_anthropic_thinking_to_reasoning_effort(
|
||||
thinking
|
||||
thinking, model=model
|
||||
)
|
||||
if reasoning_effort:
|
||||
return {
|
||||
|
|
@ -1147,6 +1153,7 @@ class LiteLLMAnthropicMessagesAdapter:
|
|||
*,
|
||||
custom_llm_provider: str | None = None,
|
||||
drop_params: bool = True,
|
||||
report_reasoning_drops: bool = False,
|
||||
) -> None:
|
||||
"""Translate Anthropic thinking to either thinking or reasoning_effort.
|
||||
|
||||
|
|
@ -1207,7 +1214,10 @@ class LiteLLMAnthropicMessagesAdapter:
|
|||
return
|
||||
|
||||
reasoning_effort: Final = declared_effort or self.translate_anthropic_thinking_to_reasoning_effort(
|
||||
cast(AnthropicThinkingParam, thinking), drop_params=drop_params
|
||||
cast(AnthropicThinkingParam, thinking),
|
||||
model=model,
|
||||
drop_params=drop_params,
|
||||
report_reasoning_drops=report_reasoning_drops,
|
||||
)
|
||||
if not reasoning_effort:
|
||||
return
|
||||
|
|
@ -1267,6 +1277,7 @@ class LiteLLMAnthropicMessagesAdapter:
|
|||
api_base: str | None = None,
|
||||
preserve_midturn_system: bool = False,
|
||||
drop_params: bool = True,
|
||||
report_reasoning_drops: bool = False,
|
||||
) -> tuple[ChatCompletionRequest, dict[str, str]]:
|
||||
"""
|
||||
This is used by the beta Anthropic Adapter, for translating anthropic `/v1/messages` requests to the openai format.
|
||||
|
|
@ -1321,6 +1332,7 @@ class LiteLLMAnthropicMessagesAdapter:
|
|||
new_kwargs=new_kwargs,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
drop_params=drop_params,
|
||||
report_reasoning_drops=report_reasoning_drops,
|
||||
)
|
||||
## CONVERT STOP_SEQUENCES
|
||||
self._translate_stop_sequences_to_openai(
|
||||
|
|
|
|||
|
|
@ -20,6 +20,7 @@ from litellm.litellm_core_utils.core_helpers import as_str_mapping
|
|||
from litellm.litellm_core_utils.exception_mapping_utils import exception_type
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
from litellm.llms.anthropic.common_utils import (
|
||||
AnthropicModelInfo,
|
||||
strip_provider_specific_fields_from_anthropic_messages,
|
||||
tools_marked_for_a_tool_config_point,
|
||||
)
|
||||
|
|
@ -36,7 +37,6 @@ from litellm.router_utils.reasoning_effort_capability import (
|
|||
keeps_effort,
|
||||
model_group_from_metadata,
|
||||
resolve_deployment_reasoning_effort,
|
||||
with_adaptive_thinking_as_effort,
|
||||
)
|
||||
from litellm.types.llms.anthropic import AllAnthropicToolsValues
|
||||
from litellm.types.llms.anthropic_messages.anthropic_request import AnthropicMetadata
|
||||
|
|
@ -185,9 +185,11 @@ def _with_deployment_output_effort(
|
|||
).value
|
||||
|
||||
def translate(source: Mapping[str, object]) -> Mapping[str, object]:
|
||||
return with_adaptive_thinking_as_effort(
|
||||
source, model=model, custom_llm_provider=custom_llm_provider, drop_params=drop_params
|
||||
translated: Final = dict(source)
|
||||
AnthropicModelInfo.translate_legacy_thinking_for_adaptive_model(
|
||||
model=model, optional_params=translated, custom_llm_provider=custom_llm_provider, drop_params=drop_params
|
||||
)
|
||||
return translated
|
||||
|
||||
requested_output: Final = as_str_mapping(params.get("output_config"))
|
||||
caller_output: Final = None if requested_output is None else resolve(requested_output)
|
||||
|
|
|
|||
|
|
@ -21,6 +21,7 @@ from litellm.constants import ANTHROPIC_BILLING_METADATA_PREFIX
|
|||
from litellm.exceptions import AuthenticationError
|
||||
from litellm.litellm_core_utils.core_helpers import drops_unsupported_params
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
from litellm.litellm_core_utils.reasoning_effort_utils import OPAQUE_VALUE, warn_non_string_reasoning_effort
|
||||
from litellm.litellm_core_utils.url_utils import ensure_api_route
|
||||
from litellm.llms.anthropic.messages.convertible_blocks import drop_non_string_tool_result_texts
|
||||
from litellm.llms.anthropic.messages.safeguards import proxy_serves_safeguards
|
||||
|
|
@ -40,6 +41,7 @@ from litellm.types.llms.anthropic_tool_search import get_tool_search_beta_header
|
|||
from litellm.types.router import GenericLiteLLMParams
|
||||
|
||||
from ...common_utils import (
|
||||
NATIVE_COMPACTION_PROVIDERS,
|
||||
OBJECT_LIST_ADAPTER,
|
||||
AnthropicError,
|
||||
AnthropicModelInfo,
|
||||
|
|
@ -103,6 +105,7 @@ _MESSAGES_ADAPTER: Final = TypeAdapter(list[dict[str, object]])
|
|||
|
||||
class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
|
||||
_workload_identity_eligible: ClassVar[bool] = True
|
||||
_report_reasoning_drops: ClassVar[bool] = True
|
||||
|
||||
@property
|
||||
def custom_llm_provider(self) -> str | None:
|
||||
|
|
@ -126,7 +129,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
|
|||
"tool_choice",
|
||||
"thinking",
|
||||
"context_management",
|
||||
*(("compaction",) if self._resolved_provider == "anthropic" else ()),
|
||||
*(("compaction",) if self._resolved_provider in NATIVE_COMPACTION_PROVIDERS else ()),
|
||||
"output_format",
|
||||
"inference_geo",
|
||||
"speed",
|
||||
|
|
@ -695,12 +698,17 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
|
|||
):
|
||||
anthropic_messages_optional_request_params.pop("output_format") # pyright: ignore[reportUnknownMemberType] # the request params arrive as a bare dict
|
||||
|
||||
requested_reasoning_effort: Final = OPAQUE_VALUE.validate_python(
|
||||
anthropic_messages_optional_request_params.get("reasoning_effort") # pyright: ignore[reportUnknownMemberType] # the request params arrive as a bare dict
|
||||
)
|
||||
self._translate_reasoning_effort_to_anthropic(
|
||||
model=model,
|
||||
optional_params=anthropic_messages_optional_request_params,
|
||||
max_tokens=max_tokens,
|
||||
custom_llm_provider=self._resolved_provider,
|
||||
)
|
||||
if self._report_reasoning_drops:
|
||||
warn_non_string_reasoning_effort(requested_reasoning_effort, model)
|
||||
|
||||
AnthropicModelInfo.maybe_drop_disabled_thinking(
|
||||
model=model,
|
||||
|
|
@ -708,12 +716,14 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
|
|||
custom_llm_provider=self._resolved_provider,
|
||||
)
|
||||
|
||||
AnthropicModelInfo.translate_legacy_thinking_for_adaptive_model(
|
||||
dropped_budget: Final = AnthropicModelInfo.translate_legacy_thinking_for_adaptive_model_silently(
|
||||
model=model,
|
||||
optional_params=anthropic_messages_optional_request_params,
|
||||
custom_llm_provider=self._resolved_provider,
|
||||
drop_params=getattr(litellm_params, "drop_params", None) is True,
|
||||
)
|
||||
if dropped_budget is not None and self._report_reasoning_drops:
|
||||
dropped_budget.warn(model)
|
||||
|
||||
self._translate_adaptive_effort_for_non_adaptive_model(
|
||||
model=model,
|
||||
|
|
|
|||
|
|
@ -534,6 +534,7 @@ class LiteLLMAnthropicToResponsesAPIAdapter:
|
|||
output_config: dict[str, object] | None = None,
|
||||
auto_summary_level: str = "detailed",
|
||||
*,
|
||||
model: str,
|
||||
drop_params: bool = True,
|
||||
) -> dict[str, object] | None:
|
||||
"""
|
||||
|
|
@ -555,7 +556,7 @@ class LiteLLMAnthropicToResponsesAPIAdapter:
|
|||
if isinstance(output_config, dict) and output_config.get("effort"):
|
||||
effort = output_config["effort"]
|
||||
elif thinking_type == "enabled":
|
||||
budget_effort: Final = reasoning_effort_from_thinking(thinking, drop_params=drop_params)
|
||||
budget_effort: Final = reasoning_effort_from_thinking(thinking, model=model, drop_params=drop_params)
|
||||
if budget_effort is None:
|
||||
return None
|
||||
effort = budget_effort
|
||||
|
|
@ -667,6 +668,7 @@ class LiteLLMAnthropicToResponsesAPIAdapter:
|
|||
thinking,
|
||||
output_config=cast(dict[str, object] | None, output_config),
|
||||
auto_summary_level=auto_summary_level,
|
||||
model=model,
|
||||
drop_params=drop_params,
|
||||
)
|
||||
if reasoning:
|
||||
|
|
|
|||
|
|
@ -9,7 +9,7 @@ from dataclasses import dataclass, field
|
|||
from functools import reduce
|
||||
from itertools import accumulate, groupby
|
||||
from types import MappingProxyType
|
||||
from typing import Annotated, Final, Literal, Protocol, TypeAlias
|
||||
from typing import Annotated, ClassVar, Final, Literal, Protocol, TypeAlias
|
||||
|
||||
import httpx
|
||||
from pydantic import ConfigDict, Field, JsonValue, StrictInt, TypeAdapter, ValidationError
|
||||
|
|
@ -81,6 +81,10 @@ _DEPLOYMENT_OPTIONS: Final = frozenset(
|
|||
)
|
||||
|
||||
|
||||
class _BaselineAnthropicMessagesConfig(AnthropicMessagesConfig):
|
||||
_report_reasoning_drops: ClassVar[bool] = False
|
||||
|
||||
|
||||
class _StrictModel(LiteLLMBaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, strict=True)
|
||||
|
||||
|
|
@ -661,7 +665,7 @@ def prepare_native_baseline_body(request: Mapping[str, object], model: str) -> M
|
|||
TypeAdapter(tuple[str, ...]).validate_python(owned.get("additional_drop_params") or ()),
|
||||
dict(options),
|
||||
)
|
||||
config: Final = AnthropicMessagesConfig()
|
||||
config: Final = _BaselineAnthropicMessagesConfig()
|
||||
body: Final = config.transform_anthropic_messages_request(
|
||||
model=resolved_model,
|
||||
messages=strip_provider_specific_fields_from_anthropic_messages(messages),
|
||||
|
|
|
|||
|
|
@ -14,6 +14,7 @@ from litellm.constants import AZURE_DEFAULT_RESPONSES_API_VERSION
|
|||
from litellm.litellm_core_utils.url_utils import encode_url_path_segment
|
||||
from litellm.llms.azure.chat.gpt_reasoning_transformation import AzureOpenAIGPT5Config
|
||||
from litellm.llms.azure.common_utils import BaseAzureLLM
|
||||
from litellm.llms.base_llm.responses.transformation import SUMMARY_ONLY_REASONING_FIELD_VALUES, ReasoningFieldValues
|
||||
from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig
|
||||
from litellm.types.llms.openai import *
|
||||
from litellm.types.responses.main import *
|
||||
|
|
@ -92,6 +93,12 @@ class AzureOpenAIResponsesAPIConfig(OpenAIResponsesAPIConfig):
|
|||
def _effort_resolves_to_none(model: str, effort: str | None) -> bool:
|
||||
return AzureOpenAIGPT5Config.effort_resolves_to_none(model, effort)
|
||||
|
||||
def deployment_names_wrap_openai_models(self) -> bool:
|
||||
return True
|
||||
|
||||
def unresolved_model_reasoning_field_values(self, model: str) -> ReasoningFieldValues | None:
|
||||
return SUMMARY_ONLY_REASONING_FIELD_VALUES if self._is_gpt_5_model(model) else None
|
||||
|
||||
def get_supported_openai_params(self, model: str) -> list:
|
||||
"""
|
||||
Azure Responses API does not support context_management (compaction).
|
||||
|
|
|
|||
|
|
@ -84,6 +84,7 @@ class AzureAIAnthropicCountTokensHandler(AzureAIAnthropicCountTokensConfig):
|
|||
headers: Final = self.get_required_headers(
|
||||
api_key=api_key,
|
||||
litellm_params=litellm_params,
|
||||
messages=messages,
|
||||
)
|
||||
if not {"x-api-key", "Authorization"} & headers.keys():
|
||||
raise AnthropicError(
|
||||
|
|
|
|||
|
|
@ -4,11 +4,13 @@ Azure AI Anthropic CountTokens API transformation logic.
|
|||
Extends the base Anthropic CountTokens transformation with Azure authentication.
|
||||
"""
|
||||
|
||||
from collections.abc import Sequence
|
||||
from typing import Final
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
from litellm.constants import ANTHROPIC_TOKEN_COUNTING_BETA_VERSION
|
||||
from litellm.litellm_core_utils.url_utils import join_api_base
|
||||
from litellm.llms.anthropic.common_utils import count_tokens_feature_betas, merge_anthropic_beta_headers
|
||||
from litellm.llms.anthropic.count_tokens.transformation import (
|
||||
AnthropicCountTokensConfig,
|
||||
)
|
||||
|
|
@ -28,6 +30,7 @@ class AzureAIAnthropicCountTokensConfig(AnthropicCountTokensConfig):
|
|||
self,
|
||||
api_key: str | None,
|
||||
litellm_params: dict[str, object] | None = None,
|
||||
messages: Sequence[object] = (),
|
||||
) -> dict[str, str]:
|
||||
"""
|
||||
Get the required headers for the Azure AI Anthropic CountTokens API.
|
||||
|
|
@ -38,6 +41,7 @@ class AzureAIAnthropicCountTokensConfig(AnthropicCountTokensConfig):
|
|||
Args:
|
||||
api_key: The Azure AI API key
|
||||
litellm_params: Optional LiteLLM parameters for additional auth config
|
||||
messages: The request messages, read for betas the counted features need
|
||||
|
||||
Returns:
|
||||
Dictionary of required headers with both x-api-key and Azure authentication
|
||||
|
|
@ -46,7 +50,9 @@ class AzureAIAnthropicCountTokensConfig(AnthropicCountTokensConfig):
|
|||
headers: Final = {
|
||||
"Content-Type": "application/json",
|
||||
"anthropic-version": "2023-06-01",
|
||||
"anthropic-beta": ANTHROPIC_TOKEN_COUNTING_BETA_VERSION,
|
||||
"anthropic-beta": merge_anthropic_beta_headers(
|
||||
ANTHROPIC_TOKEN_COUNTING_BETA_VERSION, count_tokens_feature_betas("azure_ai", {}, messages)
|
||||
),
|
||||
**({"x-api-key": api_key} if api_key else {}),
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -20,6 +20,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
|||
prompt_cache_breakpoint_for_wire,
|
||||
with_prompt_cache_breakpoint,
|
||||
)
|
||||
from litellm.litellm_core_utils.reasoning_effort_utils import warn_dropped_reasoning_params
|
||||
from litellm.llms.azure.common_utils import BaseAzureLLM
|
||||
from litellm.llms.azure_ai.common_utils import (
|
||||
api_key_header_for_base,
|
||||
|
|
@ -40,6 +41,14 @@ if TYPE_CHECKING:
|
|||
from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
|
||||
|
||||
|
||||
_REQUEST_BODY: Final = TypeAdapter(dict[str, object])
|
||||
|
||||
|
||||
def _request_model(request: Mapping[str, object]) -> str | None:
|
||||
model: Final = request.get("model")
|
||||
return model if isinstance(model, str) else None
|
||||
|
||||
|
||||
class AzureFoundryErrorStrings(str, enum.Enum):
|
||||
SET_EXTRA_PARAMETERS_TO_PASS_THROUGH = "Set extra-parameters to 'pass-through'"
|
||||
|
||||
|
|
@ -510,7 +519,13 @@ class AzureAIStudioConfig(OpenAIConfig):
|
|||
request_data = self._drop_extra_params_from_request_data(request_data, error_text)
|
||||
if "Extra inputs are not permitted" in error_text and self._error_has_tool_level_extra_fields(error_text):
|
||||
request_data = self._drop_tool_level_extra_fields(request_data, error_text)
|
||||
data: Final = drop_params_from_unprocessable_entity_error(e=e, data=request_data)
|
||||
data: Final = drop_params_from_unprocessable_entity_error(
|
||||
e=e,
|
||||
data=request_data,
|
||||
model=_request_model(_REQUEST_BODY.validate_python(request_data)),
|
||||
drop_params_gated="Extra inputs are not permitted" in error_text
|
||||
and not self._error_has_tool_level_extra_fields(error_text),
|
||||
)
|
||||
return data
|
||||
|
||||
def _drop_tool_level_extra_fields(self, request_data: dict, error_text: str) -> dict:
|
||||
|
|
@ -526,6 +541,14 @@ class AzureAIStudioConfig(OpenAIConfig):
|
|||
def _drop_extra_params_from_request_data(self, request_data: dict, error_text: str) -> dict:
|
||||
params_to_drop: Final = self._extract_params_to_drop_from_error_text(error_text)
|
||||
if params_to_drop:
|
||||
request: Final = _REQUEST_BODY.validate_python(request_data)
|
||||
warn_dropped_reasoning_params(
|
||||
params_to_drop,
|
||||
request,
|
||||
_request_model(request),
|
||||
"Azure AI rejected it as an extra parameter its deployment does not pass through.",
|
||||
drop_params_gated=False,
|
||||
)
|
||||
for param in params_to_drop:
|
||||
if param in request_data:
|
||||
request_data.pop(param, None)
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
import types
|
||||
from abc import ABC, abstractmethod
|
||||
from collections.abc import Mapping
|
||||
from typing import TYPE_CHECKING, Any, Final, Literal, TypeAlias, cast
|
||||
from typing import TYPE_CHECKING, Any, Final, Literal, TypeAlias, cast, get_args
|
||||
|
||||
import httpx
|
||||
|
||||
|
|
@ -27,6 +27,20 @@ else:
|
|||
BaseLLMException = Any
|
||||
|
||||
ReasoningSummaryLevel: TypeAlias = Literal["auto", "concise", "detailed"]
|
||||
ReasoningFieldValues: TypeAlias = Mapping[str, frozenset[str | None] | None]
|
||||
REASONING_SUMMARY_VALUES: Final = frozenset((*get_args(ReasoningSummaryLevel), None))
|
||||
SUMMARY_ONLY_REASONING_FIELD_VALUES: Final[ReasoningFieldValues] = types.MappingProxyType(
|
||||
{"summary": REASONING_SUMMARY_VALUES}
|
||||
)
|
||||
STANDARD_MODE_WITHOUT_CONTEXT_REASONING_FIELD_VALUES: Final[ReasoningFieldValues] = types.MappingProxyType(
|
||||
{**SUMMARY_ONLY_REASONING_FIELD_VALUES, "mode": frozenset({"standard"})}
|
||||
)
|
||||
STANDARD_MODE_REASONING_FIELD_VALUES: Final[ReasoningFieldValues] = types.MappingProxyType(
|
||||
{**STANDARD_MODE_WITHOUT_CONTEXT_REASONING_FIELD_VALUES, "context": None}
|
||||
)
|
||||
ANY_REASONING_FIELD_VALUES: Final[ReasoningFieldValues] = types.MappingProxyType(
|
||||
{**SUMMARY_ONLY_REASONING_FIELD_VALUES, "mode": None, "context": None}
|
||||
)
|
||||
|
||||
|
||||
class BaseResponsesAPIConfig(ABC):
|
||||
|
|
@ -100,6 +114,18 @@ class BaseResponsesAPIConfig(ABC):
|
|||
"""The ``reasoning.summary`` LiteLLM's opt-in auto summary asks this provider for."""
|
||||
return "detailed"
|
||||
|
||||
def reasoning_field_values(self) -> ReasoningFieldValues | None:
|
||||
return None
|
||||
|
||||
def deployment_names_wrap_openai_models(self) -> bool:
|
||||
return False
|
||||
|
||||
def unresolved_model_reasoning_field_values(self, model: str) -> ReasoningFieldValues | None:
|
||||
return None
|
||||
|
||||
def ignores_unsupported_reasoning_fields(self) -> bool:
|
||||
return False
|
||||
|
||||
@abstractmethod
|
||||
def map_openai_params(
|
||||
self,
|
||||
|
|
|
|||
|
|
@ -33,6 +33,7 @@ from litellm.litellm_core_utils.prompt_templates.image_handling import (
|
|||
inline_remote_image_urls,
|
||||
inline_remote_media,
|
||||
)
|
||||
from litellm.litellm_core_utils.reasoning_effort_utils import warn_dropped_reasoning
|
||||
from litellm.litellm_core_utils.url_utils import ensure_api_route, join_api_base
|
||||
from litellm.llms.base_llm.chat.transformation import BaseLLMException
|
||||
from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM, bedrock_bearer_token
|
||||
|
|
@ -42,6 +43,7 @@ from litellm.llms.bedrock.common_utils import (
|
|||
bedrock_rejects_stop_sequences,
|
||||
bedrock_runtime_chat_completions_serves_reasoning_inline,
|
||||
split_bedrock_region_path,
|
||||
strip_bedrock_routing_prefix,
|
||||
)
|
||||
from litellm.llms.openai.chat.gpt_transformation import OpenAIChatCompletionStreamingHandler
|
||||
from litellm.llms.openai_like.chat.transformation import OpenAILikeChatConfig
|
||||
|
|
@ -121,6 +123,13 @@ def without_refused_reasoning_effort(model: str, params: Mapping[str, object]) -
|
|||
effort: Final = params.get("reasoning_effort")
|
||||
if not isinstance(effort, str) or effort not in chat_completions_reasoning_efforts_refused_for(model):
|
||||
return params
|
||||
warn_dropped_reasoning(
|
||||
"reasoning_effort",
|
||||
effort,
|
||||
strip_bedrock_routing_prefix(model),
|
||||
"AWS's Chat Completions endpoint rejects that level for this model family, so AWS applies its default effort.",
|
||||
drop_params_gated=False,
|
||||
)
|
||||
return _without_params(params, frozenset(("reasoning_effort",)))
|
||||
|
||||
|
||||
|
|
@ -445,6 +454,14 @@ class AmazonBedrockRuntimeChatCompletionsConfig(OpenAILikeChatConfig):
|
|||
),
|
||||
status_code=400,
|
||||
)
|
||||
if malformed_effort:
|
||||
warn_dropped_reasoning(
|
||||
"reasoning_effort",
|
||||
raw_params["reasoning_effort"],
|
||||
strip_bedrock_routing_prefix(model),
|
||||
"Bedrock's Chat Completions endpoint takes it only as a string, so AWS applies its default effort.",
|
||||
drop_params_gated=True,
|
||||
)
|
||||
rejected_stop: Final = frozenset(("stop",)) if bedrock_rejects_stop_sequences(model) else frozenset[str]()
|
||||
return dict(
|
||||
without_refused_reasoning_effort(
|
||||
|
|
|
|||
|
|
@ -50,6 +50,7 @@ from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import
|
|||
message_field,
|
||||
parts_of,
|
||||
)
|
||||
from litellm.litellm_core_utils.reasoning_effort_utils import OPAQUE_VALUE, warn_dropped_reasoning
|
||||
from litellm.llms.anthropic.chat.transformation import (
|
||||
DROP_UNSUPPORTED_ADAPTIVE_THINKING_WARNING,
|
||||
DROP_UNSUPPORTED_OUTPUT_CONFIG_WARNING,
|
||||
|
|
@ -63,6 +64,7 @@ from litellm.llms.bedrock.chat.tool_result_images import place_tool_result_image
|
|||
from litellm.llms.bedrock.common_utils import (
|
||||
bedrock_chat_uses_native_structured_output,
|
||||
bedrock_cleaned_output_schema,
|
||||
bedrock_model_is_unknown,
|
||||
bedrock_model_supports_regex_lookaround,
|
||||
bedrock_native_output_format,
|
||||
bedrock_native_output_is_opt_in,
|
||||
|
|
@ -186,6 +188,8 @@ def _tools_the_model_accepts(
|
|||
|
||||
|
||||
_OUTPUT_CONFIG_BLOCK: Final = TypeAdapter(OutputConfigBlock)
|
||||
_DEEPSEEK_THINKING_REASON: Final = "DeepSeek on Bedrock reasons natively and takes no thinking field."
|
||||
_DEEPSEEK_R1_EFFORT_REASON: Final = "DeepSeek R1 on Bedrock always reasons and rejects reasoning_effort."
|
||||
|
||||
|
||||
def _has_native_text_format(output_config: object) -> bool:
|
||||
|
|
@ -529,13 +533,14 @@ class AmazonConverseConfig(BaseConfig):
|
|||
"""
|
||||
Handle the reasoning_effort parameter based on the model type.
|
||||
|
||||
- GPT-OSS and DeepSeek V3 models: passed through unchanged via additionalModelRequestFields.
|
||||
- GPT-OSS, DeepSeek V3 and unknown models (``bedrock_model_is_unknown``): passed through unchanged
|
||||
via additionalModelRequestFields.
|
||||
- OpenAI GPT-5.x and GPT-6 models: mapped to ``reasoning.effort`` via additionalModelRequestFields.
|
||||
- Nova 2 models: transformed to reasoningConfig.
|
||||
- Anthropic models: mapped to ``thinking`` (and ``output_config.effort`` on
|
||||
adaptive Claude 4.6 / 4.7).
|
||||
"""
|
||||
if "gpt-oss" in model or "deepseek" in model:
|
||||
if "gpt-oss" in model or "deepseek" in model or bedrock_model_is_unknown(model):
|
||||
optional_params["reasoning_effort"] = reasoning_effort
|
||||
elif self._is_openai_gpt_reasoning_model(model):
|
||||
reasoning: Final[BedrockConverseGptReasoningEffortBlock] = {"effort": reasoning_effort}
|
||||
|
|
@ -638,6 +643,10 @@ class AmazonConverseConfig(BaseConfig):
|
|||
or supports_reasoning(model=base_model, custom_llm_provider=self.custom_llm_provider)
|
||||
)
|
||||
|
||||
def _warn_native_reasoning_drop(self, param: str, value: object, model: str, reason: str) -> None:
|
||||
if param in self.get_supported_openai_params(model):
|
||||
warn_dropped_reasoning(param, value, model, reason, drop_params_gated=False)
|
||||
|
||||
def _model_rejects_reasoning_effort_param(self, model: str, base_model: str) -> bool:
|
||||
"""Whether the model returns a 400 for every ``reasoning_effort`` shape on Converse.
|
||||
|
||||
|
|
@ -681,7 +690,7 @@ class AmazonConverseConfig(BaseConfig):
|
|||
"parallel_tool_calls",
|
||||
]
|
||||
|
||||
if "arn" in model: # we can't infer the model from the arn, so just add all params
|
||||
if "arn" in model or bedrock_model_is_unknown(model):
|
||||
supported_params.append("tools")
|
||||
supported_params.append("tool_choice")
|
||||
supported_params.append("thinking")
|
||||
|
|
@ -1141,9 +1150,8 @@ class AmazonConverseConfig(BaseConfig):
|
|||
"tool_choice": {"type": "auto", "disable_parallel_tool_use": not value}
|
||||
}
|
||||
if param == "thinking" and drop_thinking_param:
|
||||
verbose_logger.debug(
|
||||
"Dropping unsupported `thinking` param for Bedrock model=%s; it reasons natively.",
|
||||
model,
|
||||
self._warn_native_reasoning_drop(
|
||||
param, OPAQUE_VALUE.validate_python(value), model, _DEEPSEEK_THINKING_REASON
|
||||
)
|
||||
elif param == "thinking" and not self._is_openai_gpt_reasoning_model(model):
|
||||
if (
|
||||
|
|
@ -1177,10 +1185,7 @@ class AmazonConverseConfig(BaseConfig):
|
|||
drop_params=drop_params,
|
||||
)
|
||||
elif param == "reasoning_effort" and isinstance(value, str) and drop_reasoning_effort_param:
|
||||
verbose_logger.debug(
|
||||
"Dropping unsupported `reasoning_effort` param for Bedrock model=%s; it always reasons and rejects it.",
|
||||
model,
|
||||
)
|
||||
self._warn_native_reasoning_drop(param, value, model, _DEEPSEEK_R1_EFFORT_REASON)
|
||||
elif (
|
||||
param == "reasoning_effort"
|
||||
and isinstance(value, str)
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@ import copy
|
|||
import json
|
||||
import time
|
||||
from collections.abc import Mapping, Sequence
|
||||
from typing import TYPE_CHECKING, Any, Final, cast, get_args
|
||||
from typing import TYPE_CHECKING, Any, Final
|
||||
|
||||
import httpx
|
||||
from pydantic import ConfigDict, TypeAdapter, ValidationError, with_config
|
||||
|
|
@ -74,10 +74,6 @@ _GUARDRAIL_CONFIG_EXPECTED_FORMAT: Final = (
|
|||
"{'guardrailIdentifier': str, 'guardrailVersion': str, 'trace': 'enabled'|'disabled'|'enabled_full'}"
|
||||
)
|
||||
|
||||
_INVOKE_PROVIDERS: Final = TypeAdapter(tuple[litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL, ...]).validate_python(
|
||||
get_args(litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL)
|
||||
)
|
||||
|
||||
|
||||
def _response_header(headers: Mapping[str, str], name: str) -> str | None:
|
||||
return headers.get(name, None)
|
||||
|
|
@ -562,62 +558,6 @@ class AmazonInvokeConfig(BaseConfig, BaseAWSLLM):
|
|||
"""
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
def get_bedrock_invoke_provider(
|
||||
model: str,
|
||||
) -> litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL | None:
|
||||
"""
|
||||
Helper function to get the bedrock provider from the model
|
||||
|
||||
handles 4 scenarios:
|
||||
1. model=invoke/anthropic.claude-3-5-sonnet-20240620-v1:0 -> Returns `anthropic`
|
||||
2. model=anthropic.claude-3-5-sonnet-20240620-v1:0 -> Returns `anthropic`
|
||||
3. model=llama/arn:aws:bedrock:us-east-1:086734376398:imported-model/r4c4kewx2s0n -> Returns `llama`
|
||||
4. model=us.amazon.nova-pro-v1:0 -> Returns `nova`
|
||||
"""
|
||||
if model.startswith("invoke/"):
|
||||
model = model.replace("invoke/", "", 1)
|
||||
|
||||
# Special case: Check for "nova" in model name first (before "amazon")
|
||||
# This handles amazon.nova-* models which would otherwise match "amazon" (Titan)
|
||||
if "nova" in model.lower():
|
||||
if "nova" in get_args(litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL):
|
||||
return cast(litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL, "nova")
|
||||
|
||||
_split_model: Final = model.split(".")[0]
|
||||
if _split_model in get_args(litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL):
|
||||
return cast(litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL, _split_model)
|
||||
|
||||
# If not a known provider, check for pattern with two slashes
|
||||
provider = AmazonInvokeConfig._get_provider_from_model_path(model)
|
||||
if provider is not None:
|
||||
return provider
|
||||
|
||||
for provider in _INVOKE_PROVIDERS:
|
||||
if provider in model:
|
||||
return provider
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _get_provider_from_model_path(
|
||||
model_path: str,
|
||||
) -> litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL | None:
|
||||
"""
|
||||
Helper function to get the provider from a model path with format: provider/model-name
|
||||
|
||||
Args:
|
||||
model_path (str): The model path (e.g., 'llama/arn:aws:bedrock:us-east-1:086734376398:imported-model/r4c4kewx2s0n' or 'anthropic/model-name')
|
||||
|
||||
Returns:
|
||||
Optional[str]: The provider name, or None if no valid provider found
|
||||
"""
|
||||
parts: Final = model_path.split("/")
|
||||
if len(parts) >= 1:
|
||||
provider: Final = parts[0]
|
||||
if provider in get_args(litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL):
|
||||
return cast(litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL, provider)
|
||||
return None
|
||||
|
||||
def convert_messages_to_prompt(self, model: str, messages, provider, custom_prompt_dict) -> tuple[str, list | None]:
|
||||
# handle anthropic prompts and amazon titan prompts
|
||||
prompt = ""
|
||||
|
|
|
|||
|
|
@ -1098,6 +1098,11 @@ def _model_cost_entry(key: str) -> Mapping[str, object] | None:
|
|||
return None if raw is None else _MODEL_COST_ENTRY_ADAPTER.validate_python(raw)
|
||||
|
||||
|
||||
def _cost_map_row_names_a_provider(key: str) -> bool:
|
||||
entry: Final = _model_cost_entry(key)
|
||||
return entry is not None and entry.get("litellm_provider") is not None
|
||||
|
||||
|
||||
def _bedrock_price_map_entries(model: str) -> tuple[Mapping[str, object] | None, ...]:
|
||||
return tuple(
|
||||
_model_cost_entry(key)
|
||||
|
|
@ -1567,7 +1572,8 @@ def bedrock_model_supports_regex_lookaround(model: str, litellm_params: Mapping[
|
|||
|
||||
The deployment's ``model_info.supports_regex_lookaround`` wins, then the
|
||||
``model_prices_and_context_window.json`` entry of its ``base_model``, then the
|
||||
entry of ``model`` itself. A model nobody flagged keeps its schema as sent.
|
||||
entry of ``model`` itself. A model nobody flagged keeps its schema as sent, unless
|
||||
no candidate names a model LiteLLM knows, whatever route prefix it carries.
|
||||
"""
|
||||
params: Final = litellm_params or {}
|
||||
deployment_flag: Final = deployment_model_info(params).get("supports_regex_lookaround")
|
||||
|
|
@ -1576,7 +1582,41 @@ def bedrock_model_supports_regex_lookaround(model: str, litellm_params: Mapping[
|
|||
base_model: Final = params.get("base_model")
|
||||
candidates: Final = (*((base_model,) if isinstance(base_model, str) else ()), model)
|
||||
flags: Final = (_bedrock_converse_model_flag(candidate, "supports_regex_lookaround") for candidate in candidates)
|
||||
return next((flag for flag in flags if flag is not None), True)
|
||||
flag: Final = next((flag for flag in flags if flag is not None), None)
|
||||
if flag is not None:
|
||||
return flag
|
||||
return not all(bedrock_model_is_unknown(split_bedrock_region_path(candidate)[1]) for candidate in candidates)
|
||||
|
||||
|
||||
def _bedrock_model_id_without_region(model: str) -> str:
|
||||
routed_id: Final = model.removeprefix("bedrock/")
|
||||
region, separator, regional_id = routed_id.partition("/")
|
||||
return regional_id if separator and region in get_all_bedrock_regions() else routed_id
|
||||
|
||||
|
||||
def bedrock_model_is_unknown(model: str) -> bool:
|
||||
model_id: Final = _bedrock_model_id_without_region(model)
|
||||
if "/" in model_id:
|
||||
return False
|
||||
base: Final = get_bedrock_base_model(model_id)
|
||||
regional_ids: Final = (f"{region}.{base}" for region in get_bedrock_cross_region_inference_regions())
|
||||
if any(_cost_map_row_names_a_provider(key) for key in (*_bedrock_model_info_candidates(model_id), *regional_ids)):
|
||||
return False
|
||||
from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM
|
||||
|
||||
return BaseAWSLLM.get_bedrock_invoke_provider(model_id) is None
|
||||
|
||||
|
||||
def _names_no_invoke_provider(model: str) -> bool:
|
||||
from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM
|
||||
|
||||
model_id: Final = _bedrock_model_id_without_region(model)
|
||||
names_one_model: Final = bool(model_id) and ("/" not in model_id or model_id.startswith("arn:"))
|
||||
return names_one_model and BaseAWSLLM.get_bedrock_invoke_provider(model) is None
|
||||
|
||||
|
||||
def bedrock_model_reaches_converse_by_default(model: str) -> bool:
|
||||
return BedrockModelInfo.named_bedrock_route(model, None) is None and _names_no_invoke_provider(model)
|
||||
|
||||
|
||||
_BedrockConverseModelFlag: TypeAlias = Literal[
|
||||
|
|
@ -1585,10 +1625,13 @@ _BedrockConverseModelFlag: TypeAlias = Literal[
|
|||
]
|
||||
|
||||
|
||||
def _bedrock_converse_model_flag(model: str, key: _BedrockConverseModelFlag) -> bool | None:
|
||||
def _bedrock_model_info_candidates(model: str) -> tuple[str, ...]:
|
||||
base: Final = get_bedrock_base_model(model)
|
||||
candidates: Final = dict.fromkeys((model, base, _BEDROCK_MODEL_VERSION_SUFFIX_RE.sub("", base)))
|
||||
for candidate in candidates:
|
||||
return tuple(dict.fromkeys((model, base, _BEDROCK_MODEL_VERSION_SUFFIX_RE.sub("", base))))
|
||||
|
||||
|
||||
def _bedrock_converse_model_flag(model: str, key: _BedrockConverseModelFlag) -> bool | None:
|
||||
for candidate in _bedrock_model_info_candidates(model):
|
||||
with contextlib.suppress(Exception):
|
||||
model_info = get_cached_model_info()(
|
||||
model=candidate,
|
||||
|
|
@ -1766,8 +1809,19 @@ class BedrockModelInfo(BaseLLMModelInfo):
|
|||
default (``bedrock_runtime_chat_completions_is_default``) and ``chat_completions/`` opts any other model in;
|
||||
``request_params`` (the caller's chat params) sends such a request to Converse when it needs a
|
||||
feature only Converse serves, and ``converse/`` pins a model to Converse. Every other OpenAI-family
|
||||
model stays on Converse without the prefix.
|
||||
model stays on Converse without the prefix. A model the cost map does not list as Converse goes to
|
||||
Invoke only when its id names an Invoke provider, and to Converse otherwise.
|
||||
"""
|
||||
named_route: Final = BedrockModelInfo.named_bedrock_route(model, request_params)
|
||||
if named_route is not None:
|
||||
return named_route
|
||||
return "converse" if _names_no_invoke_provider(model) else "invoke"
|
||||
|
||||
@staticmethod
|
||||
def named_bedrock_route(
|
||||
model: str,
|
||||
request_params: Mapping[str, object] | None,
|
||||
) -> BedrockRoute | None:
|
||||
route_mappings: dict[
|
||||
str,
|
||||
Literal[
|
||||
|
|
@ -1822,7 +1876,7 @@ class BedrockModelInfo(BaseLLMModelInfo):
|
|||
return "converse"
|
||||
if _OPENAI_FAMILY_MODEL_RE.search(base_model):
|
||||
return "converse"
|
||||
return "invoke"
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _explicit_converse_route(model: str) -> bool:
|
||||
|
|
|
|||
|
|
@ -122,6 +122,12 @@ class Authenticator:
|
|||
return tokens["access_token"]
|
||||
|
||||
def get_account_id(self, access_token: str | None = None) -> str | None:
|
||||
return self._account_id(access_token, persist_derived=True)
|
||||
|
||||
def read_account_id(self, access_token: str | None = None) -> str | None:
|
||||
return self._account_id(access_token, persist_derived=False)
|
||||
|
||||
def _account_id(self, access_token: str | None, persist_derived: bool) -> str | None:
|
||||
auth_data: Final = self._read_auth_file()
|
||||
if access_token and (not auth_data or auth_data.get("access_token") != access_token):
|
||||
return extract_chatgpt_account_id(access_token)
|
||||
|
|
@ -133,7 +139,7 @@ class Authenticator:
|
|||
id_token: Final = auth_data.get("id_token")
|
||||
saved_access_token: Final = auth_data.get("access_token")
|
||||
derived: Final = extract_chatgpt_account_id(_optional_str(id_token or saved_access_token))
|
||||
if derived:
|
||||
if derived and persist_derived:
|
||||
self._write_auth_file({**auth_data, "account_id": derived})
|
||||
return derived
|
||||
|
||||
|
|
|
|||
|
|
@ -11,6 +11,7 @@ from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response impo
|
|||
safe_convert_created_field,
|
||||
)
|
||||
from litellm.litellm_core_utils.url_utils import join_api_base
|
||||
from litellm.llms.base_llm.responses.transformation import STANDARD_MODE_REASONING_FIELD_VALUES, ReasoningFieldValues
|
||||
from litellm.llms.custom_httpx.sse_lines import split_sse_lines
|
||||
from litellm.llms.openai.common_utils import OpenAIError
|
||||
from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig
|
||||
|
|
@ -295,3 +296,6 @@ class ChatGPTResponsesAPIConfig(OpenAIResponsesAPIConfig):
|
|||
def supports_native_websocket(self) -> bool:
|
||||
"""ChatGPT does not support native WebSocket for Responses API"""
|
||||
return False
|
||||
|
||||
def reasoning_field_values(self) -> ReasoningFieldValues:
|
||||
return STANDARD_MODE_REASONING_FIELD_VALUES
|
||||
|
|
|
|||
|
|
@ -249,7 +249,7 @@ if TYPE_CHECKING:
|
|||
FakeAnthropicMessagesStreamIterator,
|
||||
)
|
||||
from litellm.llms.base_llm.passthrough.transformation import BasePassthroughConfig
|
||||
from litellm.responses.streaming_iterator import ResponsesWsFrameGuard
|
||||
from litellm.responses.streaming_iterator import ResponsesWsFrameGuard, ResponsesWsTurnAdmission
|
||||
from litellm.types.google_genai.main import GenerateContentResponse
|
||||
from litellm.types.llms.openai_evals import (
|
||||
CancelEvalResponse,
|
||||
|
|
@ -7269,7 +7269,7 @@ class BaseLLMHTTPHandler:
|
|||
custom_llm_provider: str | None = None,
|
||||
first_message: str | None = None,
|
||||
request_defaults: ResponsesWebSocketRequestDefaults | None = None,
|
||||
turn_admission: Callable[[], Awaitable[None]] | None = None,
|
||||
turn_admission: "ResponsesWsTurnAdmission | None" = None,
|
||||
frame_guard: "ResponsesWsFrameGuard | None" = None,
|
||||
**kwargs: object,
|
||||
) -> Exception | None:
|
||||
|
|
|
|||
|
|
@ -6,10 +6,12 @@ from collections.abc import Coroutine, Mapping, Sequence
|
|||
from typing import Final, Literal, cast, overload
|
||||
|
||||
import litellm
|
||||
from litellm.litellm_core_utils.core_helpers import as_str_mapping
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
||||
convert_content_list_to_str,
|
||||
extract_search_results_text,
|
||||
)
|
||||
from litellm.litellm_core_utils.reasoning_effort_utils import OPAQUE_VALUE, warn_dropped_reasoning
|
||||
from litellm.litellm_core_utils.url_utils import ensure_api_route
|
||||
from litellm.secret_managers.main import get_secret_str
|
||||
from litellm.types.llms.openai import AllMessageValues
|
||||
|
|
@ -18,6 +20,16 @@ from litellm.utils import supports_reasoning, supports_vision
|
|||
from ...openai.chat.gpt_transformation import OpenAIGPTConfig
|
||||
|
||||
SUPPORTED_REASONING_EFFORTS: Final = ("minimal", "low", "medium", "high", "xhigh", "max", "ultra")
|
||||
_NO_BUDGET_REASON: Final = "DeepSeek takes thinking only as enabled or disabled, with no budget."
|
||||
|
||||
|
||||
def _warn_on_dropped_budget(thinking: object, model: str) -> None:
|
||||
fields: Final = as_str_mapping(thinking)
|
||||
if fields is None or fields.get("type") != "enabled" or "budget_tokens" not in fields:
|
||||
return
|
||||
warn_dropped_reasoning(
|
||||
"thinking.budget_tokens", fields["budget_tokens"], model, _NO_BUDGET_REASON, drop_params_gated=False
|
||||
)
|
||||
|
||||
|
||||
class DeepSeekChatConfig(OpenAIGPTConfig):
|
||||
|
|
@ -53,6 +65,7 @@ class DeepSeekChatConfig(OpenAIGPTConfig):
|
|||
# Then re-add only if valid for DeepSeek
|
||||
thinking_value: Final = optional_params.pop("thinking", None)
|
||||
reasoning_effort: Final = optional_params.pop("reasoning_effort", None)
|
||||
_warn_on_dropped_budget(OPAQUE_VALUE.validate_python(thinking_value), model)
|
||||
|
||||
# Handle thinking parameter - accept both enabled and disabled, ignore budget_tokens
|
||||
if isinstance(thinking_value, dict) and thinking_value.get("type") in ("enabled", "disabled"):
|
||||
|
|
|
|||
|
|
@ -12,7 +12,9 @@ from typing import TYPE_CHECKING, Final
|
|||
import httpx
|
||||
from pydantic import ConfigDict, TypeAdapter
|
||||
|
||||
import litellm
|
||||
from litellm.litellm_core_utils.core_helpers import map_finish_reason
|
||||
from litellm.litellm_core_utils.reasoning_effort_utils import warn_dropped_reasoning
|
||||
from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException
|
||||
from litellm.secret_managers.main import get_secret_str
|
||||
from litellm.types.llms.base import LiteLLMBaseModel
|
||||
|
|
@ -29,6 +31,10 @@ PROVIDER_PREFIX: Final[str] = "fal_ai/"
|
|||
PASSTHROUGH_PARAMS: Final[frozenset[str]] = frozenset(("reasoning", "temperature", "top_p"))
|
||||
REASONING_DISABLED_EFFORTS: Final[frozenset[str]] = frozenset(("none", "minimal"))
|
||||
REASONING_ENABLED_EFFORTS: Final[frozenset[str]] = frozenset(("low", "medium", "high"))
|
||||
_UNSUPPORTED_EFFORT_REASON: Final = (
|
||||
f"fal only turns reasoning off for {sorted(REASONING_DISABLED_EFFORTS)} "
|
||||
f"and on for {sorted(REASONING_ENABLED_EFFORTS)}."
|
||||
)
|
||||
|
||||
|
||||
class _FalUsage(LiteLLMBaseModel):
|
||||
|
|
@ -120,9 +126,10 @@ class FalAIChatConfig(BaseConfig):
|
|||
return False
|
||||
if isinstance(value, str) and value in REASONING_ENABLED_EFFORTS:
|
||||
return True
|
||||
if drop_params:
|
||||
return None
|
||||
raise FalAIError(status_code=400, message=f"Unsupported reasoning_effort {value!r} for {model}")
|
||||
if not (drop_params or litellm.drop_params is True):
|
||||
raise FalAIError(status_code=400, message=f"Unsupported reasoning_effort {value!r} for {model}")
|
||||
warn_dropped_reasoning("reasoning_effort", value, model, _UNSUPPORTED_EFFORT_REASON, drop_params_gated=True)
|
||||
return None
|
||||
|
||||
def _translate_param(self, param: str, value: object, model: str, drop_params: bool) -> tuple[str, object] | None:
|
||||
if param in ("temperature", "top_p"):
|
||||
|
|
|
|||
|
|
@ -17,6 +17,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
|||
strip_litellm_response_only_fields,
|
||||
unpack_legacy_defs,
|
||||
)
|
||||
from litellm.litellm_core_utils.reasoning_effort_utils import OPAQUE_VALUE, warn_dropped_reasoning
|
||||
from litellm.litellm_core_utils.url_utils import join_api_base
|
||||
from litellm.secret_managers.main import get_secret_str
|
||||
from litellm.types.llms.openai import (
|
||||
|
|
@ -97,6 +98,7 @@ def _json_schema_response_format(schema: object, name: str) -> Mapping[str, obje
|
|||
|
||||
|
||||
EFFORT_KWARG_KEYS: Final = frozenset({"enable_thinking", "thinking", "reasoning_budget", "low_effort"})
|
||||
_NOT_A_REASONING_MODEL_REASON: Final = "litellm's model map does not list it as a reasoning model."
|
||||
|
||||
|
||||
def _bool_from_kwargs(kwargs: Mapping[str, object], keys: tuple[str, ...]) -> bool | None:
|
||||
|
|
@ -107,6 +109,13 @@ def _bool_from_kwargs(kwargs: Mapping[str, object], keys: tuple[str, ...]) -> bo
|
|||
return None
|
||||
|
||||
|
||||
def warn_dropped_effort_kwargs(chat_template_kwargs: object, effort: object, model: str) -> None:
|
||||
if effort != "none":
|
||||
warn_dropped_reasoning(
|
||||
"chat_template_kwargs", chat_template_kwargs, model, _NOT_A_REASONING_MODEL_REASON, drop_params_gated=False
|
||||
)
|
||||
|
||||
|
||||
def effort_from_chat_template_kwargs(kwargs: Mapping[str, object]) -> object:
|
||||
enable_thinking: Final = _bool_from_kwargs(kwargs, ("enable_thinking", "thinking"))
|
||||
if enable_thinking is False:
|
||||
|
|
@ -438,10 +447,7 @@ class FireworksAIConfig(FireworksAIMixin, OpenAIGPTConfig):
|
|||
if effort is None:
|
||||
return ()
|
||||
if not supports_reasoning(model=model, custom_llm_provider="fireworks_ai"):
|
||||
verbose_logger.debug(
|
||||
"fireworks_ai model %r does not support reasoning; dropping chat_template_kwargs effort keys.",
|
||||
model,
|
||||
)
|
||||
warn_dropped_effort_kwargs(OPAQUE_VALUE.validate_python(chat_template_kwargs), effort, model)
|
||||
return ()
|
||||
return (("reasoning_effort", effort),)
|
||||
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@ from collections.abc import Mapping
|
|||
from typing import Final
|
||||
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.litellm_core_utils.reasoning_effort_utils import OPAQUE_VALUE
|
||||
from litellm.types.llms.openai import AllMessageValues, OpenAITextCompletionUserMessage
|
||||
from litellm.utils import supports_reasoning
|
||||
|
||||
|
|
@ -12,6 +13,7 @@ from ..chat.transformation import (
|
|||
NIM_VLLM_STRIP_PARAMS,
|
||||
FireworksAIConfig,
|
||||
effort_from_chat_template_kwargs,
|
||||
warn_dropped_effort_kwargs,
|
||||
)
|
||||
from ..common_utils import FireworksAIMixin, resolve_fireworks_resource_name
|
||||
|
||||
|
|
@ -129,10 +131,7 @@ class FireworksAITextCompletionConfig(FireworksAIMixin, BaseTextCompletionConfig
|
|||
)
|
||||
return result
|
||||
if not supports_reasoning(model=model, custom_llm_provider="fireworks_ai"):
|
||||
verbose_logger.debug(
|
||||
"fireworks_ai model %r does not support reasoning; dropping chat_template_kwargs effort keys.",
|
||||
model,
|
||||
)
|
||||
warn_dropped_effort_kwargs(OPAQUE_VALUE.validate_python(chat_template_kwargs), effort, model)
|
||||
return result
|
||||
return {**result, "reasoning_effort": effort}
|
||||
|
||||
|
|
|
|||
|
|
@ -6,6 +6,10 @@ from urllib.parse import unquote
|
|||
import httpx
|
||||
|
||||
from litellm.litellm_core_utils.url_utils import join_api_base
|
||||
from litellm.llms.base_llm.responses.transformation import (
|
||||
STANDARD_MODE_WITHOUT_CONTEXT_REASONING_FIELD_VALUES,
|
||||
ReasoningFieldValues,
|
||||
)
|
||||
from litellm.llms.fireworks_ai.common_utils import (
|
||||
FIREWORKS_FORWARD_USER_ID_PARAM,
|
||||
get_fireworks_forwarded_user_id,
|
||||
|
|
@ -129,3 +133,9 @@ class FireworksAIResponsesAPIConfig(OpenAIResponsesAPIConfig):
|
|||
|
||||
def supports_native_file_search(self) -> bool:
|
||||
return False
|
||||
|
||||
def reasoning_field_values(self) -> ReasoningFieldValues:
|
||||
return STANDARD_MODE_WITHOUT_CONTEXT_REASONING_FIELD_VALUES
|
||||
|
||||
def ignores_unsupported_reasoning_fields(self) -> bool:
|
||||
return True
|
||||
|
|
|
|||
|
|
@ -24,6 +24,7 @@ from litellm._logging import verbose_logger
|
|||
from litellm.constants import DEFAULT_MAX_RECURSE_DEPTH
|
||||
from litellm.exceptions import AuthenticationError
|
||||
from litellm.litellm_core_utils.url_utils import join_api_base
|
||||
from litellm.llms.base_llm.responses.transformation import STANDARD_MODE_REASONING_FIELD_VALUES, ReasoningFieldValues
|
||||
from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig
|
||||
from litellm.types.llms.openai import (
|
||||
ResponseInputParam,
|
||||
|
|
@ -437,6 +438,12 @@ class GithubCopilotResponsesAPIConfig(OpenAIResponsesAPIConfig):
|
|||
|
||||
return False
|
||||
|
||||
def reasoning_field_values(self) -> ReasoningFieldValues:
|
||||
return STANDARD_MODE_REASONING_FIELD_VALUES
|
||||
|
||||
def ignores_unsupported_reasoning_fields(self) -> bool:
|
||||
return True
|
||||
|
||||
def supports_native_websocket(self) -> bool:
|
||||
"""GitHub Copilot does not support native WebSocket for Responses API"""
|
||||
return False
|
||||
|
|
|
|||
|
|
@ -99,7 +99,7 @@ class HostedVLLMChatConfig(OpenAIGPTConfig):
|
|||
thinking_effort: Final = (
|
||||
None
|
||||
if "reasoning_effort" in non_default_params
|
||||
else reasoning_effort_from_thinking(thinking, drop_params=drop_params)
|
||||
else reasoning_effort_from_thinking(thinking, model=model, drop_params=drop_params)
|
||||
)
|
||||
if thinking_effort is not None:
|
||||
non_default_params["reasoning_effort"] = thinking_effort
|
||||
|
|
|
|||
|
|
@ -12,6 +12,7 @@ from pydantic import TypeAdapter, ValidationError
|
|||
|
||||
from litellm.litellm_core_utils.url_utils import join_api_base
|
||||
from litellm.llms.base_llm.chat.transformation import BaseLLMException
|
||||
from litellm.llms.base_llm.responses.transformation import ANY_REASONING_FIELD_VALUES, ReasoningFieldValues
|
||||
from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig
|
||||
from litellm.secret_managers.main import get_secret_str
|
||||
from litellm.types.utils import LlmProviders
|
||||
|
|
@ -42,6 +43,9 @@ class LiteLLMProxyResponsesAPIConfig(OpenAIResponsesAPIConfig):
|
|||
def custom_llm_provider(self) -> LlmProviders:
|
||||
return LlmProviders.LITELLM_PROXY
|
||||
|
||||
def reasoning_field_values(self) -> ReasoningFieldValues:
|
||||
return ANY_REASONING_FIELD_VALUES
|
||||
|
||||
def get_complete_url(
|
||||
self,
|
||||
api_base: str | None,
|
||||
|
|
|
|||
|
|
@ -25,7 +25,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
|||
strip_litellm_internal_message_fields,
|
||||
strip_none_values_from_message,
|
||||
)
|
||||
from litellm.litellm_core_utils.reasoning_effort_utils import reasoning_effort_from_thinking
|
||||
from litellm.litellm_core_utils.reasoning_effort_utils import reasoning_effort_from_thinking, warn_dropped_reasoning
|
||||
from litellm.litellm_core_utils.url_utils import ensure_api_route
|
||||
from litellm.llms.openai.chat.gpt_transformation import (
|
||||
OpenAIChatCompletionStreamingHandler,
|
||||
|
|
@ -70,12 +70,12 @@ def _accepted_reasoning_effort(model: str, requested: str, custom_llm_provider:
|
|||
return accepted
|
||||
|
||||
|
||||
def _thinking_as_reasoning_effort(thinking: object, *, drop_params: bool) -> str | None:
|
||||
def _thinking_as_reasoning_effort(thinking: object, *, model: str, drop_params: bool) -> str | None:
|
||||
match thinking:
|
||||
case {"type": "disabled"}:
|
||||
return "none"
|
||||
case _:
|
||||
return reasoning_effort_from_thinking(thinking, drop_params=drop_params)
|
||||
return reasoning_effort_from_thinking(thinking, model=model, drop_params=drop_params)
|
||||
|
||||
|
||||
_NO_TOOL_CALL_TURN: Final[tuple[int, frozenset[str]]] = (-1, frozenset())
|
||||
|
|
@ -83,6 +83,9 @@ _MISTRAL_REQUEST_ROUTING: Final = TypeAdapter(MistralRequestRouting)
|
|||
_MISTRAL_CACHE_KEY_PARAMS: Final = TypeAdapter(MistralCacheKeyParams)
|
||||
_MISTRAL_REASONING_EFFORT_PARAMS: Final = TypeAdapter(MistralReasoningEffortParams)
|
||||
_MISTRAL_FILE_PART: Final = TypeAdapter(MistralFilePart)
|
||||
_NON_LEVEL_EFFORT_REASON: Final = (
|
||||
"the Mistral API takes it only as a level such as 'high', so Mistral applies its default effort."
|
||||
)
|
||||
|
||||
|
||||
def _reasoning_level_and_summary(reasoning_effort: object) -> tuple[object, object]:
|
||||
|
|
@ -234,7 +237,8 @@ class MistralConfig(OpenAIGPTConfig):
|
|||
if (
|
||||
param == "thinking"
|
||||
and "reasoning_effort" not in non_default_params
|
||||
and (thinking_effort := _thinking_as_reasoning_effort(value, drop_params=drop_params)) is not None
|
||||
and (thinking_effort := _thinking_as_reasoning_effort(value, model=model, drop_params=drop_params))
|
||||
is not None
|
||||
):
|
||||
optional_params["reasoning_effort"] = _accepted_reasoning_effort(
|
||||
model, thinking_effort, self.custom_llm_provider
|
||||
|
|
@ -246,13 +250,18 @@ class MistralConfig(OpenAIGPTConfig):
|
|||
def _reasoning_level_to_send(
|
||||
self, model: str, requested: MistralReasoningEffortParams, drop_params: bool
|
||||
) -> str | None:
|
||||
level, summary = _reasoning_level_and_summary(requested.get("reasoning_effort"))
|
||||
requested_effort: Final = requested.get("reasoning_effort")
|
||||
level, summary = _reasoning_level_and_summary(requested_effort)
|
||||
sendable: Final = (
|
||||
_accepted_reasoning_effort(model, level, self.custom_llm_provider) if isinstance(level, str) else None
|
||||
)
|
||||
if sendable is not None and summary is None:
|
||||
return sendable
|
||||
if drop_params or litellm.drop_params is True:
|
||||
if sendable is None and level is not None:
|
||||
warn_dropped_reasoning(
|
||||
"reasoning_effort", requested_effort, model, _NON_LEVEL_EFFORT_REASON, drop_params_gated=True
|
||||
)
|
||||
return sendable
|
||||
raise UnsupportedParamsError(
|
||||
message=(
|
||||
|
|
|
|||
|
|
@ -21,6 +21,7 @@ from litellm._logging import verbose_logger
|
|||
from litellm.constants import DEFAULT_OCI_CHAT_MAX_TOKENS
|
||||
from litellm.litellm_core_utils.logging_utils import track_llm_api_timing
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import messages_with_file_parts_as_image_urls
|
||||
from litellm.litellm_core_utils.reasoning_effort_utils import warn_dropped_reasoning
|
||||
from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper
|
||||
from litellm.litellm_core_utils.url_utils import join_api_base
|
||||
from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException
|
||||
|
|
@ -428,6 +429,13 @@ class OCIChatConfig(BaseConfig):
|
|||
alias = param_map.get(key)
|
||||
if key == "reasoning_effort" and alias and not _model_supports_reasoning_effort(model):
|
||||
if drop_params or litellm.drop_params:
|
||||
warn_dropped_reasoning(
|
||||
key,
|
||||
value,
|
||||
model,
|
||||
"the model map lists no reasoning_effort levels for it, so OCI would reject reasoningEffort.",
|
||||
drop_params_gated=True,
|
||||
)
|
||||
continue
|
||||
raise OCIError(
|
||||
status_code=400,
|
||||
|
|
@ -451,6 +459,14 @@ class OCIChatConfig(BaseConfig):
|
|||
if key == "n" and (value is None or value == 1):
|
||||
continue
|
||||
if drop_params or litellm.drop_params:
|
||||
if key == "reasoning_effort":
|
||||
warn_dropped_reasoning(
|
||||
key,
|
||||
value,
|
||||
model,
|
||||
"OCI's Cohere chat API has no reasoning effort field.",
|
||||
drop_params_gated=True,
|
||||
)
|
||||
continue
|
||||
raise OCIError(
|
||||
status_code=400,
|
||||
|
|
|
|||
|
|
@ -14,6 +14,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
|||
extract_images_from_message,
|
||||
extract_reasoning_content,
|
||||
)
|
||||
from litellm.litellm_core_utils.reasoning_effort_utils import OPAQUE_VALUE
|
||||
from litellm.litellm_core_utils.url_utils import ensure_api_route
|
||||
from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator
|
||||
from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException
|
||||
|
|
@ -29,7 +30,7 @@ from litellm.types.llms.openai import (
|
|||
)
|
||||
from litellm.types.utils import ModelResponse, ModelResponseStream
|
||||
|
||||
from ..common_utils import OllamaError
|
||||
from ..common_utils import OllamaError, ollama_think
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
|
||||
|
|
@ -172,10 +173,7 @@ class OllamaChatConfig(BaseConfig):
|
|||
if value.get("json_schema") and value["json_schema"].get("schema"):
|
||||
optional_params["format"] = value["json_schema"]["schema"]
|
||||
if param == "reasoning_effort" and value is not None:
|
||||
if model.startswith("gpt-oss"):
|
||||
optional_params["think"] = value
|
||||
else:
|
||||
optional_params["think"] = value in {"low", "medium", "high"}
|
||||
optional_params["think"] = ollama_think(model, OPAQUE_VALUE.validate_python(value))
|
||||
### FUNCTION CALLING LOGIC ###
|
||||
# Ollama 0.4+ supports native tool calling - pass tools directly
|
||||
# and let Ollama handle model capability detection
|
||||
|
|
|
|||
|
|
@ -8,11 +8,14 @@ from pydantic import BaseModel, TypeAdapter
|
|||
|
||||
from litellm import verbose_logger
|
||||
from litellm._uuid import uuid
|
||||
from litellm.litellm_core_utils.reasoning_effort_utils import warn_dropped_reasoning
|
||||
from litellm.litellm_core_utils.url_utils import join_api_base
|
||||
from litellm.llms.base_llm.chat.transformation import BaseLLMException
|
||||
from litellm.types.utils import ModelResponse, ModelResponseStream
|
||||
|
||||
_PROVIDER_FIELDS: Final = TypeAdapter(dict[str, object])
|
||||
_THINK_ON_EFFORTS: Final = frozenset({"low", "medium", "high"})
|
||||
_THINK_OFF_REASON: Final = "Ollama takes think only as on or off, and litellm maps this level to off."
|
||||
_DECODED_JSON: Final = TypeAdapter(object)
|
||||
|
||||
|
||||
|
|
@ -296,3 +299,12 @@ class OllamaModelInfo(BaseLLMModelInfo):
|
|||
Return the base model name for Ollama (no-op).
|
||||
"""
|
||||
return model
|
||||
|
||||
|
||||
def ollama_think(model: str, reasoning_effort: object) -> object:
|
||||
if model.startswith("gpt-oss"):
|
||||
return reasoning_effort
|
||||
think: Final = reasoning_effort in _THINK_ON_EFFORTS
|
||||
if isinstance(reasoning_effort, str) and not think and reasoning_effort != "none":
|
||||
warn_dropped_reasoning("reasoning_effort", reasoning_effort, model, _THINK_OFF_REASON, drop_params_gated=False)
|
||||
return think
|
||||
|
|
|
|||
|
|
@ -23,6 +23,7 @@ from litellm.litellm_core_utils.prompt_templates.image_handling import (
|
|||
async_inline_remote_media,
|
||||
inline_remote_image_urls,
|
||||
)
|
||||
from litellm.litellm_core_utils.reasoning_effort_utils import OPAQUE_VALUE
|
||||
from litellm.litellm_core_utils.url_utils import ensure_api_route
|
||||
from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator
|
||||
from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException
|
||||
|
|
@ -43,7 +44,7 @@ from litellm.types.utils import (
|
|||
generate_id,
|
||||
)
|
||||
|
||||
from ..common_utils import OllamaError, OllamaModelInfo, convert_image
|
||||
from ..common_utils import OllamaError, OllamaModelInfo, convert_image, ollama_think
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
|
||||
|
|
@ -249,10 +250,7 @@ class OllamaConfig(BaseConfig):
|
|||
elif param == "stop":
|
||||
optional_params["stop"] = value
|
||||
elif param == "reasoning_effort" and value is not None:
|
||||
if model.startswith("gpt-oss"):
|
||||
optional_params["think"] = value
|
||||
else:
|
||||
optional_params["think"] = value in {"low", "medium", "high"}
|
||||
optional_params["think"] = ollama_think(model, OPAQUE_VALUE.validate_python(value))
|
||||
elif param == "response_format" and isinstance(value, dict):
|
||||
if value["type"] == "json_object":
|
||||
optional_params["format"] = "json"
|
||||
|
|
|
|||
|
|
@ -32,6 +32,7 @@ import litellm
|
|||
from litellm.exceptions import error_response_without_credentials
|
||||
from litellm.litellm_core_utils.completion_timeout import CompletionTimeout
|
||||
from litellm.litellm_core_utils.core_helpers import as_str_mapping, decoded_json_object
|
||||
from litellm.litellm_core_utils.reasoning_effort_utils import warn_dropped_reasoning_params
|
||||
from litellm.litellm_core_utils.token_counter import token_counter, usage_tokenizer_model
|
||||
from litellm.llms.base_llm.chat.transformation import BaseLLMException
|
||||
from litellm.llms.custom_httpx.http_handler import (
|
||||
|
|
@ -165,17 +166,11 @@ _DECODED_JSON: Final = TypeAdapter(object)
|
|||
def drop_params_from_unprocessable_entity_error(
|
||||
e: openai.UnprocessableEntityError | httpx.HTTPStatusError,
|
||||
data: Mapping[str, object],
|
||||
*,
|
||||
model: str | None,
|
||||
drop_params_gated: bool,
|
||||
) -> dict[str, object]:
|
||||
"""
|
||||
Helper function to read OpenAI UnprocessableEntityError and drop the params that raised an error from the error message.
|
||||
|
||||
Args:
|
||||
e (UnprocessableEntityError): The UnprocessableEntityError exception
|
||||
data (Dict[str, Any]): The original data dictionary containing all parameters
|
||||
|
||||
Returns:
|
||||
Dict[str, Any]: A new dictionary with invalid parameters removed
|
||||
"""
|
||||
"""Return a copy of ``data`` without the params a 422 response names as invalid."""
|
||||
invalid_params: Final[list[str]] = []
|
||||
if isinstance(e, httpx.HTTPStatusError):
|
||||
error_json: Final = decoded_json_object(_DECODED_JSON.validate_python(e.response.json()))
|
||||
|
|
@ -203,6 +198,13 @@ def drop_params_from_unprocessable_entity_error(
|
|||
invalid_params.append(error_dict["loc"][1])
|
||||
|
||||
new_data: Final = {k: v for k, v in data.items() if k not in invalid_params}
|
||||
warn_dropped_reasoning_params(
|
||||
invalid_params,
|
||||
data,
|
||||
model,
|
||||
"the provider answered 422 naming it an invalid param.",
|
||||
drop_params_gated=drop_params_gated,
|
||||
)
|
||||
|
||||
return new_data
|
||||
|
||||
|
|
|
|||
|
|
@ -940,7 +940,9 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
|
|||
except openai.UnprocessableEntityError as e:
|
||||
## check if body contains unprocessable params - related issue https://github.com/BerriAI/litellm/issues/4800
|
||||
if litellm.drop_params is True or drop_params is True:
|
||||
inference_params = drop_params_from_unprocessable_entity_error(e, inference_params)
|
||||
inference_params = drop_params_from_unprocessable_entity_error(
|
||||
e, inference_params, model=model, drop_params_gated=True
|
||||
)
|
||||
else:
|
||||
raise e
|
||||
# e.message
|
||||
|
|
@ -1097,7 +1099,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
|
|||
except openai.UnprocessableEntityError as e:
|
||||
## check if body contains unprocessable params - related issue https://github.com/BerriAI/litellm/issues/4800
|
||||
if litellm.drop_params is True or drop_params is True:
|
||||
data = drop_params_from_unprocessable_entity_error(e, data)
|
||||
data = drop_params_from_unprocessable_entity_error(e, data, model=model, drop_params_gated=True)
|
||||
else:
|
||||
raise e
|
||||
# e.message
|
||||
|
|
@ -1252,7 +1254,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
|
|||
except openai.UnprocessableEntityError as e:
|
||||
## check if body contains unprocessable params - related issue https://github.com/BerriAI/litellm/issues/4800
|
||||
if litellm.drop_params is True or drop_params is True:
|
||||
data = drop_params_from_unprocessable_entity_error(e, data)
|
||||
data = drop_params_from_unprocessable_entity_error(e, data, model=model, drop_params_gated=True)
|
||||
else:
|
||||
raise e
|
||||
except (
|
||||
|
|
|
|||
|
|
@ -13,6 +13,7 @@ from typing import TYPE_CHECKING, Final, cast
|
|||
import httpx
|
||||
from pydantic import ConfigDict, TypeAdapter
|
||||
|
||||
from litellm.litellm_core_utils.core_helpers import finite_cost
|
||||
from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR, set_hidden_params
|
||||
from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator
|
||||
from litellm.llms.base_llm.chat.transformation import BaseLLMException
|
||||
|
|
@ -225,8 +226,11 @@ class OpenrouterConfig(OpenAIGPTConfig):
|
|||
additional_headers: Final = cast( # cast-ok: preserve mapping operations on response metadata
|
||||
dict[str, object], hidden_params["additional_headers"]
|
||||
)
|
||||
if isinstance(response_cost, str | int | float):
|
||||
additional_headers["llm_provider-x-litellm-response-cost"] = float(response_cost)
|
||||
reported_cost: Final = (
|
||||
finite_cost(float(response_cost)) if isinstance(response_cost, str | int | float) else None
|
||||
)
|
||||
if reported_cost is not None:
|
||||
additional_headers["llm_provider-x-litellm-response-cost"] = reported_cost
|
||||
except Exception:
|
||||
# If we can't extract cost, continue without it - don't fail the response
|
||||
pass
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ import httpx
|
|||
from pydantic import TypeAdapter, ValidationError
|
||||
|
||||
import litellm
|
||||
from litellm.litellm_core_utils.core_helpers import finite_reported_cost
|
||||
from litellm.litellm_core_utils.decisions_pricing import apply_decisions_pricing
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
from litellm.litellm_core_utils.litellm_logging import (
|
||||
|
|
@ -100,9 +101,10 @@ class TypeSafePassthroughLoggingHandler:
|
|||
f"{custom_llm_provider}/{model}" for model in candidate_models if model is not None
|
||||
)
|
||||
pricing: Final = _pricing_for(candidate_model_keys)
|
||||
reported_cost: Final = finite_reported_cost(usage.cost) if custom_llm_provider == "openrouter" else None
|
||||
response_cost: Final = (
|
||||
usage.cost
|
||||
if custom_llm_provider == "openrouter" and usage.cost is not None
|
||||
reported_cost
|
||||
if reported_cost is not None
|
||||
else input_tokens * pricing.input_cost_per_token + output_tokens * pricing.output_cost_per_token
|
||||
)
|
||||
usage_object: Final = Usage(
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ Helper util for handling perplexity-specific cost calculation
|
|||
from datetime import datetime
|
||||
from typing import Final
|
||||
|
||||
from litellm.litellm_core_utils.core_helpers import finite_reported_cost
|
||||
from litellm.litellm_core_utils.llm_cost_calc.utils import TokenRates, apply_off_peak_pricing
|
||||
from litellm.types.utils import Usage
|
||||
from litellm.utils import get_model_info
|
||||
|
|
@ -34,10 +35,9 @@ def cost_per_token(model: str, usage: Usage, current_time: datetime | None = Non
|
|||
total_cost = cost_info.get("total_cost")
|
||||
elif isinstance(cost_info, (int, float)) and not isinstance(cost_info, bool):
|
||||
total_cost = float(cost_info)
|
||||
if total_cost is not None:
|
||||
# Return total cost as completion_cost (prompt_cost=0) since Perplexity
|
||||
# doesn't break down by input/output in their cost object
|
||||
return (0.0, float(total_cost))
|
||||
reported_cost: Final = None if total_cost is None else finite_reported_cost(float(total_cost))
|
||||
if reported_cost is not None:
|
||||
return (0.0, reported_cost)
|
||||
|
||||
## FALLBACK: Calculate cost manually if Perplexity doesn't provide it
|
||||
## GET MODEL INFO
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@ The only provider quirks:
|
|||
Ref: https://docs.perplexity.ai/api-reference/responses-post
|
||||
"""
|
||||
|
||||
from types import MappingProxyType
|
||||
from typing import Any, Final
|
||||
|
||||
import httpx
|
||||
|
|
@ -17,6 +18,7 @@ from pydantic import TypeAdapter
|
|||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
from litellm.litellm_core_utils.url_utils import join_api_base
|
||||
from litellm.llms.base_llm.chat.transformation import BaseLLMException
|
||||
from litellm.llms.base_llm.responses.transformation import STANDARD_MODE_REASONING_FIELD_VALUES, ReasoningFieldValues
|
||||
from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig, drops_params
|
||||
from litellm.secret_managers.main import get_secret_str
|
||||
from litellm.types.llms.openai import ResponseInputParam, ResponsesAPIResponse
|
||||
|
|
@ -24,6 +26,9 @@ from litellm.types.router import GenericLiteLLMParams
|
|||
from litellm.types.utils import LlmProviders
|
||||
|
||||
_DECODED_JSON: Final = TypeAdapter(object)
|
||||
PERPLEXITY_REASONING_FIELD_VALUES: Final[ReasoningFieldValues] = MappingProxyType(
|
||||
{**STANDARD_MODE_REASONING_FIELD_VALUES, "context": frozenset({"auto", "current_turn", "all_turns"})}
|
||||
)
|
||||
|
||||
|
||||
class PerplexityResponsesConfig(OpenAIResponsesAPIConfig):
|
||||
|
|
@ -127,3 +132,9 @@ class PerplexityResponsesConfig(OpenAIResponsesAPIConfig):
|
|||
def supports_native_websocket(self) -> bool:
|
||||
"""Perplexity does not support native WebSocket for Responses API"""
|
||||
return False
|
||||
|
||||
def reasoning_field_values(self) -> ReasoningFieldValues:
|
||||
return PERPLEXITY_REASONING_FIELD_VALUES
|
||||
|
||||
def ignores_unsupported_reasoning_fields(self) -> bool:
|
||||
return True
|
||||
|
|
|
|||
|
|
@ -28,7 +28,7 @@ from litellm.types.llms.openai import AllMessageValues
|
|||
from litellm.types.utils import LlmProviders
|
||||
|
||||
from ...openai.chat.gpt_transformation import OpenAIGPTConfig
|
||||
from ..common_utils import AWSEventStreamDecoder, SagemakerError
|
||||
from ..common_utils import INFERENCE_COMPONENT_HEADER, OBJECT_KEYED_MAPPING, AWSEventStreamDecoder, SagemakerError
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
|
||||
|
|
@ -59,7 +59,12 @@ class SagemakerChatConfig(OpenAIGPTConfig, BaseAWSLLM):
|
|||
inference_component_name: Final = optional_params.get("model_id")
|
||||
if not isinstance(inference_component_name, str):
|
||||
return headers
|
||||
return {**headers, "X-Amzn-SageMaker-Inference-Component": inference_component_name}
|
||||
other_headers: Final = {
|
||||
name: value
|
||||
for name, value in OBJECT_KEYED_MAPPING.validate_python(headers).items()
|
||||
if not isinstance(name, str) or name.lower() != INFERENCE_COMPONENT_HEADER.lower()
|
||||
}
|
||||
return {**other_headers, INFERENCE_COMPONENT_HEADER: inference_component_name}
|
||||
|
||||
def transform_request(
|
||||
self,
|
||||
|
|
@ -76,9 +81,8 @@ class SagemakerChatConfig(OpenAIGPTConfig, BaseAWSLLM):
|
|||
litellm_params=litellm_params,
|
||||
headers=headers,
|
||||
)
|
||||
container_request: Final = {
|
||||
key: value for key, value in request.items() if key not in self.aws_authentication_params
|
||||
}
|
||||
header_only_params: Final = frozenset((*self.aws_authentication_params, "model_id"))
|
||||
container_request: Final = {key: value for key, value in request.items() if key not in header_only_params}
|
||||
served_model_name: Final = litellm_params.get("hf_model_name")
|
||||
if not isinstance(served_model_name, str):
|
||||
return container_request
|
||||
|
|
|
|||
|
|
@ -14,9 +14,10 @@ from typing_extensions import ReadOnly, Required, TypedDict
|
|||
from litellm import verbose_logger
|
||||
from litellm.llms.base_llm.chat.transformation import BaseLLMException
|
||||
from litellm.types.utils import GenericStreamingChunk as GChunk
|
||||
from litellm.types.utils import StreamingChatCompletionChunk
|
||||
from litellm.types.utils import LlmProviders, StreamingChatCompletionChunk
|
||||
|
||||
STR_KEYED_MAPPING: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(hide_input_in_errors=True))
|
||||
OBJECT_KEYED_MAPPING: Final = TypeAdapter(Mapping[object, object], config=ConfigDict(hide_input_in_errors=True))
|
||||
|
||||
|
||||
@with_config(ConfigDict(extra="allow", strict=True, hide_input_in_errors=True))
|
||||
|
|
@ -167,6 +168,36 @@ def get_sagemaker_response_stream_shape():
|
|||
return _load_sagemaker_response_stream_shape()
|
||||
|
||||
|
||||
INFERENCE_COMPONENT_HEADER: Final = "X-Amzn-SageMaker-Inference-Component"
|
||||
|
||||
|
||||
_SAGEMAKER_PROVIDERS: Final = frozenset(
|
||||
provider.value for provider in (LlmProviders.SAGEMAKER, LlmProviders.SAGEMAKER_CHAT, LlmProviders.SAGEMAKER_NOVA)
|
||||
)
|
||||
|
||||
|
||||
def _deployment_provider(litellm_params: Mapping[str, object]) -> str | None:
|
||||
custom_llm_provider: Final = litellm_params.get("custom_llm_provider")
|
||||
if isinstance(custom_llm_provider, str) and custom_llm_provider:
|
||||
return custom_llm_provider
|
||||
model: Final = litellm_params.get("model")
|
||||
return model.split("/", 1)[0] if isinstance(model, str) else None
|
||||
|
||||
|
||||
def deployment_inference_component(litellm_params: Mapping[str, object]) -> str | None:
|
||||
headers: Final = litellm_params.get("extra_headers")
|
||||
if _deployment_provider(litellm_params) not in _SAGEMAKER_PROVIDERS or not isinstance(headers, Mapping):
|
||||
return None
|
||||
return next(
|
||||
(
|
||||
value
|
||||
for name, value in OBJECT_KEYED_MAPPING.validate_python(headers).items()
|
||||
if isinstance(name, str) and name.lower() == INFERENCE_COMPONENT_HEADER.lower() and isinstance(value, str)
|
||||
),
|
||||
None,
|
||||
)
|
||||
|
||||
|
||||
def with_inference_component_hint(error_message: str) -> str:
|
||||
if "Inference Component Name header is required" in error_message:
|
||||
return f"{error_message}\n pass in via `litellm.embedding(..., model_id={{InferenceComponentName}})`"
|
||||
|
|
|
|||
|
|
@ -23,7 +23,7 @@ from litellm.types.llms.openai import AllMessageValues
|
|||
from litellm.types.utils import ModelResponse, Usage
|
||||
from litellm.utils import token_counter
|
||||
|
||||
from ..common_utils import SagemakerError
|
||||
from ..common_utils import STR_KEYED_MAPPING, SagemakerError
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
|
||||
|
|
@ -153,7 +153,9 @@ class SagemakerConfig(BaseConfig):
|
|||
litellm_params: dict,
|
||||
headers: dict,
|
||||
) -> dict:
|
||||
inference_params: Final = optional_params.copy()
|
||||
inference_params: Final = {
|
||||
key: value for key, value in STR_KEYED_MAPPING.validate_python(optional_params).items() if key != "model_id"
|
||||
}
|
||||
stream: Final = inference_params.pop("stream", False)
|
||||
data: Final[dict] = {"parameters": inference_params}
|
||||
if stream is True:
|
||||
|
|
|
|||
37
litellm/llms/subscription_accounts.py
Normal file
37
litellm/llms/subscription_accounts.py
Normal file
|
|
@ -0,0 +1,37 @@
|
|||
"""
|
||||
Providers whose requests are covered by a flat subscription, each with the way to read
|
||||
the account id of the login a deployment is signed into.
|
||||
"""
|
||||
|
||||
from collections.abc import Callable, Mapping
|
||||
from types import MappingProxyType
|
||||
from typing import Final
|
||||
|
||||
from litellm.llms.chatgpt.authenticator import Authenticator, get_cached_authenticator, get_chatgpt_auth_file
|
||||
from litellm.secret_managers.main import UnresolvableSecretReference, resolve_local_oidc_reference
|
||||
from litellm.utils import with_named_credential
|
||||
|
||||
|
||||
def _chatgpt_account_id(litellm_params: Mapping[str, object]) -> str | None:
|
||||
params: Final = with_named_credential(litellm_params)
|
||||
auth_file: Final = get_chatgpt_auth_file(params)
|
||||
if auth_file:
|
||||
return get_cached_authenticator(auth_file).read_account_id()
|
||||
api_key: Final = params.get("api_key")
|
||||
if not isinstance(api_key, str) or not api_key:
|
||||
return Authenticator().read_account_id()
|
||||
match resolve_local_oidc_reference(api_key):
|
||||
case UnresolvableSecretReference():
|
||||
return None
|
||||
case str() as access_token:
|
||||
return Authenticator().read_account_id(access_token)
|
||||
|
||||
|
||||
SUBSCRIPTION_ACCOUNT_ID_RESOLVERS: Final[Mapping[str, Callable[[Mapping[str, object]], str | None]]] = MappingProxyType(
|
||||
{"chatgpt": _chatgpt_account_id}
|
||||
)
|
||||
|
||||
|
||||
def resolve_subscription_account_id(custom_llm_provider: str, litellm_params: Mapping[str, object]) -> str | None:
|
||||
resolver: Final = SUBSCRIPTION_ACCOUNT_ID_RESOLVERS.get(custom_llm_provider)
|
||||
return None if resolver is None else resolver(litellm_params)
|
||||
|
|
@ -9,12 +9,15 @@ from typing import Final, TypedDict
|
|||
from typing_extensions import ReadOnly
|
||||
|
||||
import litellm
|
||||
from litellm.litellm_core_utils.reasoning_effort_utils import warn_dropped_reasoning
|
||||
from litellm.litellm_core_utils.url_utils import ensure_api_route, join_api_base
|
||||
from litellm.secret_managers.main import get_secret_str
|
||||
from litellm.utils import supports_reasoning
|
||||
|
||||
from ...openai.chat.gpt_transformation import OpenAIGPTConfig
|
||||
|
||||
_ADAPTIVE_BUDGET_REASON: Final = "it is not an integer, so litellm sends adaptive thinking with no budget."
|
||||
|
||||
|
||||
class ThinkingPayload(TypedDict, total=False):
|
||||
"""Tencent TokenHub `thinking` object.
|
||||
|
|
@ -102,6 +105,10 @@ class TencentChatConfig(OpenAIGPTConfig):
|
|||
if isinstance(budget, int):
|
||||
coerced_with_budget: Final[ThinkingPayload] = {"type": "adaptive", "budget_tokens": budget}
|
||||
return coerced_with_budget
|
||||
if budget is not None:
|
||||
warn_dropped_reasoning(
|
||||
"thinking.budget_tokens", budget, model, _ADAPTIVE_BUDGET_REASON, drop_params_gated=False
|
||||
)
|
||||
coerced: Final[ThinkingPayload] = {"type": "adaptive"}
|
||||
return coerced
|
||||
|
||||
|
|
|
|||
|
|
@ -1257,7 +1257,7 @@ def get_vertex_model_id_from_url(url: str) -> str | None:
|
|||
|
||||
`https://${LOCATION}-aiplatform.googleapis.com/v1/projects/${PROJECT_ID}/locations/${LOCATION}/publishers/google/models/${MODEL_ID}:streamGenerateContent`
|
||||
"""
|
||||
match: Final = re.search(r"/models/([^:]+)", url)
|
||||
match: Final = re.search(r"/models/([^:?#]+)", url)
|
||||
return match.group(1) if match else None
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -34,6 +34,7 @@ from litellm.litellm_core_utils.json_fragment_accumulator import JSONFragmentAcc
|
|||
from litellm.litellm_core_utils.prompt_templates.factory import (
|
||||
encode_tool_call_id_with_signature,
|
||||
)
|
||||
from litellm.litellm_core_utils.reasoning_effort_utils import warn_dropped_reasoning
|
||||
from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException
|
||||
from litellm.llms.custom_httpx.http_handler import (
|
||||
AsyncHTTPHandler,
|
||||
|
|
@ -201,6 +202,13 @@ def _unsupported_reasoning_effort(reasoning_effort: str) -> UnsupportedParamsErr
|
|||
|
||||
|
||||
_SAMPLING_DEPRECATION_UNMAPPED_MODELS: Final = "models outside litellm's model map"
|
||||
_NON_INTEGER_BUDGET_REASON: Final = (
|
||||
"Gemini takes budget_tokens only as an integer, so it uses its default thinking budget."
|
||||
)
|
||||
_LEVEL_ONLY_BUDGET_REASON: Final = (
|
||||
"Gemini 3 and later set reasoning by thinking level, not budget, so litellm does not send the budget. "
|
||||
"Send reasoning_effort to pick a level."
|
||||
)
|
||||
_sampling_deprecation_lock: Final = threading.Lock()
|
||||
|
||||
|
||||
|
|
@ -1466,6 +1474,14 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
|
|||
params["includeThoughts"] = False
|
||||
else:
|
||||
params["includeThoughts"] = sends_thoughts
|
||||
if thinking_budget is not None:
|
||||
warn_dropped_reasoning(
|
||||
"thinking.budget_tokens",
|
||||
thinking_budget,
|
||||
model,
|
||||
_LEVEL_ONLY_BUDGET_REASON,
|
||||
drop_params_gated=False,
|
||||
)
|
||||
# Follow provider defaults unless explicitly opted into legacy behavior.
|
||||
if litellm.enable_gemini_default_thinking_level_low is True:
|
||||
params["thinkingLevel"] = (
|
||||
|
|
@ -1478,8 +1494,16 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
|
|||
# For older Gemini models, use thinkingBudget
|
||||
if thinking_enabled and not VertexGeminiConfig._is_thinking_budget_zero(thinking_budget):
|
||||
params["includeThoughts"] = sends_thoughts
|
||||
if thinking_budget is not None and isinstance(thinking_budget, int):
|
||||
if isinstance(thinking_budget, int):
|
||||
params["thinkingBudget"] = thinking_budget
|
||||
elif thinking_budget is not None:
|
||||
warn_dropped_reasoning(
|
||||
"thinking.budget_tokens",
|
||||
thinking_budget,
|
||||
model,
|
||||
_NON_INTEGER_BUDGET_REASON,
|
||||
drop_params_gated=False,
|
||||
)
|
||||
|
||||
return params
|
||||
|
||||
|
|
|
|||
|
|
@ -1,9 +1,15 @@
|
|||
from collections.abc import Mapping, Sequence
|
||||
from typing import Final
|
||||
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
from litellm.exceptions import UnsupportedParamsError
|
||||
from litellm.litellm_core_utils.core_helpers import drops_unsupported_params
|
||||
from litellm.llms.anthropic.common_utils import AnthropicModelInfo, context_management_betas
|
||||
from litellm.llms.anthropic.common_utils import (
|
||||
AnthropicModelInfo,
|
||||
context_management_betas,
|
||||
requires_native_compaction_beta,
|
||||
)
|
||||
from litellm.llms.anthropic.messages.native.transformation import (
|
||||
AnthropicMessagesConfig,
|
||||
messages_carry_output_config,
|
||||
|
|
@ -11,6 +17,7 @@ from litellm.llms.anthropic.messages.native.transformation import (
|
|||
from litellm.types.llms.anthropic import (
|
||||
ANTHROPIC_BETA_HEADER_VALUES,
|
||||
ANTHROPIC_HOSTED_TOOLS,
|
||||
ANTHROPIC_MID_CONVERSATION_TOOL_CHANGES_BETA_HEADER,
|
||||
)
|
||||
from litellm.types.llms.anthropic_tool_search import get_tool_search_beta_header
|
||||
from litellm.types.llms.vertex_ai import VertexPartnerProvider
|
||||
|
|
@ -19,6 +26,8 @@ from litellm.types.router import GenericLiteLLMParams
|
|||
from ....vertex_llm_base import VertexBase
|
||||
from ..output_params_utils import sanitize_vertex_anthropic_output_params
|
||||
|
||||
_OPTIONAL_PARAMS: Final = TypeAdapter(Mapping[str, object])
|
||||
|
||||
_VERSION_HEADER_VERTEX_READS_FROM_BODY: Final = "anthropic-version"
|
||||
_VERTEX_DEFAULT_VERSION: Final = "vertex-2023-10-16"
|
||||
_VERSIONS_SERVED_AS_VERTEX_DEFAULT: Final = frozenset({"", "2023-01-01", "2023-06-01"})
|
||||
|
|
@ -190,14 +199,23 @@ class VertexAIPartnerModelsAnthropicMessagesConfig(AnthropicMessagesConfig, Vert
|
|||
if messages_carry_output_config(messages):
|
||||
beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.PER_TURN_CONTROL_2026_07_01.value)
|
||||
|
||||
beta_values.update(
|
||||
typed_params: Final = _OPTIONAL_PARAMS.validate_python(optional_params)
|
||||
if requires_native_compaction_beta("vertex_ai", typed_params, messages):
|
||||
beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_09_04.value)
|
||||
|
||||
tool_change_betas: Final = (
|
||||
(ANTHROPIC_MID_CONVERSATION_TOOL_CHANGES_BETA_HEADER,)
|
||||
if anthropic_model_info.is_mid_conversation_tool_change_used(messages)
|
||||
else ()
|
||||
)
|
||||
all_beta_values: Final = beta_values.union(
|
||||
anthropic_model_info.thinking_betas(
|
||||
optional_params.get("thinking"), # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # optional_params is the base config's bare dict
|
||||
)
|
||||
),
|
||||
tool_change_betas,
|
||||
)
|
||||
|
||||
if beta_values:
|
||||
request_headers["anthropic-beta"] = ",".join(beta_values)
|
||||
if all_beta_values:
|
||||
request_headers["anthropic-beta"] = ",".join(sorted(all_beta_values))
|
||||
|
||||
return request_headers, api_base
|
||||
|
||||
|
|
|
|||
|
|
@ -2,19 +2,23 @@
|
|||
## Handler file for calling claude-3 on vertex ai
|
||||
import base64
|
||||
import json
|
||||
from collections.abc import Mapping
|
||||
from collections.abc import Mapping, Sequence
|
||||
from typing import TYPE_CHECKING, Final
|
||||
|
||||
import httpx
|
||||
from pydantic import TypeAdapter
|
||||
from pydantic import TypeAdapter, ValidationError
|
||||
|
||||
import litellm
|
||||
from litellm.anthropic_beta_headers_manager import get_header_only_betas
|
||||
from litellm.litellm_core_utils.core_helpers import drops_unsupported_params
|
||||
from litellm.litellm_core_utils.prompt_templates.image_handling import RemoteMedia, inline_remote_image_urls
|
||||
from litellm.llms.base_llm.chat.transformation import LiteLLMLoggingObj
|
||||
from litellm.types.llms.anthropic import ANTHROPIC_BETA_HEADER_VALUES
|
||||
from litellm.types.llms.openai import AllMessageValues
|
||||
from litellm.types.utils import ModelResponse
|
||||
|
||||
from ....anthropic.chat.transformation import AnthropicConfig
|
||||
from ....anthropic.common_utils import AnthropicError, requires_native_compaction_beta
|
||||
from .output_params_utils import sanitize_vertex_anthropic_output_params
|
||||
|
||||
if TYPE_CHECKING:
|
||||
|
|
@ -36,6 +40,32 @@ def _vertex_labels_header(labels: object, header_names: Mapping[str, object]) ->
|
|||
return base64.b64encode(json.dumps(string_labels).encode()).decode() if string_labels else None
|
||||
|
||||
|
||||
_BETA_VALUE: Final[TypeAdapter[str | tuple[str, ...] | None]] = TypeAdapter(str | tuple[str, ...] | None)
|
||||
|
||||
_REQUEST_MESSAGES: Final = TypeAdapter(Sequence[object])
|
||||
|
||||
|
||||
def _beta_names(value: object) -> frozenset[str] | None:
|
||||
try:
|
||||
parsed: Final = _BETA_VALUE.validate_python(value)
|
||||
except ValidationError:
|
||||
return None
|
||||
names: Final = parsed.split(",") if isinstance(parsed, str) else parsed or ()
|
||||
return frozenset(name.strip() for name in names if name.strip())
|
||||
|
||||
|
||||
def _client_betas(value: object, field: str, drop_params: bool) -> frozenset[str]:
|
||||
names: Final = _beta_names(value)
|
||||
if names is not None:
|
||||
return names
|
||||
if drop_params:
|
||||
return frozenset()
|
||||
raise AnthropicError(
|
||||
status_code=400,
|
||||
message=f"{field} must be a string or a list of strings, got {type(value).__name__}",
|
||||
)
|
||||
|
||||
|
||||
class VertexAIError(Exception):
|
||||
def __init__(self, status_code, message):
|
||||
self.status_code = status_code
|
||||
|
|
@ -157,21 +187,43 @@ class VertexAIAnthropicConfig(AnthropicConfig):
|
|||
if context_management:
|
||||
self._add_context_management_beta_headers(beta_set, context_management)
|
||||
|
||||
extra_headers: Final = optional_params.get("extra_headers") or {}
|
||||
anthropic_beta_value: Final = extra_headers.get("anthropic-beta", "")
|
||||
if isinstance(anthropic_beta_value, str) and anthropic_beta_value:
|
||||
for beta in anthropic_beta_value.split(","):
|
||||
beta = beta.strip()
|
||||
if beta:
|
||||
beta_set.add(beta)
|
||||
elif isinstance(anthropic_beta_value, list):
|
||||
beta_set.update(anthropic_beta_value)
|
||||
|
||||
request_params: Final = _NON_DEFAULT_PARAMS.validate_python(optional_params)
|
||||
compaction_betas: Final = (
|
||||
frozenset({ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_09_04.value})
|
||||
if requires_native_compaction_beta(
|
||||
"vertex_ai", request_params, _REQUEST_MESSAGES.validate_python(data["messages"])
|
||||
)
|
||||
else frozenset[str]()
|
||||
)
|
||||
drop_params: Final = drops_unsupported_params(litellm_params.get("drop_params"))
|
||||
extra_headers: Final = _NON_DEFAULT_PARAMS.validate_python(request_params.get("extra_headers") or {})
|
||||
data.pop("extra_headers", None)
|
||||
injected_betas: Final = frozenset(beta_set) | _client_betas(
|
||||
extra_headers.get("anthropic-beta"), "The anthropic-beta header", drop_params
|
||||
)
|
||||
client_body_betas: Final = _client_betas(
|
||||
_NON_DEFAULT_PARAMS.validate_python(data).get("anthropic_beta"), "anthropic_beta", drop_params
|
||||
)
|
||||
|
||||
if beta_set:
|
||||
data["anthropic_beta"] = list(beta_set)
|
||||
headers["anthropic-beta"] = ",".join(beta_set)
|
||||
requested_betas: Final = injected_betas | client_body_betas
|
||||
header_only_betas: Final = get_header_only_betas("vertex_ai") if requested_betas else frozenset[str]()
|
||||
body_betas: Final = requested_betas - header_only_betas
|
||||
if body_betas:
|
||||
data["anthropic_beta"] = sorted(body_betas)
|
||||
elif "anthropic_beta" in data:
|
||||
del data["anthropic_beta"]
|
||||
header_betas: Final = (
|
||||
injected_betas
|
||||
| compaction_betas
|
||||
| (client_body_betas & header_only_betas)
|
||||
| _client_betas(
|
||||
_NON_DEFAULT_PARAMS.validate_python(headers).get("anthropic-beta"),
|
||||
"The anthropic-beta header",
|
||||
drop_params,
|
||||
)
|
||||
)
|
||||
if header_betas:
|
||||
headers["anthropic-beta"] = ",".join(sorted(header_betas))
|
||||
|
||||
return data
|
||||
|
||||
|
|
|
|||
|
|
@ -6,12 +6,19 @@ Unlike Gemini models which use Google's token counting API, partner models use
|
|||
their respective publisher-specific count-tokens endpoints.
|
||||
"""
|
||||
|
||||
from collections.abc import Mapping
|
||||
from collections.abc import Mapping, Sequence
|
||||
from typing import Any, Final
|
||||
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
from litellm.litellm_core_utils.core_helpers import drops_unsupported_params
|
||||
from litellm.litellm_core_utils.url_utils import join_api_base
|
||||
from litellm.llms.anthropic.common_utils import OBJECT_LIST_ADAPTER, is_claude_model_target, tools_without_custom_field
|
||||
from litellm.llms.anthropic.common_utils import (
|
||||
OBJECT_LIST_ADAPTER,
|
||||
count_tokens_feature_betas,
|
||||
is_claude_model_target,
|
||||
tools_without_custom_field,
|
||||
)
|
||||
from litellm.llms.custom_httpx.http_handler import get_async_httpx_client
|
||||
from litellm.llms.vertex_ai.common_utils import get_vertex_base_url
|
||||
from litellm.llms.vertex_ai.vertex_llm_base import VertexBase
|
||||
|
|
@ -23,6 +30,9 @@ _COUNT_TOKENS_PUBLISHERS: Final = (
|
|||
)
|
||||
|
||||
|
||||
_REQUEST_MESSAGES: Final = TypeAdapter(Sequence[object])
|
||||
|
||||
|
||||
def count_tokens_request_body(
|
||||
request_data: Mapping[str, object], model: str, drop_params: object
|
||||
) -> Mapping[str, object]:
|
||||
|
|
@ -43,6 +53,14 @@ def count_tokens_publisher(model: str) -> str | None:
|
|||
return next((publisher for marker, publisher in _COUNT_TOKENS_PUBLISHERS if marker in model), None)
|
||||
|
||||
|
||||
def count_tokens_headers(access_token: str, request_data: Mapping[str, object]) -> Mapping[str, str]:
|
||||
authorization: Final = {"Authorization": f"Bearer {access_token}"}
|
||||
betas: Final = count_tokens_feature_betas(
|
||||
"vertex_ai", request_data, _REQUEST_MESSAGES.validate_python(request_data["messages"])
|
||||
)
|
||||
return {**authorization, "anthropic-beta": ",".join(betas)} if betas else authorization
|
||||
|
||||
|
||||
class VertexAIPartnerModelsTokenCounter(VertexBase):
|
||||
"""
|
||||
Token counter for Vertex AI Partner Models.
|
||||
|
|
@ -175,8 +193,7 @@ class VertexAIPartnerModelsTokenCounter(VertexBase):
|
|||
api_base=litellm_params.get("api_base"),
|
||||
)
|
||||
|
||||
# Prepare headers
|
||||
headers: Final = {"Authorization": f"Bearer {access_token}"}
|
||||
headers: Final = dict(count_tokens_headers(access_token, request_data))
|
||||
|
||||
# Get async HTTP client
|
||||
from litellm import LlmProviders
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
from collections.abc import Mapping
|
||||
from typing import Final
|
||||
|
||||
import httpx
|
||||
|
|
@ -10,6 +11,14 @@ from litellm.types.llms.openai import AllMessageValues
|
|||
from litellm.utils import add_path_to_api_base
|
||||
|
||||
|
||||
def endpoint_under_api_base(endpoint: str, api_base: str | None) -> str:
|
||||
return (
|
||||
endpoint.lstrip("/").removeprefix("v1/")
|
||||
if api_base is not None and api_base.rstrip("/").endswith("/v1")
|
||||
else endpoint
|
||||
)
|
||||
|
||||
|
||||
class VLLMError(BaseLLMException):
|
||||
def __init__(
|
||||
self,
|
||||
|
|
@ -39,9 +48,14 @@ class VLLMModelInfo(BaseLLMModelInfo):
|
|||
api_key: str | None = None,
|
||||
api_base: str | None = None,
|
||||
) -> dict:
|
||||
if api_key is not None:
|
||||
headers["x-api-key"] = api_key
|
||||
return headers
|
||||
upstream_key: Final = self.upstream_api_key(api_key, api_base)
|
||||
return {**headers, **self.upstream_auth_headers(upstream_key)} if upstream_key else headers
|
||||
|
||||
def upstream_api_key(self, api_key: str | None, api_base: str | None) -> str | None:
|
||||
return api_key
|
||||
|
||||
def upstream_auth_headers(self, api_key: str) -> Mapping[str, str]:
|
||||
return {"Authorization": f"Bearer {api_key}", "x-api-key": api_key}
|
||||
|
||||
@staticmethod
|
||||
def get_api_base(api_base: str | None = None) -> str | None:
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ from typing import TYPE_CHECKING, Final
|
|||
|
||||
from httpx import Response
|
||||
|
||||
from litellm.constants import PLACEHOLDER_API_KEY
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging
|
||||
from litellm.llms.base_llm.passthrough.transformation import BasePassthroughConfig
|
||||
from litellm.llms.openai_like.passthrough.transformation import (
|
||||
|
|
@ -12,8 +13,14 @@ from litellm.llms.openai_like.passthrough.transformation import (
|
|||
logged_text_completion_stream,
|
||||
relays_text_completion,
|
||||
)
|
||||
from litellm.secret_managers.main import (
|
||||
GOOGLE_OIDC_SECRET_PREFIX,
|
||||
UnresolvableSecretReference,
|
||||
get_secret_str,
|
||||
resolve_local_oidc_reference,
|
||||
)
|
||||
|
||||
from ..common_utils import VLLMModelInfo
|
||||
from ..common_utils import VLLMModelInfo, endpoint_under_api_base
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from httpx import URL
|
||||
|
|
@ -22,6 +29,30 @@ if TYPE_CHECKING:
|
|||
|
||||
|
||||
class VLLMPassthroughConfig(VLLMModelInfo, BasePassthroughConfig):
|
||||
@staticmethod
|
||||
def get_api_key(api_key: str | None = None) -> str | None:
|
||||
return None if api_key == PLACEHOLDER_API_KEY else api_key
|
||||
|
||||
def upstream_api_key(self, api_key: str | None, api_base: str | None) -> str | None:
|
||||
if api_key is not None or not self._is_vllm_api_base(api_base):
|
||||
return api_key
|
||||
vllm_api_key: Final = get_secret_str("VLLM_API_KEY")
|
||||
if vllm_api_key is None or vllm_api_key.startswith(GOOGLE_OIDC_SECRET_PREFIX):
|
||||
return None
|
||||
match resolve_local_oidc_reference(vllm_api_key):
|
||||
case UnresolvableSecretReference():
|
||||
return None
|
||||
case str() as resolved:
|
||||
return resolved
|
||||
|
||||
@staticmethod
|
||||
def _is_vllm_api_base(api_base: str | None) -> bool:
|
||||
vllm_api_base: Final = get_secret_str("VLLM_API_BASE")
|
||||
return api_base is not None and vllm_api_base is not None and api_base.rstrip("/") == vllm_api_base.rstrip("/")
|
||||
|
||||
def upstream_auth_headers(self, api_key: str) -> Mapping[str, str]:
|
||||
return {"Authorization": f"Bearer {api_key}"}
|
||||
|
||||
def is_streaming_request(self, endpoint: str, request_data: dict) -> bool:
|
||||
return "stream" in request_data
|
||||
|
||||
|
|
@ -40,7 +71,7 @@ class VLLMPassthroughConfig(VLLMModelInfo, BasePassthroughConfig):
|
|||
raise Exception("VLLM api base not found")
|
||||
|
||||
return (
|
||||
self.format_url(endpoint, base_target_url, request_query_params),
|
||||
self.format_url(endpoint_under_api_base(endpoint, base_target_url), base_target_url, request_query_params),
|
||||
base_target_url,
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -10,6 +10,10 @@ from litellm._logging import verbose_logger
|
|||
from litellm.constants import XAI_API_BASE
|
||||
from litellm.exceptions import AuthenticationError
|
||||
from litellm.litellm_core_utils.url_utils import join_api_base
|
||||
from litellm.llms.base_llm.responses.transformation import (
|
||||
STANDARD_MODE_WITHOUT_CONTEXT_REASONING_FIELD_VALUES,
|
||||
ReasoningFieldValues,
|
||||
)
|
||||
from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig
|
||||
from litellm.llms.xai.common_utils import XAIModelInfo, xai_reported_cost_in_usd
|
||||
from litellm.secret_managers.main import get_secret_str
|
||||
|
|
@ -290,3 +294,9 @@ class XAIResponsesAPIConfig(OpenAIResponsesAPIConfig):
|
|||
def supports_native_websocket(self) -> bool:
|
||||
"""XAI does not support native WebSocket for Responses API"""
|
||||
return False
|
||||
|
||||
def reasoning_field_values(self) -> ReasoningFieldValues:
|
||||
return STANDARD_MODE_WITHOUT_CONTEXT_REASONING_FIELD_VALUES
|
||||
|
||||
def ignores_unsupported_reasoning_fields(self) -> bool:
|
||||
return True
|
||||
|
|
|
|||
|
|
@ -681,7 +681,7 @@ async def acompletion(
|
|||
- The `completion` function is called using `run_in_executor` to execute synchronously in the event loop.
|
||||
- If `stream` is True, the function returns an async generator that yields completion lines.
|
||||
"""
|
||||
fallbacks = kwargs.get("fallbacks", None)
|
||||
fallbacks: Final = kwargs.get("fallbacks", None) or litellm.model_fallbacks
|
||||
mock_timeout = kwargs.get("mock_timeout", None)
|
||||
|
||||
if mock_timeout is True:
|
||||
|
|
@ -697,6 +697,7 @@ async def acompletion(
|
|||
loop: Final = asyncio.get_event_loop()
|
||||
custom_llm_provider = kwargs.get("custom_llm_provider", None)
|
||||
_ = _resolve_control_options(kwargs, model)
|
||||
caller_kwargs: Final = cast("Mapping[str, object]", {**kwargs}) # cast-ok: acompletion takes untyped kwargs
|
||||
|
||||
## PROMPT MANAGEMENT HOOKS ##
|
||||
#########################################################
|
||||
|
|
@ -715,15 +716,34 @@ async def acompletion(
|
|||
custom_llm_provider=cast(str | None, custom_llm_provider), # cast-ok: read from untyped kwargs
|
||||
tools=tools,
|
||||
enable_prompt_caching=cast(bool | None, kwargs.get("enable_prompt_caching")), # cast-ok: untyped kwargs
|
||||
api_base=kwargs.get("api_base") or base_url,
|
||||
api_base=base_url or kwargs.get("api_base"),
|
||||
)
|
||||
|
||||
if isinstance(litellm_logging_obj, LiteLLMLoggingObj) and (
|
||||
litellm_logging_obj.should_run_prompt_management_hooks(
|
||||
prompt_id=kwargs.get("prompt_id", None),
|
||||
prompt_id: Final = cast(str | None, kwargs.get("prompt_id", None)) # cast-ok: read from untyped kwargs
|
||||
prompt_management_logger: Final = (
|
||||
litellm_logging_obj.get_custom_logger_for_prompt_management(
|
||||
model=model,
|
||||
non_default_params=kwargs,
|
||||
tools=tools,
|
||||
prompt_id=prompt_id,
|
||||
dynamic_callback_params=litellm_logging_obj.standard_callback_dynamic_params,
|
||||
)
|
||||
if isinstance(litellm_logging_obj, LiteLLMLoggingObj)
|
||||
and litellm_logging_obj.should_run_prompt_management_hooks(
|
||||
prompt_id=prompt_id,
|
||||
non_default_params=kwargs,
|
||||
tools=tools,
|
||||
)
|
||||
else None
|
||||
)
|
||||
each_fallback_writes_its_own_cache_dialect: Final = fallbacks is not None and isinstance(
|
||||
prompt_management_logger, AnthropicCacheControlHook
|
||||
)
|
||||
|
||||
if (
|
||||
isinstance(litellm_logging_obj, LiteLLMLoggingObj)
|
||||
and prompt_management_logger is not None
|
||||
and not each_fallback_writes_its_own_cache_dialect
|
||||
):
|
||||
(
|
||||
model,
|
||||
|
|
@ -733,8 +753,9 @@ async def acompletion(
|
|||
model=model,
|
||||
messages=messages,
|
||||
non_default_params=kwargs,
|
||||
prompt_id=kwargs.get("prompt_id", None),
|
||||
prompt_id=prompt_id,
|
||||
prompt_variables=kwargs.get("prompt_variables", None),
|
||||
prompt_management_logger=prompt_management_logger,
|
||||
tools=tools,
|
||||
prompt_label=kwargs.get("prompt_label", None),
|
||||
prompt_version=kwargs.get("prompt_version", None),
|
||||
|
|
@ -826,9 +847,10 @@ async def acompletion(
|
|||
litellm_params=(GenericLiteLLMParams.model_validate(_supplemental) if _supplemental else None),
|
||||
)
|
||||
|
||||
fallbacks = fallbacks or litellm.model_fallbacks
|
||||
if fallbacks is not None:
|
||||
response = await async_completion_with_fallbacks(**completion_kwargs, kwargs={"fallbacks": fallbacks, **kwargs})
|
||||
response = await async_completion_with_fallbacks(
|
||||
**completion_kwargs, kwargs={"fallbacks": fallbacks, **caller_kwargs}
|
||||
)
|
||||
if response is None:
|
||||
raise Exception(
|
||||
"No response from fallbacks. Got none. Turn on `litellm.set_verbose=True` to see more details."
|
||||
|
|
@ -5765,7 +5787,7 @@ def completion(
|
|||
custom_llm_provider=cast(str | None, kwargs.get("custom_llm_provider")), # cast-ok: untyped kwargs
|
||||
tools=tools,
|
||||
enable_prompt_caching=cast(bool | None, kwargs.get("enable_prompt_caching")), # cast-ok: untyped kwargs
|
||||
api_base=kwargs.get("api_base") or base_url,
|
||||
api_base=base_url or kwargs.get("api_base"),
|
||||
)
|
||||
|
||||
if isinstance(litellm_logging_obj, LiteLLMLoggingObj) and (
|
||||
|
|
@ -6267,6 +6289,9 @@ def completion(
|
|||
# reasoningSummary/reasoning_summary without tools (AI SDK) that the first
|
||||
# (early) check doesn't cover.
|
||||
_reasoning_summary_for_bridge: Final = peek_reasoning_summary_aliases(optional_params)
|
||||
typed_deployment_model_info: Final = str_keyed_mapping(
|
||||
cast(object, model_info) # cast-ok: model_info is the untyped dict kwarg
|
||||
)
|
||||
if responses_api_model_info.get("mode") != "responses":
|
||||
responses_api_model_info, model = responses_api_bridge_check(
|
||||
model=model,
|
||||
|
|
@ -6276,9 +6301,7 @@ def completion(
|
|||
reasoning_effort=bridge_reasoning_effort(resolved_reasoning_effort, optional_params.get("extra_body")),
|
||||
reasoning_summary=_reasoning_summary_for_bridge,
|
||||
api_base=api_base,
|
||||
deployment_model_info=str_keyed_mapping(
|
||||
cast(object, model_info) # cast-ok: model_info is the untyped dict kwarg
|
||||
),
|
||||
deployment_model_info=typed_deployment_model_info,
|
||||
)
|
||||
|
||||
# Use base_model (the true underlying model) for Azure model-type
|
||||
|
|
@ -6290,19 +6313,28 @@ def completion(
|
|||
|
||||
optional_params, rs_val = strip_reasoning_summary_aliases_from_optional_params(optional_params)
|
||||
|
||||
if custom_llm_provider == "openai" and isinstance(resolved_reasoning_effort, dict):
|
||||
from litellm.responses.utils import openai_bridge_reasoning
|
||||
if isinstance(resolved_reasoning_effort, dict):
|
||||
from litellm.responses.utils import (
|
||||
RefusedReasoningFields,
|
||||
bridge_reasoning,
|
||||
raise_refused_reasoning_fields,
|
||||
)
|
||||
|
||||
optional_params["reasoning_effort"] = openai_bridge_reasoning(
|
||||
bridged_reasoning: Final = bridge_reasoning(
|
||||
requested=resolved_reasoning_effort,
|
||||
effort_kept="reasoning_effort" in optional_params,
|
||||
summary_alias=rs_val,
|
||||
model=model,
|
||||
base_model=base_model if isinstance(base_model, str) else None,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
api_base=api_base,
|
||||
deployment_model_info=typed_deployment_model_info,
|
||||
drop_params=drop_unsupported_reasoning,
|
||||
additional_drop_params=routing_kwargs.get("additional_drop_params"),
|
||||
)
|
||||
elif isinstance(resolved_reasoning_effort, dict) and "summary" in resolved_reasoning_effort:
|
||||
optional_params["reasoning_effort"] = resolved_reasoning_effort
|
||||
if isinstance(bridged_reasoning, RefusedReasoningFields):
|
||||
raise_refused_reasoning_fields(bridged_reasoning, model, custom_llm_provider)
|
||||
optional_params["reasoning_effort"] = bridged_reasoning
|
||||
elif rs_val is not None:
|
||||
eff: Final = optional_params.get("reasoning_effort", resolved_reasoning_effort)
|
||||
if isinstance(eff, dict):
|
||||
|
|
|
|||
|
|
@ -8715,7 +8715,7 @@
|
|||
"search_context_size_low": 0.014,
|
||||
"search_context_size_medium": 0.014
|
||||
},
|
||||
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'",
|
||||
"source": "https://azure.microsoft.com/en-us/pricing/details/cognitive-services/openai-service/",
|
||||
"supported_endpoints": [
|
||||
"/v1/chat/completions",
|
||||
"/v1/responses"
|
||||
|
|
@ -8742,7 +8742,11 @@
|
|||
"supports_tool_choice": true,
|
||||
"supports_vision": true,
|
||||
"supports_web_search": true,
|
||||
"supports_xhigh_reasoning_effort": true
|
||||
"supports_xhigh_reasoning_effort": true,
|
||||
"cache_creation_input_token_cost_above_272k_tokens_flex": 1.25e-05,
|
||||
"cache_read_input_token_cost_above_272k_tokens_flex": 1e-06,
|
||||
"input_cost_per_token_above_272k_tokens_flex": 1e-05,
|
||||
"output_cost_per_token_above_272k_tokens_flex": 3.75e-05
|
||||
},
|
||||
"azure/gpt-6-astra-2026-09-03": {
|
||||
"cache_creation_input_token_cost": 1.25e-05,
|
||||
|
|
@ -8763,7 +8767,7 @@
|
|||
"search_context_size_low": 0.014,
|
||||
"search_context_size_medium": 0.014
|
||||
},
|
||||
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'",
|
||||
"source": "https://azure.microsoft.com/en-us/pricing/details/cognitive-services/openai-service/",
|
||||
"supported_endpoints": [
|
||||
"/v1/chat/completions",
|
||||
"/v1/responses"
|
||||
|
|
@ -8790,7 +8794,11 @@
|
|||
"supports_tool_choice": true,
|
||||
"supports_vision": true,
|
||||
"supports_web_search": true,
|
||||
"supports_xhigh_reasoning_effort": true
|
||||
"supports_xhigh_reasoning_effort": true,
|
||||
"cache_creation_input_token_cost_above_272k_tokens_flex": 1.25e-05,
|
||||
"cache_read_input_token_cost_above_272k_tokens_flex": 1e-06,
|
||||
"input_cost_per_token_above_272k_tokens_flex": 1e-05,
|
||||
"output_cost_per_token_above_272k_tokens_flex": 3.75e-05
|
||||
},
|
||||
"azure/gpt-6-luna": {
|
||||
"cache_creation_input_token_cost": 1.25e-07,
|
||||
|
|
@ -30856,8 +30864,9 @@
|
|||
"output_cost_per_token": 0,
|
||||
"output_vector_size": 3072,
|
||||
"rpm": 10000,
|
||||
"source": "https://ai.google.dev/gemini-api/docs/embeddings#model-versions",
|
||||
"tpm": 10000000
|
||||
"source": "https://ai.google.dev/gemini-api/docs/pricing",
|
||||
"tpm": 10000000,
|
||||
"input_cost_per_token_batches": 7.5e-08
|
||||
},
|
||||
"gemini/gemini-embedding-2": {
|
||||
"input_cost_per_audio_token": 6.5e-06,
|
||||
|
|
@ -31713,7 +31722,10 @@
|
|||
"input_cost_per_token_flex": 6.25e-07,
|
||||
"output_cost_per_token_batches": 5e-06,
|
||||
"output_cost_per_token_flex": 5e-06,
|
||||
"supports_url_context": true
|
||||
"supports_url_context": true,
|
||||
"cache_read_input_token_cost_above_200k_tokens_batches": 2.5e-07,
|
||||
"input_cost_per_token_above_200k_tokens_batches": 1.25e-06,
|
||||
"output_cost_per_token_above_200k_tokens_batches": 7.5e-06
|
||||
},
|
||||
"gemini/gemini-2.5-computer-use-preview-10-2025": {
|
||||
"deprecation_date": "2026-07-28",
|
||||
|
|
@ -31871,7 +31883,8 @@
|
|||
},
|
||||
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
|
||||
"web_search_billing_unit": "per_query",
|
||||
"google_maps_grounding_cost_per_query": 0.014
|
||||
"google_maps_grounding_cost_per_query": 0.014,
|
||||
"supports_computer_use": true
|
||||
},
|
||||
"gemini/gemini-3-flash-preview": {
|
||||
"cache_read_input_audio_token_cost": 1e-07,
|
||||
|
|
@ -31937,7 +31950,8 @@
|
|||
"input_cost_per_token_flex": 2.5e-07,
|
||||
"output_cost_per_token_batches": 1.5e-06,
|
||||
"output_cost_per_token_flex": 1.5e-06,
|
||||
"supports_audio_input": true
|
||||
"supports_audio_input": true,
|
||||
"supports_computer_use": true
|
||||
},
|
||||
"gemini/gemini-3.5-flash": {
|
||||
"prompt_cache_min_tokens": 4096,
|
||||
|
|
@ -32184,7 +32198,8 @@
|
|||
},
|
||||
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
|
||||
"web_search_billing_unit": "per_query",
|
||||
"google_maps_grounding_cost_per_query": 0.014
|
||||
"google_maps_grounding_cost_per_query": 0.014,
|
||||
"supports_computer_use": true
|
||||
},
|
||||
"gemini/gemini-omni-flash-preview": {
|
||||
"input_cost_per_audio_token": 1.5e-06,
|
||||
|
|
@ -32280,7 +32295,10 @@
|
|||
"web_search_billing_unit": "per_query",
|
||||
"google_maps_grounding_cost_per_query": 0.014,
|
||||
"input_cost_per_token_flex": 1e-06,
|
||||
"output_cost_per_token_flex": 6e-06
|
||||
"output_cost_per_token_flex": 6e-06,
|
||||
"cache_read_input_token_cost_above_200k_tokens_batches": 4e-07,
|
||||
"input_cost_per_token_above_200k_tokens_batches": 2e-06,
|
||||
"output_cost_per_token_above_200k_tokens_batches": 9e-06
|
||||
},
|
||||
"gemini/gemini-3.1-pro-preview-customtools": {
|
||||
"prompt_cache_min_tokens": 4096,
|
||||
|
|
@ -32343,7 +32361,10 @@
|
|||
"web_search_billing_unit": "per_query",
|
||||
"google_maps_grounding_cost_per_query": 0.014,
|
||||
"input_cost_per_token_flex": 1e-06,
|
||||
"output_cost_per_token_flex": 6e-06
|
||||
"output_cost_per_token_flex": 6e-06,
|
||||
"cache_read_input_token_cost_above_200k_tokens_batches": 4e-07,
|
||||
"input_cost_per_token_above_200k_tokens_batches": 2e-06,
|
||||
"output_cost_per_token_above_200k_tokens_batches": 9e-06
|
||||
},
|
||||
"gemini-3-flash-preview": {
|
||||
"cache_read_input_audio_token_cost": 1e-07,
|
||||
|
|
@ -72236,6 +72257,7 @@
|
|||
},
|
||||
"aiand/deepseek-ai/deepseek-v4-flash": {
|
||||
"cache_read_input_token_cost": 8e-08,
|
||||
"deprecation_date": "2026-10-27",
|
||||
"input_cost_per_token": 1.5e-07,
|
||||
"litellm_provider": "aiand",
|
||||
"max_input_tokens": 1048576,
|
||||
|
|
@ -72265,6 +72287,22 @@
|
|||
"supports_vision": false
|
||||
},
|
||||
"aiand/deepseek-ai/deepseek-v4.1-flash": {
|
||||
"cache_read_input_token_cost": 2e-08,
|
||||
"deprecation_date": "2026-11-05",
|
||||
"input_cost_per_token": 3e-07,
|
||||
"litellm_provider": "aiand",
|
||||
"max_input_tokens": 1048576,
|
||||
"max_output_tokens": 384000,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 6e-07,
|
||||
"source": "https://api.aiand.com/v1/models",
|
||||
"supports_function_calling": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": true
|
||||
},
|
||||
"aiand/deepseek-ai/deepseek-v4.1-flash-1017": {
|
||||
"cache_read_input_token_cost": 2e-08,
|
||||
"input_cost_per_token": 3e-07,
|
||||
"litellm_provider": "aiand",
|
||||
|
|
@ -72294,8 +72332,24 @@
|
|||
"supports_tool_choice": true,
|
||||
"supports_vision": true
|
||||
},
|
||||
"aiand/meta/muse-spark-open": {
|
||||
"cache_read_input_token_cost": 1.5e-07,
|
||||
"input_cost_per_token": 1.25e-06,
|
||||
"litellm_provider": "aiand",
|
||||
"max_input_tokens": 1048576,
|
||||
"max_output_tokens": 943718,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 4.25e-06,
|
||||
"source": "https://api.aiand.com/v1/models",
|
||||
"supports_function_calling": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": true
|
||||
},
|
||||
"aiand/moonshotai/kimi-k2.7-code": {
|
||||
"cache_read_input_token_cost": 2e-07,
|
||||
"deprecation_date": "2026-10-27",
|
||||
"input_cost_per_token": 7.5e-07,
|
||||
"litellm_provider": "aiand",
|
||||
"max_input_tokens": 262144,
|
||||
|
|
@ -72386,6 +72440,7 @@
|
|||
},
|
||||
"aiand/zai-org/glm-5.2": {
|
||||
"cache_read_input_token_cost": 3e-07,
|
||||
"deprecation_date": "2026-10-27",
|
||||
"input_cost_per_token": 1e-06,
|
||||
"litellm_provider": "aiand",
|
||||
"max_input_tokens": 1048576,
|
||||
|
|
@ -76548,6 +76603,7 @@
|
|||
],
|
||||
"supports_audio_input": true,
|
||||
"supports_audio_output": false,
|
||||
"supports_computer_use": true,
|
||||
"supports_function_calling": true,
|
||||
"supports_minimal_reasoning_effort": false,
|
||||
"supports_native_streaming": true,
|
||||
|
|
|
|||
|
|
@ -30,6 +30,7 @@ from litellm.models.organization_membership import LiteLLM_OrganizationMembershi
|
|||
from litellm.models.project import LiteLLM_ProjectTable
|
||||
from litellm.models.skills import LiteLLM_SkillsTable
|
||||
from litellm.models.spend_logs import LiteLLM_ErrorLogs, LiteLLM_SpendLogs
|
||||
from litellm.models.subscription_account import LiteLLM_SubscriptionAccountTable
|
||||
from litellm.models.tag import LiteLLM_TagTable
|
||||
from litellm.models.team import LiteLLM_TeamTable
|
||||
from litellm.models.team_membership import LiteLLM_TeamMembership
|
||||
|
|
@ -59,6 +60,7 @@ __all__ = [
|
|||
"LiteLLM_ProxyModelTable",
|
||||
"LiteLLM_SkillsTable",
|
||||
"LiteLLM_SpendLogs",
|
||||
"LiteLLM_SubscriptionAccountTable",
|
||||
"LiteLLM_TagTable",
|
||||
"LiteLLM_TeamMemberTable",
|
||||
"LiteLLM_TeamMembership",
|
||||
|
|
|
|||
28
litellm/models/subscription_account.py
Normal file
28
litellm/models/subscription_account.py
Normal file
|
|
@ -0,0 +1,28 @@
|
|||
"""
|
||||
Subscription account table model.
|
||||
|
||||
Canonical definition for ``litellm_subscriptionaccounttable``: the operator-entered
|
||||
recurring fee of a provider account whose requests are covered by a flat subscription.
|
||||
"""
|
||||
|
||||
from datetime import datetime
|
||||
|
||||
from pydantic import ConfigDict
|
||||
|
||||
from litellm.types.llms.base import LiteLLMPydanticObjectBase
|
||||
|
||||
|
||||
class LiteLLM_SubscriptionAccountTable(LiteLLMPydanticObjectBase):
|
||||
model_config = ConfigDict(protected_namespaces=(), frozen=True)
|
||||
|
||||
subscription_account_id: str
|
||||
custom_llm_provider: str
|
||||
account_id: str
|
||||
label: str | None = None
|
||||
monthly_fee: float
|
||||
currency: str
|
||||
billing_period_start: str
|
||||
created_at: datetime
|
||||
created_by: str
|
||||
updated_at: datetime
|
||||
updated_by: str
|
||||
|
|
@ -1648,7 +1648,7 @@
|
|||
},
|
||||
"/access_group/{access_group}/update": {
|
||||
"put": {
|
||||
"description": "Update an access group's model names.\n\nThis will:\n1. Remove the access group from all current deployments\n2. Add the access group to all deployments for the new model_names list\n\nExample:\n```bash\ncurl -X PUT 'http://localhost:4000/access_group/production-models/update' \\\n -H \"Authorization: Bearer $LITELLM_MASTER_KEY\" \\\n -H 'Content-Type: application/json' \\\n -d '{\n \"model_names\": [\"gpt-4\", \"claude-3-sonnet\"]\n }'\n```\n\nParameters:\n- access_group: str - The access group name (URL path parameter)\n- model_names: List[str] - New list of model groups to include\n\nReturns:\n- NewModelGroupResponse with the updated access group details\n\nRaises:\n- HTTPException 400: If any model names don't exist\n- HTTPException 404: If access group not found",
|
||||
"description": "Update an access group's model names.\n\nThis will:\n1. Add the access group to all deployments for the new model_names list\n2. Remove the access group from every other deployment\n\nExample:\n```bash\ncurl -X PUT 'http://localhost:4000/access_group/production-models/update' \\\n -H \"Authorization: Bearer $LITELLM_MASTER_KEY\" \\\n -H 'Content-Type: application/json' \\\n -d '{\n \"model_names\": [\"gpt-4\", \"claude-3-sonnet\"]\n }'\n```\n\nParameters:\n- access_group: str - The access group name (URL path parameter)\n- model_names: List[str] - New list of model groups to include\n\nReturns:\n- NewModelGroupResponse with the updated access group details\n\nRaises:\n- HTTPException 400: If any model names don't exist\n- HTTPException 404: If access group not found\n- HTTPException 409: If another edit holds the group or one of its models for 30 s",
|
||||
"operationId": "update_access_group_access_group__access_group__update_put",
|
||||
"parameters": [
|
||||
{
|
||||
|
|
@ -3491,6 +3491,7 @@
|
|||
"additionalProperties": {
|
||||
"$ref": "#/components/schemas/GuardrailMetric"
|
||||
},
|
||||
"description": "Spend by guardrail ID, already included in key and model spend. Populated on user, team and organization aggregated activity only, without a model filter",
|
||||
"title": "Guardrails",
|
||||
"type": "object"
|
||||
},
|
||||
|
|
@ -3812,10 +3813,12 @@
|
|||
"GuardrailKeyMetric": {
|
||||
"properties": {
|
||||
"metadata": {
|
||||
"$ref": "#/components/schemas/KeyMetadata"
|
||||
"$ref": "#/components/schemas/KeyMetadata",
|
||||
"description": "API key alias and existence information"
|
||||
},
|
||||
"metrics": {
|
||||
"$ref": "#/components/schemas/GuardrailSpendMetrics"
|
||||
"$ref": "#/components/schemas/GuardrailSpendMetrics",
|
||||
"description": "Guardrail spend and unpriced units for this API key"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
|
|
@ -3828,6 +3831,7 @@
|
|||
"GuardrailMetadata": {
|
||||
"properties": {
|
||||
"guardrail_name": {
|
||||
"description": "Guardrail display name, falling back to the guardrail ID",
|
||||
"title": "Guardrail Name",
|
||||
"type": "string"
|
||||
}
|
||||
|
|
@ -3844,14 +3848,17 @@
|
|||
"additionalProperties": {
|
||||
"$ref": "#/components/schemas/GuardrailKeyMetric"
|
||||
},
|
||||
"description": "Per-key guardrail spend and unpriced units, limited to keys in the day's API key breakdown",
|
||||
"title": "Api Key Breakdown",
|
||||
"type": "object"
|
||||
},
|
||||
"metadata": {
|
||||
"$ref": "#/components/schemas/GuardrailMetadata"
|
||||
"$ref": "#/components/schemas/GuardrailMetadata",
|
||||
"description": "Guardrail name information"
|
||||
},
|
||||
"metrics": {
|
||||
"$ref": "#/components/schemas/GuardrailSpendMetrics"
|
||||
"$ref": "#/components/schemas/GuardrailSpendMetrics",
|
||||
"description": "Guardrail spend and unpriced units across all matching API keys"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
|
|
@ -3865,11 +3872,13 @@
|
|||
"properties": {
|
||||
"spend": {
|
||||
"default": 0.0,
|
||||
"description": "Guardrail spend in USD, already included in key and model spend",
|
||||
"title": "Spend",
|
||||
"type": "number"
|
||||
},
|
||||
"untracked_units": {
|
||||
"default": 0,
|
||||
"description": "Guardrail units with no known price, excluded from spend",
|
||||
"title": "Untracked Units",
|
||||
"type": "integer"
|
||||
}
|
||||
|
|
|
|||
|
|
@ -667,6 +667,10 @@ class LiteLLMRoutes(enum.Enum):
|
|||
"/team/spend/by_user",
|
||||
# gateway request counts (SGR); deployment-wide, admin-only
|
||||
"/gateway/daily/activity",
|
||||
"/subscription_accounts/usage",
|
||||
"/subscription_accounts/new",
|
||||
"/subscription_accounts/update",
|
||||
"/subscription_accounts/delete",
|
||||
# model
|
||||
"/model/new",
|
||||
"/model/update",
|
||||
|
|
|
|||
|
|
@ -84,6 +84,7 @@ from litellm.proxy._types import (
|
|||
UserAPIKeyAuth,
|
||||
)
|
||||
from litellm.proxy.agent_endpoints.auth.agent_access_groups import (
|
||||
AgentAccessGroupCeiling,
|
||||
CeilingResolver,
|
||||
resolve_agent_access_group_ceiling,
|
||||
)
|
||||
|
|
@ -1684,6 +1685,7 @@ def get_actual_routes(allowed_routes: list) -> list:
|
|||
|
||||
KEY_END_USER_BUDGET_ID_METADATA_FIELD: Final = "end_user_budget_id"
|
||||
_KEY_METADATA_ADAPTER: Final[TypeAdapter[Mapping[str, object] | None]] = TypeAdapter(Mapping[str, object] | None)
|
||||
_OBJECTS_ADAPTER: Final[TypeAdapter[tuple[object, ...]]] = TypeAdapter(tuple[object, ...])
|
||||
|
||||
|
||||
def get_key_end_user_budget_id(key_metadata: Mapping[str, object] | None) -> str | None:
|
||||
|
|
@ -3455,28 +3457,35 @@ async def delete_cache_key_objects(
|
|||
proxy_logging_obj: ProxyLogging | None,
|
||||
) -> None:
|
||||
"""
|
||||
Evict a batch of key objects, for callers that delete keys in bulk rather than through
|
||||
`/key/delete`. Auth resolves a cached key object without re-reading its team, so a key left
|
||||
cached after its row is gone keeps buying access until its TTL expires.
|
||||
|
||||
Evicting locally only reaches this worker; `_delete_cache_key_object` itself broadcasts each
|
||||
token, so a deleted key left in a peer worker's in-memory cache still authenticates there until
|
||||
its TTL expires.
|
||||
Evict a batch of deleted key objects here and on every other worker. Auth resolves a cached key
|
||||
object without re-reading its team, so a key left cached after its row is gone keeps buying
|
||||
access until its TTL expires.
|
||||
|
||||
Best-effort per key: the rows are already deleted by the time this runs, so an unreachable
|
||||
cache backend must not abort the caller partway through its own cascade.
|
||||
|
||||
Every token is dropped from this worker's memory first, then from the auth cache and the
|
||||
internal usage cache in one batch each, so a deleted key stops authenticating here the instant
|
||||
its rows are gone instead of refilling from Redis on the next miss. Every other worker then
|
||||
gets the eviction in one pipelined broadcast: a publish per key would pass the publisher's
|
||||
pending cap on a bulk delete, and the dropped keys would keep authenticating there until their
|
||||
TTL.
|
||||
Every token is dropped from this worker's memory first, then from the auth cache's Redis layer,
|
||||
so a deleted key stops authenticating here the instant its rows are gone instead of refilling
|
||||
from Redis on the next miss. Every other worker then gets the eviction in one pipelined
|
||||
broadcast: a publish per key would pass the publisher's pending cap on a bulk delete, and the
|
||||
dropped keys would keep authenticating there until their TTL. The internal usage cache is not
|
||||
an auth refill source, so its batch delete runs beside that chain and a hung Redis costs one
|
||||
bounded wait there instead of another in line.
|
||||
"""
|
||||
evict_local(cache_keys=hashed_tokens, user_api_key_cache=user_api_key_cache)
|
||||
usage_eviction: Final = (
|
||||
()
|
||||
if proxy_logging_obj is None
|
||||
else (evict_shared(cache_keys=hashed_tokens, cache=proxy_logging_obj.internal_usage_cache.dual_cache),)
|
||||
)
|
||||
_ = await asyncio.gather(
|
||||
_evict_auth_cache_then_broadcast(hashed_tokens=hashed_tokens, user_api_key_cache=user_api_key_cache),
|
||||
*usage_eviction,
|
||||
)
|
||||
|
||||
|
||||
async def _evict_auth_cache_then_broadcast(hashed_tokens: Sequence[str], user_api_key_cache: UserApiKeyCache) -> None:
|
||||
await evict_shared(cache_keys=hashed_tokens, cache=user_api_key_cache)
|
||||
if proxy_logging_obj is not None:
|
||||
await evict_shared(cache_keys=hashed_tokens, cache=proxy_logging_obj.internal_usage_cache.dual_cache)
|
||||
await broadcast_evictions(cache_keys=hashed_tokens)
|
||||
|
||||
|
||||
|
|
@ -5085,6 +5094,36 @@ async def can_agent_call_model(
|
|||
)
|
||||
|
||||
|
||||
async def _agent_access_group_ceilings(
|
||||
valid_token: UserAPIKeyAuth, resolve_ceiling: CeilingResolver
|
||||
) -> tuple[AgentAccessGroupCeiling, ...]:
|
||||
if not valid_token.agent_id:
|
||||
return ()
|
||||
|
||||
from litellm.proxy.agent_endpoints.auth.agent_access_groups import resolve_managed_agent_ceilings
|
||||
|
||||
managed: Final = managed_agent_policy(valid_token)
|
||||
if managed is not None:
|
||||
return await resolve_managed_agent_ceilings(managed)
|
||||
unmanaged: Final = await resolve_ceiling(valid_token.agent_id)
|
||||
return (unmanaged,) if unmanaged is not None else ()
|
||||
|
||||
|
||||
def _managed_agent_model_grants(valid_token: UserAPIKeyAuth) -> frozenset[str]:
|
||||
managed: Final = managed_agent_policy(valid_token)
|
||||
granted: Final = (managed.object_permission or {}).get("models") if managed is not None else None
|
||||
if not isinstance(granted, (list, tuple)):
|
||||
return frozenset()
|
||||
return frozenset(name for name in _OBJECTS_ADAPTER.validate_python(granted) if isinstance(name, str))
|
||||
|
||||
|
||||
async def agent_model_grants(
|
||||
valid_token: UserAPIKeyAuth, resolve_ceiling: CeilingResolver = resolve_agent_access_group_ceiling
|
||||
) -> frozenset[str]:
|
||||
ceilings: Final = await _agent_access_group_ceilings(valid_token, resolve_ceiling)
|
||||
return _managed_agent_model_grants(valid_token).union(*(ceiling.models for ceiling in ceilings))
|
||||
|
||||
|
||||
async def _check_agent_access_group_model_access(
|
||||
model: str | list[str] | None, # mutable-ok: _can_object_call_model and the client message helper take list[str]
|
||||
valid_token: UserAPIKeyAuth | None,
|
||||
|
|
@ -5092,20 +5131,9 @@ async def _check_agent_access_group_model_access(
|
|||
resolve_ceiling: CeilingResolver = resolve_agent_access_group_ceiling,
|
||||
) -> Literal[True]:
|
||||
"""Attached groups naming no model deny every model; the empty allowlist in ``_can_object_call_model`` allows."""
|
||||
if not model or valid_token is None or not valid_token.agent_id:
|
||||
if not model or valid_token is None:
|
||||
return True
|
||||
|
||||
from litellm.proxy.agent_endpoints.auth.agent_access_groups import resolve_managed_agent_ceilings
|
||||
|
||||
managed: Final = managed_agent_policy(valid_token)
|
||||
unmanaged: Final = await resolve_ceiling(valid_token.agent_id) if managed is None else None
|
||||
ceilings: Final = (
|
||||
await resolve_managed_agent_ceilings(managed)
|
||||
if managed is not None
|
||||
else (unmanaged,)
|
||||
if unmanaged is not None
|
||||
else ()
|
||||
)
|
||||
ceilings: Final = await _agent_access_group_ceilings(valid_token, resolve_ceiling)
|
||||
dispatched: Final = _resolve_team_alias(
|
||||
model, team_model_aliases_for_auth_check(valid_token), valid_token.team_id, llm_router
|
||||
)
|
||||
|
|
|
|||
|
|
@ -2199,15 +2199,19 @@ class JWTAuthManager:
|
|||
# not mistaken for claimless and left with stale DB memberships the fallback
|
||||
# could later attribute. With the flag off, keep the upstream plural-only
|
||||
# reconciliation so existing deployments are unchanged.
|
||||
jwt_team_ids: Final = set(
|
||||
jwt_handler.get_all_jwt_team_ids(jwt_valid_token)
|
||||
if jwt_handler.litellm_jwtauth.fallback_to_db_teams
|
||||
else jwt_handler.get_team_ids_from_jwt(jwt_valid_token)
|
||||
claim_team_ids: Final = tuple(
|
||||
dict.fromkeys(
|
||||
jwt_handler.get_all_jwt_team_ids(jwt_valid_token)
|
||||
if jwt_handler.litellm_jwtauth.fallback_to_db_teams
|
||||
else jwt_handler.get_team_ids_from_jwt(jwt_valid_token)
|
||||
)
|
||||
)
|
||||
existing_teams: Final = set(user_object.teams or [])
|
||||
teams_to_add: Final = jwt_team_ids - existing_teams
|
||||
jwt_team_ids: Final = frozenset(claim_team_ids)
|
||||
existing_team_order: Final = tuple(dict.fromkeys(user_object.teams or []))
|
||||
existing_teams: Final = frozenset(existing_team_order)
|
||||
teams_to_add: Final = tuple(team_id for team_id in claim_team_ids if team_id not in existing_teams)
|
||||
preserve_db_teams_without_claims: Final = jwt_handler.litellm_jwtauth.fallback_to_db_teams and not jwt_team_ids
|
||||
teams_to_remove: Final = set() if preserve_db_teams_without_claims else existing_teams - jwt_team_ids
|
||||
teams_to_remove: Final = frozenset() if preserve_db_teams_without_claims else existing_teams - jwt_team_ids
|
||||
if teams_to_add or teams_to_remove:
|
||||
from litellm.proxy.management_endpoints.scim.scim_v2 import (
|
||||
patch_team_membership,
|
||||
|
|
@ -2218,7 +2222,10 @@ class JWTAuthManager:
|
|||
teams_ids_to_add_user_to=list(teams_to_add),
|
||||
teams_ids_to_remove_user_from=list(teams_to_remove),
|
||||
)
|
||||
user_object.teams = list(jwt_team_ids)
|
||||
user_object.teams = [
|
||||
*(team_id for team_id in existing_team_order if team_id not in teams_to_remove),
|
||||
*teams_to_add,
|
||||
]
|
||||
if user_api_key_cache is not None:
|
||||
await user_api_key_cache.async_set_cache(
|
||||
key=user_object.user_id,
|
||||
|
|
|
|||
|
|
@ -60,6 +60,7 @@ from litellm.litellm_core_utils.bug_report import (
|
|||
strip_bug_report_notice,
|
||||
)
|
||||
from litellm.litellm_core_utils.core_helpers import (
|
||||
finite_cost,
|
||||
get_or_create_metadata_bucket,
|
||||
independent_snapshot,
|
||||
is_expected_client_error,
|
||||
|
|
@ -140,6 +141,7 @@ from litellm.proxy.utils import ( # noqa: F401 # legacy module exports
|
|||
_check_and_merge_model_level_guardrails, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export
|
||||
check_and_merge_model_level_guardrails,
|
||||
)
|
||||
from litellm.responses.mcp.parked_stream_dispatch import in_flight_round_biller, parked_success
|
||||
from litellm.responses.streaming_iterator import partial_responses_incomplete_event_on_disconnect, spawn_logging_task
|
||||
from litellm.router import Router
|
||||
from litellm.router_api import RoutingContext
|
||||
|
|
@ -535,6 +537,16 @@ async def _bill_partial_streamed_spend_on_disconnect(request_data: dict, respons
|
|||
return await _dispatch_disconnect_billing_event(logging_obj, partial_response)
|
||||
|
||||
|
||||
async def _bill_in_flight_round_on_disconnect(request_data: Mapping[str, object]) -> None:
|
||||
bill_in_flight_round: Final = in_flight_round_biller(request_data.get("litellm_logging_obj"))
|
||||
if bill_in_flight_round is None:
|
||||
return
|
||||
try:
|
||||
await bill_in_flight_round()
|
||||
except Exception as e: # noqa: BLE001 # disconnect billing is best-effort; never break stream teardown
|
||||
verbose_proxy_logger.debug("Failed to bill the in-flight round after client disconnect: %s", e)
|
||||
|
||||
|
||||
async def _dispatch_disconnect_billing_event(logging_obj: LiteLLMLoggingObj, partial_response: object) -> bool:
|
||||
try:
|
||||
await logging_obj.dispatch_success_handlers(
|
||||
|
|
@ -1801,14 +1813,16 @@ def _get_cost_breakdown_from_logging_obj(
|
|||
|
||||
|
||||
def _header_value(value: float | None) -> str | None:
|
||||
return str(value) if value is not None else None
|
||||
return str(value) if finite_cost(value) is not None else None
|
||||
|
||||
|
||||
def _response_cost_header_value(hidden_params: Mapping[str, object]) -> float | str:
|
||||
if hidden_params.get("cache_hit") is True:
|
||||
return 0.0
|
||||
response_cost: Final = hidden_params.get("response_cost")
|
||||
return response_cost if isinstance(response_cost, (int, float, str)) and response_cost else ""
|
||||
if isinstance(response_cost, str):
|
||||
return response_cost
|
||||
return response_cost if isinstance(response_cost, int | float) and finite_cost(response_cost) else ""
|
||||
|
||||
|
||||
def cost_headers(
|
||||
|
|
@ -2083,6 +2097,13 @@ class _ClientStreamFailureReport:
|
|||
)
|
||||
|
||||
|
||||
def _metadata_logged_on(logged_on: LiteLLMLoggingObj) -> Mapping[str, object]:
|
||||
logged_params: Final = object_mapping(logged_on.model_call_details.get("litellm_params")) or MappingProxyType({})
|
||||
return MappingProxyType(
|
||||
{key: logged_params[key] for key in ("metadata", "litellm_metadata") if key in logged_params}
|
||||
)
|
||||
|
||||
|
||||
class ProxyBaseLLMRequestProcessing:
|
||||
def __init__(self, data: dict, served_model: str | None = None):
|
||||
self.data = data
|
||||
|
|
@ -4151,11 +4172,21 @@ class ProxyBaseLLMRequestProcessing:
|
|||
_captured_user_api_key_dict: Final = user_api_key_dict
|
||||
_captured_logging_obj: Final = logging_obj
|
||||
|
||||
async def _on_deferred_stream_complete(assembled_response: object, cache_hit: object) -> None:
|
||||
async def _on_deferred_stream_complete(
|
||||
assembled_response: object, cache_hit: object, logged_on: LiteLLMLoggingObj | None = None
|
||||
) -> None:
|
||||
await ProxyBaseLLMRequestProcessing._run_deferred_stream_guardrails(
|
||||
captured_data=_captured_data,
|
||||
captured_data=_captured_data
|
||||
if logged_on is None
|
||||
else {
|
||||
**_captured_data,
|
||||
**{
|
||||
key: value for key, value in _metadata_logged_on(logged_on).items() if key in _captured_data
|
||||
},
|
||||
"litellm_logging_obj": logged_on,
|
||||
},
|
||||
captured_user_api_key_dict=_captured_user_api_key_dict,
|
||||
captured_logging_obj=_captured_logging_obj,
|
||||
captured_logging_obj=_captured_logging_obj if logged_on is None else logged_on,
|
||||
assembled_response=assembled_response,
|
||||
cache_hit=cache_hit,
|
||||
)
|
||||
|
|
@ -4180,14 +4211,8 @@ class ProxyBaseLLMRequestProcessing:
|
|||
GLOBAL_LOGGING_WORKER.ensure_initialized_and_enqueue(async_coroutine=logging_coroutine)
|
||||
case (logging_coroutine,) if asyncio.iscoroutine(logging_coroutine):
|
||||
GLOBAL_LOGGING_WORKER.ensure_initialized_and_enqueue(async_coroutine=logging_coroutine)
|
||||
case (assembled_response, cache_hit):
|
||||
await _as_success_dispatcher(_captured_native_logging_obj).dispatch_success_handlers(
|
||||
assembled_response,
|
||||
cache_hit=cache_hit,
|
||||
start_time=None,
|
||||
end_time=None,
|
||||
prefer_async_handlers=True,
|
||||
)
|
||||
case _ if (success := parked_success(args)) is not None:
|
||||
await success.dispatch(_captured_native_logging_obj)
|
||||
case _:
|
||||
verbose_proxy_logger.error(
|
||||
"Deferred stream logging dropped: unexpected stored args shape %s",
|
||||
|
|
@ -4487,7 +4512,7 @@ class ProxyBaseLLMRequestProcessing:
|
|||
"""
|
||||
if isinstance(chunk, dict):
|
||||
# Use safe_dumps for proper JSON serialization with circular reference detection
|
||||
chunk_str: Final = safe_dumps(chunk)
|
||||
chunk_str: Final = safe_dumps(chunk, non_finite_as_null=True)
|
||||
return f"{STREAM_SSE_DATA_PREFIX}{chunk_str}\n\n"
|
||||
else:
|
||||
return chunk
|
||||
|
|
@ -4524,6 +4549,7 @@ class ProxyBaseLLMRequestProcessing:
|
|||
client_disconnected,
|
||||
)
|
||||
if recorded_client_disconnect:
|
||||
await _bill_in_flight_round_on_disconnect(_REQUEST_FIELDS.validate_python(request_data))
|
||||
deferred_stream_logging_armed: Final = _deferred_stream_logging_is_armed(request_data)
|
||||
ProxyLogging.fire_deferred_stream_logging(request_data)
|
||||
# A disconnect-time success event (the deferred-guardrail flush
|
||||
|
|
@ -4862,7 +4888,11 @@ class ProxyBaseLLMRequestProcessing:
|
|||
obj, model_name, litellm_logging_obj
|
||||
)
|
||||
if maybe_modified is not None:
|
||||
lines[idx] = "data: " + safe_dumps(maybe_modified) + ("\r" if ln.endswith("\r") else "")
|
||||
lines[idx] = (
|
||||
"data: "
|
||||
+ safe_dumps(maybe_modified, non_finite_as_null=True)
|
||||
+ ("\r" if ln.endswith("\r") else "")
|
||||
)
|
||||
return "\n".join(lines)
|
||||
return None
|
||||
except Exception:
|
||||
|
|
|
|||
|
|
@ -2,9 +2,12 @@ import asyncio
|
|||
import json
|
||||
import random
|
||||
import time
|
||||
from collections.abc import Awaitable, Callable
|
||||
from collections.abc import AsyncIterator, Awaitable, Callable
|
||||
from dataclasses import asdict, dataclass
|
||||
from typing import TYPE_CHECKING, Final, Protocol, cast # noqa: TID251 # untyped prisma/redis boundary needs cast
|
||||
from itertools import chain
|
||||
from typing import TYPE_CHECKING, Final, Protocol, TypeAlias, cast # noqa: TID251 # untyped prisma/redis boundary
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError
|
||||
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
from litellm.repositories.prisma_protocols import RowT_co, TableActions
|
||||
|
|
@ -106,24 +109,69 @@ _pubsub_capable_client: Final = pubsub_capable_client
|
|||
@dataclass(frozen=True, slots=True)
|
||||
class _ConfigChangeMessage:
|
||||
object_type: str
|
||||
model_ids: tuple[str, ...] = ()
|
||||
|
||||
|
||||
def _config_change_message_json(object_type: str) -> str:
|
||||
return json.dumps(asdict(_ConfigChangeMessage(object_type=object_type)))
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _PubSubEnvelope:
|
||||
data: str | bytes
|
||||
|
||||
|
||||
async def publish_config_change(redis_cache: "RedisCache | None", object_type: str) -> None:
|
||||
class _WrittenModelRow(BaseModel):
|
||||
model_config = ConfigDict(from_attributes=True, frozen=True)
|
||||
|
||||
model_id: str
|
||||
|
||||
|
||||
_CONFIG_CHANGE_MESSAGE: Final = TypeAdapter(_ConfigChangeMessage)
|
||||
_PUBSUB_ENVELOPE: Final = TypeAdapter(_PubSubEnvelope)
|
||||
_MODEL_TABLE_NAME: Final = "litellm_proxymodeltable"
|
||||
_TARGETED_WRITE_ACTION_NAMES: Final[frozenset[str]] = frozenset({"create", "update", "upsert"})
|
||||
|
||||
PublishConfigChange: TypeAlias = Callable[[str, tuple[str, ...]], Awaitable[None]]
|
||||
|
||||
|
||||
def _config_change_message_json(object_type: str, model_ids: tuple[str, ...]) -> str:
|
||||
return json.dumps(asdict(_ConfigChangeMessage(object_type=object_type, model_ids=model_ids)))
|
||||
|
||||
|
||||
def parse_config_change(message: object) -> _ConfigChangeMessage | None:
|
||||
try:
|
||||
payload: Final = (
|
||||
message if isinstance(message, (str, bytes)) else _PUBSUB_ENVELOPE.validate_python(message).data
|
||||
)
|
||||
return _CONFIG_CHANGE_MESSAGE.validate_json(payload)
|
||||
except ValidationError:
|
||||
return None
|
||||
|
||||
|
||||
def written_model_ids(object_type: str, action_name: str, result: object) -> tuple[str, ...]:
|
||||
if object_type != _MODEL_TABLE_NAME or action_name not in _TARGETED_WRITE_ACTION_NAMES:
|
||||
return ()
|
||||
try:
|
||||
return (_WrittenModelRow.model_validate(result).model_id,)
|
||||
except ValidationError:
|
||||
return ()
|
||||
|
||||
|
||||
async def publish_config_change(
|
||||
redis_cache: "RedisCache | None", object_type: str, model_ids: tuple[str, ...] = ()
|
||||
) -> None:
|
||||
if redis_cache is None:
|
||||
return
|
||||
try:
|
||||
client: Final = pubsub_capable_client(redis_cache)
|
||||
await client.publish(config_sync_channel(redis_cache), _config_change_message_json(object_type))
|
||||
await client.publish(config_sync_channel(redis_cache), _config_change_message_json(object_type, model_ids))
|
||||
except Exception as e: # noqa: BLE001 # best-effort publish; writes must never fail on redis errors
|
||||
verbose_proxy_logger.warning("config sync publish for %s failed: %s", object_type, e)
|
||||
|
||||
|
||||
async def publish_config_change_for_object_type(object_type: str) -> None:
|
||||
await publish_config_change(redis_cache=coordination_redis_cache(), object_type=object_type)
|
||||
async def publish_config_change_for_object_type(object_type: str, model_ids: tuple[str, ...] = ()) -> None:
|
||||
await publish_config_change(redis_cache=coordination_redis_cache(), object_type=object_type, model_ids=model_ids)
|
||||
|
||||
|
||||
async def publish_model_rows_written(model_ids: tuple[str, ...]) -> None:
|
||||
await publish_config_change_for_object_type(_MODEL_TABLE_NAME, model_ids)
|
||||
|
||||
|
||||
async def publish_config_param_change(param_name: str) -> None:
|
||||
|
|
@ -139,7 +187,7 @@ async def publish_config_param_change(param_name: str) -> None:
|
|||
class _PublishOnWriteActions:
|
||||
__slots__ = ("_actions", "_object_type", "_publish")
|
||||
|
||||
def __init__(self, actions: object, object_type: str, publish: Callable[[str], Awaitable[None]]) -> None:
|
||||
def __init__(self, actions: object, object_type: str, publish: PublishConfigChange) -> None:
|
||||
self._actions = actions
|
||||
self._object_type = object_type
|
||||
self._publish = publish
|
||||
|
|
@ -157,7 +205,7 @@ class _PublishOnWriteActions:
|
|||
**kwargs: object, # kwargs-ok: transparent passthrough to untyped prisma action
|
||||
) -> object:
|
||||
result: Final = await write_action(*args, **kwargs)
|
||||
await publish(object_type)
|
||||
await publish(object_type, written_model_ids(object_type, name, result))
|
||||
return result
|
||||
|
||||
return _write_then_publish
|
||||
|
|
@ -166,7 +214,7 @@ class _PublishOnWriteActions:
|
|||
def wrap_table_actions_for_config_sync(
|
||||
actions: "TableActions[RowT_co]",
|
||||
table_name: str,
|
||||
publish: Callable[[str], Awaitable[None]] = publish_config_change_for_object_type,
|
||||
publish: PublishConfigChange = publish_config_change_for_object_type,
|
||||
) -> "TableActions[RowT_co]":
|
||||
if table_name not in _CONFIG_SYNCED_TABLE_NAMES:
|
||||
return actions
|
||||
|
|
@ -174,8 +222,14 @@ def wrap_table_actions_for_config_sync(
|
|||
return cast("TableActions[RowT_co]", wrapped) # cast-ok: dynamic write-through proxy keeps the wrapped row type
|
||||
|
||||
|
||||
def _published_model_ids(messages: tuple[object, ...]) -> tuple[str, ...]:
|
||||
changes: Final = (parse_config_change(message) for message in messages)
|
||||
return tuple(dict.fromkeys(chain.from_iterable(change.model_ids for change in changes if change is not None)))
|
||||
|
||||
|
||||
class ConfigSyncSubscriber:
|
||||
__slots__ = (
|
||||
"_apply_model_ids",
|
||||
"_backoff_initial_seconds",
|
||||
"_backoff_max_seconds",
|
||||
"_debounce_seconds",
|
||||
|
|
@ -195,6 +249,7 @@ class ConfigSyncSubscriber:
|
|||
self,
|
||||
redis_cache: "RedisCache",
|
||||
resync_callbacks: tuple[Callable[[], Awaitable[None]], ...],
|
||||
apply_model_ids: Callable[[tuple[str, ...]], Awaitable[None]] | None = None,
|
||||
debounce_seconds: float = CONFIG_SYNC_DEBOUNCE_SECONDS,
|
||||
jitter_max_seconds: float = CONFIG_SYNC_JITTER_MAX_SECONDS,
|
||||
min_resync_interval_seconds: float = CONFIG_SYNC_MIN_RESYNC_INTERVAL_SECONDS,
|
||||
|
|
@ -206,6 +261,7 @@ class ConfigSyncSubscriber:
|
|||
) -> None:
|
||||
self._redis_cache = redis_cache
|
||||
self._resync_callbacks = resync_callbacks
|
||||
self._apply_model_ids = apply_model_ids
|
||||
self._debounce_seconds = debounce_seconds
|
||||
self._jitter_max_seconds = jitter_max_seconds
|
||||
self._min_resync_interval_seconds = min_resync_interval_seconds
|
||||
|
|
@ -266,32 +322,64 @@ class ConfigSyncSubscriber:
|
|||
message = await pubsub.get_message(ignore_subscribe_messages=True, timeout=_POLL_TIMEOUT_SECONDS)
|
||||
if message is None:
|
||||
continue
|
||||
await self._apply(pubsub, message)
|
||||
await self._debounce_and_resync(pubsub)
|
||||
|
||||
async def _debounce_and_resync(self, pubsub: _ConfigSyncPubSub) -> None:
|
||||
await self._sleep(self._debounce_seconds + self._rng.uniform(0.0, self._jitter_max_seconds))
|
||||
await self._wait_for_min_resync_interval()
|
||||
async def _debounce_and_resync(self, pubsub: ConfigSyncPubSub) -> None:
|
||||
await self._apply_until(
|
||||
pubsub, self._monotonic() + self._debounce_seconds + self._rng.uniform(0.0, self._jitter_max_seconds)
|
||||
)
|
||||
await self._apply_until(pubsub, self._next_resync_at())
|
||||
await self._drain_pending(pubsub)
|
||||
await self._run_resync_callbacks()
|
||||
self._last_resync_at = self._monotonic()
|
||||
|
||||
async def _wait_for_min_resync_interval(self) -> None:
|
||||
def _next_resync_at(self) -> float:
|
||||
if self._last_resync_at is None:
|
||||
return
|
||||
seconds_until_next_resync = self._min_resync_interval_seconds - (self._monotonic() - self._last_resync_at)
|
||||
if seconds_until_next_resync <= 0:
|
||||
return
|
||||
verbose_proxy_logger.debug(
|
||||
"config sync resync throttled for %.1fs to cap fleet-wide reload rate",
|
||||
seconds_until_next_resync,
|
||||
)
|
||||
await self._sleep(seconds_until_next_resync)
|
||||
return self._monotonic()
|
||||
next_resync_at: Final = self._last_resync_at + self._min_resync_interval_seconds
|
||||
throttled_for: Final = next_resync_at - self._monotonic()
|
||||
if throttled_for > 0:
|
||||
verbose_proxy_logger.debug(
|
||||
"config sync resync throttled for %.1fs to cap fleet-wide reload rate",
|
||||
throttled_for,
|
||||
)
|
||||
return next_resync_at
|
||||
|
||||
async def _apply_until(self, pubsub: ConfigSyncPubSub, deadline: float) -> None:
|
||||
while True:
|
||||
remaining = deadline - self._monotonic()
|
||||
if remaining <= 0:
|
||||
return
|
||||
message = await pubsub.get_message(
|
||||
ignore_subscribe_messages=True, timeout=min(remaining, _POLL_TIMEOUT_SECONDS)
|
||||
)
|
||||
if message is not None:
|
||||
await self._apply(pubsub, message)
|
||||
|
||||
@staticmethod
|
||||
async def _drain_pending(pubsub: ConfigSyncPubSub) -> None:
|
||||
while await pubsub.get_message(ignore_subscribe_messages=True, timeout=0) is not None:
|
||||
pass
|
||||
|
||||
@staticmethod
|
||||
async def _ready_messages(pubsub: ConfigSyncPubSub) -> AsyncIterator[object]:
|
||||
while (message := await pubsub.get_message(ignore_subscribe_messages=True, timeout=0)) is not None:
|
||||
yield message
|
||||
|
||||
async def _apply(self, pubsub: ConfigSyncPubSub, message: object) -> None:
|
||||
apply: Final = self._apply_model_ids
|
||||
if apply is None:
|
||||
return
|
||||
ready: Final = (message, *[ready_message async for ready_message in self._ready_messages(pubsub)])
|
||||
model_ids: Final = _published_model_ids(ready)
|
||||
if not model_ids:
|
||||
return
|
||||
try:
|
||||
await apply(model_ids)
|
||||
except Exception as e: # noqa: BLE001 # the debounced full resync retries what a targeted apply could not
|
||||
verbose_proxy_logger.warning("config sync targeted apply of %s failed: %s", model_ids, e)
|
||||
|
||||
async def _run_resync_callbacks(self) -> None:
|
||||
for callback in self._resync_callbacks:
|
||||
try:
|
||||
|
|
|
|||
|
|
@ -219,7 +219,7 @@ def caller_alias_maps(
|
|||
|
||||
def alias_map(aliases: object) -> Mapping[str, str]:
|
||||
try:
|
||||
entries: Final = _ALIAS_ENTRIES.validate_python(aliases, strict=True)
|
||||
entries: Final = _ALIAS_ENTRIES.validate_python(aliases)
|
||||
except ValidationError:
|
||||
return _NO_ALIASES
|
||||
return MappingProxyType(
|
||||
|
|
@ -287,6 +287,15 @@ def _dispatch(model_id: str, layers: Sequence[_AliasLayer]) -> tuple[str, bool]:
|
|||
return reduce(_through_layer, layers, (model_id, False))
|
||||
|
||||
|
||||
def dispatched_model(model_id: str, aliases: CallerAliases) -> str:
|
||||
target, _ = _dispatch(model_id, _alias_layers(aliases))
|
||||
return target
|
||||
|
||||
|
||||
def requestable_alias_names(aliases: CallerAliases) -> frozenset[str]:
|
||||
return frozenset(chain.from_iterable(layer.aliases for layer in _alias_layers(aliases)))
|
||||
|
||||
|
||||
def _lookup_id(name: str, lookup_by_response: Mapping[str, str], lookup_ids: Container[str]) -> str | None:
|
||||
if name in lookup_by_response:
|
||||
return lookup_by_response[name]
|
||||
|
|
|
|||
|
|
@ -28,6 +28,8 @@ _OPENAI_ERROR_TYPE_BY_STATUS: Final[Mapping[int, str]] = MappingProxyType(
|
|||
|
||||
_AUTH_FAILURE_STATUS_CODES: Final = frozenset({status.HTTP_401_UNAUTHORIZED, status.HTTP_403_FORBIDDEN})
|
||||
|
||||
_HTTP_ERROR_STATUSES: Final = range(status.HTTP_400_BAD_REQUEST, 600)
|
||||
|
||||
_ERROR_BODY: Final = TypeAdapter(dict[str, object])
|
||||
|
||||
_FAILOVER_SQLSTATES: Final = frozenset({"25006", "57P01", "57P02", "57P03"})
|
||||
|
|
@ -47,13 +49,26 @@ def attribute_of(value: object, name: str, default: object = None) -> object:
|
|||
|
||||
|
||||
def error_status_code(exc: object, default: int) -> int:
|
||||
"""The HTTP status an exception carries as ``status_code`` or, the way ``ProxyException``
|
||||
stores it, as a stringified ``code``; ``default`` when it carries neither."""
|
||||
carried: Final = attribute_of(exc, "status_code")
|
||||
if isinstance(carried, int) and not isinstance(carried, bool):
|
||||
"""The 4xx or 5xx HTTP status an exception carries as ``status_code`` (an int or a decimal string) or,
|
||||
the way ``ProxyException`` stores it, as a stringified ``code``; ``default`` when it carries neither."""
|
||||
carried: Final = _http_error_status(attribute_of(exc, "status_code"))
|
||||
if carried is not None:
|
||||
return carried
|
||||
stringified: Final = attribute_of(exc, "code")
|
||||
return int(stringified) if isinstance(stringified, str) and stringified.isdecimal() else default
|
||||
from_code: Final = _http_error_status(stringified) if isinstance(stringified, str) else None
|
||||
return default if from_code is None else from_code
|
||||
|
||||
|
||||
def _http_error_status(value: object) -> int | None:
|
||||
match value:
|
||||
case bool():
|
||||
return None
|
||||
case int() if value in _HTTP_ERROR_STATUSES:
|
||||
return value
|
||||
case str() if len(value) == 3 and value.isascii() and value.isdigit() and int(value) in _HTTP_ERROR_STATUSES:
|
||||
return int(value)
|
||||
case _:
|
||||
return None
|
||||
|
||||
|
||||
def openai_error_type(exc: object, status_code: int) -> str:
|
||||
|
|
@ -105,9 +120,9 @@ def proxy_exception_for(exc: Exception, default_status_code: int) -> ProxyExcept
|
|||
code=status.HTTP_503_SERVICE_UNAVAILABLE,
|
||||
)
|
||||
case HTTPException(status_code=status_code, detail=detail):
|
||||
return _openai_shaped_proxy_exception(exc, str(detail), status_code)
|
||||
return openai_shaped_proxy_exception(exc, str(detail), status_code)
|
||||
case _:
|
||||
return _openai_shaped_proxy_exception(
|
||||
return openai_shaped_proxy_exception(
|
||||
exc, str(exc) or type(exc).__name__, error_status_code(exc, default_status_code)
|
||||
)
|
||||
|
||||
|
|
@ -176,14 +191,19 @@ def _route_failure_error_type(exc: Exception, raised_status: int, answered_statu
|
|||
return openai_error_type(exc, answered_status)
|
||||
|
||||
|
||||
def _openai_shaped_proxy_exception(exc: Exception, message: str, status_code: int) -> ProxyException:
|
||||
def openai_shaped_proxy_exception(exc: Exception, message: str, status_code: int) -> ProxyException:
|
||||
answered_status: Final = answered_status_code(message, status_code)
|
||||
return ProxyException(
|
||||
message=message,
|
||||
type=_catch_all_error_type(exc, answered_status),
|
||||
param=openai_error_param(exc),
|
||||
code=answered_status,
|
||||
)
|
||||
return _proxy_exception(exc, message, answered_status, _catch_all_error_type(exc, answered_status))
|
||||
|
||||
|
||||
def status_typed_proxy_exception(exc: HTTPException) -> ProxyException:
|
||||
message: Final = str(exc.detail)
|
||||
answered_status: Final = answered_status_code(message, exc.status_code)
|
||||
return _proxy_exception(exc, message, answered_status, openai_error_type(exc, answered_status))
|
||||
|
||||
|
||||
def _proxy_exception(exc: Exception, message: str, answered_status: int, error_type: str) -> ProxyException:
|
||||
return ProxyException(message=message, type=error_type, param=openai_error_param(exc), code=answered_status)
|
||||
|
||||
|
||||
def _catch_all_error_type(exc: Exception, status_code: int) -> str:
|
||||
|
|
|
|||
|
|
@ -15,12 +15,13 @@ or the first routed request on this replica would pick a tier it has not loaded.
|
|||
import asyncio
|
||||
import time
|
||||
from collections.abc import Awaitable, Callable, Mapping, Sequence
|
||||
from typing import TYPE_CHECKING, Final
|
||||
from typing import TYPE_CHECKING, Final, Literal, TypeAlias
|
||||
|
||||
from pydantic import TypeAdapter, ValidationError
|
||||
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
from litellm.caching.in_memory_cache import InMemoryCache
|
||||
from litellm.litellm_core_utils.credential_accessor import CredentialAccessor
|
||||
from litellm.proxy.common_utils.encrypt_decrypt_utils import decrypt_value_helper
|
||||
|
||||
if TYPE_CHECKING:
|
||||
|
|
@ -33,8 +34,14 @@ if TYPE_CHECKING:
|
|||
)
|
||||
|
||||
from litellm.integrations.custom_guardrail import CustomGuardrail
|
||||
from litellm.repositories.prisma_protocols import TableActions
|
||||
from litellm.types.agents import AgentResponse
|
||||
|
||||
_ModelTable: TypeAlias = TableActions[prisma_models.LiteLLM_ProxyModelTable]
|
||||
_ModelRows: TypeAlias = Sequence[prisma_models.LiteLLM_ProxyModelTable]
|
||||
|
||||
ModelLoadOutcome: TypeAlias = Literal["loaded", "missing", "no_router", "auto_router"]
|
||||
|
||||
READ_THROUGH_MISS_TTL_SECONDS: Final = 2.0
|
||||
READ_THROUGH_RESYNC_WINDOW_SECONDS: Final = 5.0
|
||||
READ_THROUGH_MAX_RESYNCS_PER_WINDOW: Final = 20
|
||||
|
|
@ -123,47 +130,114 @@ async def _resync_model_deployments(model_name: str) -> bool:
|
|||
return False
|
||||
prisma_client: Final = proxy_server.prisma_client
|
||||
assert prisma_client is not None
|
||||
table: Final = ModelRepository(prisma_client).table
|
||||
name_filter: Final[LiteLLM_ProxyModelTableWhereInput] = {"model_name": model_name}
|
||||
id_filter: Final[LiteLLM_ProxyModelTableWhereInput] = {"model_id": model_name}
|
||||
rows: Final = await table.find_many(where=name_filter) or await table.find_many(where=id_filter)
|
||||
rows: Final = await _find_rows_by_name_or_id(model_name, ModelRepository(prisma_client).table)
|
||||
if not rows:
|
||||
return False
|
||||
router: Final = proxy_server.llm_router
|
||||
if router is None:
|
||||
await proxy_server.proxy_config.add_deployment(
|
||||
prisma_client=prisma_client, proxy_logging_obj=proxy_server.proxy_logging_obj
|
||||
)
|
||||
return proxy_server.llm_router is not None
|
||||
if _holds_an_auto_router(rows):
|
||||
await proxy_server.proxy_config.add_deployment(
|
||||
prisma_client=prisma_client, proxy_logging_obj=proxy_server.proxy_logging_obj
|
||||
)
|
||||
return _model_is_loaded(model_name)
|
||||
async with proxy_server.MODEL_RECONCILE_LOCK:
|
||||
outcome: Final = await _install_model_rows(rows, _always)
|
||||
if outcome in ("no_router", "auto_router"):
|
||||
await _reload_every_model()
|
||||
return outcome == "loaded" or _model_is_loaded(model_name)
|
||||
|
||||
|
||||
async def resync_model_deployments_by_ids(model_ids: tuple[str, ...]) -> None:
|
||||
from litellm.proxy import proxy_server
|
||||
from litellm.proxy.db.routing_prisma_wrapper import WriterPinnedClient
|
||||
from litellm.repositories.model_repository import ModelRepository
|
||||
|
||||
if not _db_backed_registries_enabled("models"):
|
||||
return
|
||||
prisma_client: Final = proxy_server.prisma_client
|
||||
assert prisma_client is not None
|
||||
writer_table: Final = ModelRepository(WriterPinnedClient(prisma_client.db)).table
|
||||
async with proxy_server.MODEL_RECONCILE_LOCK:
|
||||
rows: Final = await _find_rows_by_ids(model_ids, writer_table)
|
||||
_ = await _install_model_rows(_without_auto_routers(rows), _names_an_unloaded_credential)
|
||||
|
||||
|
||||
async def _find_rows_by_name_or_id(model_name: str, table: "_ModelTable") -> "_ModelRows":
|
||||
name_filter: Final[LiteLLM_ProxyModelTableWhereInput] = {"model_name": model_name}
|
||||
id_filter: Final[LiteLLM_ProxyModelTableWhereInput] = {"model_id": model_name}
|
||||
return await table.find_many(where=name_filter) or await table.find_many(where=id_filter)
|
||||
|
||||
|
||||
async def _find_rows_by_ids(model_ids: tuple[str, ...], table: "_ModelTable") -> "_ModelRows":
|
||||
from litellm.repositories.chunked_in import find_many_in
|
||||
|
||||
return list(await find_many_in(table, "model_id", model_ids))
|
||||
|
||||
|
||||
async def _install_model_rows(
|
||||
rows: "_ModelRows",
|
||||
needs_credentials: Callable[["_ModelRows"], bool],
|
||||
) -> ModelLoadOutcome:
|
||||
from litellm.proxy import proxy_server
|
||||
|
||||
router: Final = proxy_server.llm_router
|
||||
if not rows:
|
||||
return "missing"
|
||||
if router is None:
|
||||
return "no_router"
|
||||
if _holds_an_auto_router(rows):
|
||||
return "auto_router"
|
||||
if needs_credentials(rows):
|
||||
prisma_client: Final = proxy_server.prisma_client
|
||||
assert prisma_client is not None
|
||||
await proxy_server.proxy_config.get_credentials(prisma_client=prisma_client)
|
||||
proxy_server.proxy_config._add_deployment(db_models=rows)
|
||||
proxy_server.llm_model_list = router.registry().deployments()
|
||||
proxy_server.proxy_config._add_deployment(db_models=rows)
|
||||
proxy_server.llm_model_list = router.registry().deployments()
|
||||
return "loaded"
|
||||
|
||||
|
||||
async def _reload_every_model() -> None:
|
||||
from litellm.proxy import proxy_server
|
||||
|
||||
prisma_client: Final = proxy_server.prisma_client
|
||||
assert prisma_client is not None
|
||||
await proxy_server.proxy_config.add_deployment(
|
||||
prisma_client=prisma_client, proxy_logging_obj=proxy_server.proxy_logging_obj
|
||||
)
|
||||
|
||||
|
||||
def _always(rows: "_ModelRows") -> bool:
|
||||
return True
|
||||
|
||||
|
||||
def _holds_an_auto_router(rows: Sequence["prisma_models.LiteLLM_ProxyModelTable"]) -> bool:
|
||||
def _holds_an_auto_router(rows: "_ModelRows") -> bool:
|
||||
return any(_names_an_auto_router(row.litellm_params) for row in rows)
|
||||
|
||||
|
||||
def _without_auto_routers(rows: "_ModelRows") -> "_ModelRows":
|
||||
return list(row for row in rows if not _names_an_auto_router(row.litellm_params))
|
||||
|
||||
|
||||
def _names_an_unloaded_credential(rows: "_ModelRows") -> bool:
|
||||
return any(_names_an_unloaded_credential_param(row.litellm_params) for row in rows)
|
||||
|
||||
|
||||
_STORED_LITELLM_PARAMS: Final = TypeAdapter(Mapping[str, object])
|
||||
|
||||
|
||||
def _names_an_auto_router(stored_litellm_params: object) -> bool:
|
||||
def _decrypted_param(stored_litellm_params: object, key: str) -> str | None:
|
||||
try:
|
||||
params: Final = _STORED_LITELLM_PARAMS.validate_python(stored_litellm_params)
|
||||
except ValidationError:
|
||||
return False
|
||||
stored_model: Final = params.get("model")
|
||||
if not isinstance(stored_model, str):
|
||||
return False
|
||||
model: Final = decrypt_value_helper(value=stored_model, key="model", return_original_value=True)
|
||||
return isinstance(model, str) and model.startswith("auto_router/")
|
||||
return None
|
||||
stored_value: Final = params.get(key)
|
||||
if not isinstance(stored_value, str):
|
||||
return None
|
||||
value: Final = decrypt_value_helper(value=stored_value, key=key, return_original_value=True)
|
||||
return value if isinstance(value, str) else None
|
||||
|
||||
|
||||
def _names_an_auto_router(stored_litellm_params: object) -> bool:
|
||||
model: Final = _decrypted_param(stored_litellm_params, "model")
|
||||
return model is not None and model.startswith("auto_router/")
|
||||
|
||||
|
||||
def _names_an_unloaded_credential_param(stored_litellm_params: object) -> bool:
|
||||
credential_name: Final = _decrypted_param(stored_litellm_params, "litellm_credential_name")
|
||||
return credential_name is not None and CredentialAccessor.find_credential(credential_name) is None
|
||||
|
||||
|
||||
async def _resync_guardrails(guardrail_name: str) -> bool:
|
||||
|
|
|
|||
|
|
@ -34,6 +34,7 @@ from litellm.constants import (
|
|||
DB_SPEND_UPDATE_JOB_NAME,
|
||||
INTERNAL_CALL_ORIGIN_METADATA_KEY,
|
||||
)
|
||||
from litellm.litellm_core_utils.core_helpers import finite_cost, single_log_line
|
||||
from litellm.litellm_core_utils.litellm_logging import coerce_model_access_groups
|
||||
from litellm.litellm_core_utils.safe_json_loads import safe_json_loads
|
||||
from litellm.proxy._types import (
|
||||
|
|
@ -652,7 +653,14 @@ class DBSpendUpdateWriter:
|
|||
end_time=end_time,
|
||||
llm_router=llm_router,
|
||||
)
|
||||
payload["spend"] = response_cost or 0.0
|
||||
billable_cost: Final = finite_cost(response_cost)
|
||||
if response_cost is not None and billable_cost is None:
|
||||
verbose_proxy_logger.warning(
|
||||
"Spend tracking - recording a non-finite response_cost=%s as 0 for request_id=%s",
|
||||
response_cost,
|
||||
single_log_line(payload["request_id"]),
|
||||
)
|
||||
payload["spend"] = billable_cost or 0.0
|
||||
if isinstance(payload["startTime"], datetime):
|
||||
payload["startTime"] = payload["startTime"].isoformat()
|
||||
if isinstance(payload["endTime"], datetime):
|
||||
|
|
@ -702,7 +710,7 @@ class DBSpendUpdateWriter:
|
|||
spawn_logging_task(
|
||||
asyncio.get_running_loop(),
|
||||
self._batch_database_updates(
|
||||
response_cost=response_cost,
|
||||
response_cost=billable_cost,
|
||||
user_id=user_id,
|
||||
hashed_token=hashed_token,
|
||||
team_id=team_id,
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@ import asyncio
|
|||
import json
|
||||
from collections.abc import Mapping, Sequence
|
||||
from datetime import datetime
|
||||
from functools import reduce
|
||||
from functools import partial, reduce
|
||||
from itertools import chain
|
||||
from typing import TYPE_CHECKING, Any, Final, Literal, TypeAlias, TypeVar, cast
|
||||
|
||||
|
|
@ -109,6 +109,16 @@ _ValueT = TypeVar("_ValueT")
|
|||
_DECODED_JSON: Final = TypeAdapter(object)
|
||||
|
||||
|
||||
def _zero_for_non_finite_constant(constant: str) -> float:
|
||||
verbose_proxy_logger.warning(
|
||||
"Redis spend buffer - reading a parked %s spend as 0, the spend it held is lost", constant
|
||||
)
|
||||
return 0.0
|
||||
|
||||
|
||||
_load_buffered_transaction: Final = partial(json.loads, parse_constant=_zero_for_non_finite_constant)
|
||||
|
||||
|
||||
def _spend_log_json_default(value: object) -> str:
|
||||
return value.isoformat() if isinstance(value, datetime) else str(value)
|
||||
|
||||
|
|
@ -118,7 +128,7 @@ def _encode_spend_log_row(row: SpendLogRow) -> str:
|
|||
|
||||
|
||||
def _decode_spend_log_row(encoded: str) -> dict[str, object] | None:
|
||||
decoded: Final = _DECODED_JSON.validate_python(json.loads(encoded))
|
||||
decoded: Final = _DECODED_JSON.validate_python(_load_buffered_transaction(encoded))
|
||||
return decoded if isinstance(decoded, dict) else None
|
||||
|
||||
|
||||
|
|
@ -825,13 +835,13 @@ class RedisUpdateBuffer:
|
|||
if slot_result is None:
|
||||
daily_results.append(None)
|
||||
else:
|
||||
list_of_daily = [json.loads(t) for t in slot_result]
|
||||
list_of_daily = [_load_buffered_transaction(t) for t in slot_result]
|
||||
aggregated = DailySpendUpdateQueue.get_aggregated_daily_spend_update_transactions(list_of_daily)
|
||||
daily_results.append(aggregated)
|
||||
|
||||
window_spend: Final = (
|
||||
WindowSpendUpdateQueue.get_aggregated_window_spend_transactions(
|
||||
tuple(json.loads(transaction) for transaction in raw_results[6])
|
||||
tuple(_load_buffered_transaction(transaction) for transaction in raw_results[6])
|
||||
)
|
||||
if raw_results[6] is not None
|
||||
else None
|
||||
|
|
@ -880,7 +890,7 @@ class RedisUpdateBuffer:
|
|||
)
|
||||
if list_of_transactions is None:
|
||||
return None
|
||||
return [json.loads(transaction) for transaction in list_of_transactions]
|
||||
return [_load_buffered_transaction(transaction) for transaction in list_of_transactions]
|
||||
|
||||
async def get_all_daily_spend_update_transactions_from_redis_buffer(
|
||||
self,
|
||||
|
|
@ -1010,9 +1020,9 @@ class RedisUpdateBuffer:
|
|||
Parses the list of transactions from Redis
|
||||
"""
|
||||
if isinstance(list_of_transactions, list):
|
||||
return [json.loads(transaction) for transaction in list_of_transactions]
|
||||
return [_load_buffered_transaction(transaction) for transaction in list_of_transactions]
|
||||
else:
|
||||
return [json.loads(list_of_transactions)]
|
||||
return [_load_buffered_transaction(list_of_transactions)]
|
||||
|
||||
@staticmethod
|
||||
def _combine_list_of_transactions(
|
||||
|
|
|
|||
|
|
@ -22,6 +22,7 @@ from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
|
|||
from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth
|
||||
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
|
||||
from litellm.proxy.common_utils.callback_utils import CALLBACK_VAR_ENCRYPTED_PREFIX
|
||||
from litellm.proxy.common_utils.openai_error_payload import status_typed_proxy_exception
|
||||
from litellm.proxy.common_utils.path_utils import is_within, safe_join
|
||||
from litellm.proxy.guardrails.content_filter_data import CATEGORIES_DIR, DATA_ROOTS, category_dirs, find_category_file
|
||||
from litellm.proxy.guardrails.guardrail_hooks.custom_code.bounded_execution import (
|
||||
|
|
@ -2533,7 +2534,7 @@ async def apply_guardrail(
|
|||
e = transformed_exception
|
||||
except Exception:
|
||||
verbose_proxy_logger.exception("apply_guardrail: post_call_failure_hook failed")
|
||||
raise handle_exception_on_proxy(e)
|
||||
raise handle_exception_on_proxy(status_typed_proxy_exception(e) if isinstance(e, HTTPException) else e)
|
||||
|
||||
# Success logging outside except so a hook error never triggers failure handlers.
|
||||
response = await _emit_guardrail_success_logs(
|
||||
|
|
|
|||
|
|
@ -32,7 +32,7 @@ from litellm.router_strategy.auto_router.auto_router_model_naming import (
|
|||
classify_strategy_router_model,
|
||||
strategy_router_dependencies,
|
||||
)
|
||||
from litellm.router_utils.common_utils import is_model_capability_field
|
||||
from litellm.router_utils.common_utils import deployment_model_id_pin, is_model_capability_field
|
||||
from litellm.types.utils import secret_bearing_wif_litellm_params, server_owned_wif_litellm_params
|
||||
|
||||
# Provider routing and server-owned federation or OAuth token-exchange fields. Allowed for proxy
|
||||
|
|
@ -855,6 +855,8 @@ def update_litellm_params_for_health_check(
|
|||
`deployment_model` (default: the incoming `litellm_params` model), so a `health_check_model` or a `model`
|
||||
in `health_check_params` is judged on its own. Keys already set through `health_check_params` are kept,
|
||||
and `modalities` set there without "audio" turns the audio fields off
|
||||
- sends a SageMaker deployment's `X-Amzn-SageMaker-Inference-Component` extra header as `model_id` when the
|
||||
deployment sets none, so legacy `sagemaker/` probes reach that component the way routed requests do
|
||||
"""
|
||||
incoming_model: Final = _PROBE_PARAMS_ADAPTER.validate_python(litellm_params).get("model")
|
||||
base_model_owner: Final = deployment_model or (incoming_model if isinstance(incoming_model, str) else None)
|
||||
|
|
@ -935,9 +937,11 @@ def update_litellm_params_for_health_check(
|
|||
audio_output_params: Final = _health_check_audio_output_params(
|
||||
_MODEL_INFO_ADAPTER.validate_python(model_info), probe_params, mode, base_model_owner
|
||||
)
|
||||
model_id_pin: Final = deployment_model_id_pin(probe_params)
|
||||
model_id_params: Final = {} if model_id_pin is None else {"model_id": model_id_pin}
|
||||
return {
|
||||
key: value
|
||||
for key, value in {**audio_output_params, **probe_params}.items()
|
||||
for key, value in {**audio_output_params, **model_id_params, **probe_params}.items()
|
||||
if not is_model_capability_field(key)
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -7,13 +7,15 @@ import secrets
|
|||
import time
|
||||
import traceback
|
||||
from collections.abc import Iterable, Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from itertools import chain, groupby
|
||||
from types import MappingProxyType
|
||||
from typing import Final, Literal, TypedDict
|
||||
|
||||
import fastapi
|
||||
from fastapi import APIRouter, Depends, HTTPException, Request, Response, status
|
||||
from pydantic import TypeAdapter
|
||||
from pydantic import BaseModel, ConfigDict, TypeAdapter
|
||||
from typing_extensions import ReadOnly
|
||||
|
||||
import litellm
|
||||
|
|
@ -31,7 +33,9 @@ from litellm.proxy._types import (
|
|||
CallInfo,
|
||||
EnterpriseLicenseData,
|
||||
Litellm_EntityType,
|
||||
LiteLLM_UserTable,
|
||||
LitellmUserRoles,
|
||||
ModelAccessDeniedProxyException,
|
||||
ProxyException,
|
||||
SpecialModelNames,
|
||||
UserAPIKeyAuth,
|
||||
|
|
@ -39,13 +43,28 @@ from litellm.proxy._types import (
|
|||
)
|
||||
from litellm.proxy.auth.auth_checks import (
|
||||
_resolve_key_models_for_auth_check, # pyright: ignore[reportPrivateUsage] # the auth layer's sentinel resolution, reused so /health scopes exactly like a request
|
||||
agent_model_grants,
|
||||
can_agent_call_model,
|
||||
can_key_call_resolved_model,
|
||||
can_user_call_model,
|
||||
get_user_object,
|
||||
key_model_aliases_for_auth_check,
|
||||
model_matches_any_wildcard_pattern_in_list,
|
||||
team_model_aliases_for_auth_check,
|
||||
)
|
||||
from litellm.proxy.auth.auth_exception_handler import (
|
||||
_as_proxy_exception, # pyright: ignore[reportPrivateUsage] # one auth-failure mapping, shared with user_api_key_auth
|
||||
)
|
||||
from litellm.proxy.auth.auth_utils import (
|
||||
_BANNED_REQUEST_BODY_PARAMS, # pyright: ignore[reportPrivateUsage] # one canonical list, shared with the request-body check
|
||||
)
|
||||
from litellm.proxy.auth.model_checks import get_key_models
|
||||
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
|
||||
from litellm.proxy.common_utils import http_parsing_utils
|
||||
from litellm.proxy.common_utils.model_listing_utils import (
|
||||
CallerAliases,
|
||||
dispatched_model,
|
||||
requestable_alias_names,
|
||||
)
|
||||
from litellm.proxy.common_utils.openai_error_payload import proxy_exception_for
|
||||
from litellm.proxy.db.db_lookup_gate import db_lookup_stall_tracker
|
||||
from litellm.proxy.db.exception_handler import PrismaDBExceptionHandler
|
||||
|
|
@ -79,6 +98,7 @@ from litellm.router_utils.clientside_credential_handler import (
|
|||
_ADMIN_CONFIG_FIELDS_TO_CLEAR_ON_BASE_OVERRIDE, # pyright: ignore[reportPrivateUsage] # one canonical list, shared with the router path
|
||||
clientside_credential_keys,
|
||||
)
|
||||
from litellm.router_utils.common_utils import resolve_model_group_alias
|
||||
from litellm.secret_managers.main import get_secret_bool
|
||||
from litellm.types.proxy.litellm_pre_call_utils import RedactedDict, SecretFields
|
||||
from litellm.types.router import reject_server_owned_wif_fields
|
||||
|
|
@ -253,6 +273,28 @@ def get_callback_identifier(callback):
|
|||
|
||||
router: Final = APIRouter()
|
||||
_OBJECT_MAPPING: Final = TypeAdapter(Mapping[str, object])
|
||||
_OBJECTS: Final = TypeAdapter(tuple[object, ...])
|
||||
_DEPLOYMENTS: Final = TypeAdapter(tuple[Mapping[str, object], ...])
|
||||
|
||||
|
||||
class _ModelGrants(BaseModel):
|
||||
model_config = ConfigDict(frozen=True, from_attributes=True)
|
||||
|
||||
models: tuple[object, ...] = ()
|
||||
|
||||
|
||||
class _ModelInfoAccessGroups(BaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
|
||||
access_groups: tuple[object, ...]
|
||||
|
||||
|
||||
class _DeploymentAccessGroups(BaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
|
||||
model_info: _ModelInfoAccessGroups
|
||||
|
||||
|
||||
services = (
|
||||
Literal[
|
||||
"slack_budget_alerts",
|
||||
|
|
@ -1045,42 +1087,339 @@ def _strip_admin_only_fields_from_health_result(result: dict) -> dict:
|
|||
return out
|
||||
|
||||
|
||||
def _health_accessible_model_names(
|
||||
user_api_key_dict: UserAPIKeyAuth, llm_router: Router | None
|
||||
) -> frozenset[str] | None:
|
||||
"""Model names the caller may health-check, or None when the key is unrestricted."""
|
||||
def _key_models_unrestricted(user_api_key_dict: UserAPIKeyAuth) -> bool:
|
||||
granted_models: Final = _resolve_key_models_for_auth_check(user_api_key_dict)
|
||||
if not granted_models or SpecialModelNames.all_proxy_models.value in granted_models:
|
||||
return not granted_models or SpecialModelNames.all_proxy_models.value in granted_models
|
||||
|
||||
|
||||
def _own_team_names(deployment: object, team_id: str | None) -> tuple[str, str] | None:
|
||||
match deployment:
|
||||
case {
|
||||
"model_name": str(internal_name),
|
||||
"model_info": {"team_id": str(owner_team_id), "team_public_model_name": str(public_name)},
|
||||
} if owner_team_id == team_id:
|
||||
return (internal_name, public_name)
|
||||
case _:
|
||||
return None
|
||||
|
||||
|
||||
def _names_caller_may_request(deployment: object, team_id: str | None) -> frozenset[str]:
|
||||
team_names: Final = _own_team_names(deployment, team_id)
|
||||
if team_names is not None:
|
||||
return frozenset(team_names)
|
||||
match deployment:
|
||||
case {"model_name": str(name)}:
|
||||
return frozenset((name,))
|
||||
case _:
|
||||
return frozenset()
|
||||
|
||||
|
||||
def _public_names_by_internal_name(llm_model_list: Sequence[object], team_id: str | None) -> Mapping[str, str]:
|
||||
team_names: Final = (_own_team_names(deployment, team_id) for deployment in llm_model_list)
|
||||
return MappingProxyType(dict(names for names in team_names if names is not None))
|
||||
|
||||
|
||||
async def _key_owner(user_api_key_dict: UserAPIKeyAuth) -> LiteLLM_UserTable | None:
|
||||
if user_api_key_dict.team_id is not None or user_api_key_dict.user_id is None:
|
||||
return None
|
||||
from litellm.proxy.proxy_server import prisma_client, proxy_logging_obj, user_api_key_cache
|
||||
|
||||
try:
|
||||
return await get_user_object(
|
||||
user_id=user_api_key_dict.user_id,
|
||||
prisma_client=prisma_client,
|
||||
user_api_key_cache=user_api_key_cache,
|
||||
user_id_upsert=False,
|
||||
parent_otel_span=user_api_key_dict.parent_otel_span,
|
||||
proxy_logging_obj=proxy_logging_obj,
|
||||
)
|
||||
except Exception: # noqa: BLE001 # auth reads an unloadable user as no user limit, so /health does too
|
||||
return None
|
||||
|
||||
|
||||
async def _key_may_call_model(
|
||||
model_name: str,
|
||||
user_api_key_dict: UserAPIKeyAuth,
|
||||
owner: LiteLLM_UserTable | None,
|
||||
llm_router: Router | None,
|
||||
) -> bool:
|
||||
from litellm.proxy.proxy_server import prisma_client, proxy_logging_obj, user_api_key_cache
|
||||
|
||||
await asyncio.sleep(0)
|
||||
try:
|
||||
await can_key_call_resolved_model(
|
||||
model=model_name, llm_model_list=None, valid_token=user_api_key_dict, llm_router=llm_router
|
||||
)
|
||||
await can_agent_call_model(
|
||||
model=model_name,
|
||||
valid_token=user_api_key_dict,
|
||||
llm_router=llm_router,
|
||||
prisma_client=prisma_client,
|
||||
user_api_key_cache=user_api_key_cache,
|
||||
proxy_logging_obj=proxy_logging_obj,
|
||||
)
|
||||
await can_user_call_model(
|
||||
model=model_name,
|
||||
llm_router=llm_router,
|
||||
user_object=owner,
|
||||
key_model_aliases=key_model_aliases_for_auth_check(user_api_key_dict),
|
||||
)
|
||||
except ModelAccessDeniedProxyException:
|
||||
return False
|
||||
except ProxyException:
|
||||
raise
|
||||
except Exception as failure:
|
||||
if isinstance(failure, HTTPException) and failure.status_code == status.HTTP_403_FORBIDDEN:
|
||||
return False
|
||||
raise _as_proxy_exception(failure) from failure
|
||||
return True
|
||||
|
||||
|
||||
async def _caller_aliases(user_api_key_dict: UserAPIKeyAuth) -> CallerAliases:
|
||||
from litellm.proxy.proxy_server import prisma_client, proxy_config, proxy_logging_obj
|
||||
|
||||
router_settings: Final = await proxy_config.get_hierarchical_router_settings(
|
||||
user_api_key_dict, prisma_client, proxy_logging_obj
|
||||
)
|
||||
return CallerAliases(
|
||||
team_model_aliases_for_auth_check(user_api_key_dict),
|
||||
key_model_aliases_for_auth_check(user_api_key_dict),
|
||||
router_settings.get("model_group_alias") if router_settings is not None else None,
|
||||
)
|
||||
|
||||
|
||||
def _model_auth_checks(model_name: str, aliases: CallerAliases) -> str:
|
||||
return resolve_model_group_alias(aliases.router, model_name) or model_name
|
||||
|
||||
|
||||
def _deployment_id(deployment: object) -> str | None:
|
||||
match deployment:
|
||||
case {"model_info": {"id": str(model_id)}}:
|
||||
return model_id
|
||||
case _:
|
||||
return None
|
||||
|
||||
|
||||
def _granted_names(user_api_key_dict: UserAPIKeyAuth) -> frozenset[str]:
|
||||
grants: Final = (
|
||||
*_ModelGrants.model_validate(user_api_key_dict).models,
|
||||
*_OBJECTS.validate_python(user_api_key_dict.team_models),
|
||||
)
|
||||
return frozenset(name for name in grants if isinstance(name, str))
|
||||
|
||||
|
||||
async def _names_any_limit_grants(user_api_key_dict: UserAPIKeyAuth, owner: LiteLLM_UserTable | None) -> frozenset[str]:
|
||||
owner_grants: Final = _ModelGrants.model_validate(owner).models if owner is not None else ()
|
||||
return (
|
||||
_granted_names(user_api_key_dict)
|
||||
| frozenset(name for name in owner_grants if isinstance(name, str))
|
||||
| await agent_model_grants(user_api_key_dict)
|
||||
)
|
||||
|
||||
|
||||
def _alias_names_caller_may_request(aliases: CallerAliases, llm_router: Router | None) -> frozenset[str]:
|
||||
router_aliases: Final = frozenset(llm_router.registry().aliases()) if llm_router is not None else frozenset[str]()
|
||||
return requestable_alias_names(aliases) | router_aliases
|
||||
|
||||
|
||||
def _served_group(model_name: str, aliases: CallerAliases, llm_router: Router | None) -> str:
|
||||
dispatched: Final = dispatched_model(model_name, aliases)
|
||||
return llm_router.registry().resolve(dispatched) if llm_router is not None else dispatched
|
||||
|
||||
|
||||
def _name_id_pairs(deployment: object, team_id: str | None) -> tuple[tuple[str, str], ...]:
|
||||
model_id: Final = _deployment_id(deployment)
|
||||
if model_id is None:
|
||||
return ()
|
||||
return tuple((name, model_id) for name in _names_caller_may_request(deployment, team_id) | {model_id})
|
||||
|
||||
|
||||
def _ids_by_requested_name(llm_model_list: Sequence[object], team_id: str | None) -> Mapping[str, frozenset[str]]:
|
||||
pairs: Final = sorted(chain.from_iterable(_name_id_pairs(deployment, team_id) for deployment in llm_model_list))
|
||||
return MappingProxyType(
|
||||
{name: frozenset(model_id for _, model_id in group) for name, group in groupby(pairs, key=lambda pair: pair[0])}
|
||||
)
|
||||
|
||||
|
||||
def _ids_of(deployments: Iterable[object]) -> frozenset[str]:
|
||||
return frozenset(model_id for model_id in map(_deployment_id, deployments) if model_id is not None)
|
||||
|
||||
|
||||
def _access_groups_of(deployment: object) -> frozenset[str]:
|
||||
match deployment:
|
||||
case {"model_info": {"access_groups": list()}}:
|
||||
access_groups: Final = _DeploymentAccessGroups.model_validate(deployment).model_info.access_groups
|
||||
return frozenset(access_group for access_group in access_groups if isinstance(access_group, str))
|
||||
case _:
|
||||
return frozenset()
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _AccessGroupGrants:
|
||||
granted_names: frozenset[str]
|
||||
ids_in_granted_groups: frozenset[str]
|
||||
|
||||
def requested_name(self, name: str, public_names: Mapping[str, str], admitted_names: frozenset[str]) -> str:
|
||||
public_name: Final = public_names.get(name)
|
||||
if public_name is None or public_name not in admitted_names or self._grants(name):
|
||||
return name
|
||||
return public_name
|
||||
|
||||
def _grants(self, name: str) -> bool:
|
||||
return name in self.granted_names or model_matches_any_wildcard_pattern_in_list(name, tuple(self.granted_names))
|
||||
|
||||
def narrow(self, serving_ids: frozenset[str], served_names: frozenset[str]) -> frozenset[str]:
|
||||
in_granted_groups: Final = serving_ids & self.ids_in_granted_groups
|
||||
granted_directly: Final = not self.granted_names.isdisjoint(served_names)
|
||||
return serving_ids if granted_directly or not in_granted_groups else in_granted_groups
|
||||
|
||||
|
||||
def _access_group_grants(llm_model_list: Sequence[object], user_api_key_dict: UserAPIKeyAuth) -> _AccessGroupGrants:
|
||||
granted: Final = _granted_names(user_api_key_dict)
|
||||
in_granted_groups: Final = (
|
||||
deployment for deployment in llm_model_list if not granted.isdisjoint(_access_groups_of(deployment))
|
||||
)
|
||||
return _AccessGroupGrants(granted, _ids_of(in_granted_groups))
|
||||
|
||||
|
||||
def _ids_serving_group(
|
||||
group: str,
|
||||
ids_by_name: Mapping[str, frozenset[str]],
|
||||
user_api_key_dict: UserAPIKeyAuth,
|
||||
llm_router: Router | None,
|
||||
access_group_grants: _AccessGroupGrants,
|
||||
) -> frozenset[str]:
|
||||
named: Final = ids_by_name.get(group)
|
||||
if named is not None:
|
||||
return access_group_grants.narrow(named, frozenset((group,)))
|
||||
if llm_router is None:
|
||||
return frozenset(granted_models)
|
||||
registry: Final = llm_router.registry()
|
||||
return frozenset()
|
||||
team_id: Final = user_api_key_dict.team_id
|
||||
wildcard_served: Final = llm_router.registry().deployments(group, source="wildcard", team_id=team_id)
|
||||
served_names: Final = frozenset((group,)).union(
|
||||
*(_names_caller_may_request(deployment, team_id) for deployment in wildcard_served)
|
||||
)
|
||||
return access_group_grants.narrow(_ids_of(wildcard_served), served_names)
|
||||
|
||||
|
||||
async def _names_key_may_request(
|
||||
names: Iterable[str],
|
||||
aliases: CallerAliases,
|
||||
user_api_key_dict: UserAPIKeyAuth,
|
||||
owner: LiteLLM_UserTable | None,
|
||||
llm_router: Router | None,
|
||||
) -> frozenset[str]:
|
||||
return frozenset(
|
||||
get_key_models(
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
proxy_model_list=registry.names(team_id=user_api_key_dict.team_id),
|
||||
model_access_groups=registry.access_groups(),
|
||||
[
|
||||
name
|
||||
for name in names
|
||||
if await _key_may_call_model(_model_auth_checks(name, aliases), user_api_key_dict, owner, llm_router)
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
def _ids_serving(
|
||||
groups: Iterable[str],
|
||||
ids_by_name: Mapping[str, frozenset[str]],
|
||||
user_api_key_dict: UserAPIKeyAuth,
|
||||
llm_router: Router | None,
|
||||
access_group_grants: _AccessGroupGrants,
|
||||
) -> frozenset[str]:
|
||||
return frozenset[str]().union(
|
||||
*(
|
||||
_ids_serving_group(group, ids_by_name, user_api_key_dict, llm_router, access_group_grants)
|
||||
for group in groups
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _caller_may_probe_deployment(
|
||||
deployment: Mapping[str, object],
|
||||
allowed_models: frozenset[str] | None,
|
||||
def _shadowed_ids(
|
||||
llm_model_list: Sequence[object], admitted_names: frozenset[str], reached_ids: frozenset[str], team_id: str | None
|
||||
) -> frozenset[str]:
|
||||
ids_of_admitted_names: Final = (
|
||||
_deployment_id(deployment)
|
||||
for deployment in llm_model_list
|
||||
if not admitted_names.isdisjoint(_names_caller_may_request(deployment, team_id))
|
||||
)
|
||||
return frozenset(model_id for model_id in ids_of_admitted_names if model_id is not None) - reached_ids
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _ReachableDeployments:
|
||||
model_ids: frozenset[str]
|
||||
groups: frozenset[str]
|
||||
|
||||
def include(self, deployment: object, team_id: str | None) -> bool:
|
||||
model_id: Final = _deployment_id(deployment)
|
||||
if model_id is None:
|
||||
return not self.groups.isdisjoint(_names_caller_may_request(deployment, team_id))
|
||||
return model_id in self.model_ids
|
||||
|
||||
|
||||
_NOTHING_REACHABLE: Final = _ReachableDeployments(frozenset(), frozenset())
|
||||
|
||||
|
||||
async def _deployments_key_may_reach(
|
||||
llm_model_list: Sequence[object],
|
||||
aliases: CallerAliases,
|
||||
user_api_key_dict: UserAPIKeyAuth,
|
||||
owner: LiteLLM_UserTable | None,
|
||||
llm_router: Router | None,
|
||||
) -> _ReachableDeployments:
|
||||
candidate_names: Final = (await _names_any_limit_grants(user_api_key_dict, owner)).union(
|
||||
_alias_names_caller_may_request(aliases, llm_router),
|
||||
*(_names_caller_may_request(deployment, user_api_key_dict.team_id) for deployment in llm_model_list),
|
||||
)
|
||||
admitted_names: Final = await _names_key_may_request(candidate_names, aliases, user_api_key_dict, owner, llm_router)
|
||||
access_group_grants: Final = _access_group_grants(llm_model_list, user_api_key_dict)
|
||||
public_names: Final = _public_names_by_internal_name(llm_model_list, user_api_key_dict.team_id)
|
||||
groups: Final = frozenset(
|
||||
_served_group(access_group_grants.requested_name(name, public_names, admitted_names), aliases, llm_router)
|
||||
for name in admitted_names
|
||||
)
|
||||
ids_by_name: Final = _ids_by_requested_name(llm_model_list, user_api_key_dict.team_id)
|
||||
reached_ids: Final = _ids_serving(groups, ids_by_name, user_api_key_dict, llm_router, access_group_grants)
|
||||
admitted_ids: Final = await _names_key_may_request(
|
||||
_shadowed_ids(llm_model_list, admitted_names, reached_ids, user_api_key_dict.team_id),
|
||||
aliases,
|
||||
user_api_key_dict,
|
||||
owner,
|
||||
llm_router,
|
||||
)
|
||||
id_groups: Final = frozenset(_served_group(model_id, aliases, llm_router) for model_id in admitted_ids)
|
||||
return _ReachableDeployments(
|
||||
reached_ids | _ids_serving(id_groups, ids_by_name, user_api_key_dict, llm_router, access_group_grants), groups
|
||||
)
|
||||
|
||||
|
||||
def _model_param_target(
|
||||
model: str,
|
||||
aliases: CallerAliases,
|
||||
llm_model_list: Sequence[Mapping[str, object]],
|
||||
team_id: str | None,
|
||||
caller_is_admin: bool,
|
||||
llm_router: Router | None,
|
||||
) -> str:
|
||||
group: Final = _served_group(model, aliases, llm_router)
|
||||
if llm_router is None or deployments_targeted_by_name(llm_model_list, group, team_id):
|
||||
return group
|
||||
patterns: Final = frozenset(
|
||||
deployment.get("model_name")
|
||||
for deployment in llm_router.registry().deployments(group, source="wildcard", team_id=team_id)
|
||||
)
|
||||
match tuple(patterns):
|
||||
case (str(pattern),):
|
||||
return pattern
|
||||
case _:
|
||||
return group
|
||||
|
||||
|
||||
def _caller_may_probe_deployment(
|
||||
deployment: Mapping[str, object], reachable: _ReachableDeployments, team_id: str | None, caller_is_admin: bool
|
||||
) -> bool:
|
||||
"""Same deployment visibility rule as routing: another team's deployment is never in scope, team-less callers included."""
|
||||
if not caller_is_admin and not deployment_usable_by_team(deployment, team_id):
|
||||
return False
|
||||
if allowed_models is None:
|
||||
return True
|
||||
if llm_router is None:
|
||||
return deployment.get("model_name") in allowed_models
|
||||
model: Final = dict(deployment)
|
||||
registry: Final = llm_router.registry()
|
||||
return any(registry.includes(model, model_name=name, team_id=team_id) for name in allowed_models)
|
||||
return reachable.include(deployment, team_id)
|
||||
|
||||
|
||||
def _resolve_targeted_model_ids(
|
||||
|
|
@ -1137,6 +1476,7 @@ def _filter_health_check_results_by_model_ids(results: dict, allowed_model_ids:
|
|||
async def _perform_health_check_and_save(
|
||||
model_list,
|
||||
target_model,
|
||||
recorded_model_name: str | None,
|
||||
cli_model,
|
||||
details,
|
||||
prisma_client,
|
||||
|
|
@ -1159,8 +1499,7 @@ async def _perform_health_check_and_save(
|
|||
|
||||
# Optionally save health check result to database (non-blocking)
|
||||
if prisma_client is not None:
|
||||
# For CLI model, use cli_model name; for router models, use target_model
|
||||
model_name_for_db: Final = cli_model if cli_model is not None else target_model
|
||||
model_name_for_db: Final = cli_model if cli_model is not None else recorded_model_name
|
||||
if model_name_for_db is not None:
|
||||
asyncio.create_task(
|
||||
_save_health_check_to_db(
|
||||
|
|
@ -1290,6 +1629,7 @@ async def health_endpoint(
|
|||
cli_result: Final = await _perform_health_check_and_save(
|
||||
model_list=[],
|
||||
target_model=None,
|
||||
recorded_model_name=None,
|
||||
cli_model=user_model,
|
||||
details=health_check_details,
|
||||
prisma_client=prisma_client,
|
||||
|
|
@ -1304,15 +1644,39 @@ async def health_endpoint(
|
|||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
detail={"error": "Model list not initialized"},
|
||||
)
|
||||
allowed_models: Final = _health_accessible_model_names(user_api_key_dict, llm_router)
|
||||
restrict_to_allowed_models: Final = not is_admin or allowed_models is not None
|
||||
restrict_to_allowed_models: Final = not is_admin or not _key_models_unrestricted(user_api_key_dict)
|
||||
aliases: Final = await _caller_aliases(user_api_key_dict) if restrict_to_allowed_models else None
|
||||
reachable: Final = (
|
||||
await _deployments_key_may_reach(
|
||||
_OBJECTS.validate_python(llm_model_list),
|
||||
aliases,
|
||||
user_api_key_dict,
|
||||
await _key_owner(user_api_key_dict),
|
||||
llm_router,
|
||||
)
|
||||
if aliases is not None
|
||||
else _NOTHING_REACHABLE
|
||||
)
|
||||
requested_model: Final = (
|
||||
_model_param_target(
|
||||
model,
|
||||
aliases,
|
||||
_DEPLOYMENTS.validate_python(llm_model_list),
|
||||
user_api_key_dict.team_id,
|
||||
llm_router,
|
||||
)
|
||||
if model is not None and aliases is not None
|
||||
else model
|
||||
)
|
||||
_llm_model_list: Final = [
|
||||
m
|
||||
for m in copy.deepcopy(llm_model_list)
|
||||
if not restrict_to_allowed_models
|
||||
or _caller_may_probe_deployment(m, allowed_models, llm_router, user_api_key_dict.team_id, is_admin)
|
||||
or _caller_may_probe_deployment(m, reachable, user_api_key_dict.team_id, is_admin)
|
||||
]
|
||||
targeted_ids: Final = _resolve_targeted_model_ids(_llm_model_list, model, model_id, user_api_key_dict.team_id)
|
||||
targeted_ids: Final = _resolve_targeted_model_ids(
|
||||
_llm_model_list, requested_model, model_id, user_api_key_dict.team_id
|
||||
)
|
||||
if restrict_to_allowed_models and targeted_ids is not None and not targeted_ids:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_403_FORBIDDEN,
|
||||
|
|
@ -1370,7 +1734,8 @@ async def health_endpoint(
|
|||
else:
|
||||
router_result: Final = await _perform_health_check_and_save(
|
||||
model_list=_llm_model_list,
|
||||
target_model=target_model,
|
||||
target_model=target_model if model_id else requested_model,
|
||||
recorded_model_name=target_model,
|
||||
cli_model=None,
|
||||
details=health_check_details,
|
||||
prisma_client=prisma_client,
|
||||
|
|
|
|||
|
|
@ -16,6 +16,7 @@ from litellm.constants import BACKGROUND_INTERACTION_COST_POLLING_ENABLED, BILLE
|
|||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.litellm_core_utils.core_helpers import (
|
||||
budget_reservation_from_metadata,
|
||||
finite_cost,
|
||||
get_litellm_metadata_from_kwargs,
|
||||
get_metadata_variable_name_from_kwargs,
|
||||
get_parent_otel_span_from_kwargs,
|
||||
|
|
@ -703,7 +704,7 @@ class _ProxyDBLogger(CustomLogger):
|
|||
raw_cost: Final = _str_keyed_fields(
|
||||
sl_object if sl_object is not None else kwargs # pyright: ignore[reportUnknownArgumentType] # untyped success-callback payload
|
||||
).get("response_cost")
|
||||
priced_cost: Final = raw_cost if isinstance(raw_cost, (int, float)) else None
|
||||
priced_cost: Final = finite_cost(raw_cost)
|
||||
response_cost: Final = (
|
||||
await apply_search_free_tier(
|
||||
kwargs=kwargs, # pyright: ignore[reportUnknownArgumentType] # untyped success-callback payload
|
||||
|
|
|
|||
|
|
@ -3188,9 +3188,9 @@ def _fallback_lists(
|
|||
) -> tuple[Sequence[object], ...]:
|
||||
"""Every fallback list the router can route this request through: the three
|
||||
router-level lists, the same three keys from the request body (which replace
|
||||
the router's when present, so the union is the safe superset), and the
|
||||
key/team ``router_settings.fallbacks`` the proxy prefers over router-level
|
||||
fallbacks in ``_configured_fallbacks``."""
|
||||
the router's when present, so the union is the safe superset), and the same
|
||||
three keys from key/team ``router_settings``, which the proxy forwards into
|
||||
the request after this discovery runs."""
|
||||
lists: Final[list[Sequence[object]]] = [] # mutable-ok: accumulates each fallback list found
|
||||
configured: Final = llm_router.settings()
|
||||
for router_list in (
|
||||
|
|
@ -3205,11 +3205,12 @@ def _fallback_lists(
|
|||
if isinstance(entries, list):
|
||||
lists.append(cast("Sequence[object]", entries)) # cast-ok: request fallback lists hold mixed entry shapes
|
||||
for settings in (router_settings, team_router_settings):
|
||||
key_fallbacks: object = settings.get("fallbacks") if settings is not None else None
|
||||
if isinstance(key_fallbacks, list):
|
||||
lists.append(
|
||||
cast("Sequence[object]", key_fallbacks) # cast-ok: settings fallback lists hold mixed entry shapes
|
||||
)
|
||||
for key in _REQUEST_FALLBACK_KEYS:
|
||||
key_fallbacks: object = settings.get(key) if settings is not None else None
|
||||
if isinstance(key_fallbacks, list):
|
||||
lists.append(
|
||||
cast("Sequence[object]", key_fallbacks) # cast-ok: settings fallback lists hold mixed entry shapes
|
||||
)
|
||||
return tuple(lists)
|
||||
|
||||
|
||||
|
|
|
|||
Some files were not shown because too many files have changed in this diff Show more
Loading…
Add table
Reference in a new issue